Initial pve cloud backup app
This commit is contained in:
Codex
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,489 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from . import commands
|
||||
from .db import db, row_to_dict, utc_now
|
||||
from .jobs import advance_job, due_jobs
|
||||
from .notifications import notify
|
||||
from .settings_store import get_settings
|
||||
|
||||
|
||||
ACTIVE_STATES = {
|
||||
"queued",
|
||||
"pve_running",
|
||||
"local_ready",
|
||||
"uploading",
|
||||
"remote_ready",
|
||||
"local_deleting",
|
||||
"deleting",
|
||||
}
|
||||
RUNNING_STATES = ACTIVE_STATES - {"queued"}
|
||||
RECOVERABLE_STATES = {
|
||||
"pve_running",
|
||||
"local_ready",
|
||||
"uploading",
|
||||
"remote_ready",
|
||||
"local_deleting",
|
||||
"deleting",
|
||||
}
|
||||
TERMINAL_STATES = {"completed", "failed", "deleted"}
|
||||
|
||||
|
||||
def _active_for_guest(conn, vmid: int) -> bool:
|
||||
row = conn.execute(
|
||||
f"SELECT 1 FROM backups WHERE guest_vmid = ? AND state IN ({','.join('?' for _ in ACTIVE_STATES)}) LIMIT 1",
|
||||
(vmid, *sorted(ACTIVE_STATES)),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def _backup_row(backup_id: str) -> dict:
|
||||
with db() as conn:
|
||||
row = conn.execute("SELECT * FROM backups WHERE id = ?", (backup_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(f"Backup {backup_id} not found")
|
||||
return dict(row)
|
||||
|
||||
|
||||
def _job_row(job_id: int | None) -> dict | None:
|
||||
if job_id is None:
|
||||
return None
|
||||
with db() as conn:
|
||||
row = conn.execute("SELECT * FROM backup_jobs WHERE id = ?", (job_id,)).fetchone()
|
||||
return row_to_dict(row)
|
||||
|
||||
|
||||
def update_backup(backup_id: str, **values) -> dict:
|
||||
values["updated_at"] = utc_now()
|
||||
assignments = ", ".join(f"{key} = ?" for key in values)
|
||||
with db() as conn:
|
||||
conn.execute(
|
||||
f"UPDATE backups SET {assignments} WHERE id = ?",
|
||||
(*values.values(), backup_id),
|
||||
)
|
||||
row = conn.execute("SELECT * FROM backups WHERE id = ?", (backup_id,)).fetchone()
|
||||
return dict(row)
|
||||
|
||||
|
||||
def fail_backup(backup_id: str, error: str) -> dict:
|
||||
return update_backup(backup_id, state="failed", error_message=error, completed_at=utc_now())
|
||||
|
||||
|
||||
def _claim_queued_backup(backup_id: str) -> dict | None:
|
||||
now = utc_now()
|
||||
with db() as conn:
|
||||
cur = conn.execute(
|
||||
"UPDATE backups SET state = 'pve_running', updated_at = ? WHERE id = ? AND state = 'queued'",
|
||||
(now, backup_id),
|
||||
)
|
||||
if cur.rowcount == 0:
|
||||
return None
|
||||
row = conn.execute("SELECT * FROM backups WHERE id = ?", (backup_id,)).fetchone()
|
||||
return dict(row)
|
||||
|
||||
|
||||
def queue_backup_for_job(job: dict) -> dict | None:
|
||||
now = utc_now()
|
||||
backup_id = str(uuid.uuid4())
|
||||
settings = get_settings()
|
||||
max_active = max(1, int(settings["max_concurrent_backups"]))
|
||||
with db() as conn:
|
||||
active_count = conn.execute(
|
||||
f"SELECT COUNT(*) AS count FROM backups WHERE state IN ({','.join('?' for _ in ACTIVE_STATES)})",
|
||||
tuple(sorted(ACTIVE_STATES)),
|
||||
).fetchone()["count"]
|
||||
if active_count >= max_active:
|
||||
return None
|
||||
if _active_for_guest(conn, int(job["guest_vmid"])):
|
||||
return None
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO backups(
|
||||
id, job_id, guest_vmid, guest_name, guest_type, node, state,
|
||||
started_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 'queued', ?, ?)
|
||||
""",
|
||||
(
|
||||
backup_id,
|
||||
job["id"],
|
||||
job["guest_vmid"],
|
||||
job["guest_name"],
|
||||
job["guest_type"],
|
||||
job["node"],
|
||||
now,
|
||||
now,
|
||||
),
|
||||
)
|
||||
row = conn.execute("SELECT * FROM backups WHERE id = ?", (backup_id,)).fetchone()
|
||||
return dict(row)
|
||||
|
||||
|
||||
def list_backups(guest_vmid: int | None = None, state: str | None = None) -> list[dict]:
|
||||
query = "SELECT * FROM backups"
|
||||
args: list = []
|
||||
where = []
|
||||
if guest_vmid is not None:
|
||||
where.append("guest_vmid = ?")
|
||||
args.append(guest_vmid)
|
||||
if state:
|
||||
where.append("state = ?")
|
||||
args.append(state)
|
||||
if where:
|
||||
query += " WHERE " + " AND ".join(where)
|
||||
query += " ORDER BY started_at DESC LIMIT 500"
|
||||
with db() as conn:
|
||||
return [dict(row) for row in conn.execute(query, args).fetchall()]
|
||||
|
||||
|
||||
def get_backup(backup_id: str) -> dict:
|
||||
return _backup_row(backup_id)
|
||||
|
||||
|
||||
def purge_deleted_backup(backup_id: str) -> dict:
|
||||
backup = _backup_row(backup_id)
|
||||
if backup["state"] != "deleted":
|
||||
raise RuntimeError(f"Backup {backup_id} is in state {backup['state']} and is not ready to purge")
|
||||
with db() as conn:
|
||||
conn.execute("DELETE FROM backups WHERE id = ?", (backup_id,))
|
||||
return {"id": backup_id, "purged": True}
|
||||
|
||||
|
||||
def dashboard() -> dict:
|
||||
with db() as conn:
|
||||
latest = row_to_dict(conn.execute("SELECT * FROM backups ORDER BY started_at DESC LIMIT 1").fetchone())
|
||||
active = [dict(row) for row in conn.execute(
|
||||
f"SELECT * FROM backups WHERE state IN ({','.join('?' for _ in ACTIVE_STATES)}) ORDER BY started_at DESC",
|
||||
tuple(sorted(ACTIVE_STATES)),
|
||||
).fetchall()]
|
||||
failed = conn.execute("SELECT COUNT(*) AS count FROM backups WHERE state = 'failed'").fetchone()["count"]
|
||||
remote = conn.execute("SELECT COUNT(*) AS count FROM backups WHERE state = 'completed' AND remote_path IS NOT NULL").fetchone()["count"]
|
||||
next_jobs = [dict(row) for row in conn.execute(
|
||||
"SELECT * FROM backup_jobs WHERE enabled = 1 ORDER BY next_run_at ASC LIMIT 10"
|
||||
).fetchall()]
|
||||
settings = get_settings()
|
||||
local_dir = Path(settings["local_backup_dir"])
|
||||
usage = {"path": str(local_dir), "bytes": 0}
|
||||
if local_dir.exists():
|
||||
usage["bytes"] = sum(p.stat().st_size for p in local_dir.glob("*") if p.is_file())
|
||||
return {
|
||||
"latest_backup": latest,
|
||||
"active_tasks": active,
|
||||
"failed_backups": failed,
|
||||
"remote_backup_count": remote,
|
||||
"local_staging_usage": usage,
|
||||
"next_scheduled_jobs": next_jobs,
|
||||
}
|
||||
|
||||
|
||||
async def _wait_for_pve_task(backup: dict) -> dict:
|
||||
while True:
|
||||
status = await asyncio.to_thread(commands.get_task_status, backup["node"], backup["upid"])
|
||||
if status.get("status") == "stopped":
|
||||
exitstatus = status.get("exitstatus")
|
||||
if exitstatus == "OK":
|
||||
return status
|
||||
raise RuntimeError(f"Proxmox task failed with exitstatus={exitstatus}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
|
||||
def _remote_object(settings: dict, backup: dict, archive: Path) -> str:
|
||||
remote = str(settings["rclone_remote"]).rstrip(":")
|
||||
base_path = str(settings.get("rclone_remote_path") or "pve-cloud-backup").strip("/")
|
||||
object_path = f"{backup['id']}/{archive.name}"
|
||||
if base_path:
|
||||
object_path = f"{base_path}/{object_path}"
|
||||
return f"{remote}:{object_path}"
|
||||
|
||||
|
||||
def _remote_parent_dir(remote_object: str) -> str | None:
|
||||
if ":" not in remote_object:
|
||||
return None
|
||||
remote, object_path = remote_object.split(":", 1)
|
||||
object_path = object_path.strip("/")
|
||||
if "/" not in object_path:
|
||||
return None
|
||||
parent_path = object_path.rsplit("/", 1)[0]
|
||||
if not parent_path:
|
||||
return None
|
||||
return f"{remote}:{parent_path}"
|
||||
|
||||
|
||||
async def _remove_empty_remote_parent(settings: dict, remote_object: str) -> None:
|
||||
parent = _remote_parent_dir(remote_object)
|
||||
if not parent:
|
||||
return
|
||||
try:
|
||||
await asyncio.to_thread(commands.rclone_rmdir, settings["rclone_path"], parent)
|
||||
except commands.CommandError:
|
||||
# Non-empty folders and backend-specific "directory not found" behavior
|
||||
# should not turn an already successful exact file delete into a failed
|
||||
# backup deletion.
|
||||
return
|
||||
|
||||
|
||||
async def _delete_remote_file_and_empty_parent(settings: dict, remote_object: str) -> None:
|
||||
await asyncio.to_thread(commands.rclone_delete_file, settings["rclone_path"], remote_object)
|
||||
await _remove_empty_remote_parent(settings, remote_object)
|
||||
|
||||
|
||||
async def process_backup(backup_id: str) -> None:
|
||||
backup = _backup_row(backup_id)
|
||||
settings = get_settings()
|
||||
job = _job_row(backup.get("job_id"))
|
||||
try:
|
||||
if backup["state"] == "queued":
|
||||
await notify("backup_started", backup=backup)
|
||||
backup = _claim_queued_backup(backup_id)
|
||||
if backup is None:
|
||||
return
|
||||
upid = await asyncio.to_thread(
|
||||
commands.start_proxmox_backup,
|
||||
node=backup["node"],
|
||||
vmid=int(backup["guest_vmid"]),
|
||||
storage=(job or {}).get("proxmox_storage") or settings["proxmox_storage"],
|
||||
mode=(job or {}).get("backup_mode") or settings["default_backup_mode"],
|
||||
compression=(job or {}).get("compression") or settings["default_compression"],
|
||||
)
|
||||
backup = update_backup(backup_id, upid=upid)
|
||||
|
||||
if backup["state"] == "pve_running":
|
||||
if not backup.get("upid"):
|
||||
raise RuntimeError("Backup is pve_running but has no Proxmox UPID recorded; refusing to poll task status")
|
||||
await _wait_for_pve_task(backup)
|
||||
backup = update_backup(backup_id, state="local_ready", pve_completed_at=utc_now())
|
||||
await notify("pve_completed", backup=backup)
|
||||
|
||||
if backup["state"] == "local_ready":
|
||||
archive = await asyncio.to_thread(
|
||||
commands.discover_archive,
|
||||
settings["local_backup_dir"],
|
||||
int(backup["guest_vmid"]),
|
||||
backup["guest_type"],
|
||||
backup["started_at"],
|
||||
)
|
||||
size = archive.stat().st_size
|
||||
remote_object = _remote_object(settings, backup, archive)
|
||||
backup = update_backup(
|
||||
backup_id,
|
||||
local_path=str(archive),
|
||||
remote_path=remote_object,
|
||||
size_bytes=size,
|
||||
state="uploading",
|
||||
upload_started_at=utc_now(),
|
||||
)
|
||||
await notify("upload_started", backup=backup)
|
||||
|
||||
if backup["state"] == "uploading":
|
||||
await asyncio.to_thread(
|
||||
commands.rclone_copyto,
|
||||
settings["rclone_path"],
|
||||
backup["local_path"],
|
||||
backup["remote_path"],
|
||||
)
|
||||
exists = await asyncio.to_thread(
|
||||
commands.rclone_object_exists,
|
||||
settings["rclone_path"],
|
||||
backup["remote_path"],
|
||||
)
|
||||
if not exists:
|
||||
raise RuntimeError("Upload finished but remote object was not found")
|
||||
backup = update_backup(backup_id, state="remote_ready")
|
||||
|
||||
if backup["state"] == "remote_ready":
|
||||
backup = update_backup(backup_id, state="local_deleting")
|
||||
|
||||
if backup["state"] == "local_deleting":
|
||||
if backup.get("local_path"):
|
||||
local_path = Path(backup["local_path"])
|
||||
if local_path.exists():
|
||||
local_path.unlink()
|
||||
backup = update_backup(backup_id, state="completed", completed_at=utc_now())
|
||||
await notify("backup_completed", backup=backup)
|
||||
await apply_retention_for_job(backup["job_id"])
|
||||
except Exception as exc:
|
||||
failed = fail_backup(backup_id, str(exc))
|
||||
await notify("backup_failed", backup=failed)
|
||||
|
||||
|
||||
async def delete_remote_backup(backup: dict, reason: str) -> None:
|
||||
if not backup.get("remote_path"):
|
||||
update_backup(backup["id"], state="deleted", deleted_at=utc_now())
|
||||
return
|
||||
settings = get_settings()
|
||||
update_backup(backup["id"], state="deleting")
|
||||
await _delete_remote_file_and_empty_parent(settings, backup["remote_path"])
|
||||
deleted = update_backup(backup["id"], state="deleted", deleted_at=utc_now())
|
||||
await notify("remote_deleted", backup=deleted, message=reason)
|
||||
|
||||
|
||||
async def delete_backup_manually(backup_id: str, reason: str = "manual deletion") -> dict:
|
||||
backup = _backup_row(backup_id)
|
||||
if backup["state"] == "deleted":
|
||||
return purge_deleted_backup(backup_id)
|
||||
if backup["state"] in ACTIVE_STATES:
|
||||
raise RuntimeError(f"Backup {backup_id} is active in state {backup['state']} and cannot be deleted")
|
||||
|
||||
settings = get_settings()
|
||||
if backup.get("remote_path") and backup["state"] != "deleted":
|
||||
exists = await asyncio.to_thread(
|
||||
commands.rclone_object_exists,
|
||||
settings["rclone_path"],
|
||||
backup["remote_path"],
|
||||
)
|
||||
if exists:
|
||||
update_backup(backup_id, state="deleting")
|
||||
await _delete_remote_file_and_empty_parent(settings, backup["remote_path"])
|
||||
else:
|
||||
await _remove_empty_remote_parent(settings, backup["remote_path"])
|
||||
|
||||
if backup.get("local_path"):
|
||||
local_path = Path(backup["local_path"])
|
||||
if local_path.exists() and local_path.is_file():
|
||||
local_path.unlink()
|
||||
|
||||
deleted = update_backup(backup_id, state="deleted", deleted_at=utc_now())
|
||||
await notify("remote_deleted", backup=deleted, message=reason)
|
||||
return deleted
|
||||
|
||||
|
||||
async def delete_job_with_backups(job_id: int) -> dict:
|
||||
with db() as conn:
|
||||
job = row_to_dict(conn.execute("SELECT * FROM backup_jobs WHERE id = ?", (job_id,)).fetchone())
|
||||
if not job:
|
||||
raise KeyError(f"Backup job {job_id} not found")
|
||||
active = conn.execute(
|
||||
f"SELECT COUNT(*) AS count FROM backups WHERE job_id = ? AND state IN ({','.join('?' for _ in ACTIVE_STATES)})",
|
||||
(job_id, *sorted(ACTIVE_STATES)),
|
||||
).fetchone()["count"]
|
||||
if active:
|
||||
raise RuntimeError(f"Backup job {job_id} has {active} active backup(s) and cannot be deleted")
|
||||
backups = [
|
||||
dict(row)
|
||||
for row in conn.execute(
|
||||
"SELECT * FROM backups WHERE job_id = ? AND state != 'deleted' ORDER BY started_at ASC",
|
||||
(job_id,),
|
||||
).fetchall()
|
||||
]
|
||||
|
||||
deleted_backup_ids: list[str] = []
|
||||
for backup in backups:
|
||||
deleted = await delete_backup_manually(backup["id"], reason=f"job {job_id} deleted")
|
||||
deleted_backup_ids.append(deleted["id"])
|
||||
|
||||
with db() as conn:
|
||||
conn.execute("DELETE FROM backup_jobs WHERE id = ?", (job_id,))
|
||||
return {"job_id": job_id, "deleted_backups": deleted_backup_ids}
|
||||
|
||||
|
||||
async def apply_retention_for_job(job_id: int | None) -> None:
|
||||
if job_id is None:
|
||||
return
|
||||
with db() as conn:
|
||||
job = row_to_dict(conn.execute("SELECT * FROM backup_jobs WHERE id = ?", (job_id,)).fetchone())
|
||||
backups = [dict(row) for row in conn.execute(
|
||||
"""
|
||||
SELECT * FROM backups
|
||||
WHERE job_id = ? AND state = 'completed' AND remote_path IS NOT NULL
|
||||
ORDER BY completed_at DESC
|
||||
""",
|
||||
(job_id,),
|
||||
).fetchall()]
|
||||
if not job:
|
||||
return
|
||||
delete: list[dict] = []
|
||||
if job["retention_type"] == "latest":
|
||||
delete = backups[int(job["retention_value"]):]
|
||||
else:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=int(job["retention_value"]))
|
||||
for backup in backups:
|
||||
completed_at = backup.get("completed_at")
|
||||
if completed_at and datetime.fromisoformat(completed_at) < cutoff:
|
||||
delete.append(backup)
|
||||
for backup in delete:
|
||||
try:
|
||||
await delete_remote_backup(backup, f"retention {job['retention_type']}={job['retention_value']}")
|
||||
except Exception as exc:
|
||||
failed = fail_backup(backup["id"], f"Retention delete failed: {exc}")
|
||||
await notify("backup_failed", backup=failed)
|
||||
|
||||
|
||||
async def retention_sweep() -> None:
|
||||
with db() as conn:
|
||||
job_ids = [
|
||||
row["id"]
|
||||
for row in conn.execute(
|
||||
"""
|
||||
SELECT DISTINCT backup_jobs.id
|
||||
FROM backup_jobs
|
||||
JOIN backups ON backups.job_id = backup_jobs.id
|
||||
WHERE backups.state = 'completed' AND backups.remote_path IS NOT NULL
|
||||
"""
|
||||
).fetchall()
|
||||
]
|
||||
for job_id in job_ids:
|
||||
await apply_retention_for_job(job_id)
|
||||
|
||||
|
||||
async def recover_unfinished() -> None:
|
||||
with db() as conn:
|
||||
rows = conn.execute(
|
||||
f"SELECT * FROM backups WHERE state IN ({','.join('?' for _ in RECOVERABLE_STATES)})",
|
||||
tuple(sorted(RECOVERABLE_STATES)),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
backup = dict(row)
|
||||
await notify("recovery_action", backup=backup, message=f"resuming state {backup['state']}")
|
||||
if backup["state"] == "deleting":
|
||||
await delete_remote_backup(backup, "recovery delete resume")
|
||||
else:
|
||||
await process_backup(backup["id"])
|
||||
|
||||
|
||||
async def scheduler_tick() -> None:
|
||||
settings = get_settings()
|
||||
limit = max(1, int(settings["max_concurrent_backups"]))
|
||||
with db() as conn:
|
||||
active_count = conn.execute(
|
||||
f"SELECT COUNT(*) AS count FROM backups WHERE state IN ({','.join('?' for _ in ACTIVE_STATES)})",
|
||||
tuple(sorted(ACTIVE_STATES)),
|
||||
).fetchone()["count"]
|
||||
free_slots = max(0, limit - active_count)
|
||||
if free_slots <= 0:
|
||||
return
|
||||
for job in due_jobs(free_slots):
|
||||
queued = queue_backup_for_job(job)
|
||||
advance_job(job)
|
||||
if queued:
|
||||
asyncio.create_task(process_backup(queued["id"]))
|
||||
|
||||
|
||||
async def worker_loop() -> None:
|
||||
await recover_unfinished()
|
||||
retention_sweep_interval_seconds = 3600
|
||||
last_retention_sweep = 0.0
|
||||
while True:
|
||||
await scheduler_tick()
|
||||
now = asyncio.get_running_loop().time()
|
||||
if now - last_retention_sweep >= retention_sweep_interval_seconds:
|
||||
await retention_sweep()
|
||||
last_retention_sweep = now
|
||||
settings = get_settings()
|
||||
max_active = max(1, int(settings["max_concurrent_backups"]))
|
||||
with db() as conn:
|
||||
active_count = conn.execute(
|
||||
f"SELECT COUNT(*) AS count FROM backups WHERE state IN ({','.join('?' for _ in RUNNING_STATES)})",
|
||||
tuple(sorted(RUNNING_STATES)),
|
||||
).fetchone()["count"]
|
||||
free_slots = max(0, max_active - active_count)
|
||||
queued = [dict(row) for row in conn.execute(
|
||||
"SELECT * FROM backups WHERE state = 'queued' ORDER BY started_at ASC LIMIT ?",
|
||||
(free_slots,),
|
||||
).fetchall()]
|
||||
for backup in queued:
|
||||
asyncio.create_task(process_backup(backup["id"]))
|
||||
await asyncio.sleep(30)
|
||||
@@ -0,0 +1,199 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommandResult:
|
||||
stdout: str
|
||||
stderr: str
|
||||
|
||||
|
||||
class CommandError(RuntimeError):
|
||||
def __init__(self, command: list[str], result: subprocess.CompletedProcess[str]):
|
||||
super().__init__(
|
||||
f"Command failed ({result.returncode}): {' '.join(command)}\n{result.stderr.strip()}"
|
||||
)
|
||||
self.command = command
|
||||
self.returncode = result.returncode
|
||||
self.stdout = result.stdout
|
||||
self.stderr = result.stderr
|
||||
|
||||
|
||||
def run(command: list[str], timeout: int | None = None) -> CommandResult:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise CommandError(command, result)
|
||||
return CommandResult(stdout=result.stdout, stderr=result.stderr)
|
||||
|
||||
|
||||
def find_executable(name: str) -> str:
|
||||
return shutil.which(name) or ""
|
||||
|
||||
|
||||
def pvesh_json(args: list[str], timeout: int | None = 120):
|
||||
result = run(["pvesh", *args, "--output-format", "json"], timeout=timeout)
|
||||
text = result.stdout.strip()
|
||||
return json.loads(text) if text else None
|
||||
|
||||
|
||||
def _extract_upid(text: str):
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return None
|
||||
if text.startswith("UPID:"):
|
||||
return text
|
||||
match = re.search(r"UPID:[^\s\"']+", text)
|
||||
if match:
|
||||
return match.group(0)
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
if isinstance(data, str) and data.startswith("UPID:"):
|
||||
return data
|
||||
if isinstance(data, dict):
|
||||
for key in ("upid", "data"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, str) and value.startswith("UPID:"):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def list_guests() -> list[dict]:
|
||||
resources = pvesh_json(["get", "/cluster/resources", "--type", "vm"]) or []
|
||||
guests: list[dict] = []
|
||||
for item in resources:
|
||||
raw_type = item.get("type")
|
||||
guest_type = "lxc" if raw_type == "lxc" else "vm"
|
||||
guests.append(
|
||||
{
|
||||
"vmid": int(item["vmid"]),
|
||||
"name": item.get("name") or str(item["vmid"]),
|
||||
"type": guest_type,
|
||||
"node": item.get("node") or "",
|
||||
"status": item.get("status") or "unknown",
|
||||
}
|
||||
)
|
||||
return sorted(guests, key=lambda row: (row["node"], row["vmid"]))
|
||||
|
||||
|
||||
def list_storages(node: str | None = None) -> list[str]:
|
||||
path = f"/nodes/{node}/storage" if node else "/storage"
|
||||
data = pvesh_json(["get", path]) or []
|
||||
return sorted({item.get("storage") for item in data if item.get("storage")})
|
||||
|
||||
|
||||
def start_proxmox_backup(
|
||||
*,
|
||||
node: str,
|
||||
vmid: int,
|
||||
storage: str,
|
||||
mode: str,
|
||||
compression: str,
|
||||
) -> str:
|
||||
command = [
|
||||
"pvesh",
|
||||
"create",
|
||||
f"/nodes/{node}/vzdump",
|
||||
"--vmid",
|
||||
str(vmid),
|
||||
"--storage",
|
||||
storage,
|
||||
"--mode",
|
||||
mode,
|
||||
"--compress",
|
||||
compression,
|
||||
"--output-format",
|
||||
"json",
|
||||
]
|
||||
result = run(command, timeout=300)
|
||||
upid = _extract_upid(result.stdout)
|
||||
if not upid:
|
||||
upid = _extract_upid(result.stderr)
|
||||
if upid:
|
||||
return upid
|
||||
|
||||
# Some pvesh versions ignore --output-format for create actions and return
|
||||
# plain task text. Include both streams so failures are diagnosable without
|
||||
# assuming JSON.
|
||||
raise RuntimeError(
|
||||
"Could not read UPID from Proxmox response: "
|
||||
f"stdout={result.stdout.strip()!r} stderr={result.stderr.strip()!r}"
|
||||
)
|
||||
|
||||
|
||||
def get_task_status(node: str, upid: str) -> dict:
|
||||
return pvesh_json(["get", f"/nodes/{node}/tasks/{upid}/status"]) or {}
|
||||
|
||||
|
||||
def discover_archive(local_backup_dir: str, vmid: int, guest_type: str, started_at: str) -> Path:
|
||||
base = Path(local_backup_dir)
|
||||
prefix = "vzdump-lxc" if guest_type == "lxc" else "vzdump-qemu"
|
||||
archive_suffixes = (
|
||||
".tar",
|
||||
".tar.gz",
|
||||
".tar.lzo",
|
||||
".tar.zst",
|
||||
".vma",
|
||||
".vma.gz",
|
||||
".vma.lzo",
|
||||
".vma.zst",
|
||||
)
|
||||
started = datetime.fromisoformat(started_at)
|
||||
candidates = []
|
||||
for path in base.glob(f"{prefix}-{vmid}-*"):
|
||||
if path.is_file() and path.name.endswith(archive_suffixes):
|
||||
mtime = datetime.fromtimestamp(path.stat().st_mtime, tz=started.tzinfo)
|
||||
if mtime >= started:
|
||||
candidates.append(path)
|
||||
if not candidates:
|
||||
raise FileNotFoundError(f"No backup archive found for VMID {vmid} in {base}")
|
||||
return max(candidates, key=lambda p: p.stat().st_mtime)
|
||||
|
||||
|
||||
def rclone_list_remotes(rclone_path: str) -> list[str]:
|
||||
result = run([rclone_path, "listremotes"], timeout=60)
|
||||
return [line.rstrip(":") for line in result.stdout.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def rclone_remote_type(rclone_path: str, remote_name: str) -> str:
|
||||
remote = remote_name.rstrip(":")
|
||||
result = run([rclone_path, "config", "show", remote])
|
||||
for line in result.stdout.splitlines():
|
||||
if line.strip().startswith("type"):
|
||||
_, value = line.split("=", 1)
|
||||
return value.strip()
|
||||
return ""
|
||||
|
||||
|
||||
def rclone_copyto(rclone_path: str, local_path: str, remote_object: str) -> None:
|
||||
run([rclone_path, "copyto", local_path, remote_object], timeout=None)
|
||||
|
||||
|
||||
def rclone_object_exists(rclone_path: str, remote_object: str) -> bool:
|
||||
try:
|
||||
run([rclone_path, "lsjson", remote_object], timeout=120)
|
||||
return True
|
||||
except CommandError:
|
||||
return False
|
||||
|
||||
|
||||
def rclone_delete_file(rclone_path: str, remote_object: str) -> None:
|
||||
run([rclone_path, "deletefile", remote_object], timeout=300)
|
||||
|
||||
|
||||
def rclone_rmdir(rclone_path: str, remote_dir: str) -> None:
|
||||
run([rclone_path, "rmdir", remote_dir], timeout=120)
|
||||
@@ -0,0 +1,16 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
APP_DIR = Path(__file__).resolve().parents[2]
|
||||
BACKEND_DIR = APP_DIR / "backend"
|
||||
FRONTEND_DIR = APP_DIR / "frontend"
|
||||
STATIC_DIR = APP_DIR / "static"
|
||||
DATA_DIR = APP_DIR / "data"
|
||||
LOG_DIR = APP_DIR / "logs"
|
||||
SCRIPTS_DIR = APP_DIR / "scripts"
|
||||
DB_PATH = DATA_DIR / "app.db"
|
||||
|
||||
|
||||
def ensure_dirs() -> None:
|
||||
for path in (STATIC_DIR, DATA_DIR, LOG_DIR, SCRIPTS_DIR):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
@@ -0,0 +1,37 @@
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from typing import Iterator
|
||||
|
||||
from .config import DB_PATH, ensure_dirs
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def connect() -> sqlite3.Connection:
|
||||
ensure_dirs()
|
||||
conn = sqlite3.connect(DB_PATH, timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("PRAGMA journal_mode = WAL")
|
||||
conn.execute("PRAGMA busy_timeout = 30000")
|
||||
return conn
|
||||
|
||||
|
||||
@contextmanager
|
||||
def db() -> Iterator[sqlite3.Connection]:
|
||||
conn = connect()
|
||||
try:
|
||||
yield conn
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def row_to_dict(row: sqlite3.Row | None) -> dict | None:
|
||||
return None if row is None else dict(row)
|
||||
@@ -0,0 +1,152 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from croniter import croniter
|
||||
|
||||
from .db import db, row_to_dict, utc_now
|
||||
from .settings_store import get_settings
|
||||
|
||||
|
||||
def validate_cron(expr: str) -> None:
|
||||
if not croniter.is_valid(expr):
|
||||
raise ValueError("Invalid cron expression")
|
||||
|
||||
|
||||
def _scheduler_timezone(timezone_name: str | None = None) -> ZoneInfo:
|
||||
return ZoneInfo(timezone_name or get_settings().get("timezone") or "UTC")
|
||||
|
||||
|
||||
def next_run(expr: str, base: datetime | None = None, timezone_name: str | None = None) -> str:
|
||||
base = base or datetime.now(timezone.utc)
|
||||
if base.tzinfo is None:
|
||||
base = base.replace(tzinfo=timezone.utc)
|
||||
scheduler_tz = _scheduler_timezone(timezone_name)
|
||||
local_base = base.astimezone(scheduler_tz)
|
||||
local_next = croniter(expr, local_base).get_next(datetime)
|
||||
if local_next.tzinfo is None:
|
||||
local_next = local_next.replace(tzinfo=scheduler_tz)
|
||||
return local_next.astimezone(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def list_jobs() -> list[dict]:
|
||||
with db() as conn:
|
||||
return [dict(row) for row in conn.execute("SELECT * FROM backup_jobs ORDER BY id DESC")]
|
||||
|
||||
|
||||
def get_job(job_id: int) -> dict:
|
||||
with db() as conn:
|
||||
row = conn.execute("SELECT * FROM backup_jobs WHERE id = ?", (job_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(f"Backup job {job_id} not found")
|
||||
return dict(row)
|
||||
|
||||
|
||||
def create_job(payload: dict) -> dict:
|
||||
validate_cron(payload["cron_schedule"])
|
||||
now = utc_now()
|
||||
with db() as conn:
|
||||
cur = conn.execute(
|
||||
"""
|
||||
INSERT INTO backup_jobs(
|
||||
guest_vmid, guest_name, guest_type, node, enabled, cron_schedule,
|
||||
proxmox_storage, backup_mode, compression, retention_type, retention_value,
|
||||
next_run_at, created_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
payload["guest_vmid"],
|
||||
payload["guest_name"],
|
||||
payload["guest_type"],
|
||||
payload["node"],
|
||||
int(payload.get("enabled", True)),
|
||||
payload["cron_schedule"],
|
||||
payload["proxmox_storage"],
|
||||
payload["backup_mode"],
|
||||
payload["compression"],
|
||||
payload["retention_type"],
|
||||
int(payload["retention_value"]),
|
||||
next_run(payload["cron_schedule"]),
|
||||
now,
|
||||
now,
|
||||
),
|
||||
)
|
||||
row = conn.execute("SELECT * FROM backup_jobs WHERE id = ?", (cur.lastrowid,)).fetchone()
|
||||
return dict(row)
|
||||
|
||||
|
||||
def update_job(job_id: int, payload: dict) -> dict:
|
||||
existing = get_job(job_id)
|
||||
merged = {**existing, **payload}
|
||||
validate_cron(merged["cron_schedule"])
|
||||
merged["next_run_at"] = next_run(merged["cron_schedule"]) if payload.get("cron_schedule") else existing["next_run_at"]
|
||||
merged["updated_at"] = utc_now()
|
||||
with db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE backup_jobs SET
|
||||
guest_vmid = ?, guest_name = ?, guest_type = ?, node = ?, enabled = ?,
|
||||
cron_schedule = ?, proxmox_storage = ?, backup_mode = ?, compression = ?,
|
||||
retention_type = ?, retention_value = ?, next_run_at = ?, updated_at = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(
|
||||
merged["guest_vmid"],
|
||||
merged["guest_name"],
|
||||
merged["guest_type"],
|
||||
merged["node"],
|
||||
int(merged["enabled"]),
|
||||
merged["cron_schedule"],
|
||||
merged["proxmox_storage"],
|
||||
merged["backup_mode"],
|
||||
merged["compression"],
|
||||
merged["retention_type"],
|
||||
int(merged["retention_value"]),
|
||||
merged["next_run_at"],
|
||||
merged["updated_at"],
|
||||
job_id,
|
||||
),
|
||||
)
|
||||
row = conn.execute("SELECT * FROM backup_jobs WHERE id = ?", (job_id,)).fetchone()
|
||||
return dict(row)
|
||||
|
||||
|
||||
def delete_job(job_id: int) -> None:
|
||||
with db() as conn:
|
||||
conn.execute("DELETE FROM backup_jobs WHERE id = ?", (job_id,))
|
||||
|
||||
|
||||
def recalculate_next_runs() -> None:
|
||||
now = utc_now()
|
||||
with db() as conn:
|
||||
rows = conn.execute("SELECT id, cron_schedule FROM backup_jobs").fetchall()
|
||||
for row in rows:
|
||||
conn.execute(
|
||||
"UPDATE backup_jobs SET next_run_at = ?, updated_at = ? WHERE id = ?",
|
||||
(next_run(row["cron_schedule"]), now, row["id"]),
|
||||
)
|
||||
|
||||
|
||||
def due_jobs(limit: int) -> list[dict]:
|
||||
now = utc_now()
|
||||
with db() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT * FROM backup_jobs
|
||||
WHERE enabled = 1 AND next_run_at IS NOT NULL AND next_run_at <= ?
|
||||
ORDER BY next_run_at ASC
|
||||
LIMIT ?
|
||||
""",
|
||||
(now, limit),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def advance_job(job: dict) -> None:
|
||||
with db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE backup_jobs SET next_run_at = ?, updated_at = ? WHERE id = ?",
|
||||
(next_run(job["cron_schedule"]), utc_now(), job["id"]),
|
||||
)
|
||||
@@ -0,0 +1,272 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.responses import FileResponse, JSONResponse, Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from . import commands, jobs
|
||||
from .backup_service import (
|
||||
dashboard,
|
||||
delete_backup_manually,
|
||||
delete_job_with_backups,
|
||||
get_backup,
|
||||
list_backups,
|
||||
process_backup,
|
||||
queue_backup_for_job,
|
||||
)
|
||||
from .config import STATIC_DIR, ensure_dirs
|
||||
from .migrations import run_migrations
|
||||
from .settings_store import get_settings, set_settings
|
||||
|
||||
|
||||
class SettingsPayload(BaseModel):
|
||||
proxmox_node: str = ""
|
||||
proxmox_storage: str = ""
|
||||
local_backup_dir: str = ""
|
||||
rclone_path: str = ""
|
||||
rclone_remote: str = ""
|
||||
rclone_remote_path: str = "pve-cloud-backup"
|
||||
discord_webhook_url: str = ""
|
||||
allowed_cors_origins: list[str] = Field(default_factory=list)
|
||||
default_compression: str = "zstd"
|
||||
default_backup_mode: str = "snapshot"
|
||||
default_retention_type: str = "days"
|
||||
default_retention_value: int = 30
|
||||
max_concurrent_backups: int = 1
|
||||
timezone: str = "UTC"
|
||||
setup_complete: bool = False
|
||||
|
||||
|
||||
class JobPayload(BaseModel):
|
||||
guest_vmid: int
|
||||
guest_name: str
|
||||
guest_type: str
|
||||
node: str
|
||||
enabled: bool = True
|
||||
cron_schedule: str
|
||||
proxmox_storage: str
|
||||
backup_mode: str
|
||||
compression: str
|
||||
retention_type: str
|
||||
retention_value: int
|
||||
|
||||
|
||||
class PartialJobPayload(BaseModel):
|
||||
guest_vmid: int | None = None
|
||||
guest_name: str | None = None
|
||||
guest_type: str | None = None
|
||||
node: str | None = None
|
||||
enabled: bool | None = None
|
||||
cron_schedule: str | None = None
|
||||
proxmox_storage: str | None = None
|
||||
backup_mode: str | None = None
|
||||
compression: str | None = None
|
||||
retention_type: str | None = None
|
||||
retention_value: int | None = None
|
||||
|
||||
|
||||
app = FastAPI(title="PVE Cloud Backup")
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def dynamic_cors(request: Request, call_next: Callable):
|
||||
origin = request.headers.get("origin")
|
||||
allowed = get_settings().get("allowed_cors_origins", [])
|
||||
is_allowed = origin and ("*" in allowed or origin in allowed)
|
||||
if request.method == "OPTIONS":
|
||||
response = Response(status_code=204)
|
||||
else:
|
||||
response = await call_next(request)
|
||||
if is_allowed:
|
||||
response.headers["Access-Control-Allow-Origin"] = origin
|
||||
response.headers["Vary"] = "Origin"
|
||||
response.headers["Access-Control-Allow-Methods"] = "GET,POST,PUT,PATCH,DELETE,OPTIONS"
|
||||
response.headers["Access-Control-Allow-Headers"] = "Content-Type,Authorization"
|
||||
return response
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
async def startup() -> None:
|
||||
ensure_dirs()
|
||||
run_migrations()
|
||||
|
||||
|
||||
@app.get("/api/setup/status")
|
||||
def setup_status():
|
||||
settings = get_settings()
|
||||
return {"setup_complete": settings["setup_complete"]}
|
||||
|
||||
|
||||
@app.get("/api/discover")
|
||||
def discover():
|
||||
settings = get_settings()
|
||||
rclone_path = settings.get("rclone_path") or commands.find_executable("rclone")
|
||||
remotes: list[str] = []
|
||||
if rclone_path:
|
||||
try:
|
||||
remotes = commands.rclone_list_remotes(rclone_path)
|
||||
except Exception:
|
||||
remotes = []
|
||||
storages: list[str] = []
|
||||
try:
|
||||
storages = commands.list_storages(settings.get("proxmox_node") or None)
|
||||
except Exception:
|
||||
storages = []
|
||||
return {
|
||||
"proxmox_node": settings.get("proxmox_node") or socket.gethostname(),
|
||||
"rclone_path": rclone_path,
|
||||
"rclone_remotes": remotes,
|
||||
"proxmox_storages": storages,
|
||||
}
|
||||
|
||||
|
||||
def _validate_settings(payload: dict) -> None:
|
||||
try:
|
||||
ZoneInfo(payload.get("timezone") or "UTC")
|
||||
except ZoneInfoNotFoundError as exc:
|
||||
raise HTTPException(400, "Timezone must be a valid IANA timezone, for example Europe/Berlin or UTC") from exc
|
||||
if payload.get("setup_complete"):
|
||||
missing = [
|
||||
key
|
||||
for key in ("proxmox_node", "proxmox_storage", "local_backup_dir", "rclone_path", "rclone_remote")
|
||||
if not payload.get(key)
|
||||
]
|
||||
if missing:
|
||||
raise HTTPException(400, f"Missing required settings: {', '.join(missing)}")
|
||||
try:
|
||||
remotes = commands.rclone_list_remotes(payload["rclone_path"])
|
||||
except Exception as exc:
|
||||
raise HTTPException(400, f"rclone is not usable: {exc}") from exc
|
||||
if payload["rclone_remote"].rstrip(":") not in remotes:
|
||||
raise HTTPException(400, "Configured rclone remote was not found. Configure rclone first, then rerun setup.")
|
||||
if payload.get("max_concurrent_backups", 1) < 1:
|
||||
raise HTTPException(400, "Maximum concurrent backups must be at least 1")
|
||||
|
||||
|
||||
@app.get("/api/settings")
|
||||
def read_settings():
|
||||
return get_settings()
|
||||
|
||||
|
||||
@app.put("/api/settings")
|
||||
def write_settings(payload: SettingsPayload):
|
||||
data = payload.model_dump()
|
||||
_validate_settings(data)
|
||||
old_timezone = get_settings().get("timezone")
|
||||
try:
|
||||
saved = set_settings(data)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
if saved.get("timezone") != old_timezone:
|
||||
jobs.recalculate_next_runs()
|
||||
return get_settings()
|
||||
|
||||
|
||||
@app.get("/api/dashboard")
|
||||
def read_dashboard():
|
||||
return dashboard()
|
||||
|
||||
|
||||
@app.get("/api/guests")
|
||||
def read_guests():
|
||||
try:
|
||||
return commands.list_guests()
|
||||
except Exception as exc:
|
||||
raise HTTPException(502, f"Failed to list Proxmox guests via pvesh: {exc}") from exc
|
||||
|
||||
|
||||
@app.get("/api/jobs")
|
||||
def read_jobs():
|
||||
return jobs.list_jobs()
|
||||
|
||||
|
||||
@app.post("/api/jobs")
|
||||
def create_job(payload: JobPayload):
|
||||
try:
|
||||
return jobs.create_job(payload.model_dump())
|
||||
except ValueError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
|
||||
|
||||
@app.put("/api/jobs/{job_id}")
|
||||
def update_job(job_id: int, payload: PartialJobPayload):
|
||||
data = {key: value for key, value in payload.model_dump().items() if value is not None}
|
||||
try:
|
||||
return jobs.update_job(job_id, data)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(404, str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
|
||||
|
||||
@app.delete("/api/jobs/{job_id}")
|
||||
async def delete_job(job_id: int):
|
||||
try:
|
||||
result = await delete_job_with_backups(job_id)
|
||||
return {"ok": True, **result}
|
||||
except KeyError as exc:
|
||||
raise HTTPException(404, str(exc)) from exc
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(409, str(exc)) from exc
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/run")
|
||||
async def run_job_now(job_id: int):
|
||||
try:
|
||||
job = jobs.get_job(job_id)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(404, str(exc)) from exc
|
||||
queued = queue_backup_for_job(job)
|
||||
if not queued:
|
||||
raise HTTPException(409, "This guest already has an active backup")
|
||||
import asyncio
|
||||
|
||||
asyncio.create_task(process_backup(queued["id"]))
|
||||
return queued
|
||||
|
||||
|
||||
@app.get("/api/backups")
|
||||
def read_backups(guest_vmid: int | None = None, state: str | None = None):
|
||||
return list_backups(guest_vmid=guest_vmid, state=state)
|
||||
|
||||
|
||||
@app.get("/api/backups/{backup_id}")
|
||||
def read_backup(backup_id: str):
|
||||
try:
|
||||
return get_backup(backup_id)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(404, str(exc)) from exc
|
||||
|
||||
|
||||
@app.delete("/api/backups/{backup_id}")
|
||||
async def delete_backup(backup_id: str):
|
||||
try:
|
||||
return await delete_backup_manually(backup_id)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(404, str(exc)) from exc
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(409, str(exc)) from exc
|
||||
|
||||
|
||||
if (STATIC_DIR / "assets").exists():
|
||||
app.mount("/assets", StaticFiles(directory=STATIC_DIR / "assets"), name="assets")
|
||||
|
||||
|
||||
@app.get("/{full_path:path}")
|
||||
def spa(full_path: str):
|
||||
index = STATIC_DIR / "index.html"
|
||||
requested = STATIC_DIR / full_path
|
||||
if full_path and requested.exists() and requested.is_file():
|
||||
return FileResponse(requested)
|
||||
if index.exists():
|
||||
return FileResponse(index)
|
||||
return JSONResponse(
|
||||
{"detail": "Frontend has not been built yet. Run /opt/pve-cloud-backup/scripts/install.sh."},
|
||||
status_code=503,
|
||||
)
|
||||
@@ -0,0 +1,149 @@
|
||||
from pathlib import Path
|
||||
|
||||
from .db import db, utc_now
|
||||
|
||||
|
||||
def _system_timezone() -> str:
|
||||
timezone_file = Path("/etc/timezone")
|
||||
if timezone_file.exists():
|
||||
value = timezone_file.read_text().strip()
|
||||
if value:
|
||||
return value
|
||||
return "UTC"
|
||||
|
||||
|
||||
DEFAULT_SETTINGS = {
|
||||
"setup_complete": "false",
|
||||
"proxmox_node": "",
|
||||
"proxmox_storage": "",
|
||||
"local_backup_dir": "/var/lib/vz/dump",
|
||||
"rclone_path": "/usr/bin/rclone",
|
||||
"rclone_remote": "",
|
||||
"rclone_remote_path": "pve-cloud-backup",
|
||||
"discord_webhook_url": "",
|
||||
"allowed_cors_origins": "http://localhost:5173,http://127.0.0.1:5173",
|
||||
"default_compression": "zstd",
|
||||
"default_backup_mode": "snapshot",
|
||||
"default_retention_type": "days",
|
||||
"default_retention_value": "30",
|
||||
"max_concurrent_backups": "1",
|
||||
"timezone": _system_timezone(),
|
||||
}
|
||||
|
||||
|
||||
def run_migrations() -> None:
|
||||
with db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version INTEGER PRIMARY KEY,
|
||||
applied_at TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
applied = {
|
||||
row["version"]
|
||||
for row in conn.execute("SELECT version FROM schema_migrations").fetchall()
|
||||
}
|
||||
if 1 not in applied:
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE TABLE settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE backup_jobs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
guest_vmid INTEGER NOT NULL,
|
||||
guest_name TEXT NOT NULL,
|
||||
guest_type TEXT NOT NULL CHECK (guest_type IN ('vm','lxc')),
|
||||
node TEXT NOT NULL,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
cron_schedule TEXT NOT NULL,
|
||||
proxmox_storage TEXT NOT NULL,
|
||||
backup_mode TEXT NOT NULL,
|
||||
compression TEXT NOT NULL,
|
||||
retention_type TEXT NOT NULL CHECK (retention_type IN ('days','latest')),
|
||||
retention_value INTEGER NOT NULL,
|
||||
next_run_at TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE backups (
|
||||
id TEXT PRIMARY KEY,
|
||||
job_id INTEGER,
|
||||
guest_vmid INTEGER NOT NULL,
|
||||
guest_name TEXT NOT NULL,
|
||||
guest_type TEXT NOT NULL,
|
||||
node TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
upid TEXT,
|
||||
local_path TEXT,
|
||||
remote_path TEXT,
|
||||
size_bytes INTEGER,
|
||||
error_message TEXT,
|
||||
started_at TEXT NOT NULL,
|
||||
pve_completed_at TEXT,
|
||||
upload_started_at TEXT,
|
||||
completed_at TEXT,
|
||||
deleted_at TEXT,
|
||||
updated_at TEXT NOT NULL,
|
||||
FOREIGN KEY (job_id) REFERENCES backup_jobs(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
CREATE TABLE notification_logs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
event_type TEXT NOT NULL,
|
||||
backup_id TEXT,
|
||||
payload TEXT NOT NULL,
|
||||
success INTEGER NOT NULL,
|
||||
error_message TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
FOREIGN KEY (backup_id) REFERENCES backups(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_backup_jobs_enabled_next_run ON backup_jobs(enabled, next_run_at);
|
||||
CREATE INDEX idx_backups_guest_state ON backups(guest_vmid, state);
|
||||
CREATE INDEX idx_backups_state ON backups(state);
|
||||
"""
|
||||
)
|
||||
now = utc_now()
|
||||
conn.executemany(
|
||||
"INSERT INTO settings(key, value, updated_at) VALUES (?, ?, ?)",
|
||||
[(key, value, now) for key, value in DEFAULT_SETTINGS.items()],
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations(version, applied_at) VALUES (?, ?)",
|
||||
(1, now),
|
||||
)
|
||||
if 2 not in applied:
|
||||
now = utc_now()
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO settings(key, value, updated_at)
|
||||
VALUES ('rclone_remote_path', 'pve-cloud-backup', ?)
|
||||
ON CONFLICT(key) DO NOTHING
|
||||
""",
|
||||
(now,),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations(version, applied_at) VALUES (?, ?)",
|
||||
(2, now),
|
||||
)
|
||||
if 3 not in applied:
|
||||
now = utc_now()
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO settings(key, value, updated_at)
|
||||
VALUES ('timezone', ?, ?)
|
||||
ON CONFLICT(key) DO NOTHING
|
||||
""",
|
||||
(DEFAULT_SETTINGS["timezone"], now),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations(version, applied_at) VALUES (?, ?)",
|
||||
(3, now),
|
||||
)
|
||||
@@ -0,0 +1,79 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from .db import db, utc_now
|
||||
from .settings_store import get_setting
|
||||
|
||||
|
||||
COLORS = {
|
||||
"backup_started": 0x3498DB,
|
||||
"pve_completed": 0x2ECC71,
|
||||
"upload_started": 0x9B59B6,
|
||||
"backup_completed": 0x2ECC71,
|
||||
"backup_failed": 0xE74C3C,
|
||||
"remote_deleted": 0xF39C12,
|
||||
"recovery_action": 0x95A5A6,
|
||||
}
|
||||
|
||||
|
||||
def _field(name: str, value: Any) -> dict:
|
||||
return {"name": name, "value": str(value if value is not None else "-"), "inline": True}
|
||||
|
||||
|
||||
def _duration_seconds(backup: dict) -> int | None:
|
||||
started = backup.get("started_at")
|
||||
ended = backup.get("completed_at") or backup.get("deleted_at") or backup.get("pve_completed_at")
|
||||
if not started or not ended:
|
||||
return None
|
||||
try:
|
||||
return int((datetime.fromisoformat(ended) - datetime.fromisoformat(started)).total_seconds())
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
async def notify(event_type: str, *, backup: dict | None = None, message: str = "") -> None:
|
||||
webhook_url = get_setting("discord_webhook_url", "")
|
||||
fields = []
|
||||
backup_id = None
|
||||
if backup:
|
||||
backup_id = backup.get("id")
|
||||
duration = _duration_seconds(backup)
|
||||
enriched = {**backup, "duration_seconds": duration}
|
||||
for key in ("guest_name", "guest_vmid", "id", "state", "size_bytes", "duration_seconds", "error_message"):
|
||||
if enriched.get(key) not in (None, ""):
|
||||
fields.append(_field(key, enriched.get(key)))
|
||||
if message:
|
||||
fields.append(_field("message", message))
|
||||
payload = {
|
||||
"embeds": [
|
||||
{
|
||||
"title": event_type.replace("_", " ").title(),
|
||||
"color": COLORS.get(event_type, 0x3498DB),
|
||||
"timestamp": utc_now(),
|
||||
"fields": fields,
|
||||
}
|
||||
]
|
||||
}
|
||||
success = False
|
||||
error_message = None
|
||||
if webhook_url:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
response = await client.post(webhook_url, json=payload)
|
||||
response.raise_for_status()
|
||||
success = True
|
||||
except Exception as exc: # Notifications must never fail backups.
|
||||
error_message = str(exc)
|
||||
with db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO notification_logs(event_type, backup_id, payload, success, error_message, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(event_type, backup_id, json.dumps(payload), int(success), error_message, utc_now()),
|
||||
)
|
||||
@@ -0,0 +1,63 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .db import db, utc_now
|
||||
from .migrations import DEFAULT_SETTINGS
|
||||
|
||||
|
||||
INT_KEYS = {"default_retention_value", "max_concurrent_backups"}
|
||||
BOOL_KEYS = {"setup_complete"}
|
||||
|
||||
|
||||
def parse_value(key: str, value: str):
|
||||
if key in INT_KEYS:
|
||||
return int(value)
|
||||
if key in BOOL_KEYS:
|
||||
return value.lower() == "true"
|
||||
if key == "allowed_cors_origins":
|
||||
return [item.strip() for item in value.split(",") if item.strip()]
|
||||
return value
|
||||
|
||||
|
||||
def serialize_value(key: str, value) -> str:
|
||||
if key in BOOL_KEYS:
|
||||
return "true" if bool(value) else "false"
|
||||
if key == "allowed_cors_origins":
|
||||
if isinstance(value, list):
|
||||
return ",".join(str(item).strip() for item in value if str(item).strip())
|
||||
return str(value)
|
||||
|
||||
|
||||
def get_settings() -> dict:
|
||||
with db() as conn:
|
||||
rows = conn.execute("SELECT key, value FROM settings").fetchall()
|
||||
raw = {row["key"]: row["value"] for row in rows}
|
||||
for key, value in DEFAULT_SETTINGS.items():
|
||||
raw.setdefault(key, value)
|
||||
return {key: parse_value(key, value) for key, value in raw.items()}
|
||||
|
||||
|
||||
def set_settings(values: dict) -> dict:
|
||||
now = utc_now()
|
||||
valid_keys = set(DEFAULT_SETTINGS)
|
||||
unknown = set(values) - valid_keys
|
||||
if unknown:
|
||||
raise ValueError(f"Unknown settings: {', '.join(sorted(unknown))}")
|
||||
with db() as conn:
|
||||
for key, value in values.items():
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO settings(key, value, updated_at)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at
|
||||
""",
|
||||
(key, serialize_value(key, value), now),
|
||||
)
|
||||
return get_settings()
|
||||
|
||||
|
||||
def get_setting(key: str, default: str = "") -> str:
|
||||
with db() as conn:
|
||||
row = conn.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone()
|
||||
if row:
|
||||
return row["value"]
|
||||
return default
|
||||
@@ -0,0 +1,21 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from .backup_service import worker_loop
|
||||
from .config import LOG_DIR, ensure_dirs
|
||||
from .migrations import run_migrations
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ensure_dirs()
|
||||
logging.basicConfig(
|
||||
filename=LOG_DIR / "worker.log",
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
)
|
||||
run_migrations()
|
||||
asyncio.run(worker_loop())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user