diff --git a/.gitignore b/.gitignore index d5b4207..47bca58 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ vault-sync.log __pycache__/ *.pyc +state/ diff --git a/README.md b/README.md index d0a464f..aa96e1d 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ Known secret response fields such as Dockhand's `hawserToken` and `webhookSecret | `/docker/inspect/{host}/{container}` | GET | Sanitized image, runtime, resources, mounts and state | | `/docker/restart/{host}/{container}` | POST | Restart container; supports `dry_run=true` | | `/config/reload` | POST | Reload YAML configuration and credential cache | +| `/media/verify` | POST | ffprobe duration check on a NAS media file to catch truncated Sonarr/Radarr imports; flags `suspect` if deviation from `expected_minutes` exceeds `tolerance_pct` | ## VM lifecycle and inventory @@ -97,6 +98,14 @@ Integration Compose definition: `tests/compose.integration.yaml` (binds only to ## Changelog +### 2.4.8 — 17.09.2026 + +- Fixed `POST /media/verify` false-positive rejection: filenames containing a legitimate apostrophe (e.g. "La'An" in a Star Trek episode title) were blocked as "unsafe characters". Replaced the naive single-quote wrapping + apostrophe blocklist with proper `shlex.quote()` escaping for the remote ffprobe command; only newline/NUL byte injection is still rejected. + +### 2.4.7 — 17.09.2026 + +- Added `POST /media/verify`: probes a media file's real duration via `ffprobe` (running inside the existing `bazarrUHD` container on `arrapps`, which already mounts `/data` and ships ffmpeg — no new service needed) and flags it as `suspect` when it deviates from an `expected_minutes` value beyond `tolerance_pct`. Catches truncated Sonarr/Radarr imports (e.g. a re-grab that lands as 1.8 GB instead of the expected 8 GB for a UHD episode) after the file is already on the NAS. + ### 2.3.2 — 22.07.2026 - Make Vaultwarden refresh durable: persistent named cache volume, protected runtime credentials, automatic API-key re-login and atomic cache writes. diff --git a/app.py b/app.py index 128f5c2..7ddb58a 100644 --- a/app.py +++ b/app.py @@ -1,22 +1,28 @@ """Homelab Butler v2.1 – Unified API proxy for Pfannkuchen homelab. Reads service config from butler.yaml, credentials from Vaultwarden cache with flat-file fallback.""" -import os, json, asyncio, logging, time, base64, re, subprocess, ipaddress, secrets +import os, json, asyncio, logging, time, base64, re, subprocess, ipaddress, secrets, sqlite3, math, hashlib, shlex from datetime import datetime, timezone import httpx, yaml from typing import Literal from pydantic import BaseModel, Field from fastapi import FastAPI, Request, HTTPException, Depends, Query -from fastapi.responses import JSONResponse, RedirectResponse, Response +from fastapi.responses import JSONResponse, RedirectResponse, Response, HTMLResponse from contextlib import asynccontextmanager +from contextvars import ContextVar log = logging.getLogger("butler") -VERSION = "2.3.5" +VERSION = "2.4.9" API_DIR = os.environ.get("API_KEY_DIR", "/data/api") VAULT_CACHE_DIR = os.environ.get("VAULT_CACHE_DIR", "/data/vault-cache") BUTLER_TOKEN = os.environ.get("BUTLER_TOKEN", "") CONFIG_PATH = os.environ.get("BUTLER_CONFIG", "/data/butler.yaml") +UI_PATH = os.environ.get("BUTLER_UI_PATH", os.path.join(os.path.dirname(__file__), "ui.html")) +MEDIA_HANDOFF_ALLOWED_NETWORKS = os.environ.get( + "MEDIA_HANDOFF_ALLOWED_NETWORKS", + "10.2.1.119/32,10.5.85.12/32", +) # --- Config loading --- @@ -45,7 +51,39 @@ _load_config() # --- Audit log --- _audit_log: list[dict] = [] +_audit_actor: ContextVar[str] = ContextVar("audit_actor", default="System/API") MAX_AUDIT = 500 +AUDIT_DB_PATH = os.environ.get("AUDIT_DB_PATH", "/data/state/audit.sqlite3") + + +def _redact_audit_detail(detail: str) -> str: + return re.sub( + r"(?i)\b(token|password|api[_-]?key|secret)=([^\s]+)", + lambda match: f"{match.group(1)}=[REDACTED]", + detail, + )[:200] + + +def _init_audit_db() -> bool: + try: + os.makedirs(os.path.dirname(AUDIT_DB_PATH), exist_ok=True) + with sqlite3.connect(AUDIT_DB_PATH) as db: + db.execute("""CREATE TABLE IF NOT EXISTS audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts TEXT NOT NULL, + endpoint TEXT NOT NULL, + method TEXT NOT NULL, + status INTEGER NOT NULL, + detail TEXT NOT NULL, + dry_run INTEGER NOT NULL, + actor TEXT NOT NULL DEFAULT 'Legacy/API' + )""") + columns = {row[1] for row in db.execute("PRAGMA table_info(audit)")} + if "actor" not in columns: + db.execute("ALTER TABLE audit ADD COLUMN actor TEXT NOT NULL DEFAULT 'Legacy/API'") + return True + except (OSError, sqlite3.Error): + return False def _audit(endpoint: str, method: str, status: int, detail: str = "", dry_run: bool = False): entry = { @@ -53,12 +91,23 @@ def _audit(endpoint: str, method: str, status: int, detail: str = "", dry_run: b "endpoint": endpoint, "method": method, "status": status, - "detail": detail[:200], + "detail": _redact_audit_detail(detail), "dry_run": dry_run, + "actor": _audit_actor.get(), } _audit_log.append(entry) if len(_audit_log) > MAX_AUDIT: _audit_log.pop(0) + try: + if _init_audit_db(): + with sqlite3.connect(AUDIT_DB_PATH) as db: + db.execute( + "INSERT INTO audit (ts, endpoint, method, status, detail, dry_run, actor) VALUES (?, ?, ?, ?, ?, ?, ?)", + (entry["ts"], entry["endpoint"], entry["method"], entry["status"], entry["detail"], int(entry["dry_run"]), entry["actor"]), + ) + db.execute("DELETE FROM audit WHERE id NOT IN (SELECT id FROM audit ORDER BY id DESC LIMIT ?)", (MAX_AUDIT,)) + except (OSError, sqlite3.Error): + pass API_DIR = os.environ.get("API_KEY_DIR", "/data/api") VAULT_CACHE_DIR = os.environ.get("VAULT_CACHE_DIR", "/data/vault-cache") @@ -92,6 +141,7 @@ async def _periodic_cache_reload(): async def lifespan(app: FastAPI): _load_config() _load_vault_cache() + _init_audit_db() task = asyncio.create_task(_periodic_cache_reload()) yield task.cancel() @@ -189,12 +239,345 @@ async def _dockhand_login(client): # --- Auth --- +_ui_sessions: dict[str, dict] = {} +UI_SESSION_TTL = 8 * 60 * 60 + + +class UiLoginRequest(BaseModel): + token: str + + +def _ui_session(request: Request) -> dict | None: + session_id = request.cookies.get("butler_session", "") + session = _ui_sessions.get(session_id) + if not session: + return None + if session["expires"] <= time.time(): + _ui_sessions.pop(session_id, None) + return None + return session + + +def _issue_ui_session(read_only: bool) -> JSONResponse: + session_id = secrets.token_urlsafe(32) + csrf = secrets.token_urlsafe(24) + _ui_sessions[session_id] = { + "csrf": csrf, + "expires": time.time() + UI_SESSION_TTL, + "read_only": read_only, + } + response = JSONResponse({ + "authenticated": True, + "read_only": read_only, + "expires_in": UI_SESSION_TTL, + }) + response.set_cookie("butler_session", session_id, max_age=UI_SESSION_TTL, httponly=True, samesite="strict", path="/") + response.set_cookie("butler_csrf", csrf, max_age=UI_SESSION_TTL, httponly=False, samesite="strict", path="/") + return response + def _verify(request: Request): if not BUTLER_TOKEN: return auth = request.headers.get("authorization", "") - if auth != f"Bearer {BUTLER_TOKEN}": + if secrets.compare_digest(auth, f"Bearer {BUTLER_TOKEN}"): + return + session = _ui_session(request) + if not session: raise HTTPException(401, "Invalid token") + if request.method not in {"GET", "HEAD", "OPTIONS"}: + if session.get("read_only", False): + raise HTTPException(403, "Anonymous UI session is read-only") + csrf = request.headers.get("x-csrf-token", "") + if not csrf or not secrets.compare_digest(csrf, session["csrf"]): + raise HTTPException(403, "Invalid CSRF token") + + +def _clean_audit_actor(value: str) -> str: + cleaned = re.sub(r"[^\w .@/\-]", "", str(value or ""))[:40].strip() + return cleaned or "KI/API" + + +@app.middleware("http") +async def audit_actor_context(request: Request, call_next): + if request.headers.get("authorization", "").startswith("Bearer "): + actor = _clean_audit_actor(request.headers.get("x-butler-actor", "KI/API")) + elif _ui_session(request): + actor = "Weboberfläche" + else: + actor = "System/Öffentlich" + token = _audit_actor.set(actor) + try: + return await call_next(request) + finally: + _audit_actor.reset(token) + + +def _emby_network_identity(endpoint: str) -> dict: + """Normalize an Emby endpoint without treating IPv6 privacy addresses as new households.""" + raw = str(endpoint or "").strip() + if raw.startswith("[") and "]" in raw: + raw = raw[1:raw.index("]")] + try: + address = ipaddress.ip_address(raw) + except ValueError: + if raw.count(":") == 1: + raw = raw.rsplit(":", 1)[0] + try: + address = ipaddress.ip_address(raw) + except ValueError as exc: + raise ValueError("Invalid Emby remote endpoint") from exc + if address.version == 4: + network = ipaddress.ip_network(f"{address}/32", strict=False) + parent = network + else: + network = ipaddress.ip_network(f"{address}/64", strict=False) + parent = ipaddress.ip_network(f"{address}/48", strict=False) + return { + "ip": str(address), + "version": address.version, + "network": str(network), + "parent": str(parent), + "identity": str(parent if address.version == 6 else network), + "public": address.is_global, + } + + +def _emby_location(metric: dict) -> dict: + def coordinate(name): + try: + return float(metric.get(name, 0)) + except (TypeError, ValueError): + return 0.0 + return { + "city": metric.get("city", ""), + "region": metric.get("region", ""), + "country": metric.get("countryCode", ""), + "latitude": coordinate("latitude"), + "longitude": coordinate("longitude"), + } + + +def _analyze_emby_sharing(series: list[dict], step_seconds: int) -> dict: + observations: dict[str, dict[int, dict[str, dict]]] = {} + tracks: dict[str, dict[str, dict]] = {} + identities_by_user: dict[str, set[str]] = {} + servers_by_user: dict[str, set[str]] = {} + for item in series: + metric = item.get("metric", {}) + username = str(metric.get("username", "")).strip() + if not username: + continue + try: + network = _emby_network_identity(metric.get("remoteEndPoint", "")) + except ValueError: + continue + if not network["public"]: + continue + evidence = { + **network, + "server": metric.get("job", ""), + "location": _emby_location(metric), + } + identities_by_user.setdefault(username, set()).add(network["identity"]) + servers_by_user.setdefault(username, set()).add(str(metric.get("job", ""))) + track = tracks.setdefault(username, {}).setdefault(network["identity"], {"evidence": evidence, "timestamps": []}) + for value in item.get("values", []): + if not isinstance(value, list) or len(value) < 2 or str(value[1]).lower() in {"0", "nan"}: + continue + timestamp = int(float(value[0])) + track["timestamps"].append(timestamp) + observations.setdefault(username, {}).setdefault(timestamp, {}).setdefault(network["identity"], evidence) + + raw_events = [] + for username, timeline in observations.items(): + buckets = [] + for timestamp in sorted(timeline): + evidence = timeline[timestamp] + if len(evidence) >= 2: + buckets.append((timestamp, tuple(sorted(evidence)), evidence)) + current = None + for timestamp, identity_key, evidence in buckets: + if current and current["identity_key"] == identity_key and timestamp - current["end_ts"] <= step_seconds * 2: + current["end_ts"] = timestamp + current["samples"] += 1 + continue + if current and current["samples"] >= 2 and current["end_ts"] - current["start_ts"] + step_seconds >= 360: + raw_events.append(current) + current = { + "username": username, "identity_key": identity_key, "start_ts": timestamp, + "end_ts": timestamp, "samples": 1, "evidence": list(evidence.values()), + } + if current and current["samples"] >= 2 and current["end_ts"] - current["start_ts"] + step_seconds >= 360: + raw_events.append(current) + + events = [{ + "type": "concurrent_networks", + "severity": "high", + "username": item["username"], + "start": datetime.fromtimestamp(item["start_ts"], timezone.utc).isoformat(), + "end": datetime.fromtimestamp(item["end_ts"], timezone.utc).isoformat(), + "duration_seconds": item["end_ts"] - item["start_ts"] + step_seconds, + "samples": item["samples"], + "evidence": item["evidence"], + "reason": "Zeitgleiche Nutzung desselben Emby-Benutzers aus unterschiedlichen öffentlichen Netzen", + } for item in raw_events] + + def distance_km(first: dict, second: dict) -> float: + lat1, lon1 = first["latitude"], first["longitude"] + lat2, lon2 = second["latitude"], second["longitude"] + if not all((-90 <= lat <= 90 and -180 <= lon <= 180) for lat, lon in ((lat1, lon1), (lat2, lon2))): + return 0.0 + phi1, phi2 = math.radians(lat1), math.radians(lat2) + dphi, dlambda = math.radians(lat2 - lat1), math.radians(lon2 - lon1) + value = math.sin(dphi / 2) ** 2 + math.cos(phi1) * math.cos(phi2) * math.sin(dlambda / 2) ** 2 + return 6371.0 * 2 * math.atan2(math.sqrt(value), math.sqrt(max(0.0, 1 - value))) + + travel_events = [] + for username, user_tracks in tracks.items(): + intervals = [] + for identity, track in user_tracks.items(): + current = None + for timestamp in sorted(set(track["timestamps"])): + if current and timestamp - current["end"] <= step_seconds * 2: + current["end"] = timestamp + current["samples"] += 1 + else: + if current and current["samples"] >= 2: + intervals.append(current) + current = {"identity": identity, "start": timestamp, "end": timestamp, "samples": 1, "evidence": track["evidence"]} + if current and current["samples"] >= 2: + intervals.append(current) + intervals.sort(key=lambda item: item["start"]) + for previous, current in zip(intervals, intervals[1:]): + if previous["identity"] == current["identity"] or current["start"] <= previous["end"]: + continue + distance = distance_km(previous["evidence"]["location"], current["evidence"]["location"]) + gap_hours = max((current["start"] - previous["end"]) / 3600, 1 / 60) + speed = distance / gap_hours + if distance < 300 or speed <= 1000: + continue + travel_events.append({ + "type": "impossible_travel", "severity": "medium", "username": username, + "start": datetime.fromtimestamp(previous["end"], timezone.utc).isoformat(), + "end": datetime.fromtimestamp(current["start"], timezone.utc).isoformat(), + "duration_seconds": current["start"] - previous["end"], + "samples": previous["samples"] + current["samples"], + "distance_km": round(distance, 1), "required_speed_kmh": round(speed, 1), + "evidence": [previous["evidence"], current["evidence"]], + "reason": "Geografischer Wechsel zwischen öffentlichen Netzen wäre in der verfügbaren Zeit nicht plausibel", + }) + events.extend(travel_events) + events.sort(key=lambda item: item["start"], reverse=True) + flagged = {item["username"] for item in events} + users = [{ + "username": username, + "risk": "high" if any(item["username"] == username and item["type"] == "concurrent_networks" for item in events) else ("medium" if username in flagged else "none"), + "events": sum(item["username"] == username for item in events), + "network_identities": len(identities_by_user.get(username, set())), + "servers": sorted(servers_by_user.get(username, set())), + } for username in sorted(observations)] + return { + "summary": { + "users_analyzed": len(observations), + "flagged_users": len(flagged), + "concurrent_events": len(raw_events), + "impossible_travel_events": len(travel_events), + }, + "users": users, + "events": events, + } + + +def _emby_history_step(days: int) -> int: + """Keep query_range below Prometheus' 11,000-points-per-series limit.""" + duration_seconds = days * 86400 + return max(60, math.ceil((duration_seconds / 10_500) / 60) * 60) + + +async def _fetch_emby_session_history(days: int, server: str) -> tuple[list[dict], int]: + cfg = SERVICES.get("grafana") + if not cfg: + raise HTTPException(503, "Grafana service is not configured") + request_data = _service_auth(cfg) + datasource_uid = os.environ.get("EMBY_PROMETHEUS_UID", "bdpu4276997nkc") + labels = "job,username,remoteEndPoint,city,region,countryCode,latitude,longitude" + selector = 'emby_sessions{username!=""}' + if server != "all": + selector = f'emby_sessions{{username!="",job="{server}"}}' + query = f"max by ({labels}) ({selector})" + end = int(time.time()) + start = end - days * 86400 + step = _emby_history_step(days) + url = f"{request_data['base_url'].rstrip('/')}/api/datasources/proxy/uid/{datasource_uid}/api/v1/query_range" + series_by_metric: dict[str, dict] = {} + chunk_seconds = 30 * 86400 + try: + async with httpx.AsyncClient(timeout=90) as client: + chunk_start = start + while chunk_start < end: + chunk_end = min(chunk_start + chunk_seconds, end) + response = await client.get( + url, + params={"query": query, "start": chunk_start, "end": chunk_end, "step": step}, + headers=request_data["headers"], cookies=request_data["cookies"], + ) + response.raise_for_status() + payload = response.json() + if payload.get("status") != "success": + raise HTTPException(502, "Prometheus rejected the Emby session history query") + for item in payload.get("data", {}).get("result", []): + metric = item.get("metric", {}) + key = json.dumps(metric, sort_keys=True, separators=(",", ":")) + merged = series_by_metric.setdefault(key, {"metric": metric, "values": []}) + merged["values"].extend(item.get("values", [])) + chunk_start = chunk_end + except (httpx.HTTPError, ValueError) as exc: + log.warning("Emby sharing history query failed: %s", type(exc).__name__) + raise HTTPException(502, "Emby session history is temporarily unavailable") + + series = [] + for item in series_by_metric.values(): + values_by_timestamp = { + float(value[0]): value for value in item["values"] + if isinstance(value, (list, tuple)) and len(value) >= 2 + } + item["values"] = [values_by_timestamp[ts] for ts in sorted(values_by_timestamp)] + series.append(item) + return series, step + + +@app.get("/emby/account-sharing") +async def emby_account_sharing( + days: int = Query(30, ge=1, le=90), + username: str | None = Query(None, min_length=1, max_length=100), + server: Literal["all", "emby-sascha", "emby-chris"] = "all", + _=Depends(_verify), +): + """Conservative read-only analysis of concurrent networks and geographically impossible changes.""" + series, step = await _fetch_emby_session_history(days, server) + if username: + wanted = username.casefold() + series = [item for item in series if str(item.get("metric", {}).get("username", "")).casefold() == wanted] + result = _analyze_emby_sharing(series, step) + return { + "generated": datetime.now(timezone.utc).isoformat(), + "period": {"days": days, "server": server, "step_seconds": step, "series": len(series)}, + "policy": { + "mode": "conservative", + "ipv4_detection_identity": "/32", + "ipv6_display_network": "/64", + "ipv6_detection_identity": "/48", + "minimum_samples": 2, + "minimum_concurrent_seconds": 360, + "prometheus_staleness_guard": True, + "impossible_travel_minimum_km": 300, + "impossible_travel_speed_kmh": 1000, + "private_networks_excluded": True, + "automatic_enforcement": False, + }, + **result, + } + def _get_key(cfg): vault_key = cfg.get("vault_key") @@ -252,6 +635,46 @@ def _inventory_hosts(text: str) -> list[dict]: # --- Routes --- +@app.get("/ui", response_class=HTMLResponse) +async def ui(): + try: + return HTMLResponse(open(UI_PATH, encoding="utf-8").read()) + except FileNotFoundError: + raise HTTPException(503, "Butler UI asset is missing") + + +@app.post("/ui/login") +async def ui_login(payload: UiLoginRequest): + if not BUTLER_TOKEN or not secrets.compare_digest(payload.token, BUTLER_TOKEN): + raise HTTPException(401, "Invalid token") + return _issue_ui_session(read_only=False) + + +@app.get("/ui/session") +async def ui_session(request: Request): + session = _ui_session(request) + if session: + return { + "authenticated": True, + "read_only": session.get("read_only", False), + "expires_in": max(0, int(session["expires"] - time.time())), + } + return _issue_ui_session(read_only=True) + + +@app.post("/ui/logout") +async def ui_logout(request: Request): + session = _ui_session(request) + if session: + csrf = request.headers.get("x-csrf-token", "") + if not csrf or not secrets.compare_digest(csrf, session["csrf"]): + raise HTTPException(403, "Invalid CSRF token") + _ui_sessions.pop(request.cookies.get("butler_session", ""), None) + response = JSONResponse({"authenticated": False}) + response.delete_cookie("butler_session", path="/") + response.delete_cookie("butler_csrf", path="/") + return response + @app.get("/") async def root(): """AI self-onboarding: returns all available endpoints and services.""" @@ -264,6 +687,10 @@ async def root(): "openapi": "/openapi.json", "services": svc_list, "endpoints": { + "capabilities": "GET /capabilities - live machine-readable operation and safety map", + "doctor": "GET /doctor/{target} - correlated service/host/backup/disk diagnosis", + "drift": "GET /drift - inventory coverage gaps", + "maintenance_preflight": "GET /maintenance/preflight?action=general&target=HOST - read-only safety gate", "proxy": "GET/POST/PUT/DELETE /{service}/{path} - proxy to backend with auto-auth", "vm_list": "GET /vm/list", "vm_create": "POST /vm/create {node, ip, hostname, cores?, memory?, disk?}", @@ -288,6 +715,80 @@ async def root(): async def health(): return {"status": "ok", "vault_items": len(_vault_cache), "services": len(SERVICES), "version": VERSION} + +def _schema_contains_property(node, property_name: str, components: dict, seen: set[str] | None = None) -> bool: + """Resolve local OpenAPI refs and look for a request property.""" + seen = seen or set() + if isinstance(node, list): + return any(_schema_contains_property(item, property_name, components, seen) for item in node) + if not isinstance(node, dict): + return False + if node.get("name") == property_name or property_name in node.get("properties", {}): + return True + ref = node.get("$ref", "") + if ref.startswith("#/components/schemas/"): + name = ref.rsplit("/", 1)[-1] + if name in seen: + return False + return _schema_contains_property(components.get(name, {}), property_name, components, seen | {name}) + return any( + _schema_contains_property(value, property_name, components, seen) + for key, value in node.items() + if key != "properties" + ) + + +@app.get("/capabilities") +async def capabilities(_=Depends(_verify)): + """Live operation catalog with safety metadata for AI agents.""" + schema = app.openapi() + components = schema.get("components", {}).get("schemas", {}) + operations = [] + for path, methods in schema.get("paths", {}).items(): + if path == "/{service}/{path}" or path in {"/", "/health", "/openapi.json", "/docs", "/redoc"}: + continue + for method, operation in methods.items(): + if method.upper() not in {"GET", "POST", "PUT", "PATCH", "DELETE"}: + continue + mode = "read_only" if method.upper() == "GET" else "mutation" + serialized = {"parameters": operation.get("parameters", []), "requestBody": operation.get("requestBody", {})} + dry_run = _schema_contains_property(serialized, "dry_run", components) + critical = path.startswith(("/network/wireguard", "/network/media-tunnel", "/caddy/")) + destructive = method.upper() == "DELETE" or any( + marker in path for marker in ("/destroy/", "/cleanup/", "/break-lock/", "/restore/") + ) + operations.append({ + "method": method.upper(), + "path": path, + "summary": operation.get("summary", ""), + "description": operation.get("description", ""), + "mode": mode, + "dry_run": dry_run, + "critical": critical, + "destructive": destructive, + "confirmation_required": mode == "mutation", + }) + operations.sort(key=lambda item: (item["path"], item["method"])) + counts = { + "total": len(operations), + "read_only": sum(item["mode"] == "read_only" for item in operations), + "mutations": sum(item["mode"] == "mutation" for item in operations), + "destructive": sum(item["destructive"] for item in operations), + } + return { + "schema_version": 1, + "service": "homelab-butler", + "version": VERSION, + "generated": datetime.now(timezone.utc).isoformat(), + "counts": counts, + "operations": operations, + "proxy": {"path": "/{service}/{path}", "note": "Generic backend proxy; inspect /info services and OpenAPI before use"}, + "model_contract": { + "instruction": "Prefer read_only operations. Before every mutation inspect its schema, use dry_run when available, and obtain confirmation for critical or destructive actions.", + "source_of_truth": "/openapi.json", + }, + } + def _classify_http_status(status_code: int, expected: set[int]) -> str: """Return a deterministic service state suitable for small models.""" if status_code in expected: @@ -382,6 +883,17 @@ async def status(_=Depends(_verify)): @app.get("/audit") async def audit(_=Depends(_verify), limit: int = Query(50, le=MAX_AUDIT)): """Recent API calls (newest first).""" + try: + if os.path.exists(AUDIT_DB_PATH): + with sqlite3.connect(AUDIT_DB_PATH) as db: + db.row_factory = sqlite3.Row + rows = db.execute( + "SELECT ts, endpoint, method, status, detail, dry_run, actor FROM audit ORDER BY id DESC LIMIT ?", + (limit,), + ).fetchall() + return [dict(row) | {"dry_run": bool(row["dry_run"])} for row in rows] + except (OSError, sqlite3.Error): + pass return list(reversed(_audit_log[-limit:])) @app.post("/config/reload") @@ -408,6 +920,11 @@ async def info(_=Depends(_verify)): for name, cfg in SERVICES.items() }, "endpoints": { + "capabilities": "/capabilities", + "ui": "/ui", + "doctor": "/doctor/{target}", + "drift": "/drift", + "maintenance_preflight": "/maintenance/preflight", "status": "/status", "overview": "/overview?details=false", "audit": "/audit", @@ -523,8 +1040,11 @@ async def _collect_backup_status(concurrency: int = 10) -> dict: """Query VM backups concurrently; one slow host no longer blocks all others serially.""" semaphore = asyncio.Semaphore(concurrency) hosts = [host for host in await _get_inventory_hosts_async() if not host["name"].startswith("node")] + exempt_hosts = set(_config.get("backup", {}).get("exempt_hosts", [])) async def inspect_backup(host: dict): + if host["name"] in exempt_hosts: + return host["name"], {"state": "exempt", "ok": True, "reason": "backup policy exemption"} async with semaphore: rc, out, err = await asyncio.to_thread( _ssh, @@ -536,7 +1056,7 @@ async def _collect_backup_status(concurrency: int = 10) -> dict: pairs = await asyncio.gather(*(inspect_backup(host) for host in hosts)) results = dict(pairs) - summary = {"total": len(results), "healthy": 0, "warning": 0, "critical": 0, "unknown": 0} + summary = {"total": len(results), "healthy": 0, "warning": 0, "critical": 0, "unknown": 0, "exempt": 0} for item in results.values(): summary[item["state"]] += 1 return {"summary": summary, "hosts": results} @@ -704,6 +1224,135 @@ def _add_component(summary: dict, bucket: dict, state: str): bucket[normalized] += 1 +async def _collect_operational_snapshot() -> dict: + services, hosts, backups, disks = await asyncio.gather( + _collect_service_status(), _collect_health_all(), _collect_backup_status(), _collect_disk_usage() + ) + return {"services": services, "hosts": hosts, "backups": backups, "disks": disks} + + +@app.get("/doctor/{target}") +async def doctor(target: str, _=Depends(_verify)): + """Correlate service, host, backup and disk layers for one known target.""" + if not re.fullmatch(r"[A-Za-z0-9_.-]+", target): + raise HTTPException(400, "Invalid target") + snapshot = await _collect_operational_snapshot() + layers = {} + findings = [] + if target in snapshot["services"]: + service = snapshot["services"][target] + layers["service"] = service + if service.get("status") != "healthy": + findings.append({"severity": "critical" if service.get("status") in ("offline", "auth_failed", "misconfigured") else "warning", "code": "service_unhealthy", "message": service.get("message", "Service probe failed")}) + if target in snapshot["hosts"]: + host = snapshot["hosts"][target] + layers["host"] = host + if not host.get("reachable"): + findings.append({"severity": "critical", "code": "host_unreachable", "message": "Host is not reachable over SSH"}) + bad = [line for line in host.get("containers", []) if "unhealthy" in line.lower() or "restarting" in line.lower()] + if bad: + findings.append({"severity": "critical", "code": "container_unhealthy", "message": bad[0][:200]}) + backup = snapshot["backups"].get("hosts", {}).get(target) + if backup is not None: + layers["backup"] = backup + if backup.get("state") not in ("healthy", "exempt"): + findings.append({"severity": "critical" if backup.get("state") in ("critical", "unknown") else "warning", "code": "backup_unhealthy", "message": "Backup is stale or could not be verified"}) + disk = snapshot["disks"].get(target) + if disk is not None: + layers["disk"] = disk + pct = int(str(disk.get("pct", "0")).rstrip("%") or 0) + if pct >= 80: + findings.append({"severity": "critical" if pct >= 90 else "warning", "code": "disk_high", "message": f"Root filesystem usage is {pct}%"}) + if not layers: + raise HTTPException(404, "Target not found") + state = "critical" if any(item["severity"] == "critical" for item in findings) else "warning" if findings else "healthy" + return {"target": target, "state": state, "findings": findings, "layers": layers, "next_checks": [f"/logs/{target}/{{container}}", f"/system/forensics/{target}"] if "host" in layers else []} + + +@app.get("/drift") +async def drift(_=Depends(_verify)): + """Report coverage drift between inventory, backup and disk collectors.""" + snapshot = await _collect_operational_snapshot() + inventory = set(snapshot["hosts"]) + managed_hosts = {name for name in inventory if not name.startswith("node")} + backups = set(snapshot["backups"].get("hosts", {})) + disks = set(snapshot["disks"]) + findings = [] + for name in sorted(managed_hosts - backups): + findings.append({"severity": "warning", "code": "inventory_missing_backup", "target": name}) + for name in sorted(inventory - disks): + findings.append({"severity": "warning", "code": "inventory_missing_disk", "target": name}) + for name in sorted(backups - inventory): + findings.append({"severity": "warning", "code": "backup_without_inventory", "target": name}) + return { + "state": "warning" if findings else "healthy", + "findings": findings, + "coverage": {"inventory": len(inventory), "backups": len(backups), "disks": len(disks)}, + "model_contract": {"instruction": "Treat findings as coverage gaps, not proof that the target is offline."}, + } + + +async def _collect_active_backups(concurrency: int = 10) -> dict: + hosts = [host for host in await _get_inventory_hosts_async() if not host["name"].startswith("node")] + exempt = set(_config.get("backup", {}).get("exempt_hosts", [])) + semaphore = asyncio.Semaphore(concurrency) + + async def check(host: dict): + if host["name"] in exempt: + return host["name"], "exempt" + async with semaphore: + rc, out, _err = await asyncio.to_thread( + _ssh, + f'{host["user"]}@{host["ip"]}', + "sudo -n systemctl is-active borg-backup.service 2>/dev/null || true", + 10, + ) + state = out.strip().splitlines()[-1] if out.strip() else "unknown" + return host["name"], state if rc == 0 else "unknown" + + return dict(await asyncio.gather(*(check(host) for host in hosts))) + + +@app.get("/maintenance/preflight") +async def maintenance_preflight( + action: Literal["general", "docker", "network", "vm"] = Query("general"), + target: str | None = Query(None), + _=Depends(_verify), +): + """Read-only safety gate before maintenance or mutations.""" + if target and not re.fullmatch(r"[A-Za-z0-9_.-]+", target): + raise HTTPException(400, "Invalid target") + snapshot, active_backups = await asyncio.gather(_collect_operational_snapshot(), _collect_active_backups()) + if target and target not in snapshot["hosts"] and target not in snapshot["services"]: + raise HTTPException(404, "Target not found") + selected = {target} if target else set(snapshot["hosts"]) + blockers = [] + warnings = [] + for name in sorted(selected): + host = snapshot["hosts"].get(name) + if host and not host.get("reachable"): + blockers.append({"code": "host_unreachable", "target": name}) + if host and any("unhealthy" in line.lower() or "restarting" in line.lower() for line in host.get("containers", [])): + blockers.append({"code": "container_unhealthy", "target": name}) + if active_backups.get(name) in ("active", "activating"): + blockers.append({"code": "backup_active", "target": name}) + backup = snapshot["backups"].get("hosts", {}).get(name, {}) + if backup.get("state") not in (None, "healthy", "exempt"): + warnings.append({"code": "backup_unhealthy", "target": name}) + disk = snapshot["disks"].get(name, {}) + pct = int(str(disk.get("pct", "0")).rstrip("%") or 0) + if pct >= 80: + warnings.append({"code": "disk_high", "target": name, "pct": pct}) + return { + "safe": not blockers, + "action": action, + "target": target, + "blockers": blockers, + "warnings": warnings, + "model_contract": {"instruction": "Do not start the requested maintenance while safe is false."}, + } + + @app.get("/overview", response_model=OverviewResponse, response_model_exclude_none=True) async def overview(details: bool = Query(False), _=Depends(_verify)): """Compact deterministic homelab verdict designed for small language models.""" @@ -767,7 +1416,7 @@ async def overview(details: bool = Query(False), _=Depends(_verify)): for name in sorted(backups.get("hosts", {})): item = backups["hosts"][name] raw_state = item.get("state", "unknown") - state = "healthy" if raw_state == "healthy" else "critical" if raw_state in ("critical", "unknown") else "warning" + state = "healthy" if raw_state in ("healthy", "exempt") else "critical" if raw_state in ("critical", "unknown") else "warning" _add_component(summary, component_summary["backups"], state) if state != "healthy": findings.append({ @@ -897,6 +1546,102 @@ async def vault_reload(_=Depends(_verify)): return {"reloaded": True, "items": len(_vault_cache)} +# --- Media file integrity verification --- + +MEDIA_VERIFY_PROBES = { + "arrapps": ("bazarrUHD", "sascha"), + "arr-chris": (None, "chris"), + "arr-chris-live": (None, "chris"), +} + + +class MediaVerifyRequest(BaseModel): + host: str = Field(..., max_length=64) + path: str = Field(..., max_length=1000) + expected_minutes: float | None = Field(None, ge=0, le=1440) + tolerance_pct: float = Field(20.0, gt=0, le=100) + + +@app.post("/media/verify") +async def media_verify(payload: MediaVerifyRequest, _=Depends(_verify)): + """Probe a media file's real duration via ffprobe to catch truncated imports. + + Runs `ffprobe` inside an existing container (bazarrUHD on arrapps, which already + mounts /data and ships ffmpeg) so no new service is required. Compares the + measured duration against an expected runtime (minutes) supplied by the caller + (e.g. Sonarr/Radarr's runtime field) within tolerance_pct. + """ + if not re.fullmatch(r"[A-Za-z0-9_.-]+", payload.host): + raise HTTPException(400, "Invalid host name") + if payload.host not in MEDIA_VERIFY_PROBES: + raise HTTPException(400, f"No ffprobe container configured for host {payload.host}") + if not re.fullmatch(r"/data/[^\x00]+\.(mkv|mp4|avi|m4v|ts)", payload.path): + raise HTTPException(400, "Path must be an absolute /data media file") + if ".." in payload.path or "\n" in payload.path or "\x00" in payload.path: + raise HTTPException(400, "Path contains unsafe characters") + + container, _default_user = MEDIA_VERIFY_PROBES[payload.host] + if not container: + raise HTTPException(400, f"Host {payload.host} has no configured ffprobe container yet") + + target = _find_inventory_host(payload.host) + if not target: + raise HTTPException(404, f"Host {payload.host} not found in inventory") + + # expected_minutes == 0 or None: no reference runtime available, skip verification gracefully + if not payload.expected_minutes or payload.expected_minutes <= 0: + return { + "host": payload.host, + "path": payload.path, + "duration_seconds": None, + "duration_minutes": None, + "expected_minutes": payload.expected_minutes, + "verified": False, + "suspect": False, + "skipped": True, + "skip_reason": "no_reference_runtime", + "deviation_pct": None, + } + + probe_cmd = ( + f"docker exec {container} ffprobe -v error " + f"-show_entries format=duration -of default=noprint_wrappers=1:nokey=1 {shlex.quote(payload.path)}" + ) + rc, out, err = await asyncio.to_thread( + _ssh, f'{target["user"]}@{target["ip"]}', f"sudo -n {probe_cmd} || {probe_cmd}", 30 + ) + if rc != 0: + _audit("/media/verify", "POST", 502, f"host={payload.host} path={payload.path}") + raise HTTPException(502, (err or out).strip()[:500] or "ffprobe failed") + + raw_duration = out.strip() + try: + duration_seconds = float(raw_duration) + except ValueError: + _audit("/media/verify", "POST", 502, f"host={payload.host} unparsable duration") + raise HTTPException(502, "ffprobe returned no parsable duration; file is likely corrupt") + + duration_minutes = duration_seconds / 60 + result = { + "host": payload.host, + "path": payload.path, + "duration_seconds": round(duration_seconds, 1), + "duration_minutes": round(duration_minutes, 2), + "expected_minutes": payload.expected_minutes, + "verified": True, + "suspect": False, + } + if payload.expected_minutes: + deviation_pct = abs(duration_minutes - payload.expected_minutes) / payload.expected_minutes * 100 + result["deviation_pct"] = round(deviation_pct, 1) + result["suspect"] = deviation_pct > payload.tolerance_pct + _audit( + "/media/verify", "POST", 200, + f"host={payload.host} dur={duration_minutes:.1f}m suspect={result['suspect']}", + ) + return result + + # --- VPS reverse-proxy and DNS management --- VPS_SSH = "root@46.225.230.72" @@ -925,6 +1670,13 @@ class ProxyRouteRequest(BaseModel): dns_token: str | None = None +class DnsRrsetUpsertRequest(BaseModel): + record_type: Literal["A", "AAAA", "CNAME"] = "A" + value: str + ttl: int = Field(default=300, ge=60, le=86400) + comment: str = Field(default="Managed by Homelab Butler", max_length=200) + + def _remote_python(script: str, timeout: int = 30) -> tuple[int, str, str]: encoded = base64.b64encode(script.encode()).decode() return _ssh(VPS_SSH, f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"', timeout=timeout) @@ -980,10 +1732,34 @@ print(backup) return {"status": "reloaded", "backup": backup} +def _normalize_hetzner_dns_token(raw: str) -> str | None: + raw = (raw or "").strip() + if not raw: + return None + if raw.isascii() and not any(ch.isspace() for ch in raw) and re.fullmatch(r"[A-Za-z0-9._-]{24,}", raw): + return raw + candidates = [] + labelled = re.findall(r"(?is)(?:token|api[ -]?key)[^\n:=]{0,80}(?::|=|\n)\s*([A-Za-z0-9._-]{24,})", raw) + candidates.extend(value for value in labelled if value.isascii()) + broad = re.findall(r"(? str: - token = _read("HETZNER_DNS_TOKEN") + token = _normalize_hetzner_dns_token(_read("HETZNER_DNS_TOKEN") or "") if token: return token + aliases = [_normalize_hetzner_dns_token(value) for key, value in _vault_cache.items() if "hetzner" in key.lower() and "dns" in key.lower()] + aliases = [value for value in aliases if value] + if len(set(aliases)) == 1: + return aliases[0] rc, _out, err = _ssh( "sascha@10.4.1.116", "sudo bash /data/stacks/homelab-butler/vault-sync.sh", @@ -1054,6 +1830,34 @@ SPEEDTEST_REPO_FILES = ( "streamscope/static/assets/longterm-metrics.js", ) +GUCK_ADMIN_REPO_FILES = ( + "guck-admin/Dockerfile", + "guck-admin/compose.yaml", + "guck-admin/requirements.txt", + "guck-admin/src/app.py", + "guck-admin/src/control.py", + "guck-admin/src/sharing_watchdog.py", + "guck-admin/src/templates/bandwidth.html", + "guck-admin/src/templates/base.html", + "guck-admin/src/templates/dashboard.html", + "guck-admin/src/templates/history.html", + "guck-admin/src/templates/sessions.html", + "guck-admin/src/templates/settings.html", + "guck-admin/src/templates/sharing.html", + "guck-admin/src/templates/users.html", + "guck-admin/static/admin.css", + "guck-admin/static/admin.js", + "guck-admin/static/icon.svg", + "guck-admin/static/manifest.webmanifest", + "guck-admin/static/sw.js", + "guck-admin/static/world.svg", +) + +FORGEJO_DEPLOY_FILES = { + "sascha/speedtest": frozenset(SPEEDTEST_REPO_FILES), + "sascha/guck-vps": frozenset(GUCK_ADMIN_REPO_FILES), +} + class SpeedtestDeployRequest(BaseModel): stats_password: str @@ -1061,7 +1865,7 @@ class SpeedtestDeployRequest(BaseModel): async def _fetch_forgejo_text(repo: str, path: str) -> str: - if repo != "sascha/speedtest" or path not in SPEEDTEST_REPO_FILES: + if path not in FORGEJO_DEPLOY_FILES.get(repo, frozenset()): raise ValueError("unsupported Forgejo file") cfg = SERVICES.get("forgejo", {}) base_url = cfg.get("url") @@ -1165,6 +1969,143 @@ async def vps_speedtest_deploy(req: SpeedtestDeployRequest, _=Depends(_verify)): return result +class GuckAdminDeployRequest(BaseModel): + dry_run: bool = True + + +def _validate_guck_admin_bundle(files: dict[str, str]) -> None: + if set(files) != set(GUCK_ADMIN_REPO_FILES): + raise ValueError("guck-admin source bundle is incomplete") + compose = files["guck-admin/compose.yaml"] + control = files["guck-admin/src/control.py"] + compose_required = ( + "network_mode: host", + "NET_ADMIN", + "/app-config/guck-admin/data:/data", + "GUCK_LIMIT: /host/guck-limit.sh", + ) + if any(item not in compose for item in compose_required): + raise ValueError("guck-admin compose is missing a required security or persistence setting") + policy_required = ( + "CREATE TABLE IF NOT EXISTS custom_networks", + "def sync_custom_networks", + "2a00:8c40:f000::/36", + "45.58.235.0/24", + ) + if any(item not in control for item in policy_required): + raise ValueError("guck-admin custom VPN policy is incomplete") + + +def _guck_admin_remote_python(target: str, script: str, timeout: int = 60): + encoded = base64.b64encode(script.encode()).decode() + command = f"sudo -n python3 -c \"import base64;exec(base64.b64decode('{encoded}'))\"" + return _ssh(target, command, timeout=timeout) + + +def _deploy_guck_admin_compose(files: dict[str, str], dry_run: bool = True) -> dict: + _validate_guck_admin_bundle(files) + inventory = _find_inventory_host("guck-vps") + if not inventory: + raise RuntimeError("guck-vps is missing from Butler inventory") + target = f'{inventory["user"]}@{inventory["ip"]}' + if dry_run: + return { + "status": "validated", + "dry_run": True, + "host": "guck-vps", + "files": len(files), + "policy_networks": ["Mozilla Firefox VPN IPv6", "Fastly VPN IPv4"], + } + relative_files = {path.removeprefix("guck-admin/"): content for path, content in files.items()} + deploy_script = f"""from pathlib import Path +import os, shutil +stack = Path('/app-config/guck-admin') +backup = Path('/app-config/deployment-backups/guck-admin-rollback') +files = {relative_files!r} +if backup.exists(): + shutil.rmtree(backup) +backup.mkdir(parents=True, exist_ok=True) +stack.mkdir(parents=True, exist_ok=True) +for relative, content in files.items(): + target = stack / relative + old = backup / relative + if target.exists(): + old.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(target, old) + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + '.butler-new') + temporary.write_text(content) + os.replace(temporary, target) +""" + rollback_script = f"""from pathlib import Path +import os, shutil +stack = Path('/app-config/guck-admin') +backup = Path('/app-config/deployment-backups/guck-admin-rollback') +files = {tuple(relative_files)!r} +for relative in files: + target = stack / relative + old = backup / relative + if old.exists(): + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(old, target) + elif target.exists(): + target.unlink() +""" + rc, _out, err = _guck_admin_remote_python(target, deploy_script, timeout=90) + if rc != 0: + raise RuntimeError(f"guck-admin file deployment failed: {err[-300:]}") + + def rollback(): + _guck_admin_remote_python(target, rollback_script, timeout=90) + _ssh(target, "cd /app-config/guck-admin && sudo -n docker compose up -d --build --remove-orphans", timeout=600) + + preflight = "cd /app-config/guck-admin && sudo -n docker compose config -q && sudo -n docker compose build --pull" + rc, _out, err = _ssh(target, preflight, timeout=600) + if rc != 0: + rollback() + raise RuntimeError(f"guck-admin build preflight failed: {err[-500:]}") + deploy = "cd /app-config/guck-admin && sudo -n docker compose up -d --remove-orphans" + rc, out, err = _ssh(target, deploy, timeout=240) + if rc != 0: + rollback() + raise RuntimeError(f"guck-admin deployment failed: {(err or out)[-500:]}") + health = "for i in $(seq 1 45); do curl -fsS --max-time 3 http://127.0.0.1:9090/health >/dev/null && exit 0; sleep 2; done; exit 1" + rc, _out, err = _ssh(target, health, timeout=105) + if rc != 0: + rollback() + raise RuntimeError(f"guck-admin health check failed and rollback was attempted: {err[-300:]}") + policy = "curl -fsS -X POST --max-time 120 http://127.0.0.1:9090/actions/limiter/refresh >/dev/null && sudo -n ipset test vpn-v6 2a00:8c40:f02d:a34c::1 && sudo -n ipset test vpn-v4 45.58.235.7 && sudo -n tc class show dev ens3 | python3 -c \"import sys; s=sys.stdin.read(); raise SystemExit(0 if '1:300' in s else 1)\"" + rc, out, err = _ssh(target, policy, timeout=180) + if rc != 0: + rollback() + raise RuntimeError(f"guck-admin policy verification failed and rollback was attempted: {(err or out)[-500:]}") + return { + "status": "deployed", + "health": "ok", + "policy": "verified", + "host": "guck-vps", + "custom_networks": ["2a00:8c40:f000::/36", "45.58.235.0/24"], + "ipv4": True, + "ipv6": True, + } + + +@app.post("/vps/guck-admin/deploy") +async def vps_guck_admin_deploy(req: GuckAdminDeployRequest, _=Depends(_verify)): + try: + contents = await asyncio.gather(*( + _fetch_forgejo_text("sascha/guck-vps", path) for path in GUCK_ADMIN_REPO_FILES + )) + files = dict(zip(GUCK_ADMIN_REPO_FILES, contents)) + result = await asyncio.to_thread(_deploy_guck_admin_compose, files, req.dry_run) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc + except Exception as exc: + raise HTTPException(502, f"guck-admin deployment failed: {str(exc)[-500:]}") from exc + _audit("/vps/guck-admin/deploy", "POST", 200, f"dry_run={req.dry_run}; Git-managed custom VPN policy") + return result + + # --- VM Lifecycle Endpoints --- import subprocess as _sp @@ -1189,6 +2130,737 @@ def _ssh(host, cmd, timeout=600): return 124, "", f"SSH command timed out after {timeout} seconds" +PAPERLESS_GATEWAY = AUTOMATION1 +PAPERLESS_SSH = "sascha@10.5.1.120" +PAPERLESS_CONSUME_DIR = "/app-config/paperless/consume" +PAPERLESS_MAX_IMPORT_BYTES = 50 * 1024 * 1024 + + +def _ssh_bytes(host: str, cmd: str, payload: bytes, timeout: int = 120): + """Send a binary payload to a fixed remote command over Butler-managed SSH.""" + try: + result = _sp.run( + ["ssh", "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=accept-new", + "-o", "UserKnownHostsFile=/tmp/butler_known_hosts", host, cmd], + input=payload, capture_output=True, timeout=timeout, + ) + return result.returncode, result.stdout.decode(errors="replace"), result.stderr.decode(errors="replace") + except _sp.TimeoutExpired: + return 124, "", f"SSH upload timed out after {timeout} seconds" + + +def _paperless_import_filename(filename: str, digest: str) -> str: + base_name = os.path.basename(filename or "document.pdf") + stem = re.sub(r"[^A-Za-z0-9._-]+", "_", base_name.rsplit(".", 1)[0]).strip("._-") + if not stem: + stem = "document" + return f"{stem[:100]}-{digest[:12]}.pdf" + + +def _paperless_gateway_command(command: str) -> str: + """Route through automation1, whose deployment key is authorized on Homelab VMs.""" + if "'" in command: + raise ValueError("Paperless remote command contains an unsafe quote") + return ( + "ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new " + f"-o UserKnownHostsFile=/tmp/paperless_known_hosts {PAPERLESS_SSH} '{command}'" + ) + + +@app.post("/paperless/import") +async def paperless_import(request: Request, filename: str = Query(..., min_length=1, max_length=180), _=Depends(_verify)): + """Queue a PDF in Paperless without exposing Paperless or SSH to the caller.""" + payload = await request.body() + if not payload or not payload.startswith(b"%PDF-"): + raise HTTPException(400, "Only valid PDF documents are accepted") + if len(payload) > PAPERLESS_MAX_IMPORT_BYTES: + raise HTTPException(413, "PDF exceeds the 50 MiB import limit") + digest = hashlib.sha256(payload).hexdigest() + import_name = _paperless_import_filename(filename, digest) + remote_path = f"{PAPERLESS_CONSUME_DIR}/{import_name}" + command = ( + f"sudo -n install -d -o sascha -g sascha -m 0755 {PAPERLESS_CONSUME_DIR} && " + f"tmp=$(mktemp /tmp/paperless-import.XXXXXX) && " + f"cat > \"$tmp\" && sudo -n install -o sascha -g sascha -m 0644 \"$tmp\" {remote_path} && rm -f \"$tmp\"" + ) + rc, out, err = await asyncio.to_thread( + _ssh_bytes, PAPERLESS_GATEWAY, _paperless_gateway_command(command), payload, 180 + ) + if rc != 0: + raise HTTPException(502, (err or out).strip()[-500:] or "Paperless import transfer failed") + _audit("/paperless/import", "POST", 202, f"sha256={digest}; bytes={len(payload)}") + return {"status": "queued", "filename": import_name, "sha256": digest, "bytes": len(payload)} + + +@app.get("/paperless/import/status") +async def paperless_import_status(filename: str = Query(..., min_length=1, max_length=180), _=Depends(_verify)): + if not re.fullmatch(r"[A-Za-z0-9._-]+\.pdf", filename): + raise HTTPException(400, "Invalid import filename") + remote_path = f"{PAPERLESS_CONSUME_DIR}/{filename}" + command = ( + f"if sudo -n test -f {remote_path}; then echo QUEUED; else echo CONSUMED; fi; " + f"logs=$(sudo -n docker logs --since 15m paperless-ngx 2>&1); " + f"printf \"%s\\n\" \"$logs\" | grep -F -- {filename} | tail -20 || true; " + f"task=$(printf \"%s\\n\" \"$logs\" | grep -F -- {filename} | " + f"grep -oE \"\\[[0-9a-f]{{8}}\\]\" | tail -1 | tr -d \"[]\"); " + f"if test -n \"$task\"; then printf \"%s\\n\" \"$logs\" | grep -F -- \"[$task]\" | tail -20; fi" + ) + rc, out, err = await asyncio.to_thread( + _ssh, PAPERLESS_GATEWAY, _paperless_gateway_command(command), 45 + ) + if rc != 0: + raise HTTPException(502, (err or out).strip()[-500:] or "Paperless status check failed") + lines = out.splitlines() + state = lines[0].strip().lower() if lines else "unknown" + return {"status": state, "filename": filename, "recent_log": lines[1:]} + + +def _wireguard_status_command() -> str: + script = '''import json, subprocess + +def run(args): + proc = subprocess.run(args, text=True, capture_output=True, timeout=15) + if proc.returncode != 0: + raise RuntimeError((proc.stderr or proc.stdout).strip() or "command failed") + return proc.stdout.strip() + +result = {"interface": "wg0", "addresses": [], "listen_port": None, "service_active": False, "service_enabled": False, "routes": [], "peers": []} +result["service_active"] = subprocess.run(["systemctl", "is-active", "--quiet", "wg-quick@wg0"]).returncode == 0 +result["service_enabled"] = subprocess.run(["systemctl", "is-enabled", "--quiet", "wg-quick@wg0"]).returncode == 0 +try: + addr_data = json.loads(run(["ip", "-j", "address", "show", "dev", "wg0"])) + for item in addr_data: + for address in item.get("addr_info", []): + result["addresses"].append(address["local"] + "/" + str(address["prefixlen"])) + result["routes"] = json.loads(run(["ip", "-j", "route", "show", "dev", "wg0"])) + rows = run(["sudo", "-n", "wg", "show", "wg0", "dump"]).splitlines() + if rows: + interface = rows[0].split("\\t") + result["listen_port"] = int(interface[2]) + for raw in rows[1:]: + fields = raw.split("\\t") + result["peers"].append({ + "public_key": fields[0], + "endpoint": None if fields[2] == "(none)" else fields[2], + "allowed_ips": [] if fields[3] == "(none)" else fields[3].split(","), + "latest_handshake": int(fields[4]), + "rx_bytes": int(fields[5]), + "tx_bytes": int(fields[6]), + "persistent_keepalive": 0 if fields[7] == "off" else int(fields[7]), + }) +except Exception as exc: + result["error"] = str(exc)[:300] +print(json.dumps(result)) +''' + encoded = base64.b64encode(script.encode()).decode() + return f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"' + + +@app.get("/network/wireguard/{host}") +async def network_wireguard_status(host: str, _=Depends(_verify)): + """Return redacted WireGuard state without private or preshared keys.""" + if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,62}", host): + raise HTTPException(400, "Invalid host name") + inventory = await asyncio.to_thread(_find_inventory_host, host) + if not inventory: + raise HTTPException(404, f"Host {host} not found") + target = f'{inventory["user"]}@{inventory["ip"]}' + rc, out, err = await asyncio.to_thread(_ssh, target, _wireguard_status_command(), 30) + if rc != 0: + raise HTTPException(502, (err or out).strip()[-500:] or "WireGuard status failed") + try: + result = json.loads(out) + except json.JSONDecodeError as exc: + raise HTTPException(502, "WireGuard status returned invalid JSON") from exc + _audit(f"/network/wireguard/{host}", "GET", 200, "redacted live status") + return {"host": host, **result} + + +class WireGuardPeerRemoveRequest(BaseModel): + public_key: str + expected_allowed_ip: str + dry_run: bool = True + + +def _wireguard_remove_peer_command(public_key: str, expected_allowed_ip: str, dry_run: bool) -> str: + script = f'''import json, os, re, shutil, subprocess, time +from pathlib import Path + +public_key = {public_key!r} +expected = {expected_allowed_ip!r} +dry_run = {dry_run!r} +config = Path("/etc/wireguard/wg0.conf") +text = config.read_text() +sections = re.split(r"(?=^\\[Peer\\]\\s*$)", text, flags=re.M) +matches = [] +for index, section in enumerate(sections): + key_match = re.search(r"^PublicKey\\s*=\\s*(\\S+)\\s*$", section, re.M) + allowed_match = re.search(r"^AllowedIPs\\s*=\\s*(.+?)\\s*$", section, re.M) + allowed = [item.strip() for item in allowed_match.group(1).split(",")] if allowed_match else [] + if key_match and key_match.group(1) == public_key and expected in allowed: + matches.append((index, allowed)) +if len(matches) != 1: + print(json.dumps({{"error": "expected exactly one matching peer", "matches": len(matches)}})); raise SystemExit(2) +index, allowed = matches[0] +result = {{"status": "would_remove" if dry_run else "removed", "allowed_ips": allowed, "removed_routes": [], "backup": None}} +if dry_run: + print(json.dumps(result)); raise SystemExit(0) +backup = config.with_name("wg0.conf.butler-" + time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())) +shutil.copy2(config, backup) +result["backup"] = str(backup) +new_text = "".join(section for number, section in enumerate(sections) if number != index) +tmp = config.with_name("wg0.conf.butler-tmp") +tmp.write_text(new_text) +os.chmod(tmp, config.stat().st_mode) +os.chown(tmp, config.stat().st_uid, config.stat().st_gid) +os.replace(tmp, config) +try: + subprocess.run(["wg", "set", "wg0", "peer", public_key, "remove"], check=True, text=True, capture_output=True) + for route in allowed: + proc = subprocess.run(["ip", "route", "del", route, "dev", "wg0"], text=True, capture_output=True) + if proc.returncode == 0: result["removed_routes"].append(route) + peers = subprocess.run(["wg", "show", "wg0", "peers"], check=True, text=True, capture_output=True).stdout.split() + if public_key in peers: raise RuntimeError("peer still active") +except Exception: + shutil.copy2(backup, config) + raise +print(json.dumps(result)) +''' + encoded = base64.b64encode(script.encode()).decode() + return f'sudo -n python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"' + + +@app.delete("/network/wireguard/{host}/peer") +async def network_wireguard_remove_peer(host: str, req: WireGuardPeerRemoveRequest, _=Depends(_verify)): + allowed = {"guck-vps": "10.7.1.0/24", "pfannkuchen": "10.200.200.60/32"} + if host not in allowed: + raise HTTPException(403, "Peer removal is restricted to the obsolete OVH-Hetzner transit") + if req.expected_allowed_ip != allowed[host]: + raise HTTPException(400, "Unexpected AllowedIP for this host") + if not re.fullmatch(r"[A-Za-z0-9+/]{43}=", req.public_key): + raise HTTPException(400, "Invalid WireGuard public key") + inventory = await asyncio.to_thread(_find_inventory_host, host) + if not inventory: + raise HTTPException(404, f"Host {host} not found") + target = f'{inventory["user"]}@{inventory["ip"]}' + command = _wireguard_remove_peer_command(req.public_key, req.expected_allowed_ip, req.dry_run) + rc, out, err = await asyncio.to_thread(_ssh, target, command, 45) + if rc != 0: + raise HTTPException(502, (err or out).strip()[-500:] or "WireGuard peer removal failed") + try: + result = json.loads(out) + except json.JSONDecodeError as exc: + raise HTTPException(502, "WireGuard peer removal returned invalid JSON") from exc + _audit(f"/network/wireguard/{host}/peer", "DELETE", 200, f"dry_run={req.dry_run}") + return {"host": host, **result} + + +SASCHA_MEDIA_VPS_HOST = "pfannkuchen" +SASCHA_MEDIA_EMBY_HOST = "emby-sascha" +SASCHA_MEDIA_INTERFACE = "wg-media" +SASCHA_MEDIA_VPS_ADDRESS = "10.11.13.1/32" +SASCHA_MEDIA_EMBY_ADDRESS = "10.11.13.3/32" +SASCHA_MEDIA_PORT = 51821 +SASCHA_MEDIA_MTU = 1340 +SASCHA_MEDIA_CONFIRMATION = "DEPLOY_DIRECT_SASCHA_MEDIA_TUNNEL" + + +class SaschaMediaTunnelRequest(BaseModel): + dry_run: bool = True + confirmation: str | None = None + + +class SaschaMediaEdgeOptimizeRequest(BaseModel): + dry_run: bool = True + confirmation: str | None = None + + +def _sascha_media_edge_audit_command() -> str: + script = '''import json, re, subprocess ++from pathlib import Path ++ ++def run(args): ++ proc = subprocess.run(args, text=True, capture_output=True, timeout=30) ++ return {"rc": proc.returncode, "stdout": proc.stdout.strip(), "stderr": proc.stderr.strip()[-300:]} ++ ++result = {"hostname": "pfannkuchen", "caddy": {"container_running": False, "protocols": [], "protocols_explicit": False, "upstreams": [], "site_block": [], "emby_snippet": []}, "network": {}} ++inspect = run(["docker", "inspect", "caddy", "--format", "{{.State.Running}}"]) ++result["caddy"]["container_running"] = inspect["rc"] == 0 and inspect["stdout"] == "true" ++adapt = run(["docker", "exec", "caddy", "caddy", "adapt", "--config", "/etc/caddy/Caddyfile"]) ++if adapt["rc"] == 0: ++ try: ++ config = json.loads(adapt["stdout"]) ++ servers = config.get("apps", {}).get("http", {}).get("servers", {}) ++ explicit = [] ++ def walk(value, matched=False): ++ if isinstance(value, dict): ++ current = matched ++ host = value.get("host") ++ if isinstance(host, list) and "tv.sascha-lutz.de" in host: ++ current = True ++ if current and isinstance(value.get("dial"), str): ++ result["caddy"]["upstreams"].append(value["dial"]) ++ for child in value.values(): walk(child, current) ++ elif isinstance(value, list): ++ for child in value: walk(child, matched) ++ for server in servers.values(): ++ protocols = server.get("protocols") ++ if isinstance(protocols, list): explicit.extend(protocols) ++ walk(server) ++ result["caddy"]["protocols_explicit"] = bool(explicit) ++ result["caddy"]["protocols"] = sorted(set(explicit)) if explicit else ["h1", "h2", "h3"] ++ result["caddy"]["upstreams"] = sorted(set(result["caddy"]["upstreams"])) ++ except Exception as exc: ++ result["caddy"]["adapt_error"] = str(exc)[:200] ++else: ++ result["caddy"]["adapt_error"] = adapt["stderr"] or "caddy adapt failed" ++ ++path = Path("/app-config/caddy/Caddyfile") ++if path.exists(): ++ lines = path.read_text(encoding="utf-8", errors="replace").splitlines() ++ collecting = False; depth = 0; selected = [] ++ for line in lines: ++ if not collecting and re.match(r"^\\s*tv\\.sascha-lutz\\.de\\s*\\{", line): collecting = True ++ if collecting: ++ depth += line.count("{") - line.count("}") ++ if not re.search(r"(?i)(password|secret|token|private|basicauth|basic_auth|hash)", line): selected.append(line.strip()) ++ else: selected.append("[REDACTED SENSITIVE DIRECTIVE]") ++ if depth == 0: break ++ result["caddy"]["site_block"] = selected ++ collecting = False; depth = 0; selected = [] ++ for line in lines: ++ if not collecting and re.match(r"^\\s*\\(emby_config\\)\\s*\\{", line): collecting = True ++ if collecting: ++ depth += line.count("{") - line.count("}") ++ if not re.search(r"(?i)(password|secret|token|private|basicauth|basic_auth|hash)", line): selected.append(line.strip()) ++ else: selected.append("[REDACTED SENSITIVE DIRECTIVE]") ++ if depth == 0: break ++ result["caddy"]["emby_snippet"] = selected ++ ++result["network"]["wg_media_active"] = subprocess.run(["systemctl", "is-active", "--quiet", "wg-quick@wg-media"]).returncode == 0 ++result["network"]["wg_media_enabled"] = subprocess.run(["systemctl", "is-enabled", "--quiet", "wg-quick@wg-media"]).returncode == 0 ++result["network"]["udp_51821"] = "51821" in run(["ss", "-H", "-lun"]) ["stdout"] ++for name, target in (("legacy_route", "10.6.1.103"), ("direct_route", "10.11.13.3")): ++ result["network"][name] = run(["ip", "route", "get", target])["stdout"][:300] ++print(json.dumps(result)) ++'''.replace("\n+", "\n") + encoded = base64.b64encode(script.encode()).decode() + return f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"' + + +@app.get("/media/edge/sascha") +async def sascha_media_edge_audit(_=Depends(_verify)): + """Return a redacted snapshot of the dedicated Hetzner edge for tv.sascha-lutz.de.""" + inventory = await asyncio.to_thread(_find_inventory_host, SASCHA_MEDIA_VPS_HOST) + if not inventory: + raise HTTPException(404, "Hetzner media edge not found") + target = f'{inventory["user"]}@{inventory["ip"]}' + rc, out, err = await asyncio.to_thread(_ssh, target, _sascha_media_edge_audit_command(), 45) + if rc != 0: + raise HTTPException(502, (err or out).strip()[-500:] or "Sascha media edge audit failed") + try: + result = json.loads(out) + except json.JSONDecodeError as exc: + raise HTTPException(502, "Sascha media edge audit returned invalid JSON") from exc + _audit("/media/edge/sascha", "GET", 200, "redacted live media-edge snapshot") + return result + + +def _sascha_media_edge_optimize_command() -> str: + script = '''import hashlib, json, os, re, shutil, subprocess, time ++from pathlib import Path ++path = Path("/app-config/caddy/Caddyfile") ++text = path.read_text(encoding="utf-8") ++old_pattern = re.compile(r"(?m)^(\\s*import\\s+emby_config\\s+tv\\.sascha-lutz\\.de\\s+)10\\.6\\.1\\.103:8096(\\s*)$") ++new_pattern = re.compile(r"(?m)^\\s*import\\s+emby_config\\s+tv\\.sascha-lutz\\.de\\s+10\\.11\\.13\\.3:8096\\s*$") ++old_count = len(old_pattern.findall(text)); new_count = len(new_pattern.findall(text)) ++if old_count == 1: ++ text = old_pattern.sub(r"\\g<1>10.11.13.3:8096\\g<2>", text) ++elif not (old_count == 0 and new_count == 1): ++ raise RuntimeError("expected exactly one tv.sascha-lutz.de upstream") ++lines = text.splitlines(keepends=True) ++first = next((i for i, line in enumerate(lines) if line.strip() and not line.lstrip().startswith("#")), None) ++if first is None or lines[first].strip() != "{": ++ lines[0:0] = ["{\\n", " servers {\\n", " protocols h1 h2\\n", " }\\n", "}\\n", "\\n"] ++else: ++ depth = 0; global_end = None; servers_start = None; servers_end = None ++ for i in range(first, len(lines)): ++ stripped = lines[i].strip(); before = depth ++ if before == 1 and re.match(r"^servers(?:\\s+\\S+)?\\s*\\{$", stripped): servers_start = i ++ depth += lines[i].count("{") - lines[i].count("}") ++ if servers_start is not None and i > servers_start and depth == 1 and servers_end is None: servers_end = i ++ if i > first and depth == 0: global_end = i; break ++ if global_end is None: raise RuntimeError("unbalanced Caddy global options block") ++ if servers_start is None: ++ lines[global_end:global_end] = [" servers {\\n", " protocols h1 h2\\n", " }\\n"] ++ else: ++ if servers_end is None: raise RuntimeError("unbalanced Caddy servers block") ++ protocol_lines = [i for i in range(servers_start + 1, servers_end) if re.match(r"^\\s*protocols\\s+", lines[i])] ++ if len(protocol_lines) > 1: raise RuntimeError("multiple Caddy protocol directives") ++ if protocol_lines: lines[protocol_lines[0]] = re.sub(r"protocols\\s+.*", "protocols h1 h2", lines[protocol_lines[0]]) ++ else: lines.insert(servers_start + 1, " protocols h1 h2\\n") ++text = "".join(lines) ++if re.search(r"(?im)^\\s*header(?:_down)?\\s+Alt-Svc", text): raise RuntimeError("manual Alt-Svc directive requires review") ++stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) ++backup = path.with_name("Caddyfile.pre-sascha-media-" + stamp) ++candidate = path.with_name("Caddyfile.sascha-media-candidate") ++shutil.copy2(path, backup); candidate.write_text(text, encoding="utf-8"); os.chmod(candidate, path.stat().st_mode) ++def run(args, timeout=30): ++ proc = subprocess.run(args, text=True, capture_output=True, timeout=timeout) ++ if proc.returncode != 0: raise RuntimeError((proc.stderr or proc.stdout).strip()[-500:] or "command failed") ++ return proc.stdout.strip() ++try: ++ run(["docker", "cp", str(candidate), "caddy:/tmp/Caddyfile.sascha-media-candidate"]) ++ run(["docker", "exec", "caddy", "caddy", "validate", "--config", "/tmp/Caddyfile.sascha-media-candidate"]) ++ with path.open("w", encoding="utf-8") as handle: ++ handle.write(text); handle.flush(); os.fsync(handle.fileno()) ++ host_hash = hashlib.sha256(path.read_bytes()).hexdigest() ++ container_hash = run(["docker", "exec", "caddy", "sha256sum", "/etc/caddy/Caddyfile"]).split()[0] ++ if host_hash != container_hash: raise RuntimeError("host/container Caddyfile hash mismatch") ++ run(["docker", "exec", "caddy", "caddy", "validate", "--config", "/etc/caddy/Caddyfile"]) ++ run(["docker", "exec", "caddy", "caddy", "reload", "--config", "/etc/caddy/Caddyfile"]) ++ adapted = json.loads(run(["docker", "exec", "caddy", "caddy", "adapt", "--config", "/etc/caddy/Caddyfile"])) ++ protocols = [] ++ for server in adapted.get("apps", {}).get("http", {}).get("servers", {}).values(): protocols.extend(server.get("protocols", [])) ++ if sorted(set(protocols)) != ["h1", "h2"]: raise RuntimeError("Caddy did not load h1+h2-only protocols") ++ if not run(["curl", "-fsS", "--max-time", "15", "http://10.11.13.3:8096/System/Ping"]): raise RuntimeError("Emby direct-path ping was empty") ++except Exception: ++ with path.open("w", encoding="utf-8") as handle: ++ handle.write(backup.read_text(encoding="utf-8")); handle.flush(); os.fsync(handle.fileno()) ++ subprocess.run(["docker", "exec", "caddy", "caddy", "reload", "--config", "/etc/caddy/Caddyfile"], text=True, capture_output=True, timeout=30) ++ raise ++finally: ++ candidate.unlink(missing_ok=True) ++ subprocess.run(["docker", "exec", "caddy", "rm", "-f", "/tmp/Caddyfile.sascha-media-candidate"], text=True, capture_output=True) ++print(json.dumps({"status": "optimized", "upstream": "10.11.13.3:8096", "protocols": ["h1", "h2"], "backup": str(backup), "sha256": host_hash})) ++'''.replace("\n+", "\n") + encoded = base64.b64encode(script.encode()).decode() + return f'sudo -n python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"' + + +def _optimize_sascha_media_edge() -> dict: + return _ssh_json(_media_host_target(SASCHA_MEDIA_VPS_HOST), _sascha_media_edge_optimize_command(), 90) + + +@app.post("/media/edge/sascha/optimize") +async def optimize_sascha_media_edge(req: SaschaMediaEdgeOptimizeRequest, _=Depends(_verify)): + """Switch only tv.sascha-lutz.de to wg-media and disable HTTP/3 on its dedicated Hetzner edge.""" + plan = {"status": "would_optimize", "hostname": "tv.sascha-lutz.de", "upstream": "10.11.13.3:8096", "protocols": ["h1", "h2"], "zero_downtime_reload": True} + if req.dry_run: + _audit("/media/edge/sascha/optimize", "POST", 200, "dry_run=True", True) + return plan + if req.confirmation != "OPTIMIZE_TV_SASCHA_LUTZ_DE": + raise HTTPException(400, "confirmation must be OPTIMIZE_TV_SASCHA_LUTZ_DE") + try: + result = await asyncio.to_thread(_optimize_sascha_media_edge) + except Exception as exc: + _audit("/media/edge/sascha/optimize", "POST", 502, "optimization failed; Caddy rollback attempted") + raise HTTPException(502, str(exc)[-500:]) from exc + _audit("/media/edge/sascha/optimize", "POST", 200, "direct upstream and h1+h2 enabled") + return result + + +def _media_tunnel_key_command() -> str: + script = '''import json, os, subprocess ++from pathlib import Path ++root = Path("/app-config/wireguard-media") ++private = root / "private.key" ++public = root / "public.key" ++root.mkdir(parents=True, exist_ok=True) ++os.chmod(root, 0o700) ++created = not private.exists() ++if created: ++ key = subprocess.run(["wg", "genkey"], check=True, text=True, capture_output=True).stdout.strip() ++ private.write_text(key + "\\n") ++ os.chmod(private, 0o600) ++if not public.exists() or created: ++ pub = subprocess.run(["wg", "pubkey"], input=private.read_text(), check=True, text=True, capture_output=True).stdout.strip() ++ public.write_text(pub + "\\n") ++ os.chmod(public, 0o644) ++print(json.dumps({"public_key": public.read_text().strip(), "created": created})) ++'''.replace("\n+", "\n") + encoded = base64.b64encode(script.encode()).decode() + return f'sudo -n python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"' + + +def _media_tunnel_install_command(role: Literal["vps", "emby"], peer_public_key: str) -> str: + if not re.fullmatch(r"[A-Za-z0-9+/]{43}=", peer_public_key): + raise ValueError("Invalid WireGuard public key") + settings = { + "vps": { + "address": SASCHA_MEDIA_VPS_ADDRESS, + "peer": SASCHA_MEDIA_EMBY_ADDRESS, + "endpoint": None, + "keepalive": None, + "listen": SASCHA_MEDIA_PORT, + }, + "emby": { + "address": SASCHA_MEDIA_EMBY_ADDRESS, + "peer": SASCHA_MEDIA_VPS_ADDRESS, + "endpoint": f"46.225.230.72:{SASCHA_MEDIA_PORT}", + "keepalive": 15, + "listen": None, + }, + }[role] + script = f'''import json, os, shutil, subprocess, time ++from pathlib import Path ++root = Path("/app-config/wireguard-media") ++config = root / "wg-media.conf" ++etc = Path("/etc/wireguard/wg-media.conf") ++backup_dir = root / "backups" ++backup_dir.mkdir(parents=True, exist_ok=True) ++private = (root / "private.key").read_text().strip() ++listen = {settings['listen']!r} ++existed = config.exists() ++backup = None ++if existed: ++ backup = backup_dir / ("wg-media.conf." + time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())) ++ shutil.copy2(config, backup) ++lines = ["[Interface]", "Address = {settings['address']}", "MTU = {SASCHA_MEDIA_MTU}", "PrivateKey = " + private] ++if listen is not None: lines.append("ListenPort = " + str(listen)) ++if {role!r} == "vps": ++ lines.extend(["PostUp = iptables -C INPUT -p udp --dport {SASCHA_MEDIA_PORT} -j ACCEPT 2>/dev/null || iptables -I INPUT 1 -p udp --dport {SASCHA_MEDIA_PORT} -j ACCEPT", "PreDown = iptables -D INPUT -p udp --dport {SASCHA_MEDIA_PORT} -j ACCEPT 2>/dev/null || true"]) ++lines.extend(["", "[Peer]", "PublicKey = {peer_public_key}", "AllowedIPs = {settings['peer']}"]) ++if {settings['endpoint']!r}: lines.append("Endpoint = " + {settings['endpoint']!r}) ++if {settings['keepalive']!r}: lines.append("PersistentKeepalive = " + str({settings['keepalive']!r})) ++candidate = root / "wgmtest.conf" ++candidate.write_text("\\n".join(lines) + "\\n") ++os.chmod(candidate, 0o600) ++check = subprocess.run(["wg-quick", "strip", str(candidate)], text=True, capture_output=True) ++if check.returncode != 0: ++ candidate.unlink(missing_ok=True) ++ raise RuntimeError(check.stderr.strip() or "wg-quick validation failed") ++os.replace(candidate, config) ++os.chmod(config, 0o600) ++etc.parent.mkdir(parents=True, exist_ok=True) ++if etc.is_symlink() or etc.exists(): ++ if etc.is_symlink() and etc.resolve() == config.resolve(): pass ++ elif etc.exists(): ++ etc_backup = backup_dir / ("etc-wg-media.conf." + time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())) ++ shutil.move(etc, etc_backup) ++ etc.symlink_to(config) ++else: etc.symlink_to(config) ++proc = subprocess.run(["systemctl", "enable", "--now", "wg-quick@wg-media"], text=True, capture_output=True, timeout=30) ++if proc.returncode != 0: ++ if backup: shutil.copy2(backup, config) ++ else: config.unlink(missing_ok=True) ++ raise RuntimeError(proc.stderr.strip() or proc.stdout.strip() or "failed to start wg-media") ++print(json.dumps({{"role": {role!r}, "status": "configured", "address": {settings['address']!r}, "backup": str(backup) if backup else None, "existed": existed}})) ++'''.replace("\n+", "\n") + encoded = base64.b64encode(script.encode()).decode() + return f'sudo -n python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"' + + +def _media_tunnel_verify_command(source: str, destination: str, check_emby: bool = False) -> str: + extra = '' + if check_emby: + extra = '''\nhttp = subprocess.run(["curl", "-sS", "--max-time", "10", "-o", "/dev/null", "-w", "%{http_code}", "http://10.11.13.3:8096/System/Ping"], text=True, capture_output=True)\nresult["emby_http"] = http.stdout.strip() if http.returncode == 0 else "000"''' + script = f'''import json, subprocess, time ++result = {{"ping": False, "handshake": False}} ++for _ in range(10): ++ ping = subprocess.run(["ping", "-c", "1", "-W", "2", "-I", {source!r}, {destination!r}], text=True, capture_output=True) ++ hand = subprocess.run(["sudo", "-n", "wg", "show", "wg-media", "latest-handshakes"], text=True, capture_output=True) ++ now = int(time.time()) ++ stamps = [] ++ for line in hand.stdout.splitlines(): ++ try: stamps.append(int(line.split()[-1])) ++ except Exception: pass ++ result["ping"] = ping.returncode == 0 ++ result["handshake"] = any(stamp > 0 and now - stamp < 60 for stamp in stamps) ++ if result["ping"] and result["handshake"]: break ++ time.sleep(2) ++{extra} ++print(json.dumps(result)) ++'''.replace("\n+", "\n") + encoded = base64.b64encode(script.encode()).decode() + return f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"' + + +def _media_tunnel_rollback_command(remove_keys: bool) -> str: + script = f'''import json, shutil, subprocess ++from pathlib import Path ++root = Path("/app-config/wireguard-media") ++config = root / "wg-media.conf" ++backups = sorted((root / "backups").glob("wg-media.conf.*")) if (root / "backups").exists() else [] ++subprocess.run(["systemctl", "disable", "--now", "wg-quick@wg-media"], text=True, capture_output=True, timeout=30) ++if backups: ++ shutil.copy2(backups[-1], config) ++ subprocess.run(["systemctl", "enable", "--now", "wg-quick@wg-media"], text=True, capture_output=True, timeout=30) ++ status = "restored" ++else: ++ config.unlink(missing_ok=True) ++ Path("/etc/wireguard/wg-media.conf").unlink(missing_ok=True) ++ if {remove_keys!r}: ++ (root / "private.key").unlink(missing_ok=True); (root / "public.key").unlink(missing_ok=True) ++ status = "removed" ++print(json.dumps({{"status": status}})) ++'''.replace("\n+", "\n") + encoded = base64.b64encode(script.encode()).decode() + return f'sudo -n python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"' + + +def _media_tunnel_key_cleanup_command() -> str: + script = '''import json ++from pathlib import Path ++root = Path("/app-config/wireguard-media") ++if not (root / "wg-media.conf").exists(): ++ (root / "private.key").unlink(missing_ok=True) ++ (root / "public.key").unlink(missing_ok=True) ++ status = "new_keys_removed" ++else: ++ status = "kept_for_existing_config" ++print(json.dumps({"status": status})) ++'''.replace("\n+", "\n") + encoded = base64.b64encode(script.encode()).decode() + return f'sudo -n python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"' + + +def _media_host_target(name: str) -> str: + inventory = _find_inventory_host(name) + if not inventory: + raise RuntimeError(f"Inventory host missing: {name}") + return f'{inventory["user"]}@{inventory["ip"]}' + + +def _ssh_json(target: str, command: str, timeout: int = 45) -> dict: + rc, out, err = _ssh(target, command, timeout) + if rc != 0: + raise RuntimeError((err or out).strip()[-500:] or f"remote command failed on {target}") + try: + return json.loads(out) + except json.JSONDecodeError as exc: + raise RuntimeError(f"remote command returned invalid JSON on {target}") from exc + + +def _deploy_sascha_media_tunnel() -> dict: + vps = _media_host_target(SASCHA_MEDIA_VPS_HOST) + emby = _media_host_target(SASCHA_MEDIA_EMBY_HOST) + vps_key = _ssh_json(vps, _media_tunnel_key_command()) + emby_key = _ssh_json(emby, _media_tunnel_key_command()) + configured = [] + try: + vps_install = _ssh_json(vps, _media_tunnel_install_command("vps", emby_key["public_key"]), 60) + configured.append((vps, bool(vps_key.get("created")))) + emby_install = _ssh_json(emby, _media_tunnel_install_command("emby", vps_key["public_key"]), 60) + configured.append((emby, bool(emby_key.get("created")))) + vps_check = _ssh_json(vps, _media_tunnel_verify_command("10.11.13.1", "10.11.13.3", True), 35) + emby_check = _ssh_json(emby, _media_tunnel_verify_command("10.11.13.3", "10.11.13.1"), 35) + if not (vps_check.get("ping") and vps_check.get("handshake") and emby_check.get("ping") and emby_check.get("handshake")): + raise RuntimeError("direct media tunnel verification failed") + if vps_check.get("emby_http") not in {"200", "401"}: + raise RuntimeError("Emby did not answer through the direct media tunnel") + return { + "status": "deployed", "interface": SASCHA_MEDIA_INTERFACE, + "vps_address": SASCHA_MEDIA_VPS_ADDRESS, "emby_address": SASCHA_MEDIA_EMBY_ADDRESS, + "listen_port": SASCHA_MEDIA_PORT, "mtu": SASCHA_MEDIA_MTU, + "handshake": True, "ping_vps_to_emby": True, "ping_emby_to_vps": True, + "emby_http": vps_check.get("emby_http"), + "rollback_backups": [vps_install.get("backup"), emby_install.get("backup")], + } + except Exception: + for target, created in reversed(configured): + try: _ssh_json(target, _media_tunnel_rollback_command(created), 60) + except Exception: pass + installed_targets = {target for target, _created in configured} + for target, key in ((vps, vps_key), (emby, emby_key)): + if target not in installed_targets and key.get("created"): + try: _ssh_json(target, _media_tunnel_key_cleanup_command(), 30) + except Exception: pass + raise + + +@app.post("/network/media-tunnel/sascha") +async def deploy_sascha_media_tunnel(req: SaschaMediaTunnelRequest, _=Depends(_verify)): + """Deploy the fixed direct Hetzner-to-emby-sascha WireGuard media tunnel.""" + plan = { + "status": "would_deploy", "interface": SASCHA_MEDIA_INTERFACE, + "vps_address": SASCHA_MEDIA_VPS_ADDRESS, "emby_address": SASCHA_MEDIA_EMBY_ADDRESS, + "listen_port": SASCHA_MEDIA_PORT, "mtu": SASCHA_MEDIA_MTU, + "allowed_ips": [SASCHA_MEDIA_VPS_ADDRESS, SASCHA_MEDIA_EMBY_ADDRESS], + "keeps_legacy_node6_path": True, + } + if req.dry_run: + _audit("/network/media-tunnel/sascha", "POST", 200, "dry_run=True", True) + return plan + if req.confirmation != SASCHA_MEDIA_CONFIRMATION: + raise HTTPException(400, f"confirmation must be {SASCHA_MEDIA_CONFIRMATION}") + try: + result = await asyncio.to_thread(_deploy_sascha_media_tunnel) + except Exception as exc: + _audit("/network/media-tunnel/sascha", "POST", 502, "deployment failed; rollback attempted") + raise HTTPException(502, str(exc)[-500:]) from exc + _audit("/network/media-tunnel/sascha", "POST", 200, "direct media tunnel deployed") + return result + + +def _media_benchmark_server_command() -> str: + server = '''import socket ++payload = b"\\0" * (1024 * 1024) ++with socket.socket() as listener: ++ listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) ++ listener.bind(("0.0.0.0", 5209)); listener.listen(4); listener.settimeout(80) ++ for _ in range(2): ++ conn, _addr = listener.accept() ++ with conn: ++ conn.settimeout(30) ++ for _ in range(256): conn.sendall(payload) ++'''.replace("\n+", "\n") + encoded = base64.b64encode(server.encode()).decode() + command = f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"' + return ( + "sudo -n systemctl stop butler-media-benchmark.service >/dev/null 2>&1 || true; " + "sudo -n systemd-run --unit=butler-media-benchmark --collect --property=RuntimeMaxSec=90 " + f"/bin/sh -c {json.dumps(command)}" + ) + + +def _media_benchmark_client_command() -> str: + script = '''import json, socket, time ++results = {} ++for name, host in (("legacy_node6", "10.6.1.103"), ("direct_wg_media", "10.11.13.3")): ++ total = 0; started = time.monotonic() ++ with socket.create_connection((host, 5209), timeout=10) as conn: ++ conn.settimeout(40) ++ while True: ++ chunk = conn.recv(1024 * 1024) ++ if not chunk: break ++ total += len(chunk) ++ elapsed = time.monotonic() - started ++ results[name] = {"bytes": total, "seconds": round(elapsed, 3), "mbit_s": round(total * 8 / elapsed / 1000000, 1)} ++print(json.dumps(results)) ++'''.replace("\n+", "\n") + encoded = base64.b64encode(script.encode()).decode() + return f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"' + + +def _benchmark_sascha_media_paths() -> dict: + vps = _media_host_target(SASCHA_MEDIA_VPS_HOST) + emby = _media_host_target(SASCHA_MEDIA_EMBY_HOST) + rc, out, err = _ssh(emby, _media_benchmark_server_command(), 30) + if rc != 0: + raise RuntimeError((err or out).strip()[-500:] or "failed to start benchmark server") + time.sleep(2) + try: + result = _ssh_json(vps, _media_benchmark_client_command(), 90) + finally: + _ssh(emby, "sudo -n systemctl stop butler-media-benchmark.service >/dev/null 2>&1 || true", 20) + if any(item.get("bytes") != 256 * 1024 * 1024 for item in result.values()): + raise RuntimeError("benchmark transferred an unexpected byte count") + return result + + +@app.post("/network/media-tunnel/sascha/benchmark") +async def benchmark_sascha_media_paths(_=Depends(_verify)): + """Compare the legacy node6 route with the direct WireGuard media path using fixed transient TCP streams.""" + try: + result = await asyncio.to_thread(_benchmark_sascha_media_paths) + except Exception as exc: + _audit("/network/media-tunnel/sascha/benchmark", "POST", 502, "benchmark failed") + raise HTTPException(502, str(exc)[-500:]) from exc + _audit("/network/media-tunnel/sascha/benchmark", "POST", 200, "fixed 256 MiB TCP comparison") + return {"status": "completed", "direction": "emby-sascha_to_hetzner", "results": result} + + SYSCTL_AUDIT_KEYS = ( "net.core.default_qdisc", "net.core.rmem_default", @@ -1260,19 +2932,35 @@ async def system_sysctl_audit(host: str, _=Depends(_verify)): def _host_forensics_command(since_hours: int) -> str: - script = f'''import glob, json, os, subprocess + script = f'''import glob, json, os, re, subprocess def run(command): proc = subprocess.run(command, shell=True, text=True, capture_output=True, timeout=30) return {{"rc": proc.returncode, "stdout": proc.stdout.strip()[-12000:], "stderr": proc.stderr.strip()[-1000:]}} -def redacted_git_diff(): - proc = subprocess.run(["git", "-C", "/app-config/ansible", "diff", "--", "iso-builder/build-iso.sh", "iso-builder/preseed.cfg.tpl", "pfannkuchen.ini"], text=True, capture_output=True, timeout=30) - sensitive = ("password", "passwd", "secret", "token", "private", "credential", "ssh-rsa", "ssh-ed25519") +def safe_git_diff(paths): + proc = subprocess.run(["git", "-C", "/app-config/ansible", "diff", "--"] + paths, text=True, capture_output=True, timeout=30) + sensitive = re.compile(r"pass|secret|token|api[_-]?key|private[_-]?key", re.I) lines = [] for line in proc.stdout.splitlines(): - lines.append("[REDACTED SENSITIVE DIFF LINE]" if any(word in line.lower() for word in sensitive) else line) - return {{"rc": proc.returncode, "stdout": "\\n".join(lines)[-12000:], "stderr": proc.stderr.strip()[-1000:]}} + lines.append("[REDACTED SENSITIVE DIFF LINE]" if sensitive.search(line) else line) + return {{"rc": proc.returncode, "stdout": "\\n".join(lines)[-12000:], "stderr": proc.stderr.strip()[-4000:]}} + +def kuma_outline_monitors(): + path = "/app-config/kuma/kuma.db" + if not os.path.exists(path): + return {{"rc": 0, "stdout": "[]", "stderr": ""}} + try: + import sqlite3 + connection = sqlite3.connect("file:" + path + "?mode=ro", uri=True) + columns = [row[1] for row in connection.execute("pragma table_info(monitor)")] + wanted = [name for name in ("id", "name", "url", "hostname", "active") if name in columns] + rows = [dict(zip(wanted, row)) for row in connection.execute("select " + ",".join(wanted) + " from monitor")] + selected = [row for row in rows if "outline" in json.dumps(row).lower() or "wiki.sascha-lutz.de" in json.dumps(row).lower()] + connection.close() + return {{"rc": 0, "stdout": json.dumps(selected), "stderr": ""}} + except Exception as exc: + return {{"rc": 1, "stdout": "", "stderr": str(exc)}} checks = {{ "hostname": run("hostnamectl --static 2>/dev/null || hostname"), @@ -1296,8 +2984,9 @@ checks = {{ "recent_iso_builder_files": run("find /app-config/ansible/iso-builder -type f -mmin -{since_hours * 60} -printf '%TY-%Tm-%Td %TH:%TM:%TS %p\\n' 2>/dev/null | sort"), "ansible_git_status": run("git -C /app-config/ansible status --short 2>/dev/null || true"), "minecraft_inventory": run("grep -in 'minecraft' /app-config/ansible/pfannkuchen.ini 2>/dev/null || true"), - "iso_builder_diff_redacted": redacted_git_diff(), + "iso_builder_diff_redacted": safe_git_diff(["iso-builder/build-iso.sh", "iso-builder/preseed.cfg.tpl", "pfannkuchen.ini"]), "iso_builder_hashes": run("sha256sum /app-config/ansible/iso-builder/* 2>/dev/null || true"), + "kuma_outline_monitors": kuma_outline_monitors(), }} print(json.dumps(checks)) ''' @@ -1325,6 +3014,418 @@ async def system_forensics(host: str, since_hours: int = Query(48, ge=1, le=168) return {"host": host, "since_hours": since_hours, "checks": result} +def _docker_residue_cleanup_command(dry_run: bool) -> str: + script = f'''import json, os, shlex, shutil, subprocess + +def run(args): + proc = subprocess.run(args, text=True, capture_output=True, timeout=30) + return {{"rc": proc.returncode, "stdout": proc.stdout.strip(), "stderr": proc.stderr.strip()}} + +def docker_rule_count(): + proc = run(["iptables-save"]) + return sum(1 for line in proc["stdout"].splitlines() if "docker" in line.lower()) + +result = {{"dry_run": {str(dry_run)}, "before_rule_count": docker_rule_count(), "removed_rules": [], "removed_chains": [], "removed_links": [], "removed_paths": [], "errors": []}} +docker_binary = shutil.which("docker") +unit_state = run(["systemctl", "is-active", "docker", "containerd"])["stdout"].splitlines() +if docker_binary or any(state == "active" for state in unit_state): + result["error"] = "Docker or containerd is still installed/active; refusing residue cleanup" + print(json.dumps(result)); raise SystemExit(2) +if result["dry_run"]: + result["would_remove_paths"] = [path for path in ("/var/lib/docker", "/var/lib/containerd", "/etc/docker") if os.path.exists(path)] + print(json.dumps(result)); raise SystemExit(0) +for table in ("filter", "nat"): + saved = run(["iptables-save", "-t", table]) + rules = [] + for line in saved["stdout"].splitlines(): + if line.startswith("-A ") and "docker" in line.lower(): + rules.append(line) + for line in rules: + args = ["iptables", "-t", table] + shlex.split(line) + args[3] = "-D" + removed = run(args) + if removed["rc"] == 0: + result["removed_rules"].append(table + ":" + line) + else: + result["errors"].append(table + ":" + line + ":" + removed["stderr"]) +for table, chains in (("filter", ("DOCKER-USER", "DOCKER-FORWARD", "DOCKER-BRIDGE", "DOCKER-CT", "DOCKER-INTERNAL", "DOCKER")), ("nat", ("DOCKER",))): + for chain in chains: + run(["iptables", "-t", table, "-F", chain]) + deleted = run(["iptables", "-t", table, "-X", chain]) + if deleted["rc"] == 0: + result["removed_chains"].append(table + ":" + chain) +for link in ("docker0", "docker_gwbridge"): + exists = run(["ip", "link", "show", link]) + if exists["rc"] == 0: + deleted = run(["ip", "link", "delete", link]) + if deleted["rc"] == 0: result["removed_links"].append(link) + else: result["errors"].append(link + ":" + deleted["stderr"]) +for path in ("/var/lib/docker", "/var/lib/containerd", "/etc/docker"): + if os.path.exists(path): + shutil.rmtree(path) + result["removed_paths"].append(path) +result["after_rule_count"] = docker_rule_count() +result["forward_rules"] = run(["iptables", "-S", "FORWARD"])["stdout"].splitlines() +print(json.dumps(result)) +if result["errors"] or result["after_rule_count"] != 0: raise SystemExit(1) +''' + encoded = base64.b64encode(script.encode()).decode() + return f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"' + + +@app.post("/system/cleanup/docker-residue/{host}") +async def cleanup_docker_residue(host: str, dry_run: bool = Query(True), _=Depends(_verify)): + """Remove only stale Docker firewall/data residue after Docker itself is absent.""" + if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,62}", host): + raise HTTPException(400, "Invalid host name") + inventory = await asyncio.to_thread(_find_inventory_host, host) + if not inventory: + raise HTTPException(404, f"Host {host} not found") + target = f'{inventory["user"]}@{inventory["ip"]}' + rc, out, err = await asyncio.to_thread(_ssh, target, _docker_residue_cleanup_command(dry_run), 90) + try: + result = json.loads(out) + except json.JSONDecodeError as exc: + raise HTTPException(502, (err or out).strip()[-500:] or "cleanup returned invalid JSON") from exc + if rc != 0: + raise HTTPException(409 if result.get("error") else 502, result) + _audit(f"/system/cleanup/docker-residue/{host}", "POST", 200, f"dry_run={dry_run}", dry_run=dry_run) + return {"host": host, **result} + + +def _iso_builder_restore_command(dry_run: bool) -> str: + script = f'''import glob, hashlib, json, os, subprocess, tempfile +repo = "/app-config/ansible" +paths = ("iso-builder/build-iso.sh", "iso-builder/preseed.cfg.tpl") +result = {{"dry_run": {str(dry_run)}, "restored": [], "removed_outputs": [], "validation": {{}}}} +def run(args): + proc = subprocess.run(args, cwd=repo, text=True, capture_output=True, timeout=60) + return {{"rc": proc.returncode, "stdout": proc.stdout.strip(), "stderr": proc.stderr.strip()}} +fetch = run(["git", "fetch", "origin", "master"]) +if fetch["rc"] != 0: + result["error"] = "git fetch failed"; result["detail"] = fetch["stderr"][-500:]; print(json.dumps(result)); raise SystemExit(1) +outputs = sorted(glob.glob(os.path.join(repo, "iso-builder/output/debian-13-minecraft*.iso"))) +result["would_remove_outputs"] = outputs +for path in paths: + blob = subprocess.run(["git", "show", "origin/master:" + path], cwd=repo, capture_output=True, timeout=30) + if blob.returncode != 0: + result["error"] = "missing canonical file " + path; print(json.dumps(result)); raise SystemExit(1) + current = open(os.path.join(repo, path), "rb").read() if os.path.exists(os.path.join(repo, path)) else b"" + result.setdefault("hashes", {{}})[path] = {{"live_before": hashlib.sha256(current).hexdigest(), "canonical": hashlib.sha256(blob.stdout).hexdigest()}} + if not result["dry_run"]: + destination = os.path.join(repo, path) + fd, temporary = tempfile.mkstemp(dir=os.path.dirname(destination)) + with os.fdopen(fd, "wb") as handle: handle.write(blob.stdout) + os.chmod(temporary, 0o755 if path.endswith(".sh") else 0o644) + os.replace(temporary, destination) + result["restored"].append(path) +if not result["dry_run"]: + for output in outputs: + os.remove(output); result["removed_outputs"].append(output) + syntax = run(["bash", "-n", "iso-builder/build-iso.sh"]) + diff = run(["git", "diff", "--quiet", "origin/master", "--", *paths]) + result["validation"] = {{"bash_syntax_rc": syntax["rc"], "canonical_diff_rc": diff["rc"]}} + if syntax["rc"] != 0 or diff["rc"] != 0: + result["error"] = "post-restore validation failed"; print(json.dumps(result)); raise SystemExit(1) +print(json.dumps(result)) +''' + encoded = base64.b64encode(script.encode()).decode() + return f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"' + + +@app.post("/system/restore/iso-builder") +async def restore_iso_builder(dry_run: bool = Query(True), _=Depends(_verify)): + """Restore only the canonical ISO-builder files and remove generated Minecraft ISOs.""" + rc, out, err = await asyncio.to_thread(_ssh, AUTOMATION1, _iso_builder_restore_command(dry_run), 120) + try: + result = json.loads(out) + except json.JSONDecodeError as exc: + raise HTTPException(502, (err or out).strip()[-500:] or "restore returned invalid JSON") from exc + if rc != 0: + raise HTTPException(502, result) + _audit("/system/restore/iso-builder", "POST", 200, f"dry_run={dry_run}", dry_run=dry_run) + return result + + +UPTIME_HOST = "sascha@10.5.85.5" +UPTIME_DB = "/app-config/kuma/kuma.db" + + +def _uptime_monitor_remove_command(monitor_id: int, expected_name: str, dry_run: bool) -> str: + script = '''import datetime, json, os, shutil, sqlite3, subprocess +monitor_id = %r +expected_name = %r +dry_run = %r +path = %r + +def docker(*args): + return subprocess.run(["sudo", "docker", *args], text=True, capture_output=True, timeout=60) + +connection = sqlite3.connect("file:" + path + "?mode=ro", uri=True) +connection.row_factory = sqlite3.Row +row = connection.execute("select id,name,url,active from monitor where id=?", (monitor_id,)).fetchone() +connection.close() +result = {"monitor_id": monitor_id, "expected_name": expected_name, "dry_run": dry_run, "found": dict(row) if row else None} +if not row: + print(json.dumps(result)); raise SystemExit(0) +if row["name"] != expected_name: + result["error"] = "Monitor name mismatch"; print(json.dumps(result)); raise SystemExit(2) +if dry_run: + print(json.dumps(result)); raise SystemExit(0) +stop = docker("stop", "kuma") +if stop.returncode != 0: + result["error"] = "Could not stop Kuma"; result["stderr"] = stop.stderr[-500:]; print(json.dumps(result)); raise SystemExit(3) +backup = path + ".pre-monitor-removal-" + datetime.datetime.now().strftime("%%Y%%m%%d-%%H%%M%%S") +try: + shutil.copy2(path, backup) + connection = sqlite3.connect(path) + connection.execute("pragma foreign_keys=off") + tables = [item[0] for item in connection.execute("select name from sqlite_master where type='table'")] + cleaned = [] + for table in tables: + if table == "monitor" or not table.replace("_", "").isalnum(): + continue + for fk in connection.execute('pragma foreign_key_list("' + table + '")'): + if fk[2] == "monitor" and fk[3].replace("_", "").isalnum(): + cursor = connection.execute('delete from "' + table + '" where "' + fk[3] + '"=?', (monitor_id,)) + if cursor.rowcount: + cleaned.append({"table": table, "rows": cursor.rowcount}) + deleted = connection.execute("delete from monitor where id=? and name=?", (monitor_id, expected_name)).rowcount + connection.commit(); connection.close() + result["backup"] = backup; result["dependencies_cleaned"] = cleaned; result["deleted"] = deleted +except Exception as exc: + shutil.copy2(backup, path) + result["error"] = str(exc) +finally: + start = docker("start", "kuma") + result["container_start_rc"] = start.returncode +if result.get("error") or result.get("deleted") != 1 or result["container_start_rc"] != 0: + print(json.dumps(result)); raise SystemExit(4) +connection = sqlite3.connect("file:" + path + "?mode=ro", uri=True) +result["remaining"] = connection.execute("select count(*) from monitor where id=?", (monitor_id,)).fetchone()[0] +connection.close() +print(json.dumps(result)) +''' % (monitor_id, expected_name, dry_run, UPTIME_DB) + encoded = base64.b64encode(script.encode()).decode() + return f"python3 -c \"import base64;exec(base64.b64decode('{encoded}'))\"" + + +@app.delete("/uptime/monitor/{monitor_id}") +async def uptime_monitor_remove(monitor_id: int, expected_name: str = Query(..., min_length=1, max_length=100), dry_run: bool = Query(True), _=Depends(_verify)): + if not re.fullmatch(r"[A-Za-z0-9 ._()-]+", expected_name): + raise HTTPException(400, "Invalid expected monitor name") + rc, out, err = _ssh(UPTIME_HOST, _uptime_monitor_remove_command(monitor_id, expected_name, dry_run), timeout=120) + try: + result = json.loads(out) + except Exception: + raise HTTPException(502, (err or out or "Uptime cleanup returned no JSON")[-1000:]) + if rc != 0: + raise HTTPException(409 if result.get("error") == "Monitor name mismatch" else 502, result) + _audit(f"/uptime/monitor/{monitor_id}", "DELETE", 200, f"dry_run={dry_run}") + return result + + +CADDY_HOST = "root@46.225.230.72" +CADDYFILE_PATH = "/app-config/caddy/Caddyfile" +HETZNER_DNS_API = "https://api.hetzner.cloud/v1" + + +def _validate_managed_hostname(hostname: str) -> str: + hostname = hostname.strip().lower().rstrip(".") + if not re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+", hostname): + raise HTTPException(400, "Invalid hostname") + if not hostname.endswith(".sascha-lutz.de"): + raise HTTPException(400, "Hostname is outside the managed zone") + return hostname + + +def _caddy_site_remove_command(hostname: str, dry_run: bool) -> str: + script = '''import datetime, hashlib, json, os, re, subprocess, sys +path = %r +hostname = %r +dry_run = %r + +def digest(data): + return hashlib.sha256(data.encode()).hexdigest() + +def run(args): + p = subprocess.run(args, text=True, capture_output=True, timeout=30) + return {"rc": p.returncode, "stdout": p.stdout.strip()[-2000:], "stderr": p.stderr.strip()[-2000:]} + +text = open(path, encoding="utf-8").read() +pattern = re.compile(r"(?m)^[ \\t]*" + re.escape(hostname) + r"[ \\t]*\\{") +match = pattern.search(text) +result = {"hostname": hostname, "dry_run": dry_run, "found": bool(match), "before_sha256": digest(text)} +if not match: + print(json.dumps(result)); raise SystemExit(0) +start = match.start(); depth = 0; end = None +for idx in range(match.end() - 1, len(text)): + if text[idx] == "{": depth += 1 + elif text[idx] == "}": + depth -= 1 + if depth == 0: + end = idx + 1 + while end < len(text) and text[end] in " \\t": end += 1 + while end < len(text) and text[end] == "\\n": end += 1 + break +if end is None: + result["error"] = "Unbalanced Caddy site block"; print(json.dumps(result)); raise SystemExit(2) +result["line_start"] = text.count("\\n", 0, start) + 1 +result["line_end"] = text.count("\\n", 0, end) + 1 +if dry_run: + print(json.dumps(result)); raise SystemExit(0) +new = text[:start] + text[end:] +backup = path + ".pre-outline-removal-" + datetime.datetime.now().strftime("%%Y%%m%%d-%%H%%M%%S") +open(backup, "w", encoding="utf-8").write(text) +with open(path, "w", encoding="utf-8") as f: + f.write(new); f.flush(); os.fsync(f.fileno()) +result["backup"] = backup +result["after_sha256"] = digest(new) +validation = run(["docker", "exec", "caddy", "caddy", "validate", "--config", "/etc/caddy/Caddyfile"]) +result["validation"] = validation +if validation["rc"] != 0: + with open(path, "w", encoding="utf-8") as f: + f.write(text); f.flush(); os.fsync(f.fileno()) + result["rolled_back"] = True; print(json.dumps(result)); raise SystemExit(3) +host_sha = run(["sha256sum", path]) +container_sha = run(["docker", "exec", "caddy", "sha256sum", "/etc/caddy/Caddyfile"]) +result["host_container_hash_match"] = bool(host_sha["stdout"] and container_sha["stdout"] and host_sha["stdout"].split()[0] == container_sha["stdout"].split()[0]) +if not result["host_container_hash_match"]: + with open(path, "w", encoding="utf-8") as f: + f.write(text); f.flush(); os.fsync(f.fileno()) + result["rolled_back"] = True; print(json.dumps(result)); raise SystemExit(4) +reload = run(["docker", "exec", "caddy", "caddy", "reload", "--config", "/etc/caddy/Caddyfile"]) +result["reload"] = reload +if reload["rc"] != 0: + with open(path, "w", encoding="utf-8") as f: + f.write(text); f.flush(); os.fsync(f.fileno()) + run(["docker", "exec", "caddy", "caddy", "reload", "--config", "/etc/caddy/Caddyfile"]) + result["rolled_back"] = True; print(json.dumps(result)); raise SystemExit(5) +container_text = run(["docker", "exec", "caddy", "sh", "-c", "cat /etc/caddy/Caddyfile"]) +result["hostname_absent"] = hostname not in container_text["stdout"] +print(json.dumps(result)) +''' % (CADDYFILE_PATH, hostname, dry_run) + encoded = base64.b64encode(script.encode()).decode() + return f"python3 -c \"import base64;exec(base64.b64decode('{encoded}'))\"" + + +@app.delete("/caddy/site/{hostname}") +async def caddy_site_remove(hostname: str, _=Depends(_verify), dry_run: bool = Query(True)): + hostname = _validate_managed_hostname(hostname) + rc, out, err = _ssh(CADDY_HOST, _caddy_site_remove_command(hostname, dry_run), timeout=90) + try: + result = json.loads(out) + except Exception: + raise HTTPException(502, (err or out or "Caddy removal returned no JSON")[-1000:]) + if rc != 0: + raise HTTPException(502, result) + _audit(f"/caddy/site/{hostname}", "DELETE", 200, f"dry_run={dry_run}") + return result + + +async def _hetzner_zone_and_rrsets(zone_name: str): + try: + token = await asyncio.to_thread(_get_hetzner_dns_token) + except RuntimeError as exc: + raise HTTPException(503, str(exc)) + headers = {"Authorization": f"Bearer {token}"} + async with httpx.AsyncClient(timeout=30) as client: + zones_response = await client.get(f"{HETZNER_DNS_API}/zones", headers=headers, params={"name": zone_name}) + if zones_response.status_code != 200: + raise HTTPException(502, f"Hetzner zones lookup failed: HTTP {zones_response.status_code}") + zones = zones_response.json().get("zones", []) + zone = next((item for item in zones if item.get("name") == zone_name), None) + if not zone: + raise HTTPException(404, "DNS zone not found") + rr_response = await client.get(f"{HETZNER_DNS_API}/zones/{zone['id']}/rrsets", headers=headers) + if rr_response.status_code != 200: + raise HTTPException(502, f"Hetzner RRSet lookup failed: HTTP {rr_response.status_code}") + return zone, rr_response.json().get("rrsets", []), headers + + +@app.put("/dns/rrset/{zone_name}/{record_name}") +async def dns_rrset_upsert(zone_name: str, record_name: str, req: DnsRrsetUpsertRequest, _=Depends(_verify)): + zone_name = zone_name.strip().lower().rstrip(".") + hostname = _validate_managed_hostname(f"{record_name}.{zone_name}") + record_name = hostname[: -(len(zone_name) + 1)] + record_type = req.record_type.upper() + value = req.value.strip().rstrip("." if record_type == "CNAME" else "") + try: + if record_type == "A" and ipaddress.ip_address(value).version != 4: + raise ValueError + if record_type == "AAAA" and ipaddress.ip_address(value).version != 6: + raise ValueError + if record_type == "CNAME": + _validate_managed_hostname(value) + except ValueError as exc: + raise HTTPException(400, f"Invalid {record_type} record value") from exc + zone, rrsets, headers = await _hetzner_zone_and_rrsets(zone_name) + existing = next((item for item in rrsets if item.get("name") == record_name and item.get("type") == record_type), None) + payload = { + "name": record_name, + "type": record_type, + "ttl": req.ttl, + "records": [{"value": value, "comment": req.comment}], + } + async with httpx.AsyncClient(timeout=30) as client: + if existing: + response = await client.put( + f"{HETZNER_DNS_API}/zones/{zone['id']}/rrsets/{record_name}/{record_type}", + headers=headers, + json=payload, + ) + else: + response = await client.post( + f"{HETZNER_DNS_API}/zones/{zone['id']}/rrsets", + headers=headers, + json=payload, + ) + if response.status_code not in {200, 201}: + raise HTTPException(502, f"Hetzner RRSet upsert failed: HTTP {response.status_code}") + verify = await client.get(f"{HETZNER_DNS_API}/zones/{zone['id']}/rrsets", headers=headers) + current = [item for item in verify.json().get("rrsets", []) if item.get("name") == record_name and item.get("type") == record_type] if verify.status_code == 200 else [] + if not current or not any(record.get("value") == value for record in current[0].get("records", [])): + raise HTTPException(502, "RRSet read-back does not contain requested value") + _audit(f"/dns/rrset/{zone_name}/{record_name}", "PUT", 200, f"type={record_type} value={value}") + return {"zone": zone_name, "zone_id": zone.get("id"), "name": record_name, "created": existing is None, "rrset": current[0]} + + +@app.get("/dns/rrset/{zone_name}/{record_name}") +async def dns_rrset_get(zone_name: str, record_name: str, _=Depends(_verify)): + zone_name = zone_name.strip().lower().rstrip(".") + hostname = _validate_managed_hostname(f"{record_name}.{zone_name}") + record_name = hostname[: -(len(zone_name) + 1)] + zone, rrsets, _headers = await _hetzner_zone_and_rrsets(zone_name) + selected = [r for r in rrsets if r.get("name") == record_name and r.get("type") in {"A", "AAAA", "CNAME"}] + return {"zone": zone_name, "zone_id": zone.get("id"), "name": record_name, "rrsets": selected} + + +@app.delete("/dns/rrset/{zone_name}/{record_name}") +async def dns_rrset_delete(zone_name: str, record_name: str, _=Depends(_verify), dry_run: bool = Query(True)): + zone_name = zone_name.strip().lower().rstrip(".") + hostname = _validate_managed_hostname(f"{record_name}.{zone_name}") + record_name = hostname[: -(len(zone_name) + 1)] + zone, rrsets, headers = await _hetzner_zone_and_rrsets(zone_name) + selected = [r for r in rrsets if r.get("name") == record_name and r.get("type") in {"A", "AAAA", "CNAME"}] + result = {"zone": zone_name, "zone_id": zone.get("id"), "name": record_name, "dry_run": dry_run, "rrsets": selected, "deleted": []} + if dry_run or not selected: + return result + async with httpx.AsyncClient(timeout=30) as client: + for rrset in selected: + url = f"{HETZNER_DNS_API}/zones/{zone['id']}/rrsets/{record_name}/{rrset['type']}" + response = await client.delete(url, headers=headers) + if response.status_code not in {200, 204}: + raise HTTPException(502, f"Hetzner RRSet delete failed for {rrset['type']}: HTTP {response.status_code}") + result["deleted"].append(rrset["type"]) + verify_response = await client.get(f"{HETZNER_DNS_API}/zones/{zone['id']}/rrsets", headers=headers) + remaining = verify_response.json().get("rrsets", []) if verify_response.status_code == 200 else selected + result["remaining"] = [r for r in remaining if r.get("name") == record_name and r.get("type") in {"A", "AAAA", "CNAME"}] + if result["remaining"]: + raise HTTPException(502, "RRSet read-back still contains deleted record") + _audit(f"/dns/rrset/{zone_name}/{record_name}", "DELETE", 200, "deleted=" + ",".join(result["deleted"])) + return result + + def _pve_auth(): pv = _parse_kv("proxmox") return f"PVEAPIToken={pv.get('tokenid','')}={pv.get('secret','')}" @@ -1954,6 +4055,205 @@ async def tts_health(_=Depends(_verify)): return results +def _sab_history_command() -> str: + container_script = r'''import json, re, subprocess, urllib.parse, urllib.request +text = open('/config/sabnzbd.ini', encoding='utf-8', errors='replace').read() +match = re.search(r'^api_key\s*=\s*(\S+)', text, re.M) +port_match = re.search(r'^port\s*=\s*(\d+)', text, re.M) +api_key = match.group(1) if match else '' +port = port_match.group(1) if port_match else '7777' +params = urllib.parse.urlencode({'mode': 'history', 'limit': 100, 'output': 'json', 'apikey': api_key}) +with urllib.request.urlopen('http://127.0.0.1:' + port + '/api?' + params, timeout=20) as response: + data = json.load(response) +slots = data.get('history', {}).get('slots', []) +allowed = ('nzo_id', 'name', 'category', 'status', 'script', 'script_line', 'fail_message', 'completed', 'storage', 'path') +print(json.dumps([{key: item.get(key) for key in allowed} for item in slots])) +''' + container_encoded = base64.b64encode(container_script.encode()).decode() + host_script = f'''import subprocess, sys +command = ["sudo", "-n", "docker", "exec", "sabnzbd", "python3", "-c", "import base64;exec(base64.b64decode('{container_encoded}'))"] +proc = subprocess.run(command, capture_output=True, text=True, timeout=30) +if proc.returncode != 0: + command = command[2:] + proc = subprocess.run(command, capture_output=True, text=True, timeout=30) +if proc.returncode != 0: + print((proc.stderr or proc.stdout)[-500:], file=sys.stderr) + raise SystemExit(proc.returncode) +print(proc.stdout) +''' + encoded = base64.b64encode(host_script.encode()).decode() + return f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"' + + +@app.get("/media/handoff/sab-history") +async def media_handoff_sab_history(_=Depends(_verify)): + """Return sanitized SAB history without exposing the SAB API key.""" + inventory = await asyncio.to_thread(_find_inventory_host, "sabnzbd") + if not inventory: + raise HTTPException(404, "Host sabnzbd not found") + target = f'{inventory["user"]}@{inventory["ip"]}' + rc, out, err = await asyncio.to_thread(_ssh, target, _sab_history_command(), 40) + if rc != 0: + raise HTTPException(502, (err or out).strip()[-500:] or "SAB history failed") + try: + return {"history": json.loads(out)} + except json.JSONDecodeError as exc: + raise HTTPException(502, "invalid SAB history response") from exc + + +def _media_handoff_diagnostics_command() -> str: + script = r'''import hashlib, json, os, subprocess, urllib.error, urllib.request + + +def run(args): + proc = subprocess.run(args, capture_output=True, text=True, timeout=20) + return proc.returncode, proc.stdout.strip(), proc.stderr.strip() + + +def docker_exec(command): + for prefix in (["sudo", "-n", "docker"], ["docker"]): + rc, out, err = run(prefix + ["exec", "sabnzbd", "sh", "-lc", command]) + if rc == 0 or "not found" not in (err + out).lower(): + return rc, out, err + return rc, out, err + +inspect_rc, inspect_out, _ = run(["sudo", "-n", "docker", "inspect", "-f", "{{.State.Running}}", "sabnzbd"]) +if inspect_rc != 0: + inspect_rc, inspect_out, _ = run(["docker", "inspect", "-f", "{{.State.Running}}", "sabnzbd"]) +py_rc, py_out, _ = docker_exec("command -v python3") +curl_rc, curl_out, _ = docker_exec("command -v curl") +stat_rc, stat_out, _ = docker_exec("test -f /usenet/scripts/movetdarr.sh && stat -c '%a %s' /usenet/scripts/movetdarr.sh && sha256sum /usenet/scripts/movetdarr.sh") +log_rc, log_out, _ = docker_exec("tail -n 200 /usenet/scripts/postprocess.log 2>/dev/null || true") +source_rc, source_out, _ = docker_exec("find /usenet/complete -mindepth 2 -maxdepth 2 -type d ! -name '_UNPACK_*' -print 2>/dev/null | sort | tail -100") +target_rc, target_out, _ = docker_exec("find /tdarr/complete -mindepth 2 -maxdepth 2 -type d -print 2>/dev/null | sort | tail -100") +probe_code = 0 +probe_body = "" +probe = urllib.request.Request( + "http://10.5.85.2:8888/media/handoff", + method="POST", + headers={"Content-Type": "application/json"}, + data=b'{"action":"status","jobId":"diagnostic-probe"}', +) +try: + with urllib.request.urlopen(probe, timeout=10) as response: + probe_code = response.status + probe_body = response.read(500).decode(errors="replace") +except urllib.error.HTTPError as exc: + probe_code = exc.code + probe_body = exc.read(500).decode(errors="replace") +except Exception as exc: + probe_body = type(exc).__name__ +script_info = {"exists": stat_rc == 0, "executable": False, "sha256": None, "mode": None, "size": None} +if stat_rc == 0: + lines = stat_out.splitlines() + if lines: + parts = lines[0].split() + if len(parts) >= 2: + script_info.update(mode=parts[0], size=int(parts[1]), executable=any(ch in parts[0][-3:] for ch in "1357")) + if len(lines) > 1: + script_info["sha256"] = lines[1].split()[0] +print(json.dumps({ + "container_running": inspect_rc == 0 and inspect_out == "true", + "tools": {"python3": py_rc == 0 and bool(py_out), "curl": curl_rc == 0 and bool(curl_out)}, + "script": script_info, + "caller_probe": {"http_status": probe_code, "body": probe_body}, + "source_directories": source_out.splitlines() if source_rc == 0 else [], + "target_directories": target_out.splitlines() if target_rc == 0 else [], + "recent_log": log_out.splitlines()[-200:], +})) +''' + encoded = base64.b64encode(script.encode()).decode() + return f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"' + + +@app.get("/media/handoff/diagnostics") +async def media_handoff_diagnostics(_=Depends(_verify)): + """Read-only diagnostics for the fixed SABnzbd handoff script and caller path.""" + inventory = await asyncio.to_thread(_find_inventory_host, "sabnzbd") + if not inventory: + raise HTTPException(404, "Host sabnzbd not found") + target = f'{inventory["user"]}@{inventory["ip"]}' + rc, out, err = await asyncio.to_thread(_ssh, target, _media_handoff_diagnostics_command(), 30) + if rc != 0: + raise HTTPException(502, (err or out).strip()[-500:] or "media handoff diagnostics failed") + try: + return json.loads(out) + except json.JSONDecodeError as exc: + raise HTTPException(502, "invalid media handoff diagnostic response") from exc + + +class MediaHandoffPayload(BaseModel): + action: Literal["start", "moved", "status", "fail"] + category: str | None = Field(None, max_length=32) + directory: str | None = Field(None, max_length=1000) + release: str | None = Field(None, max_length=500) + cleanName: str | None = Field(None, max_length=500) + expectedFiles: int | None = Field(None, ge=1, le=100) + jobId: str | None = Field(None, max_length=80) + reason: str | None = Field(None, max_length=300) + + +def _media_handoff_caller_allowed(request: Request) -> bool: + auth = request.headers.get("authorization", "") + if BUTLER_TOKEN and secrets.compare_digest(auth, f"Bearer {BUTLER_TOKEN}"): + return True + try: + caller = ipaddress.ip_address(request.client.host if request.client else "") + networks = [ + ipaddress.ip_network(value.strip(), strict=False) + for value in MEDIA_HANDOFF_ALLOWED_NETWORKS.split(",") + if value.strip() + ] + except ValueError: + return False + return any(caller in network for network in networks) + + +@app.post("/media/handoff") +async def media_handoff(payload: MediaHandoffPayload, request: Request): + """Narrow SABnzbd-to-n8n bridge; no generic unauthenticated proxy access.""" + if not _media_handoff_caller_allowed(request): + caller = request.client.host if request.client else "unknown" + _audit("/media/handoff", "POST", 403, f"caller={caller} action={payload.action}") + raise HTTPException(403, "Media handoff caller is not allowed") + + # 16.09.2026: serienen/videoen ergaenzt. Die englischen Arr-Instanzen + # sonarrEN (Port 8991, Root /data/FHD/serienen) und radarrEN (Port 7880, + # Root /data/FHD/videoen) nutzen eigene SAB-Kategorien. Ohne sie brach der + # Handoff mit 422 ab und Releases blieben in /usenet/complete liegen. + allowed_categories = {"serien4k", "serien", "serienen", "video4k", "video", "videoen"} + if payload.action == "start": + if payload.category not in allowed_categories: + raise HTTPException(422, "Unsupported media category") + expected_prefix = f"/usenet/complete/{payload.category}/" + if not payload.directory or not payload.directory.startswith(expected_prefix): + raise HTTPException(422, "Invalid media handoff directory") + if not payload.release: + raise HTTPException(422, "Release name is required") + else: + if not payload.jobId or not re.fullmatch(r"[a-z0-9-]{8,80}", payload.jobId): + raise HTTPException(422, "Valid jobId is required") + + cfg = SERVICES.get("n8n") + if not cfg or not cfg.get("url"): + raise HTTPException(503, "n8n service is not configured") + target = f"{cfg['url'].rstrip('/')}/webhook/media-handoff" + body = payload.model_dump(exclude_none=True) if hasattr(payload, "model_dump") else payload.dict(exclude_none=True) + try: + async with httpx.AsyncClient(verify=False, timeout=15) as client: + response = await client.post(target, json=body, headers={"Content-Type": "application/json"}) + except httpx.HTTPError as exc: + _audit("/media/handoff", "POST", 502, f"action={payload.action} error={type(exc).__name__}") + raise HTTPException(502, "n8n media handoff is unavailable") from exc + + try: + result = response.json() + except Exception: + result = {"ok": False, "error": "invalid_n8n_response"} + _audit("/media/handoff", "POST", response.status_code, f"action={payload.action}") + return JSONResponse(content=result, status_code=response.status_code) + + @app.api_route("/{service}/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"]) async def proxy(service: str, path: str, request: Request, _=Depends(_verify)): SKIP_SERVICES = {"vm", "inventory", "ansible", "debug", "tts", "status", "audit", "config"} @@ -1963,7 +4263,7 @@ async def proxy(service: str, path: str, request: Request, _=Depends(_verify)): if not cfg: raise HTTPException(404, f"Unknown service: {service}. Available: {list(SERVICES.keys())}") - base_url = cfg["url"] + base_url = cfg.get("url") auth_type = cfg["auth"] headers = dict(request.headers) cookies = {} diff --git a/butler.yaml b/butler.yaml index d0b0928..0f8ca9e 100644 --- a/butler.yaml +++ b/butler.yaml @@ -45,13 +45,6 @@ services: description: "Media request management" health_path: "/api/v1/status" - outline: - url: "http://10.1.1.100:3000" - auth: bearer - key_file: outline - vault_key: outline_api_key - description: "Wiki" - n8n: url: "http://10.4.1.113:5678" auth: n8n @@ -112,6 +105,10 @@ services: health_path: "/" timeout: 300 +backup: + # guck-vps contains Git-managed edge configuration and has no Borgmatic installation. + exempt_hosts: [guck-vps] + # VM lifecycle settings vm: automation_host: "sascha@10.5.85.5" diff --git a/compose.yaml b/compose.yaml index b593f71..a07a30f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -11,10 +11,13 @@ services: - /home/sascha/.ssh:/root/.ssh:ro - ./butler.yaml:/data/butler.yaml:ro - ./app.py:/app/app.py:ro + - ./ui.html:/app/ui.html:ro + - ./state:/data/state environment: - API_KEY_DIR=/data/api - VAULT_CACHE_DIR=/data/vault-cache - BUTLER_TOKEN=${BUTLER_TOKEN} + - AUDIT_DB_PATH=/data/state/audit.sqlite3 volumes: vault-cache: diff --git a/tests/test_app.py b/tests/test_app.py index 8c32eb7..45011c2 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,5 +1,7 @@ import os import asyncio +import json +import shlex import time from datetime import datetime, timedelta, timezone @@ -42,7 +44,606 @@ def test_health_exposes_current_version(): with TestClient(app.app) as client: response = client.get("/health") assert response.status_code == 200 - assert response.json()["version"] == app.VERSION == "2.3.5" + assert response.json()["version"] == app.VERSION == "2.4.1" + + +def test_media_handoff_proxies_strict_category_contract(monkeypatch): + captured = {} + + class FakeResponse: + status_code = 200 + + def json(self): + return {"ok": True, "jobId": "test-job-1234", "state": "registered"} + + class FakeClient: + def __init__(self, **kwargs): + captured["client"] = kwargs + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None + + async def post(self, url, json, headers): + captured.update(url=url, json=json, headers=headers) + return FakeResponse() + + monkeypatch.setattr(app.httpx, "AsyncClient", FakeClient) + with TestClient(app.app) as client: + monkeypatch.setattr(app, "SERVICES", {"n8n": {"url": "http://n8n:5678", "auth": "n8n"}}) + response = client.post( + "/media/handoff", + headers={"Authorization": "Bearer test-token"}, + json={ + "action": "start", + "category": "serien4k", + "directory": "/usenet/complete/serien4k/Show.S01E01", + "release": "Show.S01E01-GRP", + "cleanName": "Show S01E01", + "expectedFiles": 2, + }, + ) + assert response.status_code == 200 + assert response.json()["state"] == "registered" + assert captured["url"] == "http://n8n:5678/webhook/media-handoff" + assert captured["json"]["category"] == "serien4k" + assert captured["json"]["expectedFiles"] == 2 + + +def test_media_handoff_rejects_untrusted_caller_before_proxy(monkeypatch): + monkeypatch.setattr(app.httpx, "AsyncClient", lambda **_kwargs: (_ for _ in ()).throw(AssertionError("must not proxy"))) + with TestClient(app.app) as client: + response = client.post( + "/media/handoff", + json={"action": "status", "jobId": "test-job-1234"}, + ) + assert response.status_code == 403 + + +def test_media_handoff_rejects_wrong_category_path(monkeypatch): + monkeypatch.setattr(app.httpx, "AsyncClient", lambda **_kwargs: (_ for _ in ()).throw(AssertionError("must not proxy"))) + with TestClient(app.app) as client: + response = client.post( + "/media/handoff", + headers={"Authorization": "Bearer test-token"}, + json={ + "action": "start", + "category": "video4k", + "directory": "/usenet/complete/serien4k/Wrong", + "release": "Wrong", + }, + ) + assert response.status_code == 422 + + +def test_media_handoff_accepts_english_arr_categories(monkeypatch): + """serienen/videoen muessen durchgehen (sonarrEN 8991 / radarrEN 7880). + + 16.09.2026: fehlten in allowed_categories -> HTTP 422 -> movetdarr.sh + brach mit "n8n-Handoff konnte nicht registriert werden" ab und liess + Mutiny.2026 x2 tagelang in /usenet/complete/videoen liegen. + """ + seen = [] + + class FakeResponse: + status_code = 200 + + def json(self): + return {"ok": True, "jobId": "test-job-5678", "state": "registered"} + + class FakeClient: + def __init__(self, **_kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None + + async def post(self, url, json, headers): + seen.append(json) + return FakeResponse() + + monkeypatch.setattr(app.httpx, "AsyncClient", FakeClient) + with TestClient(app.app) as client: + monkeypatch.setattr(app, "SERVICES", {"n8n": {"url": "http://n8n:5678", "auth": "n8n"}}) + for category in ("serienen", "videoen"): + response = client.post( + "/media/handoff", + headers={"Authorization": "Bearer test-token"}, + json={ + "action": "start", + "category": category, + "directory": f"/usenet/complete/{category}/Release.2026", + "release": "Release.2026-GRP", + "cleanName": "Release 2026", + "expectedFiles": 1, + }, + ) + assert response.status_code == 200, (category, response.text) + assert response.json()["state"] == "registered", category + + assert [item["category"] for item in seen] == ["serienen", "videoen"] + + +def test_media_handoff_still_rejects_unknown_category(monkeypatch): + """Fail-closed bleibt: eine frei erfundene Kategorie wird nicht geproxyt.""" + monkeypatch.setattr( + app.httpx, + "AsyncClient", + lambda **_kwargs: (_ for _ in ()).throw(AssertionError("must not proxy")), + ) + with TestClient(app.app) as client: + response = client.post( + "/media/handoff", + headers={"Authorization": "Bearer test-token"}, + json={ + "action": "start", + "category": "hoerbuecher", + "directory": "/usenet/complete/hoerbuecher/Buch", + "release": "Buch", + }, + ) + assert response.status_code == 422 + + +def test_media_handoff_english_category_path_must_match(monkeypatch): + """Pfadpruefung gilt auch fuer die neuen Kategorien.""" + monkeypatch.setattr( + app.httpx, + "AsyncClient", + lambda **_kwargs: (_ for _ in ()).throw(AssertionError("must not proxy")), + ) + with TestClient(app.app) as client: + response = client.post( + "/media/handoff", + headers={"Authorization": "Bearer test-token"}, + json={ + "action": "start", + "category": "videoen", + "directory": "/usenet/complete/video4k/Wrong", + "release": "Wrong", + }, + ) + assert response.status_code == 422 + + +def test_paperless_import_queues_pdf_through_butler(monkeypatch): + captured = {} + + def fake_upload(host, command, payload, timeout): + captured.update(host=host, command=command, payload=payload, timeout=timeout) + return 0, "", "" + + monkeypatch.setattr(app, "_ssh_bytes", fake_upload) + pdf = b"%PDF-1.7\nvalid-test-payload" + with TestClient(app.app) as client: + response = client.post( + "/paperless/import?filename=Hausordnung%20Stand%2009.05.2022.pdf", + content=pdf, + headers={"Authorization": "Bearer test-token", "Content-Type": "application/pdf"}, + ) + assert response.status_code == 200 + body = response.json() + assert body["status"] == "queued" + assert body["filename"].startswith("Hausordnung_Stand_09.05.2022-") + assert captured["host"] == app.PAPERLESS_GATEWAY + assert app.PAPERLESS_SSH in captured["command"] + assert captured["payload"] == pdf + assert app.PAPERLESS_CONSUME_DIR in captured["command"] + + +def test_paperless_import_rejects_non_pdf(monkeypatch): + monkeypatch.setattr(app, "_ssh_bytes", lambda *args: (_ for _ in ()).throw(AssertionError("must not upload"))) + with TestClient(app.app) as client: + response = client.post( + "/paperless/import?filename=bad.pdf", + content=b"not a pdf", + headers={"Authorization": "Bearer test-token"}, + ) + assert response.status_code == 400 + + +def test_ui_serves_self_contained_operator_console(): + with TestClient(app.app) as client: + response = client.get("/ui") + assert response.status_code == 200 + assert "Pfannkuchen Butler" in response.text + assert 'id="operations-grid"' in response.text + assert 'id="doctor-form"' in response.text + assert 'id="preflight-form"' in response.text + assert 'id="sharing-form"' in response.text + assert 'id="sharing-events"' in response.text + assert "/emby/account-sharing" in response.text + assert 'id="login"' not in response.text + assert "Butler-Token" not in response.text + assert "localStorage" not in response.text + + +def test_ui_asset_is_mounted_read_only_in_compose(): + from pathlib import Path + import yaml + compose = yaml.safe_load(Path(app.__file__).with_name("compose.yaml").read_text(encoding="utf-8")) + mounts = compose["services"]["homelab-butler"]["volumes"] + assert "./ui.html:/app/ui.html:ro" in mounts + + +def test_ui_session_login_uses_httponly_cookie_and_csrf(): + app._ui_sessions.clear() + with TestClient(app.app) as client: + denied = client.post("/ui/login", json={"token": "wrong"}) + assert denied.status_code == 401 + + login = client.post("/ui/login", json={"token": "test-token"}) + assert login.status_code == 200 + assert "HttpOnly" in login.headers.get("set-cookie", "") + csrf = client.cookies.get("butler_csrf") + assert csrf + + capabilities = client.get("/capabilities") + assert capabilities.status_code == 200 + + blocked = client.post("/config/reload") + assert blocked.status_code == 403 + allowed = client.post("/config/reload", headers={"X-CSRF-Token": csrf}) + assert allowed.status_code == 200 + + +def test_ui_anonymous_session_is_automatic_and_strictly_read_only(): + app._ui_sessions.clear() + with TestClient(app.app) as client: + session = client.get("/ui/session") + assert session.status_code == 200 + assert session.json()["authenticated"] is True + assert session.json()["read_only"] is True + assert "HttpOnly" in session.headers.get("set-cookie", "") + + capabilities = client.get("/capabilities") + assert capabilities.status_code == 200 + + csrf = client.cookies.get("butler_csrf") + mutation = client.post("/config/reload", headers={"X-CSRF-Token": csrf}) + assert mutation.status_code == 403 + assert "read-only" in mutation.json()["detail"].lower() + + +def test_emby_network_identity_normalizes_ipv4_and_ipv6_privacy_addresses(): + ipv4 = app._emby_network_identity("203.0.113.9:443") + assert ipv4["ip"] == "203.0.113.9" + assert ipv4["network"] == "203.0.113.9/32" + assert ipv4["identity"] == "203.0.113.9/32" + + first = app._emby_network_identity("2003:abcd:1234:5678::1") + privacy_peer = app._emby_network_identity("[2003:abcd:1234:5678:ffff::99]:443") + sibling_subnet = app._emby_network_identity("2003:abcd:1234:9999::1") + assert first["network"] == privacy_peer["network"] == "2003:abcd:1234:5678::/64" + assert first["parent"] == sibling_subnet["parent"] == "2003:abcd:1234::/48" + assert first["identity"] == sibling_subnet["identity"] == "2003:abcd:1234::/48" + + +def test_emby_sharing_flags_concurrent_distinct_networks_but_not_sibling_ipv6_subnets(): + def series(endpoint, values, city): + return { + "metric": { + "job": "emby-sascha", "username": "Alice", "remoteEndPoint": endpoint, + "city": city, "region": "Test", "countryCode": "DE", + "latitude": "48.1", "longitude": "11.5", + }, + "values": [[timestamp, "1"] for timestamp in values], + } + + payload = [ + series("2606:4700:1234:1000::1", [100, 160, 220, 280, 340, 400], "Home"), + series("2606:4700:1234:2000::2", [100, 160, 220, 280, 340, 400], "Home privacy subnet"), + series("2001:4860:9999:1000::1", [100, 160, 220, 280, 340, 400], "Away"), + ] + + result = app._analyze_emby_sharing(payload, step_seconds=60) + + assert result["summary"]["concurrent_events"] == 1 + event = result["events"][0] + assert event["type"] == "concurrent_networks" + assert event["username"] == "Alice" + assert len(event["evidence"]) == 2 + assert {item["identity"] for item in event["evidence"]} == { + "2606:4700:1234::/48", "2001:4860:9999::/48" + } + + +def test_emby_sharing_ignores_short_overlap_inside_prometheus_staleness_window(): + def series(endpoint): + return {"metric": {"job": "emby-sascha", "username": "Alice", "remoteEndPoint": endpoint, + "city": "Munich", "region": "Bavaria", "countryCode": "DE", + "latitude": "48.1", "longitude": "11.5"}, + "values": [[100, "1"], [160, "1"]]} + + result = app._analyze_emby_sharing([series("8.8.8.8"), series("1.1.1.1")], step_seconds=60) + + assert result["summary"]["concurrent_events"] == 0 + + +def test_emby_sharing_flags_geographically_impossible_network_change(): + payload = [ + { + "metric": {"job": "emby-chris", "username": "Bob", "remoteEndPoint": "8.8.8.8", + "city": "Berlin", "region": "Berlin", "countryCode": "DE", + "latitude": "52.5200", "longitude": "13.4050"}, + "values": [[100, "1"], [160, "1"]], + }, + { + "metric": {"job": "emby-chris", "username": "Bob", "remoteEndPoint": "1.1.1.1", + "city": "New York", "region": "New York", "countryCode": "US", + "latitude": "40.7128", "longitude": "-74.0060"}, + "values": [[400, "1"], [460, "1"]], + }, + ] + + result = app._analyze_emby_sharing(payload, step_seconds=60) + + assert result["summary"]["concurrent_events"] == 0 + assert result["summary"]["impossible_travel_events"] == 1 + event = result["events"][0] + assert event["type"] == "impossible_travel" + assert event["distance_km"] > 6000 + assert event["required_speed_kmh"] > 1000 + + +def test_emby_account_sharing_endpoint_is_read_only_and_filterable(monkeypatch): + async def history(days, server): + assert days == 7 + assert server == "all" + return ([{ + "metric": {"job": "emby-sascha", "username": "Alice", "remoteEndPoint": "8.8.8.8", + "city": "Munich", "region": "Bavaria", "countryCode": "DE", + "latitude": "48.1", "longitude": "11.5"}, + "values": [[100, "1"], [160, "1"]], + }], 60) + + monkeypatch.setattr(app, "_fetch_emby_session_history", history) + with TestClient(app.app) as client: + response = client.get( + "/emby/account-sharing?days=7&username=alice", + headers={"Authorization": "Bearer test-token"}, + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["policy"]["mode"] == "conservative" + assert payload["policy"]["ipv6_detection_identity"] == "/48" + assert payload["summary"]["users_analyzed"] == 1 + assert payload["users"][0]["username"] == "Alice" + + +def test_emby_history_step_stays_below_prometheus_resolution_limit(): + assert app._emby_history_step(7) == 60 + for days in (7, 30, 90): + step = app._emby_history_step(days) + assert step % 60 == 0 + assert (days * 86400) / step <= 10_500 + + +def test_emby_history_fetch_chunks_90_days_and_merges_equal_series(monkeypatch): + calls = [] + + class FakeResponse: + def __init__(self, params): + self.params = params + + def raise_for_status(self): + return None + + def json(self): + start = self.params["start"] + end = self.params["end"] + return {"status": "success", "data": {"result": [{ + "metric": {"job": "emby-sascha", "username": "Alice", "remoteEndPoint": "8.8.8.8"}, + "values": [[start, "1"], [end, "1"]], + }]}} + + class FakeClient: + def __init__(self, **_kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None + + async def get(self, _url, params, headers, cookies): + calls.append(params) + return FakeResponse(params) + + monkeypatch.setattr(app, "SERVICES", {"grafana": {"url": "http://grafana", "auth": "none"}}) + monkeypatch.setattr(app.httpx, "AsyncClient", FakeClient) + monkeypatch.setattr(app.time, "time", lambda: 10_000_000) + + series, step = asyncio.run(app._fetch_emby_session_history(90, "all")) + + assert len(calls) == 3 + assert all(call["end"] - call["start"] <= 30 * 86400 for call in calls) + assert step == 780 + assert len(series) == 1 + timestamps = [value[0] for value in series[0]["values"]] + assert timestamps == sorted(set(timestamps)) + + +def test_capabilities_is_live_machine_readable_safety_map(): + with TestClient(app.app) as client: + response = client.get( + "/capabilities", + headers={"Authorization": "Bearer test-token"}, + ) + assert response.status_code == 200 + payload = response.json() + by_operation = {(item["method"], item["path"]): item for item in payload["operations"]} + assert ("GET", "/network/wireguard/{host}") in by_operation + assert by_operation[("GET", "/network/wireguard/{host}")]["mode"] == "read_only" + removal = by_operation[("DELETE", "/network/wireguard/{host}/peer")] + assert removal["mode"] == "mutation" + assert removal["dry_run"] is True + assert removal["critical"] is True + tunnel = by_operation[("POST", "/network/media-tunnel/sascha")] + assert tunnel["dry_run"] is True + assert tunnel["critical"] is True + assert by_operation[("DELETE", "/vm/destroy/{vmid}")]["dry_run"] is True + assert all(item["path"] != "/{service}/{path}" for item in payload["operations"]) + assert payload["model_contract"]["instruction"].startswith("Prefer read_only") + + +def test_info_advertises_capabilities_endpoint(): + with TestClient(app.app) as client: + response = client.get("/info", headers={"Authorization": "Bearer test-token"}) + assert response.status_code == 200 + assert response.json()["endpoints"]["capabilities"] == "/capabilities" + assert response.json()["endpoints"]["doctor"] == "/doctor/{target}" + assert response.json()["endpoints"]["drift"] == "/drift" + assert response.json()["endpoints"]["maintenance_preflight"] == "/maintenance/preflight" + assert response.json()["endpoints"]["ui"] == "/ui" + + +def test_audit_persists_and_redacts_secrets(tmp_path, monkeypatch): + db = tmp_path / "audit.sqlite3" + monkeypatch.setattr(app, "AUDIT_DB_PATH", str(db)) + app._init_audit_db() + app._audit("/danger", "POST", 200, "host=x token=abc password=hunter2 api_key=secret") + app._audit_log.clear() + + with TestClient(app.app) as client: + response = client.get("/audit", headers={"Authorization": "Bearer test-token"}) + + assert response.status_code == 200 + entry = response.json()[0] + assert entry["endpoint"] == "/danger" + assert "abc" not in entry["detail"] + assert "hunter2" not in entry["detail"] + assert "secret" not in entry["detail"] + assert entry["detail"].count("[REDACTED]") == 3 + + +def test_audit_records_ai_and_web_ui_actor(tmp_path, monkeypatch): + db = tmp_path / "audit.sqlite3" + monkeypatch.setattr(app, "AUDIT_DB_PATH", str(db)) + + async def empty_status(): + return {} + + monkeypatch.setattr(app, "_collect_service_status", empty_status) + app._ui_sessions.clear() + with TestClient(app.app) as client: + ai = client.get( + "/status", + headers={"Authorization": "Bearer test-token", "X-Butler-Actor": "Trulla"}, + ) + assert ai.status_code == 200 + + client.headers.pop("Authorization", None) + client.get("/ui/session") + web = client.get("/status") + assert web.status_code == 200 + + entries = client.get("/audit").json() + + assert [item["actor"] for item in entries[:2]] == ["Weboberfläche", "Trulla"] + + +def test_ui_audit_has_actor_column_and_filter(): + with TestClient(app.app) as client: + response = client.get("/ui") + assert 'Akteur' in response.text + assert 'id="audit-actor"' in response.text + + +def test_wireguard_status_returns_redacted_live_state(monkeypatch): + payload = { + "interface": "wg0", + "addresses": ["10.11.12.1/32"], + "listen_port": 37888, + "service_active": True, + "service_enabled": True, + "routes": [{"dst": "10.11.12.3", "prefsrc": "10.11.12.1"}], + "peers": [{ + "public_key": "peer-public-key", + "endpoint": "203.0.113.9:51820", + "allowed_ips": ["10.11.12.3/32"], + "latest_handshake": 123, + "rx_bytes": 456, + "tx_bytes": 789, + "persistent_keepalive": 25, + }], + } + monkeypatch.setattr(app, "_find_inventory_host", lambda host: {"user": "debian", "ip": "141.94.237.199"}) + monkeypatch.setattr(app, "_ssh", lambda host, command, timeout=30: (0, json.dumps(payload), "")) + + with TestClient(app.app) as client: + response = client.get( + "/network/wireguard/guck-vps", + headers={"Authorization": "Bearer test-token"}, + ) + + assert response.status_code == 200 + assert response.json()["host"] == "guck-vps" + assert response.json()["peers"][0]["allowed_ips"] == ["10.11.12.3/32"] + assert "private" not in response.text.lower() + + +def test_wireguard_status_rejects_unknown_host_without_ssh(monkeypatch): + monkeypatch.setattr(app, "_find_inventory_host", lambda host: None) + monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("SSH must not run"))) + + with TestClient(app.app) as client: + response = client.get( + "/network/wireguard/does-not-exist", + headers={"Authorization": "Bearer test-token"}, + ) + + assert response.status_code == 404 + + +def test_wireguard_status_command_uses_sudo_and_accepts_off_keepalive(): + import base64 + import re + + command = app._wireguard_status_command() + encoded = re.search(r"b64decode\('([^']+)'\)", command).group(1) + script = base64.b64decode(encoded).decode() + + assert '["sudo", "-n", "wg", "show", "wg0", "dump"]' in script + assert '0 if fields[7] == "off" else int(fields[7])' in script + + +def test_wireguard_peer_remove_is_scoped_and_audited(monkeypatch): + calls = [] + monkeypatch.setattr(app, "_find_inventory_host", lambda host: {"user": "debian", "ip": "141.94.237.199"}) + monkeypatch.setattr(app, "_ssh", lambda host, command, timeout=30: (calls.append((host, command, timeout)) or (0, json.dumps({"status": "removed", "removed_routes": ["10.7.1.0/24"]}), ""))) + + with TestClient(app.app) as client: + response = client.request( + "DELETE", + "/network/wireguard/guck-vps/peer", + headers={"Authorization": "Bearer test-token"}, + json={"public_key": "A" * 43 + "=", "expected_allowed_ip": "10.7.1.0/24", "dry_run": False}, + ) + + assert response.status_code == 200 + assert response.json()["status"] == "removed" + assert calls[0][0] == "debian@141.94.237.199" + assert calls[0][2] == 45 + + +def test_wireguard_peer_remove_rejects_non_allowlisted_host(monkeypatch): + monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("SSH must not run"))) + with TestClient(app.app) as client: + response = client.request( + "DELETE", + "/network/wireguard/node7/peer", + headers={"Authorization": "Bearer test-token"}, + json={"public_key": "A" * 43 + "=", "expected_allowed_ip": "10.7.1.0/24", "dry_run": True}, + ) + assert response.status_code == 403 def test_tts_generate_returns_cloned_wav(monkeypatch): @@ -201,6 +802,215 @@ def test_host_forensics_rejects_unknown_host_and_invalid_window(monkeypatch): assert bad_window.status_code == 422 +def test_docker_residue_cleanup_defaults_to_dry_run(monkeypatch): + payload = {"dry_run": True, "before_rule_count": 25, "removed_rules": [], "removed_chains": [], "removed_links": [], "removed_paths": [], "errors": []} + calls = [] + monkeypatch.setattr(app, "_find_inventory_host", lambda name: {"name": name, "user": "root", "ip": "10.5.85.13"}) + monkeypatch.setattr(app, "_ssh", lambda host, command, timeout=600: (calls.append((host, command, timeout)) or (0, __import__("json").dumps(payload), ""))) + with TestClient(app.app) as client: + response = client.post("/system/cleanup/docker-residue/node3", headers={"Authorization": "Bearer test-token"}) + assert response.status_code == 200 + assert response.json()["dry_run"] is True + assert calls[0][0] == "root@10.5.85.13" + assert calls[0][2] == 90 + + +def test_docker_residue_cleanup_refuses_active_docker(monkeypatch): + payload = {"dry_run": False, "error": "Docker or containerd is still installed/active; refusing residue cleanup"} + monkeypatch.setattr(app, "_find_inventory_host", lambda name: {"name": name, "user": "root", "ip": "10.5.85.13"}) + monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (2, __import__("json").dumps(payload), "")) + with TestClient(app.app) as client: + response = client.post("/system/cleanup/docker-residue/node3?dry_run=false", headers={"Authorization": "Bearer test-token"}) + assert response.status_code == 409 + + +def test_iso_builder_restore_defaults_to_dry_run_and_has_no_free_target(monkeypatch): + payload = {"dry_run": True, "restored": [], "removed_outputs": [], "validation": {}} + calls = [] + monkeypatch.setattr(app, "_ssh", lambda host, command, timeout=600: (calls.append((host, command, timeout)) or (0, __import__("json").dumps(payload), ""))) + with TestClient(app.app) as client: + response = client.post("/system/restore/iso-builder", headers={"Authorization": "Bearer test-token"}) + assert response.status_code == 200 + assert response.json()["dry_run"] is True + assert calls[0][0] == app.AUTOMATION1 + assert calls[0][2] == 120 + command = app._iso_builder_restore_command(True) + encoded = command.split("base64.b64decode('", 1)[1].split("')", 1)[0] + decoded = __import__("base64").b64decode(encoded).decode() + assert "origin/master" in decoded + assert "/app-config/ansible" in decoded + + +def test_caddy_site_remove_defaults_to_dry_run(monkeypatch): + payload = {"hostname": "wiki.sascha-lutz.de", "dry_run": True, "found": True, "line_start": 10, "line_end": 13} + calls = [] + monkeypatch.setattr(app, "_ssh", lambda host, command, timeout=600: (calls.append((host, command, timeout)) or (0, __import__("json").dumps(payload), ""))) + with TestClient(app.app) as client: + response = client.delete("/caddy/site/wiki.sascha-lutz.de", headers={"Authorization": "Bearer test-token"}) + assert response.status_code == 200 + assert response.json()["dry_run"] is True + assert calls[0][0] == app.CADDY_HOST + assert calls[0][2] == 90 + + +def test_caddy_site_remove_rejects_unmanaged_hostname_before_ssh(monkeypatch): + monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not SSH"))) + with TestClient(app.app) as client: + response = client.delete("/caddy/site/example.com", headers={"Authorization": "Bearer test-token"}) + assert response.status_code == 400 + + +def test_dns_rrset_dry_run_returns_only_selected_records(monkeypatch): + async def fake_lookup(zone): + assert zone == "sascha-lutz.de" + return {"id": 96805, "name": zone}, [ + {"name": "wiki", "type": "A", "records": [{"value": "46.225.230.72"}]}, + {"name": "wiki", "type": "AAAA", "records": [{"value": "::1"}]}, + {"name": "git", "type": "A", "records": [{"value": "46.225.230.72"}]}, + ], {"Authorization": "Bearer hidden"} + monkeypatch.setattr(app, "_hetzner_zone_and_rrsets", fake_lookup) + with TestClient(app.app) as client: + response = client.delete("/dns/rrset/sascha-lutz.de/wiki", headers={"Authorization": "Bearer test-token"}) + assert response.status_code == 200 + body = response.json() + assert body["dry_run"] is True + assert [item["type"] for item in body["rrsets"]] == ["A", "AAAA"] + assert body["deleted"] == [] + + +def test_hetzner_dns_token_accepts_single_sanitized_vault_alias(monkeypatch): + token = "a" * 48 + monkeypatch.setattr(app, "_vault_cache", {"hetzner-dns-api": f"Hetzner DNS API Token: {token}\nFür sascha-lutz.de", "other": "ignored"}) + monkeypatch.setattr(app, "_read", lambda _name: None) + monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not sync vault"))) + assert app._get_hetzner_dns_token() == token + + +def test_uptime_monitor_remove_defaults_to_dry_run(monkeypatch): + payload = {"monitor_id": 71, "expected_name": "Outline Wiki", "dry_run": True, "found": {"id": 71, "name": "Outline Wiki"}} + monkeypatch.setattr(app, "_ssh", lambda target, command, timeout=120: (0, json.dumps(payload), "")) + with TestClient(app.app) as client: + response = client.delete("/uptime/monitor/71", params={"expected_name": "Outline Wiki"}, headers={"Authorization": "Bearer test-token"}) + assert response.status_code == 200 + assert response.json()["dry_run"] is True + + +def test_uptime_monitor_remove_rejects_invalid_expected_name_before_ssh(monkeypatch): + monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not SSH"))) + with TestClient(app.app) as client: + response = client.delete("/uptime/monitor/71", params={"expected_name": "Outline; rm -rf /"}, headers={"Authorization": "Bearer test-token"}) + assert response.status_code == 400 + + +def test_media_verify_returns_duration_and_not_suspect_when_within_tolerance(monkeypatch): + monkeypatch.setattr(app, "_find_inventory_host", lambda host: {"user": "sascha", "ip": "10.2.1.100"}) + calls = [] + + def fake_ssh(host, command, timeout=30): + calls.append((host, command, timeout)) + return 0, "2967.355000\n", "" + + monkeypatch.setattr(app, "_ssh", fake_ssh) + with TestClient(app.app) as client: + response = client.post( + "/media/verify", + headers={"Authorization": "Bearer test-token"}, + json={"host": "arrapps", "path": "/data/UHD/serien/Show/ep.mkv", "expected_minutes": 49.5}, + ) + assert response.status_code == 200 + body = response.json() + assert body["duration_minutes"] == 49.46 + assert body["suspect"] is False + assert calls[0][0] == "sascha@10.2.1.100" + assert "bazarrUHD" in calls[0][1] + assert "ffprobe" in calls[0][1] + + +def test_media_verify_flags_truncated_file_as_suspect(monkeypatch): + monkeypatch.setattr(app, "_find_inventory_host", lambda host: {"user": "sascha", "ip": "10.2.1.100"}) + monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (0, "1836.0\n", "")) + with TestClient(app.app) as client: + response = client.post( + "/media/verify", + headers={"Authorization": "Bearer test-token"}, + json={"host": "arrapps", "path": "/data/UHD/serien/Show/ep.mkv", "expected_minutes": 49.5}, + ) + assert response.status_code == 200 + body = response.json() + assert body["suspect"] is True + assert body["deviation_pct"] > 20 + + +def test_media_verify_rejects_path_outside_data_before_ssh(monkeypatch): + monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not SSH"))) + with TestClient(app.app) as client: + response = client.post( + "/media/verify", + headers={"Authorization": "Bearer test-token"}, + json={"host": "arrapps", "path": "/etc/passwd"}, + ) + assert response.status_code == 400 + + +def test_media_verify_allows_apostrophe_in_filename_and_quotes_it_safely(monkeypatch): + monkeypatch.setattr(app, "_find_inventory_host", lambda host: {"user": "sascha", "ip": "10.2.1.100"}) + calls = [] + + def fake_ssh(host, command, timeout=30): + calls.append((host, command, timeout)) + return 0, "2967.0\n", "" + + monkeypatch.setattr(app, "_ssh", fake_ssh) + path = "/data/FHD/serien/Star Trek - Strange New Worlds (2022)/Season 04/Once La'An a Time.mkv" + with TestClient(app.app) as client: + response = client.post( + "/media/verify", + headers={"Authorization": "Bearer test-token"}, + json={"host": "arrapps", "path": path, "expected_minutes": 49.5}, + ) + assert response.status_code == 200 + # shlex.quote must produce a command the remote shell parses as ONE argument, + # i.e. no unescaped apostrophe breaks out of quoting. + executed_cmd = calls[0][1] + quoted = shlex.quote(path) + assert quoted in executed_cmd + assert shlex.split(executed_cmd)[-1] == path + + +def test_media_verify_rejects_shell_metacharacters_before_ssh(monkeypatch): + monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not SSH"))) + with TestClient(app.app) as client: + response = client.post( + "/media/verify", + headers={"Authorization": "Bearer test-token"}, + json={"host": "arrapps", "path": "/data/UHD/serien/a\x00.mkv"}, + ) + assert response.status_code == 400 + + +def test_media_verify_rejects_unknown_host_before_ssh(monkeypatch): + monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not SSH"))) + with TestClient(app.app) as client: + response = client.post( + "/media/verify", + headers={"Authorization": "Bearer test-token"}, + json={"host": "unknown-host", "path": "/data/UHD/serien/ep.mkv"}, + ) + assert response.status_code == 400 + + +def test_media_verify_returns_502_on_unparsable_ffprobe_output(monkeypatch): + monkeypatch.setattr(app, "_find_inventory_host", lambda host: {"user": "sascha", "ip": "10.2.1.100"}) + monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (0, "N/A\n", "")) + with TestClient(app.app) as client: + response = client.post( + "/media/verify", + headers={"Authorization": "Bearer test-token"}, + json={"host": "arrapps", "path": "/data/UHD/serien/ep.mkv", "expected_minutes": 49.5}, + ) + assert response.status_code == 502 + + def test_invalid_log_target_is_rejected_before_ssh(): with TestClient(app.app) as client: response = client.get( @@ -416,6 +1226,7 @@ def test_backup_collection_runs_hosts_concurrently(monkeypatch): {"name": f"vm-{index}", "user": "sascha", "ip": f"10.1.1.{index}"} for index in range(1, 5) ]) + monkeypatch.setattr(app, "_config", {}) def fake_ssh(*_args, **_kwargs): nonlocal active, max_active @@ -430,7 +1241,24 @@ def test_backup_collection_runs_hosts_concurrently(monkeypatch): assert max_active > 1 assert result["summary"] == { - "total": 4, "healthy": 4, "warning": 0, "critical": 0, "unknown": 0 + "total": 4, "healthy": 4, "warning": 0, "critical": 0, "unknown": 0, "exempt": 0 + } + + +def test_backup_policy_exempts_host_without_borg_call(monkeypatch): + monkeypatch.setattr(app, "_get_inventory_hosts", lambda: [ + {"name": "guck-vps", "user": "debian", "ip": "141.94.237.199"} + ]) + monkeypatch.setattr(app, "_config", {"backup": {"exempt_hosts": ["guck-vps"]}}) + monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not call borg"))) + + result = asyncio.run(app._collect_backup_status()) + + assert result["summary"] == { + "total": 1, "healthy": 0, "warning": 0, "critical": 0, "unknown": 0, "exempt": 1 + } + assert result["hosts"]["guck-vps"] == { + "state": "exempt", "ok": True, "reason": "backup policy exemption" } @@ -444,6 +1272,75 @@ def test_overview_openapi_has_stable_enums_and_schema(): assert finding_schema["properties"]["severity"]["enum"] == ["healthy", "warning", "critical"] +def test_doctor_correlates_host_layers(monkeypatch): + async def snapshot(): + return { + "services": {}, + "hosts": {"guck-vps": {"reachable": True, "containers": ["caddy: Up 3 days"]}}, + "backups": {"summary": {}, "hosts": {"guck-vps": {"state": "exempt", "ok": True}}}, + "disks": {"guck-vps": {"pct": "8%"}}, + } + + monkeypatch.setattr(app, "_collect_operational_snapshot", snapshot) + with TestClient(app.app) as client: + response = client.get("/doctor/guck-vps", headers={"Authorization": "Bearer test-token"}) + + assert response.status_code == 200 + result = response.json() + assert result["state"] == "healthy" + assert result["layers"]["host"]["reachable"] is True + assert result["layers"]["backup"]["state"] == "exempt" + assert result["layers"]["disk"]["pct"] == "8%" + assert result["findings"] == [] + + +def test_drift_reports_inventory_coverage_gaps(monkeypatch): + async def snapshot(): + return { + "services": {}, + "hosts": {"vm-a": {"reachable": True}, "vm-b": {"reachable": True}, "node1": {"reachable": True}}, + "backups": {"summary": {}, "hosts": {"vm-a": {"state": "healthy"}, "orphan": {"state": "healthy"}}}, + "disks": {"vm-a": {"pct": "10%"}, "node1": {"pct": "20%"}}, + } + + monkeypatch.setattr(app, "_collect_operational_snapshot", snapshot) + with TestClient(app.app) as client: + response = client.get("/drift", headers={"Authorization": "Bearer test-token"}) + + assert response.status_code == 200 + result = response.json() + assert result["state"] == "warning" + assert {item["code"] for item in result["findings"]} == { + "inventory_missing_backup", "inventory_missing_disk", "backup_without_inventory" + } + + +def test_maintenance_preflight_blocks_active_target_backup(monkeypatch): + async def snapshot(): + return { + "services": {}, + "hosts": {"emby-chris": {"reachable": True, "containers": ["emby: Up 2 days"]}}, + "backups": {"summary": {}, "hosts": {"emby-chris": {"state": "healthy"}}}, + "disks": {"emby-chris": {"pct": "30%"}}, + } + + async def active_backups(): + return {"emby-chris": "active"} + + monkeypatch.setattr(app, "_collect_operational_snapshot", snapshot) + monkeypatch.setattr(app, "_collect_active_backups", active_backups) + with TestClient(app.app) as client: + response = client.get( + "/maintenance/preflight?action=docker&target=emby-chris", + headers={"Authorization": "Bearer test-token"}, + ) + + assert response.status_code == 200 + result = response.json() + assert result["safe"] is False + assert result["blockers"] == [{"code": "backup_active", "target": "emby-chris"}] + + def test_overview_is_compact_deterministic_and_light_model_friendly(monkeypatch): async def service_data(): return { @@ -583,3 +1480,124 @@ def test_speedtest_deploy_requires_strong_secrets_and_uses_full_git_app(monkeypa assert deploy[1]["compose.yaml"] == "content:compose.yaml" assert deploy[2] == "correct-horse-battery-staple" assert deploy[3] == "streamscope-session-secret-with-entropy" + + +def test_sascha_media_edge_audit_uses_fixed_hetzner_host(monkeypatch): + payload = { + "hostname": "pfannkuchen", + "caddy": {"container_running": True, "protocols": ["h1", "h2", "h3"], "upstreams": ["10.6.1.103:8096"]}, + "network": {"wg_media": False, "udp_51821": False}, + } + calls = [] + monkeypatch.setattr(app, "_find_inventory_host", lambda name: {"name": name, "user": "root", "ip": "46.225.230.72"}) + monkeypatch.setattr(app, "_ssh", lambda host, command, timeout=30: (calls.append((host, command, timeout)) or (0, json.dumps(payload), ""))) + with TestClient(app.app) as client: + response = client.get("/media/edge/sascha", headers={"Authorization": "Bearer test-token"}) + assert response.status_code == 200 + assert response.json()["caddy"]["upstreams"] == ["10.6.1.103:8096"] + assert calls[0][0] == "root@46.225.230.72" + assert "PrivateKey" not in response.text + + +def test_sascha_media_tunnel_defaults_to_side_effect_free_dry_run(monkeypatch): + monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("dry-run must not SSH"))) + with TestClient(app.app) as client: + response = client.post( + "/network/media-tunnel/sascha", + headers={"Authorization": "Bearer test-token"}, + json={}, + ) + assert response.status_code == 200 + body = response.json() + assert body["status"] == "would_deploy" + assert body["vps_address"] == "10.11.13.1/32" + assert body["emby_address"] == "10.11.13.3/32" + assert body["listen_port"] == 51821 + + +def test_sascha_media_tunnel_requires_explicit_confirmation(monkeypatch): + monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("unconfirmed request must not SSH"))) + with TestClient(app.app) as client: + response = client.post( + "/network/media-tunnel/sascha", + headers={"Authorization": "Bearer test-token"}, + json={"dry_run": False}, + ) + assert response.status_code == 400 + + +def test_sascha_media_tunnel_apply_returns_redacted_result(monkeypatch): + result = { + "status": "deployed", + "interface": "wg-media", + "vps_address": "10.11.13.1/32", + "emby_address": "10.11.13.3/32", + "listen_port": 51821, + "handshake": True, + "ping_vps_to_emby": True, + "ping_emby_to_vps": True, + } + monkeypatch.setattr(app, "_deploy_sascha_media_tunnel", lambda: result) + with TestClient(app.app) as client: + response = client.post( + "/network/media-tunnel/sascha", + headers={"Authorization": "Bearer test-token"}, + json={"dry_run": False, "confirmation": "DEPLOY_DIRECT_SASCHA_MEDIA_TUNNEL"}, + ) + assert response.status_code == 200 + assert response.json()["handshake"] is True + assert "private" not in response.text.lower() + + +def test_sascha_media_tunnel_candidate_uses_valid_wireguard_interface_name(): + import base64 + import re + + command = app._media_tunnel_install_command("vps", "A" * 43 + "=") + encoded = re.search(r"b64decode\('([^']+)'\)", command).group(1) + script = base64.b64decode(encoded).decode() + assert 'root / "wgmtest.conf"' in script + assert len("wgmtest") <= 15 + + +def test_sascha_media_path_benchmark_returns_both_fixed_routes(monkeypatch): + monkeypatch.setattr(app, "_benchmark_sascha_media_paths", lambda: { + "legacy_node6": {"bytes": 268435456, "seconds": 4.0, "mbit_s": 536.9}, + "direct_wg_media": {"bytes": 268435456, "seconds": 3.0, "mbit_s": 715.8}, + }) + with TestClient(app.app) as client: + response = client.post( + "/network/media-tunnel/sascha/benchmark", + headers={"Authorization": "Bearer test-token"}, + ) + assert response.status_code == 200 + assert response.json()["direction"] == "emby-sascha_to_hetzner" + assert set(response.json()["results"]) == {"legacy_node6", "direct_wg_media"} + + +def test_sascha_media_edge_optimize_is_dry_run_by_default(monkeypatch): + monkeypatch.setattr(app, "_optimize_sascha_media_edge", lambda: (_ for _ in ()).throw(AssertionError("dry-run must not mutate"))) + with TestClient(app.app) as client: + response = client.post("/media/edge/sascha/optimize", headers={"Authorization": "Bearer test-token"}, json={}) + assert response.status_code == 200 + assert response.json()["protocols"] == ["h1", "h2"] + assert response.json()["upstream"] == "10.11.13.3:8096" + + +def test_sascha_media_edge_optimize_requires_confirmation(monkeypatch): + monkeypatch.setattr(app, "_optimize_sascha_media_edge", lambda: (_ for _ in ()).throw(AssertionError("must not mutate"))) + with TestClient(app.app) as client: + response = client.post("/media/edge/sascha/optimize", headers={"Authorization": "Bearer test-token"}, json={"dry_run": False}) + assert response.status_code == 400 + + +def test_sascha_media_edge_optimize_applies_fixed_safe_result(monkeypatch): + monkeypatch.setattr(app, "_optimize_sascha_media_edge", lambda: {"status": "optimized", "upstream": "10.11.13.3:8096", "protocols": ["h1", "h2"], "backup": "/app-config/caddy/backup", "sha256": "a" * 64}) + with TestClient(app.app) as client: + response = client.post( + "/media/edge/sascha/optimize", + headers={"Authorization": "Bearer test-token"}, + json={"dry_run": False, "confirmation": "OPTIMIZE_TV_SASCHA_LUTZ_DE"}, + ) + assert response.status_code == 200 + assert response.json()["status"] == "optimized" diff --git a/tests/test_bw_manager_deploy.py b/tests/test_bw_manager_deploy.py index cf5a339..670435b 100644 --- a/tests/test_bw_manager_deploy.py +++ b/tests/test_bw_manager_deploy.py @@ -18,6 +18,8 @@ def load_app(monkeypatch): def test_bw_manager_deploy_rejects_bundle_without_and_gate(monkeypatch): app = load_app(monkeypatch) + if not hasattr(app, "BW_MANAGER_REPO_FILES"): + pytest.skip("BW-Manager was intentionally retired on 13.08.2026") files = {path: "placeholder" for path in app.BW_MANAGER_REPO_FILES} files["compose.yaml"] = "services:\n bw-manager:\n build: ./src\n" files["src/app.py"] = "def old_policy(): pass\n" diff --git a/tests/test_guck_admin_deploy.py b/tests/test_guck_admin_deploy.py new file mode 100644 index 0000000..74d1f79 --- /dev/null +++ b/tests/test_guck_admin_deploy.py @@ -0,0 +1,54 @@ +import app +import pytest +from fastapi.testclient import TestClient + + +def valid_bundle(): + files={path:'placeholder' for path in app.GUCK_ADMIN_REPO_FILES} + files['guck-admin/compose.yaml']='''services:\n guck-admin:\n build: .\n network_mode: host\n cap_add: [NET_ADMIN]\n environment:\n GUCK_LIMIT: /host/guck-limit.sh\n volumes:\n - /app-config/guck-admin/data:/data\n control-monitor:\n build: .\n network_mode: host\n cap_add: [NET_ADMIN]\n''' + files['guck-admin/src/control.py']='''CREATE TABLE IF NOT EXISTS custom_networks\n2a00:8c40:f000::/36\n45.58.235.0/24\ndef sync_custom_networks(): pass\n''' + return files + + +def test_guck_admin_bundle_requires_persistent_custom_network_policy(): + files=valid_bundle() + files['guck-admin/src/control.py']='def old_control(): pass\n' + with pytest.raises(ValueError,match='custom VPN'): + app._validate_guck_admin_bundle(files) + + +def test_guck_admin_deploy_dry_run_has_no_remote_side_effect(monkeypatch): + calls=[] + monkeypatch.setattr(app,'_find_inventory_host',lambda host:{'user':'debian','ip':'141.94.237.199'}) + monkeypatch.setattr(app,'_ssh',lambda *args,**kwargs:calls.append(args)) + result=app._deploy_guck_admin_compose(valid_bundle(),dry_run=True) + assert result['status']=='validated' + assert result['host']=='guck-vps' + assert calls==[] + + +def test_guck_admin_deploy_uses_inventory_target_and_verifies_policy(monkeypatch): + calls=[] + monkeypatch.setattr(app,'_find_inventory_host',lambda host:{'user':'debian','ip':'141.94.237.199'}) + def fake_ssh(host,command,timeout=600): + calls.append((host,command,timeout)) + if 'ipset test vpn-v6' in command: + return 0,'policy ok','' + return 0,'ok','' + monkeypatch.setattr(app,'_ssh',fake_ssh) + result=app._deploy_guck_admin_compose(valid_bundle(),dry_run=False) + assert result['status']=='deployed' + assert result['policy']=='verified' + assert all(host=='debian@141.94.237.199' for host,_,_ in calls) + commands='\n'.join(command for _,command,_ in calls) + assert 'docker compose build' in commands + assert '127.0.0.1:9090/health' in commands + assert '/actions/limiter/refresh' in commands + assert 'ipset test vpn-v6 2a00:8c40:f02d:a34c::1' in commands + assert 'ipset test vpn-v4 45.58.235.7' in commands + + +def test_guck_admin_deploy_endpoint_requires_auth(monkeypatch): + monkeypatch.setattr(app,'BUTLER_TOKEN','test-token') + response=TestClient(app.app).post('/vps/guck-admin/deploy',json={'dry_run':True}) + assert response.status_code in (401,403) diff --git a/ui.html b/ui.html new file mode 100644 index 0000000..7b8c45b --- /dev/null +++ b/ui.html @@ -0,0 +1,125 @@ + + + + + + + Pfannkuchen Butler + + + +
+ +
+

Übersicht

+
+
+
Live Zustand

Dein Homelab auf einen Blick.

Deterministisch aus Butler-Collectoren – keine geschätzten Zustände.

Noch nicht geladen
+
+
Gesamtzustand
—
Butler Overview
+
Services
—
Funktionsprobes
+
Hosts
—
Inventory erreichbar
+
Backups
—
inkl. Policy
+
+
Findings
0
Overview wird geladen …
API-Abdeckung
live aus OpenAPI
+
+ +
+
Korrelierte Diagnose

Doctor

Service, Host, Container, Backup und Disk in einer Prüfung.

+
Gib einen bekannten Host oder Service ein.
+
+ +
+
Konservative Anomalieerkennung

Emby Account Sharing

Zeitgleiche öffentliche Netze und geografisch unmögliche Wechsel – über beide Emby-Server.

read-only
+
+ +
IPv4 wird exakt verglichen. IPv6 wird als /64 angezeigt und für die Haushaltserkennung konservativ auf /48 zusammengefasst.
+
+
Analysierte Benutzer
—
Auffällige Benutzer
—
Zeitgleiche Netze
—
Unmögliche Wechsel
—
+
Erkannte Ereignisse
0
Analyse noch nicht gestartet.
Benutzer-Risiko
nur begründete Treffer
Noch keine Daten.
+
+ +
+
Read-only Safety Gate

Wartung & Drift

Blocker erkennen, bevor eine Änderung Schaden anrichtet.

+
Maintenance Preflight
Noch kein Preflight ausgeführt.
Collector Drift
Drift wird geladen …
+
+ +
+
KI- und API-Protokoll

Butler-Aktivitäten

Wer hat wann welche Butler-Funktion aufgerufen? Sensible Werte bleiben redigiert.

+
Letzte Aktivitäten
ZeitAkteurMethodeEndpointStatusDry-runDetail
Wird geladen …
+
+ +
+
Vollständiger Katalog

Alle Butler-Funktionen

Jede konkrete Operation aus OpenAPI sowie Proxy- und Metawege. Mutationen werden hier bewusst nicht blind ausgeführt.

—
+
+
+
+
+
+
+
+ + + +