sync: app.py from Butler 2.3

app.py

@@ -1,14 +1,17 @@
 """Homelab Butler v2.1 – Unified API proxy for Pfannkuchen homelab.
 Reads service config from butler.yaml, credentials from Vaultwarden cache with flat-file fallback."""
 
-import os, json, asyncio, logging, time
+import os, json, asyncio, logging, time, base64, re, subprocess, ipaddress
 from datetime import datetime, timezone
 import httpx, yaml
+from typing import Literal
+from pydantic import BaseModel
 from fastapi import FastAPI, Request, HTTPException, Depends, Query
 from fastapi.responses import JSONResponse, RedirectResponse
 from contextlib import asynccontextmanager
 
 log = logging.getLogger("butler")
+VERSION = "2.3.0"
 
 API_DIR = os.environ.get("API_KEY_DIR", "/data/api")
 VAULT_CACHE_DIR = os.environ.get("VAULT_CACHE_DIR", "/data/vault-cache")
@@ -37,6 +40,7 @@ def _load_config():
 SERVICES: dict = {}
 VM_CFG: dict = {}
 TTS_CFG: dict = {}
+_load_config()
 
 # --- Audit log ---
 
@@ -92,9 +96,45 @@ async def lifespan(app: FastAPI):
     yield
     task.cancel()
 
-app = FastAPI(title="Homelab Butler", version="2.1.0", lifespan=lifespan,
+app = FastAPI(title="Homelab Butler", version=VERSION, lifespan=lifespan,
               description="Unified API proxy + infrastructure management. AI agents: see GET / for self-onboarding.")
 
+
+OverviewState = Literal["healthy", "warning", "critical"]
+
+
+class OverviewCounts(BaseModel):
+    critical: int
+    warning: int
+    healthy: int
+
+
+class OverviewFinding(BaseModel):
+    severity: OverviewState
+    code: str
+    target: str
+    message: str
+    age_hours: float | None = None
+    pct: int | None = None
+
+
+class OverviewModelContract(BaseModel):
+    instruction: str
+    severity_order: list[OverviewState]
+
+
+class OverviewResponse(BaseModel):
+    schema_version: Literal[1]
+    generated: datetime
+    overall_state: OverviewState
+    action_required: bool
+    summary: OverviewCounts
+    components: dict[str, OverviewCounts]
+    findings: list[OverviewFinding]
+    model_contract: OverviewModelContract
+    details: dict | None = None
+
+
 # --- Credential reading (vault-first, file-fallback) ---
 
 def _read(name):
@@ -162,6 +202,54 @@ def _get_key(cfg):
         return _vault_cache[vault_key]
     return _read(cfg.get("key_file", ""))
 
+
+SENSITIVE_RESPONSE_FIELDS = {
+    "hawsertoken", "webhooksecret", "accesstoken", "refreshtoken",
+    "password", "secret", "apikey", "api_key", "privatekey",
+}
+
+
+def _redact_response(value, extra_fields=None):
+    """Recursively redact known secret fields in proxied JSON responses."""
+    sensitive = set(SENSITIVE_RESPONSE_FIELDS)
+    sensitive.update(str(x).lower() for x in (extra_fields or []))
+    if isinstance(value, dict):
+        return {
+            key: "[REDACTED]" if str(key).lower() in sensitive
+            else _redact_response(item, sensitive)
+            for key, item in value.items()
+        }
+    if isinstance(value, list):
+        return [_redact_response(item, sensitive) for item in value]
+    return value
+
+
+def _inventory_hosts(text: str) -> list[dict]:
+    """Parse Ansible inventory host lines and apply Pfannkuchen SSH defaults."""
+    hosts = []
+    seen = set()
+    for raw in text.splitlines():
+        line = raw.strip()
+        if not line or line.startswith(("#", "[")) or "ansible_host=" not in line:
+            continue
+        parts = line.split()
+        name = parts[0]
+        attrs = {k: v for k, v in (p.split("=", 1) for p in parts[1:] if "=" in p)}
+        ip = attrs.get("ansible_host")
+        if not ip or name in seen:
+            continue
+        user = attrs.get("ansible_user")
+        if not user:
+            if name.startswith("node"):
+                user = "root"
+            elif ip.startswith("10.7.1."):
+                user = "chris"
+            else:
+                user = "sascha"
+        hosts.append({"name": name, "ip": ip, "user": user})
+        seen.add(name)
+    return hosts
+
 # --- Routes ---
 
 @app.get("/")
@@ -171,7 +259,7 @@ async def root():
     for name, cfg in SERVICES.items():
         svc_list[name] = {"url": cfg.get("url", ""), "auth": cfg.get("auth", ""), "description": cfg.get("description", "")}
     return {
-        "service": "homelab-butler", "version": "2.1.0",
+        "service": "homelab-butler", "version": VERSION,
         "docs": "/docs",
         "openapi": "/openapi.json",
         "services": svc_list,
@@ -188,6 +276,7 @@ async def root():
             "tts_voices": "GET /tts/voices",
             "tts_health": "GET /tts/health",
             "status": "GET /status - health of all backends",
+            "overview": "GET /overview?details=false - deterministic homelab verdict for small models",
             "audit": "GET /audit - recent API calls",
         },
         "vault_items": len(_vault_cache),
@@ -195,23 +284,96 @@ async def root():
 
 @app.get("/health")
 async def health():
-    return {"status": "ok", "vault_items": len(_vault_cache), "services": len(SERVICES), "version": "2.1.0"}
+    return {"status": "ok", "vault_items": len(_vault_cache), "services": len(SERVICES), "version": VERSION}
+
+def _classify_http_status(status_code: int, expected: set[int]) -> str:
+    """Return a deterministic service state suitable for small models."""
+    if status_code in expected:
+        return "healthy"
+    if status_code in (401, 403):
+        return "auth_failed"
+    if status_code == 404:
+        return "misconfigured"
+    return "degraded"
+
+
+def _service_auth(cfg: dict) -> dict:
+    """Build secret-bearing request data without ever returning it from an endpoint."""
+    auth_type = cfg.get("auth", "none")
+    headers = {}
+    cookies = {}
+    base_url = cfg.get("url")
+    if auth_type == "apikey":
+        headers["X-Api-Key"] = _get_key(cfg) or ""
+    elif auth_type == "apikey_urlfile":
+        base_url, key = _parse_url_key(cfg.get("key_file", ""))
+        headers["X-Api-Key"] = key or ""
+    elif auth_type == "bearer":
+        headers["Authorization"] = f"Bearer {_get_key(cfg) or ''}"
+    elif auth_type == "n8n":
+        headers["X-N8N-API-KEY"] = _get_key(cfg) or ""
+    elif auth_type == "proxmox":
+        pv = _parse_kv("proxmox")
+        headers["Authorization"] = f"PVEAPIToken={pv.get('tokenid', '')}={pv.get('secret', '')}"
+    return {"base_url": base_url, "headers": headers, "cookies": cookies}
+
+
+async def _collect_service_status() -> dict:
+    """Run authenticated functional probes concurrently and classify their result."""
+    started = time.monotonic()
+    async with httpx.AsyncClient(verify=False, timeout=5, follow_redirects=True) as client:
+        async def probe(name: str, cfg: dict):
+            probe_started = time.monotonic()
+            try:
+                request_data = _service_auth(cfg)
+                base_url = request_data["base_url"]
+                if not base_url:
+                    return name, {
+                        "reachable": False, "status": "misconfigured", "message": "No service URL configured"
+                    }
+                if cfg.get("auth") == "session":
+                    request_data["cookies"] = await _dockhand_login(client) or {}
+                health_path = cfg.get("health_path", "")
+                target = f"{base_url.rstrip('/')}/{health_path.lstrip('/')}" if health_path else base_url
+                expected = {int(code) for code in cfg.get("health_expected", range(200, 400))}
+                response = await client.get(
+                    target,
+                    headers=request_data["headers"],
+                    cookies=request_data["cookies"],
+                )
+                state = _classify_http_status(response.status_code, expected)
+                messages = {
+                    "healthy": "Functional probe succeeded",
+                    "auth_failed": "Configured credentials were rejected",
+                    "misconfigured": "Configured health route was not found",
+                    "degraded": "Backend returned an unexpected HTTP status",
+                }
+                return name, {
+                    "reachable": True,
+                    "status": state,
+                    "http": response.status_code,
+                    "latency_ms": round((time.monotonic() - probe_started) * 1000),
+                    "message": messages[state],
+                }
+            except Exception as exc:
+                return name, {
+                    "reachable": False,
+                    "status": "offline",
+                    "latency_ms": round((time.monotonic() - probe_started) * 1000),
+                    "error": type(exc).__name__,
+                    "message": "Backend could not be reached",
+                }
+
+        pairs = await asyncio.gather(*(probe(name, cfg) for name, cfg in SERVICES.items()))
+    results = dict(pairs)
+    results["_meta"] = {"duration_ms": round((time.monotonic() - started) * 1000)}
+    return results
+
 
 @app.get("/status")
 async def status(_=Depends(_verify)):
-    """Health check all configured backend services."""
+    """Authenticated functional health check for all configured backends."""
-    results = {}
+    results = await _collect_service_status()
-    async with httpx.AsyncClient(verify=False, timeout=5) as c:
-        for name, cfg in SERVICES.items():
-            url = cfg.get("url")
-            if not url:
-                results[name] = {"status": "no_url"}
-                continue
-            try:
-                r = await c.get(url, follow_redirects=True)
-                results[name] = {"status": "ok", "http": r.status_code}
-            except Exception as e:
-                results[name] = {"status": "offline", "error": type(e).__name__}
     _audit("/status", "GET", 200)
     return results
 
@@ -227,6 +389,383 @@ async def config_reload(_=Depends(_verify)):
     _load_vault_cache()
     return {"config_services": len(SERVICES), "vault_items": len(_vault_cache)}
 
+
+@app.get("/info")
+async def info(_=Depends(_verify)):
+    """Secret-free machine-readable context for AI agents and operators."""
+    return {
+        "service": "homelab-butler",
+        "version": VERSION,
+        "generated": datetime.now(timezone.utc).isoformat(),
+        "services": {
+            name: {
+                "url": cfg.get("url"),
+                "auth": cfg.get("auth"),
+                "description": cfg.get("description", ""),
+            }
+            for name, cfg in SERVICES.items()
+        },
+        "endpoints": {
+            "status": "/status",
+            "overview": "/overview?details=false",
+            "audit": "/audit",
+            "host_health": "/health/all",
+            "backups": "/backup/status",
+            "disk": "/disk/usage",
+            "logs": "/logs/{host}/{container}?tail=200",
+            "inspect": "/docker/inspect/{host}/{container}",
+            "docs": "/docs",
+        },
+        "rules": [
+            "Backend services are accessed through Butler or dedicated MCP servers",
+            "VMs only; no LXC",
+            "Docker Compose is stored in Git; no docker run",
+            "Persistent volumes live under /app-config",
+            "Node 7 VM SSH user is chris",
+        ],
+    }
+
+
+def _get_inventory_hosts() -> list[dict]:
+    rc, out, _err = _ssh(
+        AUTOMATION1,
+        "python3 -c \"print(open('/app-config/ansible/pfannkuchen.ini').read())\"",
+        timeout=15,
+    )
+    return _inventory_hosts(out) if rc == 0 else []
+
+
+def _find_inventory_host(name: str) -> dict | None:
+    return next((host for host in _get_inventory_hosts() if host["name"] == name), None)
+
+
+async def _get_inventory_hosts_async() -> list[dict]:
+    return await asyncio.to_thread(_get_inventory_hosts)
+
+
+async def _collect_health_all(concurrency: int = 10) -> dict:
+    """Collect SSH and container health concurrently with bounded fan-out."""
+    semaphore = asyncio.Semaphore(concurrency)
+
+    async def inspect_host(host: dict):
+        async with semaphore:
+            rc, out, err = await asyncio.to_thread(
+                _ssh,
+                f'{host["user"]}@{host["ip"]}',
+                "echo __BUTLER_OK__; (sudo -n docker ps --format '{{.Names}}: {{.Status}}' 2>/dev/null || docker ps --format '{{.Names}}: {{.Status}}' 2>/dev/null) | head -30",
+                10,
+            )
+        lines = out.strip().splitlines()
+        return host["name"], {
+            "ip": host["ip"],
+            "user": host["user"],
+            "reachable": rc == 0 and bool(lines) and lines[0] == "__BUTLER_OK__",
+            "containers": lines[1:] if lines and lines[0] == "__BUTLER_OK__" else [],
+            "error": err.strip()[:200] if rc != 0 else None,
+        }
+
+    hosts = await _get_inventory_hosts_async()
+    pairs = await asyncio.gather(*(inspect_host(host) for host in hosts))
+    return dict(pairs)
+
+
+@app.get("/health/all")
+async def health_all(_=Depends(_verify)):
+    """SSH reachability and Docker status for all inventory hosts."""
+    return await _collect_health_all()
+
+
+def _parse_backup_time(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+        return parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed.astimezone(timezone.utc)
+    except (TypeError, ValueError):
+        return None
+
+
+def _backup_item(rc: int, out: str, err: str, now: datetime | None = None) -> dict:
+    """Normalize borgmatic output into one small-model-friendly state object."""
+    item = {"state": "unknown", "ok": False, "last_backup": None, "age_hours": None}
+    if rc != 0 or not out.strip():
+        if err:
+            item["error"] = err.strip()[:200]
+        return item
+    try:
+        data = json.loads(out)
+        archives = data[0].get("archives", []) if isinstance(data, list) and data else []
+        if not archives:
+            item["error"] = "no archives returned"
+            return item
+        last = archives[-1]
+        started_at = _parse_backup_time(last.get("start"))
+        if not started_at:
+            item["error"] = "invalid backup timestamp"
+            return item
+        age_hours = max(0, ((now or datetime.now(timezone.utc)) - started_at).total_seconds() / 3600)
+        state = "healthy" if age_hours <= 30 else "warning" if age_hours <= 48 else "critical"
+        return {
+            "state": state,
+            "ok": state == "healthy",
+            "last_backup": last.get("start"),
+            "age_hours": round(age_hours, 1),
+            "name": last.get("name"),
+        }
+    except (json.JSONDecodeError, TypeError, IndexError, KeyError):
+        item["error"] = "invalid borgmatic JSON"
+        return item
+
+
+async def _collect_backup_status(concurrency: int = 10) -> dict:
+    """Query VM backups concurrently; one slow host no longer blocks all others serially."""
+    semaphore = asyncio.Semaphore(concurrency)
+    hosts = [host for host in await _get_inventory_hosts_async() if not host["name"].startswith("node")]
+
+    async def inspect_backup(host: dict):
+        async with semaphore:
+            rc, out, err = await asyncio.to_thread(
+                _ssh,
+                f'{host["user"]}@{host["ip"]}',
+                "sudo -n borgmatic list --last 1 --json 2>/dev/null",
+                12,
+            )
+        return host["name"], _backup_item(rc, out, err)
+
+    pairs = await asyncio.gather(*(inspect_backup(host) for host in hosts))
+    results = dict(pairs)
+    summary = {"total": len(results), "healthy": 0, "warning": 0, "critical": 0, "unknown": 0}
+    for item in results.values():
+        summary[item["state"]] += 1
+    return {"summary": summary, "hosts": results}
+
+
+@app.get("/backup/status")
+async def backup_status(_=Depends(_verify)):
+    """Latest Borgmatic archive, age and severity for all VM inventory hosts."""
+    return await _collect_backup_status()
+
+
+async def _collect_disk_usage(concurrency: int = 10) -> dict:
+    semaphore = asyncio.Semaphore(concurrency)
+
+    async def inspect_disk(host: dict):
+        async with semaphore:
+            rc, out, _err = await asyncio.to_thread(
+                _ssh, f'{host["user"]}@{host["ip"]}', "df -P / | tail -1", 10
+            )
+        parts = out.split()
+        if rc != 0 or len(parts) < 6:
+            return host["name"], None
+        return host["name"], {
+            "size_kib": int(parts[1]), "used_kib": int(parts[2]),
+            "avail_kib": int(parts[3]), "pct": parts[4], "mount": parts[5],
+        }
+
+    hosts = await _get_inventory_hosts_async()
+    pairs = await asyncio.gather(*(inspect_disk(host) for host in hosts))
+    return {name: item for name, item in pairs if item is not None}
+
+
+@app.get("/disk/usage")
+async def disk_usage(_=Depends(_verify)):
+    """Root filesystem usage for all reachable inventory hosts."""
+    return await _collect_disk_usage()
+
+
+def _add_component(summary: dict, bucket: dict, state: str):
+    normalized = state if state in ("healthy", "warning", "critical") else "warning"
+    summary[normalized] += 1
+    bucket[normalized] += 1
+
+
+@app.get("/overview", response_model=OverviewResponse, response_model_exclude_none=True)
+async def overview(details: bool = Query(False), _=Depends(_verify)):
+    """Compact deterministic homelab verdict designed for small language models."""
+    services, hosts, backups, disks = await asyncio.gather(
+        _collect_service_status(),
+        _collect_health_all(),
+        _collect_backup_status(),
+        _collect_disk_usage(),
+    )
+    summary = {"critical": 0, "warning": 0, "healthy": 0}
+    component_summary = {
+        name: {"critical": 0, "warning": 0, "healthy": 0}
+        for name in ("services", "hosts", "backups", "disks")
+    }
+    findings = []
+
+    for name in sorted(hosts):
+        item = hosts[name]
+        containers = item.get("containers", [])
+        bad_container = next(
+            (line for line in containers if "unhealthy" in line.lower() or "restarting" in line.lower()),
+            None,
+        )
+        if not item.get("reachable"):
+            state = "critical"
+            findings.append({
+                "severity": "critical", "code": "host_unreachable", "target": name,
+                "message": "Host is not reachable over SSH",
+            })
+        elif bad_container:
+            state = "critical"
+            findings.append({
+                "severity": "critical", "code": "container_unhealthy", "target": name,
+                "message": bad_container[:200],
+            })
+        else:
+            state = "healthy"
+        _add_component(summary, component_summary["hosts"], state)
+
+    service_states = {
+        "healthy": "healthy", "degraded": "warning", "offline": "critical",
+        "auth_failed": "critical", "misconfigured": "critical",
+    }
+    service_codes = {
+        "degraded": "service_degraded", "offline": "service_offline",
+        "auth_failed": "service_auth_failed", "misconfigured": "service_misconfigured",
+    }
+    for name in sorted(key for key in services if not key.startswith("_")):
+        item = services[name]
+        raw_state = item.get("status", "degraded")
+        state = service_states.get(raw_state, "warning")
+        _add_component(summary, component_summary["services"], state)
+        if state != "healthy":
+            findings.append({
+                "severity": state,
+                "code": service_codes.get(raw_state, "service_degraded"),
+                "target": name,
+                "message": item.get("message", "Service health probe failed"),
+            })
+
+    for name in sorted(backups.get("hosts", {})):
+        item = backups["hosts"][name]
+        raw_state = item.get("state", "unknown")
+        state = "healthy" if raw_state == "healthy" else "critical" if raw_state in ("critical", "unknown") else "warning"
+        _add_component(summary, component_summary["backups"], state)
+        if state != "healthy":
+            findings.append({
+                "severity": state,
+                "code": f"backup_{raw_state}",
+                "target": name,
+                "message": "Backup is missing, stale or could not be verified",
+                "age_hours": item.get("age_hours"),
+            })
+
+    for name in sorted(disks):
+        pct = int(str(disks[name].get("pct", "0")).rstrip("%") or 0)
+        state = "critical" if pct >= 90 else "warning" if pct >= 80 else "healthy"
+        _add_component(summary, component_summary["disks"], state)
+        if state != "healthy":
+            findings.append({
+                "severity": state,
+                "code": "disk_critical" if state == "critical" else "disk_high",
+                "target": name,
+                "message": f"Root filesystem usage is {pct}%",
+                "pct": pct,
+            })
+
+    overall_state = "critical" if summary["critical"] else "warning" if summary["warning"] else "healthy"
+    response = {
+        "schema_version": 1,
+        "generated": datetime.now(timezone.utc).isoformat(),
+        "overall_state": overall_state,
+        "action_required": overall_state != "healthy",
+        "summary": summary,
+        "components": component_summary,
+        "findings": findings,
+        "model_contract": {
+            "instruction": "Report overall_state, then findings in the returned order. Do not infer missing facts.",
+            "severity_order": ["critical", "warning", "healthy"],
+        },
+    }
+    if details:
+        response["details"] = {"services": services, "hosts": hosts, "backups": backups, "disks": disks}
+    _audit("/overview", "GET", 200, f"state={overall_state} findings={len(findings)}")
+    return response
+
+
+@app.get("/logs/{host}/{container}")
+async def docker_logs(host: str, container: str, tail: int = Query(200, ge=1, le=20000), _=Depends(_verify)):
+    """Read Docker logs from an inventory host."""
+    if not re.fullmatch(r"[A-Za-z0-9_.-]+", host) or not re.fullmatch(r"[A-Za-z0-9_.-]+", container):
+        raise HTTPException(400, "Invalid host or container name")
+    target = _find_inventory_host(host)
+    if not target:
+        raise HTTPException(404, f"Host {host} not found")
+    rc, out, err = _ssh(
+        f'{target["user"]}@{target["ip"]}',
+        f"sudo -n docker logs {container} --tail {tail} 2>&1 || docker logs {container} --tail {tail} 2>&1",
+        timeout=30,
+    )
+    if rc != 0:
+        raise HTTPException(502, (err or out).strip()[:500])
+    return {"host": host, "container": container, "tail": tail, "output": out}
+
+
+@app.get("/docker/inspect/{host}/{container}")
+async def docker_inspect(host: str, container: str, _=Depends(_verify)):
+    """Return a sanitized runtime/resource summary for a Docker container."""
+    if not re.fullmatch(r"[A-Za-z0-9_.-]+", host) or not re.fullmatch(r"[A-Za-z0-9_.-]+", container):
+        raise HTTPException(400, "Invalid host or container name")
+    target = _find_inventory_host(host)
+    if not target:
+        raise HTTPException(404, f"Host {host} not found")
+    rc, out, err = _ssh(
+        f'{target["user"]}@{target["ip"]}',
+        f"sudo -n docker inspect {container}",
+        timeout=20,
+    )
+    if rc != 0:
+        raise HTTPException(502, (err or out).strip()[:500])
+    try:
+        raw = json.loads(out)[0]
+    except (json.JSONDecodeError, IndexError, TypeError):
+        raise HTTPException(502, "Invalid docker inspect response")
+    host_cfg = raw.get("HostConfig", {})
+    cfg = raw.get("Config", {})
+    state = raw.get("State", {})
+    return {
+        "name": raw.get("Name", "").lstrip("/"),
+        "image": cfg.get("Image"),
+        "state": {
+            "status": state.get("Status"), "running": state.get("Running"),
+            "started_at": state.get("StartedAt"), "exit_code": state.get("ExitCode"),
+            "oom_killed": state.get("OOMKilled"), "restart_count": raw.get("RestartCount"),
+        },
+        "runtime": host_cfg.get("Runtime"),
+        "resources": {
+            "memory": host_cfg.get("Memory"), "memory_reservation": host_cfg.get("MemoryReservation"),
+            "nano_cpus": host_cfg.get("NanoCpus"), "device_requests": host_cfg.get("DeviceRequests"),
+        },
+        "restart_policy": host_cfg.get("RestartPolicy"),
+        "log_config": host_cfg.get("LogConfig"),
+        "environment_keys": sorted(item.split("=", 1)[0] for item in cfg.get("Env", []) if "=" in item),
+        "mounts": [
+            {"type": mount.get("Type"), "source": mount.get("Source"), "destination": mount.get("Destination"), "rw": mount.get("RW")}
+            for mount in raw.get("Mounts", [])
+        ],
+    }
+
+
+@app.post("/docker/restart/{host}/{container}")
+async def docker_restart(host: str, container: str, _=Depends(_verify), dry_run: bool = Query(False)):
+    """Restart a named Docker container, with optional dry-run."""
+    if not re.fullmatch(r"[A-Za-z0-9_.-]+", host) or not re.fullmatch(r"[A-Za-z0-9_.-]+", container):
+        raise HTTPException(400, "Invalid host or container name")
+    target = _find_inventory_host(host)
+    if not target:
+        raise HTTPException(404, f"Host {host} not found")
+    if dry_run:
+        return {"dry_run": True, "host": host, "container": container}
+    rc, out, err = _ssh(f'{target["user"]}@{target["ip"]}', f"sudo -n docker restart {container}", timeout=45)
+    _audit(f"/docker/restart/{host}/{container}", "POST", 200 if rc == 0 else 502)
+    if rc != 0:
+        raise HTTPException(502, (err or out).strip()[:500])
+    return {"success": True, "output": out.strip()}
+
+
 @app.post("/vault/reload")
 async def vault_reload(_=Depends(_verify)):
     _load_vault_cache()
@@ -234,7 +773,6 @@ async def vault_reload(_=Depends(_verify)):
 
 
 # --- VM Lifecycle Endpoints ---
-from pydantic import BaseModel
 import subprocess as _sp
 
 AUTOMATION1 = VM_CFG.get("automation_host", "sascha@10.5.85.5") if VM_CFG else "sascha@10.5.85.5"
@@ -249,9 +787,12 @@ class VMCreate(BaseModel):
     disk: int = 32
 
 def _ssh(host, cmd, timeout=600):
-    r = _sp.run(["ssh","-o","ConnectTimeout=10","-o","StrictHostKeyChecking=accept-new",host,cmd],
+    try:
-                capture_output=True, text=True, timeout=timeout)
+        r = _sp.run(["ssh","-o","ConnectTimeout=10","-o","StrictHostKeyChecking=accept-new",host,cmd],
-    return r.returncode, r.stdout, r.stderr
+                    capture_output=True, text=True, timeout=timeout)
+        return r.returncode, r.stdout, r.stderr
+    except _sp.TimeoutExpired:
+        return 124, "", f"SSH command timed out after {timeout} seconds"
 
 def _pve_auth():
     pv = _parse_kv("proxmox")
@@ -497,36 +1038,67 @@ print('removed')
 
 @app.post("/inventory/host")
 async def inventory_host(request: Request, _=Depends(_verify)):
+    """Create or update an Ansible inventory host idempotently."""
     body = await request.json()
-    name, ip = body["name"], body["ip"]
+    name, ip = body.get("name", ""), body.get("ip", "")
     group = body.get("group", "auto")
     user = body.get("user", "sascha")
+    if not all(re.fullmatch(r"[A-Za-z0-9_.-]+", value) for value in (name, group, user)):
+        raise HTTPException(400, "Invalid name, group, or user")
+    try:
+        ipaddress.ip_address(ip)
+    except ValueError:
+        raise HTTPException(400, "Invalid IP address")
+
     ini = "/app-config/ansible/pfannkuchen.ini"
-    # Add host to group in pfannkuchen.ini (idempotent)
+    host_line = f"{name} ansible_host={ip} ansible_user={user}"
-    add_cmd = f"""python3 -c "
+    script = f'''lines = open({ini!r}).readlines()
-lines = open('{ini}').readlines()
+name = {name!r}
-# Check if host already exists
+host_line = {host_line!r}
-if any('{name} ' in l or '{name}\\n' in l for l in lines):
+group = {group!r}
-    print('already exists')
+updated = False
-else:
+for idx, line in enumerate(lines):
-    # Find the group and insert after it
+    parts = line.split()
-    out, found = [], False
+    if parts and parts[0] == name and "ansible_host=" in line:
-    for l in lines:
+        lines[idx] = host_line + "\\n"
-        out.append(l)
+        updated = True
-        if l.strip() == '[{group}]':
+        break
-            found = True
+if not updated:
-        elif found and (l.startswith('[') or l.strip() == ''):
+    insert_at = None
-            out.insert(-1, '{name} ansible_host={ip}\\n')
+    in_group = False
-            found = False
+    for idx, line in enumerate(lines):
-    if found:  # group was last
+        if line.strip() == "[" + group + "]":
-        out.append('{name} ansible_host={ip}\\n')
+            in_group = True
-    open('{ini}','w').writelines(out)
+            insert_at = idx + 1
-    print('added to [{group}]')
+            continue
-" """
+        if in_group and line.startswith("["):
-    rc, out, _ = _ssh(AUTOMATION1, add_cmd, timeout=30)
+            break
-    # Also create host_vars
+        if in_group:
-    _ssh(AUTOMATION1, f"mkdir -p /app-config/ansible/host_vars/{name} && printf 'ansible_host: {ip}\\nansible_user: {user}\\n' > /app-config/ansible/host_vars/{name}/vars.yml", timeout=30)
+            insert_at = idx + 1
-    return {"status": "ok", "name": name, "ip": ip, "group": group, "result": out.strip()}
+    if insert_at is None:
+        lines.extend(["\\n[" + group + "]\\n", host_line + "\\n"])
+    else:
+        lines.insert(insert_at, host_line + "\\n")
+open({ini!r}, "w").writelines(lines)
+print("updated" if updated else "added")'''
+    encoded_script = base64.b64encode(script.encode()).decode()
+    rc, out, err = _ssh(
+        AUTOMATION1,
+        f"python3 -c \"import base64;exec(base64.b64decode('{encoded_script}'))\"",
+        timeout=30,
+    )
+    if rc != 0:
+        raise HTTPException(502, err.strip()[:500])
+    vars_script = (
+        f"mkdir -p /app-config/ansible/host_vars/{name} && "
+        f"printf 'ansible_host: {ip}\\nansible_user: {user}\\n' > "
+        f"/app-config/ansible/host_vars/{name}/vars.yml"
+    )
+    rc2, _out2, err2 = _ssh(AUTOMATION1, vars_script, timeout=30)
+    if rc2 != 0:
+        raise HTTPException(502, err2.strip()[:500])
+    _audit("/inventory/host", "POST", 200, f"{name} {ip} {user}")
+    return {"status": "ok", "name": name, "ip": ip, "group": group, "user": user, "result": out.strip()}
 
 @app.post("/ansible/run")
 async def ansible_run(request: Request, _=Depends(_verify)):
@@ -749,17 +1321,20 @@ async def proxy(service: str, path: str, request: Request, _=Depends(_verify)):
     target = f"{base_url}/{path}"
     body = await request.body()
 
-    async with httpx.AsyncClient(verify=False, timeout=30.0) as client:
+    timeout = float(cfg.get("timeout", 30))
+    async with httpx.AsyncClient(verify=False, timeout=timeout) as client:
         resp = await client.request(method=request.method, url=target,
-                                    headers=headers, cookies=cookies, content=body)
+                                    headers=headers, cookies=cookies, content=body,
+                                    params=request.query_params)
         if auth_type == "session" and resp.status_code == 401:
             _dockhand_cookie = None
             await _dockhand_login(client)
             resp = await client.request(method=request.method, url=target,
-                                        headers=headers, cookies=_dockhand_cookie or {}, content=body)
+                                        headers=headers, cookies=_dockhand_cookie or {}, content=body,
+                                        params=request.query_params)
 
     try:
-        data = resp.json()
+        data = _redact_response(resp.json(), cfg.get("redact_response_fields"))
     except Exception:
         data = resp.text
     _audit(f"/{service}/{path}", request.method, resp.status_code)