v2.1: butler.yaml config, OpenAPI, /status, /audit, dry-run

app.py

@@ -1,14 +1,61 @@
-"""Homelab Butler – Unified API proxy for Pfannkuchen homelab.
+"""Homelab Butler v2.1 – Unified API proxy for Pfannkuchen homelab.
-Reads credentials from Vaultwarden cache (synced by host cron) with flat-file fallback."""
+Reads service config from butler.yaml, credentials from Vaultwarden cache with flat-file fallback."""
 
-import os, json, asyncio, logging
+import os, json, asyncio, logging, time
-import httpx
+from datetime import datetime, timezone
-from fastapi import FastAPI, Request, HTTPException, Depends
+import httpx, yaml
-from fastapi.responses import JSONResponse
+from fastapi import FastAPI, Request, HTTPException, Depends, Query
+from fastapi.responses import JSONResponse, RedirectResponse
 from contextlib import asynccontextmanager
 
 log = logging.getLogger("butler")
 
+API_DIR = os.environ.get("API_KEY_DIR", "/data/api")
+VAULT_CACHE_DIR = os.environ.get("VAULT_CACHE_DIR", "/data/vault-cache")
+BUTLER_TOKEN = os.environ.get("BUTLER_TOKEN", "")
+CONFIG_PATH = os.environ.get("BUTLER_CONFIG", "/data/butler.yaml")
+
+# --- Config loading ---
+
+_config: dict = {}
+
+def _load_config():
+    global _config, SERVICES, VM_CFG, TTS_CFG
+    try:
+        with open(CONFIG_PATH) as f:
+            _config = yaml.safe_load(f) or {}
+        SERVICES = _config.get("services", {})
+        VM_CFG = _config.get("vm", {})
+        TTS_CFG = _config.get("tts", {})
+        log.info(f"Loaded config: {len(SERVICES)} services")
+    except FileNotFoundError:
+        log.warning(f"No config at {CONFIG_PATH}, using defaults")
+        SERVICES = {}
+        VM_CFG = {}
+        TTS_CFG = {}
+
+SERVICES: dict = {}
+VM_CFG: dict = {}
+TTS_CFG: dict = {}
+
+# --- Audit log ---
+
+_audit_log: list[dict] = []
+MAX_AUDIT = 500
+
+def _audit(endpoint: str, method: str, status: int, detail: str = "", dry_run: bool = False):
+    entry = {
+        "ts": datetime.now(timezone.utc).isoformat(),
+        "endpoint": endpoint,
+        "method": method,
+        "status": status,
+        "detail": detail[:200],
+        "dry_run": dry_run,
+    }
+    _audit_log.append(entry)
+    if len(_audit_log) > MAX_AUDIT:
+        _audit_log.pop(0)
+
 API_DIR = os.environ.get("API_KEY_DIR", "/data/api")
 VAULT_CACHE_DIR = os.environ.get("VAULT_CACHE_DIR", "/data/vault-cache")
 BUTLER_TOKEN = os.environ.get("BUTLER_TOKEN", "")
@@ -39,12 +86,14 @@ async def _periodic_cache_reload():
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
+    _load_config()
     _load_vault_cache()
     task = asyncio.create_task(_periodic_cache_reload())
     yield
     task.cancel()
 
-app = FastAPI(title="Homelab Butler", version="2.0.0", lifespan=lifespan)
+app = FastAPI(title="Homelab Butler", version="2.1.0", lifespan=lifespan,
+              description="Unified API proxy + infrastructure management. AI agents: see GET / for self-onboarding.")
 
 # --- Credential reading (vault-first, file-fallback) ---
 
@@ -82,37 +131,7 @@ def _parse_url_key(name):
     lines = [l.strip() for l in raw.splitlines() if l.strip()]
     return (lines[0] if lines else None, lines[1] if len(lines) > 1 else None)
 
-# --- Service configs ---
+# --- Service configs loaded from butler.yaml ---
-
-SERVICES = {
-    "dockhand":      {"url": "http://10.4.1.116:3000", "auth": "session",
-                      "vault_key": "dockhand_password"},
-    "sonarr":        {"url": "http://10.2.1.100:8989",  "auth": "apikey", "key_file": "sonarr",
-                      "vault_key": "sonarr_uhd_key"},
-    "sonarr1080p":   {"url": None,                       "auth": "apikey_urlfile", "key_file": "sonarr1080p"},
-    "radarr":        {"url": "http://10.2.1.100:7878",  "auth": "apikey", "key_file": "radarr",
-                      "vault_key": "radarr_uhd_key"},
-    "radarr1080p":   {"url": None,                       "auth": "apikey_urlfile", "key_file": "radarr1080p"},
-    "seerr":         {"url": "http://10.2.1.100:5055",  "auth": "apikey", "key_file": "seer",
-                      "vault_key": "seerr_api_key"},
-    "outline":       {"url": "http://10.1.1.100:3000",  "auth": "bearer", "key_file": "outline",
-                      "vault_key": "outline_api_key"},
-    "n8n":           {"url": "http://10.4.1.113:5678",  "auth": "n8n", "key_file": "n8n",
-                      "vault_key": "n8n_api_key"},
-    "proxmox":       {"url": "https://10.5.85.11:8006", "auth": "proxmox"},
-    "homeassistant": {"url": "http://10.10.1.20:8123",  "auth": "bearer", "key_file": "homeassistent",
-                      "vault_key": "ha_token"},
-    "grafana":       {"url": "http://10.1.1.111:3000",  "auth": "bearer", "key_file": "grafana",
-                      "vault_key": "grafana_api_key"},
-    "uptime":        {"url": "http://159.69.245.190:3001", "auth": "bearer", "key_file": "uptime",
-                      "vault_key": "uptime_api_key"},
-    "waha":          {"url": "http://10.4.1.110:3500",  "auth": "apikey",
-                      "key_file": "waha_api_key", "vault_key": "waha_api_key"},
-    "forgejo":       {"url": "http://10.4.1.116:3001", "auth": "bearer", "key_file": "forgejo",
-                      "vault_key": "forgejo_token"},
-    "semaphore":     {"url": "http://10.4.1.116:8090", "auth": "bearer", "key_file": "semaphore",
-                      "vault_key": "semaphore_token"},
-}
 
 # --- Dockhand session ---
 
@@ -147,12 +166,65 @@ def _get_key(cfg):
 
 @app.get("/")
 async def root():
-    return {"service": "homelab-butler", "version": "2.0.0",
+    """AI self-onboarding: returns all available endpoints and services."""
-            "services": list(SERVICES.keys()), "vault_items": len(_vault_cache)}
+    svc_list = {}
+    for name, cfg in SERVICES.items():
+        svc_list[name] = {"url": cfg.get("url", ""), "auth": cfg.get("auth", ""), "description": cfg.get("description", "")}
+    return {
+        "service": "homelab-butler", "version": "2.1.0",
+        "docs": "/docs",
+        "openapi": "/openapi.json",
+        "services": svc_list,
+        "endpoints": {
+            "proxy": "GET/POST/PUT/DELETE /{service}/{path} - proxy to backend with auto-auth",
+            "vm_list": "GET /vm/list",
+            "vm_create": "POST /vm/create {node, ip, hostname, cores?, memory?, disk?}",
+            "vm_status": "GET /vm/status/{vmid}",
+            "vm_delete": "DELETE /vm/{vmid}",
+            "inventory_add": "POST /inventory/host {name, ip, group?}",
+            "ansible_run": "POST /ansible/run {hostname}",
+            "tts_speak": "POST /tts/speak {text, target: speaker|telegram}",
+            "tts_voices": "GET /tts/voices",
+            "tts_health": "GET /tts/health",
+            "status": "GET /status - health of all backends",
+            "audit": "GET /audit - recent API calls",
+        },
+        "vault_items": len(_vault_cache),
+    }
 
 @app.get("/health")
 async def health():
-    return {"status": "ok", "vault_items": len(_vault_cache)}
+    return {"status": "ok", "vault_items": len(_vault_cache), "services": len(SERVICES), "version": "2.1.0"}
+
+@app.get("/status")
+async def status(_=Depends(_verify)):
+    """Health check all configured backend services."""
+    results = {}
+    async with httpx.AsyncClient(verify=False, timeout=5) as c:
+        for name, cfg in SERVICES.items():
+            url = cfg.get("url")
+            if not url:
+                results[name] = {"status": "no_url"}
+                continue
+            try:
+                r = await c.get(url, follow_redirects=True)
+                results[name] = {"status": "ok", "http": r.status_code}
+            except Exception as e:
+                results[name] = {"status": "offline", "error": type(e).__name__}
+    _audit("/status", "GET", 200)
+    return results
+
+@app.get("/audit")
+async def audit(_=Depends(_verify), limit: int = Query(50, le=MAX_AUDIT)):
+    """Recent API calls (newest first)."""
+    return list(reversed(_audit_log[-limit:]))
+
+@app.post("/config/reload")
+async def config_reload(_=Depends(_verify)):
+    """Reload butler.yaml and vault cache."""
+    _load_config()
+    _load_vault_cache()
+    return {"config_services": len(SERVICES), "vault_items": len(_vault_cache)}
 
 @app.post("/vault/reload")
 async def vault_reload(_=Depends(_verify)):
@@ -164,8 +236,8 @@ async def vault_reload(_=Depends(_verify)):
 from pydantic import BaseModel
 import subprocess as _sp
 
-AUTOMATION1 = "sascha@10.5.85.5"
+AUTOMATION1 = VM_CFG.get("automation_host", "sascha@10.5.85.5") if VM_CFG else "sascha@10.5.85.5"
-ISO_BUILDER = "/app-config/ansible/iso-builder/build-iso.sh"
+ISO_BUILDER = VM_CFG.get("iso_builder_path", "/app-config/ansible/iso-builder/build-iso.sh") if VM_CFG else "/app-config/ansible/iso-builder/build-iso.sh"
 
 class VMCreate(BaseModel):
     node: int
@@ -198,10 +270,15 @@ async def vm_list(_=Depends(_verify)):
     return vms
 
 @app.post("/vm/create")
-async def vm_create(req: VMCreate, _=Depends(_verify)):
+async def vm_create(req: VMCreate, _=Depends(_verify), dry_run: bool = Query(False)):
+    if dry_run:
+        _audit("/vm/create", "POST", 200, f"dry_run: {req.hostname} {req.ip} node{req.node}", dry_run=True)
+        return {"dry_run": True, "would_create": {"hostname": req.hostname, "ip": req.ip, "node": req.node,
+                "cores": req.cores, "memory": req.memory, "disk": req.disk},
+                "steps": ["iso-builder", "wait ssh", "add inventory", "ansible setup"]}
     steps = []
     # Step 1: Build ISO + create VM via iso-builder on automation1
-    cmd = f"{ISO_BUILDER} --node {req.node} --ip {req.ip} --hostname {req.hostname} --cores {req.cores} --memory {req.memory} --disk {req.disk} --password 'GT500r8' --create-vm"
+    cmd = f"{ISO_BUILDER} --node {req.node} --ip {req.ip} --hostname {req.hostname} --cores {req.cores} --memory {req.memory} --disk {req.disk} --password '{VM_CFG.get('default_password', 'changeme')}' --create-vm"
     rc, out, err = _ssh(AUTOMATION1, f"cd /app-config/ansible/iso-builder && {cmd}", timeout=300)
     if rc != 0:
         return JSONResponse({"error": "iso-builder failed", "stderr": err[-500:], "stdout": out[-500:]}, status_code=500)
@@ -253,6 +330,7 @@ else:
     rc3, _, err3 = _ssh(AUTOMATION1, f"cd /app-config/ansible && bash pfannkuchen.sh setup {req.hostname}", timeout=600)
     steps.append(f"ansible: {'ok' if rc3 == 0 else 'failed (rc=' + str(rc3) + ')'}")
 
+    _audit("/vm/create", "POST", 200 if rc3 == 0 else 500, f"{req.hostname} {req.ip}")
     return {"status": "ok" if rc3 == 0 else "partial", "hostname": req.hostname, "ip": req.ip, "node": req.node, "steps": steps}
 
 @app.get("/vm/status/{vmid}")
@@ -332,8 +410,8 @@ class TTSRequest(BaseModel):
     voice: str = "deep_thought.mp3"
     language: str = "de"
 
-SPEAKER_URL = "http://10.10.1.166:10800"
+SPEAKER_URL = TTS_CFG.get("speaker_url", "http://10.10.1.166:10800") if TTS_CFG else "http://10.10.1.166:10800"
-CHATTERBOX_URL = "http://10.2.1.104:8004/tts"
+CHATTERBOX_URL = TTS_CFG.get("chatterbox_url", "http://10.2.1.104:8004/tts") if TTS_CFG else "http://10.2.1.104:8004/tts"
 
 @app.post("/tts/speak")
 async def tts_speak(req: TTSRequest, _=Depends(_verify)):
@@ -392,7 +470,7 @@ async def tts_health(_=Depends(_verify)):
 
 @app.api_route("/{service}/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"])
 async def proxy(service: str, path: str, request: Request, _=Depends(_verify)):
-    SKIP_SERVICES = {"vm", "inventory", "ansible", "debug", "tts"}
+    SKIP_SERVICES = {"vm", "inventory", "ansible", "debug", "tts", "status", "audit", "config"}
     if service in SKIP_SERVICES:
         raise HTTPException(404, f"Unknown service: {service}")
     cfg = SERVICES.get(service)
@@ -443,6 +521,7 @@ async def proxy(service: str, path: str, request: Request, _=Depends(_verify)):
         data = resp.json()
     except Exception:
         data = resp.text
+    _audit(f"/{service}/{path}", request.method, resp.status_code)
     return JSONResponse(content=data, status_code=resp.status_code)