Merge pull request 'fix: sichere Tuning-Synchronisation bei lokaler Ansible-Abweichung' (#16) from fix/ansible-tuning-sync-with-local-divergence into main

Refresh vault cache for VPS proxy route

app.py

@@ -11,7 +11,7 @@ from fastapi.responses import JSONResponse, RedirectResponse
 from contextlib import asynccontextmanager
 
 log = logging.getLogger("butler")
-VERSION = "2.3.2"
+VERSION = "2.3.5"
 
 API_DIR = os.environ.get("API_KEY_DIR", "/data/api")
 VAULT_CACHE_DIR = os.environ.get("VAULT_CACHE_DIR", "/data/vault-cache")
@@ -278,6 +278,7 @@ async def root():
             "status": "GET /status - health of all backends",
             "overview": "GET /overview?details=false - deterministic homelab verdict for small models",
             "audit": "GET /audit - recent API calls",
+            "sysctl_audit": "GET /system/sysctl/{host} - read-only live and persistent network tuning",
         },
         "vault_items": len(_vault_cache),
     }
@@ -920,6 +921,7 @@ def _validate_proxy_route(domain: str, upstream: str) -> tuple[str, str, str, st
 class ProxyRouteRequest(BaseModel):
     domain: str
     upstream: str
+    dns_token: str | None = None
 
 
 def _remote_python(script: str, timeout: int = 30) -> tuple[int, str, str]:
@@ -977,10 +979,26 @@ print(backup)
     return {"status": "reloaded", "backup": backup}
 
 
-async def _upsert_dns_records(zone: str, name: str) -> dict:
+def _get_hetzner_dns_token() -> str:
+    token = _read("HETZNER_DNS_TOKEN")
+    if token:
+        return token
+    rc, _out, err = _ssh(
+        "sascha@10.4.1.116",
+        "sudo bash /data/stacks/homelab-butler/vault-sync.sh",
+        timeout=120,
+    )
+    if rc != 0:
+        raise RuntimeError(f"Vault cache sync failed: {err[-300:]}")
+    _load_vault_cache()
     token = _read("HETZNER_DNS_TOKEN")
     if not token:
-        raise RuntimeError("HETZNER_DNS_TOKEN is unavailable")
+        raise RuntimeError("HETZNER_DNS_TOKEN is unavailable after vault sync")
+    return token
+
+
+async def _upsert_dns_records(zone: str, name: str, token_override: str | None = None) -> dict:
+    token = token_override or await asyncio.to_thread(_get_hetzner_dns_token)
     headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
     api = "https://api.hetzner.cloud/v1"
     async with httpx.AsyncClient(timeout=30) as client:
@@ -1011,7 +1029,7 @@ async def vps_proxy_route(req: ProxyRouteRequest, _=Depends(_verify)):
         raise HTTPException(400, str(exc)) from exc
     caddy = await asyncio.to_thread(_configure_caddy_route, domain, upstream)
     try:
-        dns = await _upsert_dns_records(zone, name)
+        dns = await _upsert_dns_records(zone, name, req.dns_token)
     except Exception:
         await asyncio.to_thread(_restore_caddy_backup, caddy["backup"])
         raise
@@ -1019,6 +1037,133 @@ async def vps_proxy_route(req: ProxyRouteRequest, _=Depends(_verify)):
     return {"status": "configured", "domain": domain, "upstream": upstream, "caddy": caddy, "dns": dns}
 
 
+SPEEDTEST_REPO_FILES = (
+    ".dockerignore",
+    "Dockerfile",
+    "compose.yaml",
+    "pyproject.toml",
+    "streamscope/__init__.py",
+    "streamscope/app.py",
+    "streamscope/db.py",
+    "streamscope/mtr.py",
+    "streamscope/scoring.py",
+    "streamscope/static/index.html",
+    "streamscope/static/assets/app.css",
+    "streamscope/static/assets/app.js",
+    "streamscope/static/assets/longterm-metrics.js",
+)
+
+
+class SpeedtestDeployRequest(BaseModel):
+    stats_password: str
+    session_secret: str
+
+
+async def _fetch_forgejo_text(repo: str, path: str) -> str:
+    if repo != "sascha/speedtest" or path not in SPEEDTEST_REPO_FILES:
+        raise ValueError("unsupported Forgejo file")
+    cfg = SERVICES.get("forgejo", {})
+    base_url = cfg.get("url")
+    token = _get_key(cfg)
+    if not base_url or not token:
+        raise RuntimeError("Forgejo service configuration is unavailable")
+    url = f"{base_url}/api/v1/repos/{repo}/contents/{path}"
+    async with httpx.AsyncClient(timeout=30) as client:
+        response = await client.get(url, params={"ref": "main"}, headers={"Authorization": f"token {token}"})
+        response.raise_for_status()
+    return base64.b64decode(response.json()["content"]).decode()
+
+
+def _deploy_speedtest_compose(files: dict[str, str], password: str, session_secret: str) -> dict:
+    if set(files) != set(SPEEDTEST_REPO_FILES):
+        raise ValueError("speedtest source bundle is incomplete")
+    compose = files["compose.yaml"]
+    dockerfile = files["Dockerfile"]
+    required = [
+        "build: .",
+        '127.0.0.1:8080:8080',
+        '/app-config/speedtest/data:/data',
+        'ADMIN_PASSWORD: "${ADMIN_PASSWORD:',
+        'SESSION_SECRET: "${SESSION_SECRET:',
+        "NET_RAW",
+    ]
+    if any(item not in compose for item in required):
+        raise ValueError("StreamScope compose is missing a required security or persistence setting")
+    if "python:" not in dockerfile or "mtr-tiny" not in dockerfile or "php" in dockerfile.lower():
+        raise ValueError("StreamScope image must be Python-based, MTR-capable and PHP-free")
+    secret_pattern = r"[A-Za-z0-9!@#%_+=:,.?-]{24,128}"
+    if not re.fullmatch(secret_pattern, password):
+        raise ValueError("stats password must be 24-128 safe characters")
+    if not re.fullmatch(secret_pattern, session_secret):
+        raise ValueError("session secret must be 24-128 safe characters")
+    script = f"""from pathlib import Path
+import os, shutil
+stack = Path('/app-config/github/speedtest')
+backup = Path('/app-config/deployment-backups/speedtest-rollback')
+data = Path('/app-config/speedtest/data')
+if backup.exists():
+    shutil.rmtree(backup)
+if stack.exists():
+    backup.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copytree(stack, backup)
+stack.mkdir(parents=True, exist_ok=True)
+data.mkdir(parents=True, exist_ok=True)
+files = {files!r}
+for relative, content in files.items():
+    target = stack / relative
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(content)
+env = stack / '.env'
+env.write_text('ADMIN_PASSWORD=' + {password!r} + '\\nSESSION_SECRET=' + {session_secret!r} + '\\nSTATS_PASSWORD=' + {password!r} + '\\n')
+os.chmod(env, 0o600)
+"""
+    rc, _out, err = _remote_python(script)
+    if rc != 0:
+        raise RuntimeError(f"StreamScope file deployment failed: {err[-300:]}")
+    rollback = "rm -rf /app-config/github/speedtest && cp -a /app-config/deployment-backups/speedtest-rollback /app-config/github/speedtest && cd /app-config/github/speedtest && docker compose up -d"
+    preflight = "cd /app-config/github/speedtest && docker compose config -q && docker compose build --pull"
+    rc, _out, err = _ssh(VPS_SSH, preflight, timeout=600)
+    if rc != 0:
+        _ssh(VPS_SSH, rollback, timeout=180)
+        raise RuntimeError(f"StreamScope build preflight failed: {err[-500:]}")
+    deploy = "cd /app-config/github/speedtest && (docker rm -f speedtest >/dev/null 2>&1 || true) && docker compose up -d --remove-orphans"
+    rc, out, err = _ssh(VPS_SSH, deploy, timeout=180)
+    if rc != 0:
+        _ssh(VPS_SSH, rollback, timeout=180)
+        raise RuntimeError(f"StreamScope deployment failed: {(err or out)[-500:]}")
+    health = "for i in $(seq 1 45); do curl -fsS --max-time 3 http://127.0.0.1:8080/api/health >/dev/null && exit 0; sleep 2; done; exit 1"
+    rc, _out, err = _ssh(VPS_SSH, health, timeout=105)
+    if rc != 0:
+        _ssh(VPS_SSH, rollback, timeout=180)
+        raise RuntimeError(f"StreamScope health check failed and rollback was attempted: {err[-300:]}")
+    return {
+        "status": "deployed",
+        "health": "ok",
+        "application": "streamscope",
+        "database": "/app-config/speedtest/data/streamscope.db",
+        "public_port": False,
+        "mtr": True,
+    }
+
+
+@app.post("/vps/speedtest/deploy")
+async def vps_speedtest_deploy(req: SpeedtestDeployRequest, _=Depends(_verify)):
+    secret_pattern = r"[A-Za-z0-9!@#%_+=:,.?-]{24,128}"
+    if not re.fullmatch(secret_pattern, req.stats_password):
+        raise HTTPException(400, "stats password must be 24-128 safe characters")
+    if not re.fullmatch(secret_pattern, req.session_secret):
+        raise HTTPException(400, "session secret must be 24-128 safe characters")
+    contents = await asyncio.gather(*(
+        _fetch_forgejo_text("sascha/speedtest", path) for path in SPEEDTEST_REPO_FILES
+    ))
+    files = dict(zip(SPEEDTEST_REPO_FILES, contents))
+    result = await asyncio.to_thread(
+        _deploy_speedtest_compose, files, req.stats_password, req.session_secret
+    )
+    _audit("/vps/speedtest/deploy", "POST", 200, "Git-managed StreamScope with private history and MTR")
+    return result
+
+
 # --- VM Lifecycle Endpoints ---
 import subprocess as _sp
 
@@ -1042,6 +1187,76 @@ def _ssh(host, cmd, timeout=600):
     except _sp.TimeoutExpired:
         return 124, "", f"SSH command timed out after {timeout} seconds"
 
+
+SYSCTL_AUDIT_KEYS = (
+    "net.core.default_qdisc",
+    "net.core.rmem_default",
+    "net.core.rmem_max",
+    "net.core.wmem_default",
+    "net.core.wmem_max",
+    "net.core.netdev_max_backlog",
+    "net.core.somaxconn",
+    "net.ipv4.ip_forward",
+    "net.ipv4.tcp_congestion_control",
+    "net.ipv4.tcp_fastopen",
+    "net.ipv4.tcp_mtu_probing",
+    "net.ipv4.tcp_no_metrics_save",
+    "net.ipv4.tcp_rmem",
+    "net.ipv4.tcp_slow_start_after_idle",
+    "net.ipv4.tcp_window_scaling",
+    "net.ipv4.tcp_wmem",
+)
+
+
+def _sysctl_audit_command() -> str:
+    script = f'''import glob, json
+from pathlib import Path
+keys = {SYSCTL_AUDIT_KEYS!r}
+live, errors = {{}}, {{}}
+for key in keys:
+    try:
+        live[key] = Path("/proc/sys/" + key.replace(".", "/")).read_text().strip()
+    except OSError as exc:
+        errors[key] = str(exc)[:160]
+persistent = {{}}
+for path in ["/etc/sysctl.conf", *sorted(glob.glob("/etc/sysctl.d/*.conf"))]:
+    try:
+        with open(path, encoding="utf-8", errors="replace") as handle:
+            for raw in handle:
+                line = raw.split("#", 1)[0].strip()
+                if "=" not in line:
+                    continue
+                key, value = (part.strip() for part in line.split("=", 1))
+                if key in keys:
+                    persistent.setdefault(key, []).append({{"file": path, "value": value}})
+    except (FileNotFoundError, PermissionError):
+        pass
+print(json.dumps({{"live": live, "persistent": persistent, "errors": errors}}))
+'''
+    encoded = base64.b64encode(script.encode()).decode()
+    return f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"'
+
+
+@app.get("/system/sysctl/{host}")
+async def system_sysctl_audit(host: str, _=Depends(_verify)):
+    if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,62}", host):
+        raise HTTPException(400, "Invalid host name")
+    if host == "vps":
+        target = VPS_SSH
+    else:
+        inventory = await asyncio.to_thread(_find_inventory_host, host)
+        if not inventory:
+            raise HTTPException(404, f"Host {host} not found")
+        target = f'{inventory["user"]}@{inventory["ip"]}'
+    rc, out, err = await asyncio.to_thread(_ssh, target, _sysctl_audit_command(), 30)
+    if rc != 0:
+        raise HTTPException(502, (err or out).strip()[-500:] or "sysctl audit failed")
+    try:
+        result = json.loads(out)
+    except json.JSONDecodeError as exc:
+        raise HTTPException(502, "sysctl audit returned invalid JSON") from exc
+    return {"host": host, **result}
+
 def _pve_auth():
     pv = _parse_kv("proxmox")
     return f"PVEAPIToken={pv.get('tokenid','')}={pv.get('secret','')}"
@@ -1352,10 +1567,47 @@ print("updated" if updated else "added")'''
 async def ansible_run(request: Request, _=Depends(_verify)):
     body = await request.json()
     hostname = body.get("limit", body.get("hostname", ""))
-    template_id = body.get("template_id", 10)
     if not hostname:
         return JSONResponse({"error": "limit/hostname required"}, status_code=400)
-    rc, out, err = _ssh(AUTOMATION1, f"cd /app-config/ansible && bash pfannkuchen.sh setup {hostname}", timeout=600)
+    action = body.get("action", "setup")
+    if action not in {"setup", "tune", "pvetune"}:
+        return JSONResponse({"error": "action must be setup, tune or pvetune"}, status_code=400)
+    if not re.fullmatch(r"[a-zA-Z0-9_.:-]+", hostname):
+        return JSONResponse({"error": "invalid hostname/limit"}, status_code=400)
+    if action in {"tune", "pvetune"}:
+        approved_files = (
+            "roles/sysctl/defaults/main.yml",
+            "roles/sysctl/tasks/main.yml",
+            "group_vars/vps/sysctl.yml",
+            "sysctl-proxmox.yaml",
+            "roles/sysctl_proxmox/tasks/main.yml",
+        )
+        file_sync = " && ".join(
+            f"git show origin/master:{path} > {path}" for path in approved_files
+        )
+        command = (
+            "cd /app-config/ansible && "
+            "git fetch origin master && "
+            f"{file_sync} && "
+            f"bash pfannkuchen.sh {action} {hostname}"
+        )
+    else:
+        command = (
+            "cd /app-config/ansible && "
+            "git pull --ff-only origin master && "
+            f"bash pfannkuchen.sh {action} {hostname}"
+        )
+    rc, out, err = _ssh(AUTOMATION1, command, timeout=600)
+    _audit("/ansible/run", "POST", 200 if rc == 0 else 502, f"{action} {hostname}")
+    if action != "setup":
+        return {
+            "status": "ok" if rc == 0 else "error",
+            "action": action,
+            "hostname": hostname,
+            "rc": rc,
+            "output": out[-4000:],
+            "error": err[-1000:] if rc != 0 else "",
+        }
     
     # After successful ansible run: sync Hawser token to Dockhand
     if rc == 0:

tests/test_app.py

@@ -42,7 +42,32 @@ def test_health_exposes_current_version():
     with TestClient(app.app) as client:
         response = client.get("/health")
     assert response.status_code == 200
-    assert response.json()["version"] == app.VERSION == "2.3.2"
+    assert response.json()["version"] == app.VERSION == "2.3.5"
+
+
+def test_sysctl_audit_reads_fixed_keys_from_inventory_host(monkeypatch):
+    payload = {
+        "live": {"net.ipv4.tcp_congestion_control": "bbr"},
+        "persistent": {"net.ipv4.tcp_congestion_control": [{"file": "/etc/sysctl.d/99-net-tuning.conf", "value": "bbr"}]},
+        "errors": {},
+    }
+    calls = []
+    monkeypatch.setattr(app, "_find_inventory_host", lambda name: {"name": name, "user": "root", "ip": "10.5.85.16"})
+    monkeypatch.setattr(app, "_ssh", lambda host, command, timeout=600: (calls.append((host, command, timeout)) or (0, __import__("json").dumps(payload), "")))
+    with TestClient(app.app) as client:
+        response = client.get("/system/sysctl/node6", headers={"Authorization": "Bearer test-token"})
+    assert response.status_code == 200
+    assert response.json()["live"]["net.ipv4.tcp_congestion_control"] == "bbr"
+    assert calls[0][0] == "root@10.5.85.16"
+    assert "base64.b64decode" in calls[0][1]
+
+
+def test_sysctl_audit_rejects_unknown_host_without_ssh(monkeypatch):
+    monkeypatch.setattr(app, "_find_inventory_host", lambda _name: None)
+    monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not SSH")))
+    with TestClient(app.app) as client:
+        response = client.get("/system/sysctl/not-there", headers={"Authorization": "Bearer test-token"})
+    assert response.status_code == 404
 
 
 def test_invalid_log_target_is_rejected_before_ssh():
@@ -83,6 +108,46 @@ def test_inventory_upsert_uses_base64_script(monkeypatch):
     assert "\\nname =" not in calls[0][1]
 
 
+def test_ansible_run_supports_safe_tune_action_and_syncs_approved_files(monkeypatch):
+    calls = []
+
+    def fake_ssh(host, command, timeout=600):
+        calls.append((host, command, timeout))
+        return 0, "changed=1 failed=0", ""
+
+    monkeypatch.setattr(app, "_ssh", fake_ssh)
+    with TestClient(app.app) as client:
+        response = client.post(
+            "/ansible/run",
+            headers={"Authorization": "Bearer test-token"},
+            json={"hostname": "emby-sascha", "action": "tune"},
+        )
+    assert response.status_code == 200
+    assert response.json()["action"] == "tune"
+    assert "git fetch origin master" in calls[0][1]
+    assert "git show origin/master:roles/sysctl/tasks/main.yml" in calls[0][1]
+    assert "git pull --ff-only" not in calls[0][1]
+    assert "bash pfannkuchen.sh tune emby-sascha" in calls[0][1]
+    assert len(calls) == 1
+
+
+def test_ansible_run_rejects_unknown_action_and_shell_metacharacters(monkeypatch):
+    monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not SSH")))
+    with TestClient(app.app) as client:
+        bad_action = client.post(
+            "/ansible/run",
+            headers={"Authorization": "Bearer test-token"},
+            json={"hostname": "emby-sascha", "action": "shell"},
+        )
+        bad_host = client.post(
+            "/ansible/run",
+            headers={"Authorization": "Bearer test-token"},
+            json={"hostname": "emby-sascha;id", "action": "tune"},
+        )
+    assert bad_action.status_code == 400
+    assert bad_host.status_code == 400
+
+
 def test_docker_inspect_returns_sanitized_summary(monkeypatch):
     raw = [{
         "Name": "/fileflows",
@@ -290,8 +355,8 @@ def test_proxy_route_endpoint_configures_caddy_and_dns(monkeypatch):
         calls.append(("caddy", domain, upstream))
         return {"status": "reloaded", "backup": "/app-config/caddy/Caddyfile.bak-test"}
 
-    async def fake_dns(zone, name):
-        calls.append(("dns", zone, name))
+    async def fake_dns(zone, name, token_override=None):
+        calls.append(("dns", zone, name, token_override))
         return {"zone_id": 123, "records": ["A", "AAAA"]}
 
     monkeypatch.setattr(app, "_configure_caddy_route", fake_caddy)
@@ -300,12 +365,65 @@ def test_proxy_route_endpoint_configures_caddy_and_dns(monkeypatch):
         response = client.post(
             "/vps/proxy-route",
             headers={"Authorization": "Bearer test-token"},
-            json={"domain": "speed.guck.tv", "upstream": "127.0.0.1:8080"},
+            json={"domain": "speed.guck.tv", "upstream": "127.0.0.1:8080", "dns_token": "test-dns-token"},
         )
 
     assert response.status_code == 200
     assert response.json()["status"] == "configured"
     assert calls == [
         ("caddy", "speed.guck.tv", "127.0.0.1:8080"),
-        ("dns", "guck.tv", "speed"),
+        ("dns", "guck.tv", "speed", "test-dns-token"),
     ]
+
+
+def test_hetzner_token_refreshes_vault_cache_when_missing(monkeypatch):
+    reads = iter([None, "refreshed-token"])
+    calls = []
+    monkeypatch.setattr(app, "_read", lambda _name: next(reads))
+    monkeypatch.setattr(app, "_load_vault_cache", lambda: calls.append("reload"))
+    monkeypatch.setattr(app, "_ssh", lambda host, command, timeout=600: (calls.append((host, command, timeout)) or (0, "vault-sync: ok", "")))
+
+    assert app._get_hetzner_dns_token() == "refreshed-token"
+    assert calls[0][0] == "sascha@10.4.1.116"
+    assert calls[0][1] == "sudo bash /data/stacks/homelab-butler/vault-sync.sh"
+    assert calls[1] == "reload"
+
+
+def test_speedtest_deploy_requires_strong_secrets_and_uses_full_git_app(monkeypatch):
+    calls = []
+
+    async def fake_fetch(repo, path):
+        calls.append(("fetch", repo, path))
+        return f"content:{path}"
+
+    def fake_deploy(files, password, session_secret):
+        calls.append(("deploy", files, password, session_secret))
+        return {"status": "deployed", "health": "ok", "application": "streamscope"}
+
+    monkeypatch.setattr(app, "_fetch_forgejo_text", fake_fetch)
+    monkeypatch.setattr(app, "_deploy_speedtest_compose", fake_deploy)
+    with TestClient(app.app) as client:
+        weak = client.post(
+            "/vps/speedtest/deploy",
+            headers={"Authorization": "Bearer test-token"},
+            json={"stats_password": "short", "session_secret": "long-session-secret-with-entropy"},
+        )
+        response = client.post(
+            "/vps/speedtest/deploy",
+            headers={"Authorization": "Bearer test-token"},
+            json={
+                "stats_password": "correct-horse-battery-staple",
+                "session_secret": "streamscope-session-secret-with-entropy",
+            },
+        )
+
+    assert weak.status_code == 400
+    assert response.status_code == 200
+    assert response.json()["application"] == "streamscope"
+    assert ("fetch", "sascha/speedtest", "compose.yaml") in calls
+    assert ("fetch", "sascha/speedtest", "streamscope/static/assets/app.js") in calls
+    deploy = calls[-1]
+    assert deploy[0] == "deploy"
+    assert deploy[1]["compose.yaml"] == "content:compose.yaml"
+    assert deploy[2] == "correct-horse-battery-staple"
+    assert deploy[3] == "streamscope-session-secret-with-entropy"

tests/test_streamscope_deploy.py

@@ -0,0 +1,38 @@
+import app
+
+
+def test_streamscope_deploy_replaces_legacy_container_before_compose(monkeypatch):
+    assert "streamscope/static/assets/longterm-metrics.js" in app.SPEEDTEST_REPO_FILES
+    files = {path: "placeholder" for path in app.SPEEDTEST_REPO_FILES}
+    files["compose.yaml"] = """services:
+  streamscope:
+    build: .
+    ports:
+      - '127.0.0.1:8080:8080'
+    environment:
+      ADMIN_PASSWORD: "${ADMIN_PASSWORD:?required}"
+      SESSION_SECRET: "${SESSION_SECRET:?required}"
+    volumes:
+      - /app-config/speedtest/data:/data
+    cap_add:
+      - NET_RAW
+"""
+    files["Dockerfile"] = "FROM python:3.13-slim\nRUN apt-get install -y mtr-tiny\n"
+    commands = []
+
+    monkeypatch.setattr(app, "_remote_python", lambda script: (0, "", ""))
+
+    def fake_ssh(host, command, timeout=600):
+        commands.append(command)
+        return 0, "ok", ""
+
+    monkeypatch.setattr(app, "_ssh", fake_ssh)
+    result = app._deploy_speedtest_compose(
+        files,
+        "correct-horse-battery-staple",
+        "streamscope-session-secret-with-entropy",
+    )
+
+    deploy = next(command for command in commands if "compose up -d --remove-orphans" in command)
+    assert "docker rm -f speedtest" in deploy
+    assert result["application"] == "streamscope"