Merge pull request 'fix: set TTS bridge secret ownership' (#22) from fix/tts-secret-owner-20260813130906 into main

app.py

@@ -1,17 +1,17 @@
 """Homelab Butler v2.1 – Unified API proxy for Pfannkuchen homelab.
 Reads service config from butler.yaml, credentials from Vaultwarden cache with flat-file fallback."""
 
-import os, json, asyncio, logging, time, base64, re, subprocess, ipaddress
+import os, json, asyncio, logging, time, base64, re, subprocess, ipaddress, secrets
 from datetime import datetime, timezone
 import httpx, yaml
 from typing import Literal
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 from fastapi import FastAPI, Request, HTTPException, Depends, Query
-from fastapi.responses import JSONResponse, RedirectResponse
+from fastapi.responses import JSONResponse, RedirectResponse, Response
 from contextlib import asynccontextmanager
 
 log = logging.getLogger("butler")
-VERSION = "2.3.3"
+VERSION = "2.3.5"
 
 API_DIR = os.environ.get("API_KEY_DIR", "/data/api")
 VAULT_CACHE_DIR = os.environ.get("VAULT_CACHE_DIR", "/data/vault-cache")
@@ -273,6 +273,7 @@ async def root():
             "inventory_add": "POST /inventory/host {name, ip, group?}",
             "ansible_run": "POST /ansible/run {hostname}",
             "tts_speak": "POST /tts/speak {text, target: speaker|telegram}",
+            "tts_generate": "POST /tts/generate {text, voice?, language?} - return cloned WAV audio",
             "tts_voices": "GET /tts/voices",
             "tts_health": "GET /tts/health",
             "status": "GET /status - health of all backends",
@@ -1164,6 +1165,117 @@ async def vps_speedtest_deploy(req: SpeedtestDeployRequest, _=Depends(_verify)):
     return result
 
 
+BW_MANAGER_REPO_FILES = (
+    ".env.example", ".gitignore", "README.md", "compose.yaml",
+    "src/.dockerignore", "src/Dockerfile", "src/app.py",
+    "src/remote_policy.py", "src/requirements.txt",
+    "src/templates/base.html", "src/templates/history.html",
+    "src/templates/index.html", "src/templates/users.html",
+)
+
+
+async def _fetch_bw_manager_text(path: str) -> str:
+    if path not in BW_MANAGER_REPO_FILES:
+        raise ValueError("unsupported BW Manager file")
+    cfg = SERVICES.get("forgejo", {})
+    base_url, token = cfg.get("url"), _get_key(cfg)
+    if not base_url or not token:
+        raise RuntimeError("Forgejo service configuration is unavailable")
+    url = f"{base_url}/api/v1/repos/sascha/bw-manager/contents/{path}"
+    async with httpx.AsyncClient(timeout=30) as client:
+        response = await client.get(
+            url, params={"ref": "main"},
+            headers={"Authorization": f"token {token}"},
+        )
+        response.raise_for_status()
+    return base64.b64decode(response.json()["content"]).decode()
+
+
+def _deploy_bw_manager_compose(files: dict[str, str]) -> dict:
+    if set(files) != set(BW_MANAGER_REPO_FILES):
+        raise ValueError("BW Manager source bundle is incomplete")
+    if "build: ./src" not in files["compose.yaml"]:
+        raise ValueError("BW Manager compose contract is invalid")
+    if "build_gated_targets" not in files["src/app.py"]:
+        raise ValueError("BW Manager candidate lacks the user/network AND gate")
+    rc, working_dir, err = _ssh(
+        VPS_SSH,
+        "docker inspect -f '{{ index .Config.Labels \"com.docker.compose.project.working_dir\" }}' bw-manager",
+        timeout=30,
+    )
+    working_dir = working_dir.strip()
+    if rc != 0 or not re.fullmatch(r"/app-config/[A-Za-z0-9_./-]+", working_dir):
+        raise RuntimeError(f"cannot determine safe BW Manager working directory: {(err or working_dir)[-300:]}")
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    candidate = f"/app-config/deployment-candidates/bw-manager-{timestamp}"
+    backup = f"/app-config/deployment-backups/bw-manager-{timestamp}"
+    image = "bw-manager-bw-manager"
+    rollback_image = f"{image}:rollback-{timestamp}"
+    init_script = f"""from pathlib import Path
+import shutil
+candidate = Path({candidate!r})
+if candidate.exists(): shutil.rmtree(candidate)
+candidate.mkdir(parents=True)
+live_env = Path({working_dir!r}) / '.env'
+if live_env.exists(): shutil.copy2(live_env, candidate / '.env')
+"""
+    rc, _out, err = _remote_python(init_script)
+    if rc != 0:
+        raise RuntimeError(f"BW Manager candidate initialization failed: {err[-300:]}")
+    # Stage one file per SSH call. Sending the complete repository in one
+    # command exceeds Linux's argv limit once app.py and templates are encoded.
+    for relative, content in files.items():
+        file_script = f"""from pathlib import Path
+target = Path({candidate!r}) / {relative!r}
+target.parent.mkdir(parents=True, exist_ok=True)
+target.write_text({content!r})
+"""
+        rc, _out, err = _remote_python(file_script)
+        if rc != 0:
+            raise RuntimeError(f"BW Manager staging failed for {relative}: {err[-300:]}")
+    rc, _out, err = _ssh(VPS_SSH, f"cd {candidate} && docker compose config -q && docker compose build --pull", timeout=600)
+    if rc != 0:
+        raise RuntimeError(f"BW Manager candidate build failed: {err[-500:]}")
+    deploy_script = f"""from pathlib import Path
+import shutil
+live, backup, candidate = Path({working_dir!r}), Path({backup!r}), Path({candidate!r})
+backup.parent.mkdir(parents=True, exist_ok=True)
+if backup.exists(): shutil.rmtree(backup)
+shutil.copytree(live, backup)
+for relative in {BW_MANAGER_REPO_FILES!r}:
+    source, target = candidate / relative, live / relative
+    target.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(source, target)
+"""
+    rc, _out, err = _remote_python(deploy_script)
+    if rc != 0:
+        raise RuntimeError(f"BW Manager live file switch failed: {err[-300:]}")
+    _ssh(VPS_SSH, f"docker image tag {image} {rollback_image}", timeout=60)
+    rollback = (
+        f"rm -rf {working_dir} && cp -a {backup} {working_dir} && "
+        f"docker image tag {rollback_image} {image} && cd {working_dir} && "
+        "docker compose up -d --no-build"
+    )
+    rc, out, err = _ssh(VPS_SSH, f"cd {working_dir} && docker compose up -d --build --remove-orphans", timeout=600)
+    if rc != 0:
+        _ssh(VPS_SSH, rollback, timeout=180)
+        raise RuntimeError(f"BW Manager deployment failed: {(err or out)[-500:]}")
+    health = "for i in $(seq 1 45); do curl -fsS --max-time 3 http://127.0.0.1:8870/api/status >/dev/null && exit 0; sleep 2; done; exit 1"
+    rc, _out, err = _ssh(VPS_SSH, health, timeout=105)
+    if rc != 0:
+        _ssh(VPS_SSH, rollback, timeout=180)
+        raise RuntimeError(f"BW Manager health failed; rollback attempted: {err[-300:]}")
+    return {"status": "deployed", "health": "ok", "working_dir": working_dir, "backup": backup}
+
+
+@app.post("/vps/bw-manager/deploy")
+async def vps_bw_manager_deploy(_=Depends(_verify)):
+    contents = await asyncio.gather(*(_fetch_bw_manager_text(path) for path in BW_MANAGER_REPO_FILES))
+    result = await asyncio.to_thread(_deploy_bw_manager_compose, dict(zip(BW_MANAGER_REPO_FILES, contents)))
+    _audit("/vps/bw-manager/deploy", "POST", 200, "Git-managed BW Manager deployment")
+    return result
+
+
 # --- VM Lifecycle Endpoints ---
 import subprocess as _sp
 
@@ -1209,15 +1321,15 @@ SYSCTL_AUDIT_KEYS = (
 
 
 def _sysctl_audit_command() -> str:
-    script = f'''import glob, json, subprocess
+    script = f'''import glob, json
+from pathlib import Path
 keys = {SYSCTL_AUDIT_KEYS!r}
 live, errors = {{}}, {{}}
 for key in keys:
-    result = subprocess.run(["sysctl", "-n", key], capture_output=True, text=True)
+    try:
-    if result.returncode == 0:
+        live[key] = Path("/proc/sys/" + key.replace(".", "/")).read_text().strip()
-        live[key] = result.stdout.strip()
+    except OSError as exc:
-    else:
+        errors[key] = str(exc)[:160]
-        errors[key] = result.stderr.strip()[:160]
 persistent = {{}}
 for path in ["/etc/sysctl.conf", *sorted(glob.glob("/etc/sysctl.d/*.conf"))]:
     try:
@@ -1250,7 +1362,7 @@ async def system_sysctl_audit(host: str, _=Depends(_verify)):
         target = f'{inventory["user"]}@{inventory["ip"]}'
     rc, out, err = await asyncio.to_thread(_ssh, target, _sysctl_audit_command(), 30)
     if rc != 0:
-        raise HTTPException(502, (err or out).strip()[:300] or "sysctl audit failed")
+        raise HTTPException(502, (err or out).strip()[-500:] or "sysctl audit failed")
     try:
         result = json.loads(out)
     except json.JSONDecodeError as exc:
@@ -1567,10 +1679,47 @@ print("updated" if updated else "added")'''
 async def ansible_run(request: Request, _=Depends(_verify)):
     body = await request.json()
     hostname = body.get("limit", body.get("hostname", ""))
-    template_id = body.get("template_id", 10)
     if not hostname:
         return JSONResponse({"error": "limit/hostname required"}, status_code=400)
-    rc, out, err = _ssh(AUTOMATION1, f"cd /app-config/ansible && bash pfannkuchen.sh setup {hostname}", timeout=600)
+    action = body.get("action", "setup")
+    if action not in {"setup", "tune", "pvetune"}:
+        return JSONResponse({"error": "action must be setup, tune or pvetune"}, status_code=400)
+    if not re.fullmatch(r"[a-zA-Z0-9_.:-]+", hostname):
+        return JSONResponse({"error": "invalid hostname/limit"}, status_code=400)
+    if action in {"tune", "pvetune"}:
+        approved_files = (
+            "roles/sysctl/defaults/main.yml",
+            "roles/sysctl/tasks/main.yml",
+            "group_vars/vps/sysctl.yml",
+            "sysctl-proxmox.yaml",
+            "roles/sysctl_proxmox/tasks/main.yml",
+        )
+        file_sync = " && ".join(
+            f"git show origin/master:{path} > {path}" for path in approved_files
+        )
+        command = (
+            "cd /app-config/ansible && "
+            "git fetch origin master && "
+            f"{file_sync} && "
+            f"bash pfannkuchen.sh {action} {hostname}"
+        )
+    else:
+        command = (
+            "cd /app-config/ansible && "
+            "git pull --ff-only origin master && "
+            f"bash pfannkuchen.sh {action} {hostname}"
+        )
+    rc, out, err = _ssh(AUTOMATION1, command, timeout=600)
+    _audit("/ansible/run", "POST", 200 if rc == 0 else 502, f"{action} {hostname}")
+    if action != "setup":
+        return {
+            "status": "ok" if rc == 0 else "error",
+            "action": action,
+            "hostname": hostname,
+            "rc": rc,
+            "output": out[-4000:],
+            "error": err[-1000:] if rc != 0 else "",
+        }
     
     # After successful ansible run: sync Hawser token to Dockhand
     if rc == 0:
@@ -1686,9 +1835,105 @@ class TTSRequest(BaseModel):
     voice: str = "deep_thought.mp3"
     language: str = "de"
 
+
+class TTSGenerateRequest(BaseModel):
+    text: str = Field(min_length=1, max_length=2000)
+    voice: str = Field(default="deep_thought.mp3", pattern=r"^[A-Za-z0-9_.-]+$")
+    language: str = Field(default="de", pattern=r"^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{2,8})?$")
+
+
+class TTSBridgeDeployRequest(BaseModel):
+    rotate_client_token: bool = False
+
+
 SPEAKER_URL = TTS_CFG.get("speaker_url", "http://10.10.1.166:10800") if TTS_CFG else "http://10.10.1.166:10800"
 CHATTERBOX_URL = TTS_CFG.get("chatterbox_url", "http://10.2.1.104:8004/tts") if TTS_CFG else "http://10.2.1.104:8004/tts"
 
+
+def _chatterbox_payload(text: str, voice: str, language: str) -> dict:
+    return {
+        "text": text,
+        "voice_mode": "clone",
+        "reference_audio_filename": voice,
+        "output_format": "wav",
+        "language": language,
+        "exaggeration": 0.3,
+        "cfg_weight": 0.7,
+        "temperature": 0.6,
+    }
+
+
+@app.post("/tts/generate", response_class=Response)
+async def tts_generate(req: TTSGenerateRequest, _=Depends(_verify)):
+    """Generate cloned speech and return the WAV bytes to the authenticated caller."""
+    async with httpx.AsyncClient(verify=False, timeout=180) as client:
+        try:
+            result = await client.post(CHATTERBOX_URL, json=_chatterbox_payload(req.text, req.voice, req.language))
+        except httpx.RequestError:
+            _audit("/tts/generate", "POST", 502, "chatterbox request failed")
+            raise HTTPException(status_code=502, detail="Chatterbox is unavailable")
+
+    if result.status_code != 200:
+        _audit("/tts/generate", "POST", 502, f"chatterbox_http={result.status_code}")
+        raise HTTPException(status_code=502, detail="Chatterbox generation failed")
+    if not result.content.startswith(b"RIFF"):
+        _audit("/tts/generate", "POST", 502, "invalid audio response")
+        raise HTTPException(status_code=502, detail="Chatterbox returned invalid audio")
+
+    _audit("/tts/generate", "POST", 200, f"voice={req.voice} chars={len(req.text)}")
+    return Response(
+        content=result.content,
+        media_type="audio/wav",
+        headers={
+            "Content-Disposition": 'inline; filename="voiceclone.wav"',
+            "Cache-Control": "no-store",
+            "X-Content-Type-Options": "nosniff",
+        },
+    )
+
+
+@app.post("/tts/bridge/deploy")
+async def tts_bridge_deploy(req: TTSBridgeDeployRequest, _=Depends(_verify)):
+    """Install host-local bridge secrets on automation1 without exposing them."""
+    client_token = _vault_cache.get("tts_bridge_client_token", "").strip()
+    if req.rotate_client_token or not client_token:
+        client_token = secrets.token_urlsafe(32)
+    if not BUTLER_TOKEN:
+        raise HTTPException(status_code=500, detail="Butler token is not configured")
+
+    files = {
+        "/app-config/tts-bridge/butler-token": BUTLER_TOKEN,
+        "/app-config/tts-bridge/client-token": client_token,
+    }
+    installer = """import json, os, pathlib
+files = json.loads({files_json!r})
+base = pathlib.Path('/app-config/tts-bridge')
+base.mkdir(parents=True, exist_ok=True)
+for filename, value in files.items():
+    path = pathlib.Path(filename)
+    if path.is_dir():
+        path.rmdir()
+    path.write_text(value)
+    os.chown(path, 10001, 10001)
+    os.chmod(path, 0o400)
+""".format(files_json=json.dumps(files))
+    encoded = base64.b64encode(installer.encode()).decode()
+    command = f"sudo python3 -c {__import__('shlex').quote(f'import base64;exec(base64.b64decode({encoded!r}))')}"
+    rc, _out, err = _ssh("sascha@10.5.85.5", command, timeout=30)
+    if rc != 0:
+        _audit("/tts/bridge/deploy", "POST", 500, "secret installation failed")
+        raise HTTPException(status_code=500, detail="Could not install bridge secrets")
+
+    _audit("/tts/bridge/deploy", "POST", 200, f"rotated={req.rotate_client_token or not _vault_cache.get('tts_bridge_client_token')}")
+    return {
+        "status": "ready",
+        "host": "automation1",
+        "listen": "0.0.0.0:8099",
+        "client_token": client_token,
+        "rotated": req.rotate_client_token or not _vault_cache.get("tts_bridge_client_token"),
+    }
+
+
 @app.post("/tts/speak")
 async def tts_speak(req: TTSRequest, _=Depends(_verify)):
     if req.target == "speaker":

tests/test_app.py

@@ -42,7 +42,111 @@ def test_health_exposes_current_version():
     with TestClient(app.app) as client:
         response = client.get("/health")
     assert response.status_code == 200
-    assert response.json()["version"] == app.VERSION == "2.3.3"
+    assert response.json()["version"] == app.VERSION == "2.3.5"
+
+
+def test_tts_generate_returns_cloned_wav(monkeypatch):
+    captured = {}
+
+    class FakeResponse:
+        status_code = 200
+        content = b"RIFF" + b"test-wave"
+
+    class FakeClient:
+        def __init__(self, **_kwargs):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *_args):
+            return False
+
+        async def post(self, url, json):
+            captured["url"] = url
+            captured["json"] = json
+            return FakeResponse()
+
+    monkeypatch.setattr(app.httpx, "AsyncClient", FakeClient)
+    with TestClient(app.app) as client:
+        response = client.post(
+            "/tts/generate",
+            headers={"Authorization": "Bearer test-token"},
+            json={"text": "Hallo Sascha", "voice": "deep_thought.mp3", "language": "de"},
+        )
+    assert response.status_code == 200
+    assert response.headers["content-type"].startswith("audio/wav")
+    assert response.content.startswith(b"RIFF")
+    assert captured["json"]["voice_mode"] == "clone"
+    assert captured["json"]["reference_audio_filename"] == "deep_thought.mp3"
+
+
+def test_tts_generate_validates_text_and_voice_before_backend(monkeypatch):
+    monkeypatch.setattr(
+        app.httpx,
+        "AsyncClient",
+        lambda **_kwargs: (_ for _ in ()).throw(AssertionError("backend must not be called")),
+    )
+    with TestClient(app.app) as client:
+        empty = client.post(
+            "/tts/generate",
+            headers={"Authorization": "Bearer test-token"},
+            json={"text": ""},
+        )
+        traversal = client.post(
+            "/tts/generate",
+            headers={"Authorization": "Bearer test-token"},
+            json={"text": "Hallo", "voice": "../secret.wav"},
+        )
+    assert empty.status_code == 422
+    assert traversal.status_code == 422
+
+
+def test_tts_generate_rejects_non_wav_backend_response(monkeypatch):
+    class FakeResponse:
+        status_code = 200
+        content = b"not audio"
+
+    class FakeClient:
+        def __init__(self, **_kwargs):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *_args):
+            return False
+
+        async def post(self, _url, json):
+            return FakeResponse()
+
+    monkeypatch.setattr(app.httpx, "AsyncClient", FakeClient)
+    with TestClient(app.app) as client:
+        response = client.post(
+            "/tts/generate",
+            headers={"Authorization": "Bearer test-token"},
+            json={"text": "Hallo"},
+        )
+    assert response.status_code == 502
+    assert "invalid audio" in response.text
+
+
+def test_tts_bridge_deploy_installs_secrets_without_logging_them(monkeypatch):
+    calls = []
+    monkeypatch.setattr(app, "BUTLER_TOKEN", "butler-secret")
+    monkeypatch.setattr(app, "_vault_cache", {})
+    monkeypatch.setattr(app, "_ssh", lambda host, command, timeout=30: (calls.append((host, command, timeout)) or (0, "", "")))
+    with TestClient(app.app) as client:
+        response = client.post(
+            "/tts/bridge/deploy",
+            headers={"Authorization": "Bearer butler-secret"},
+            json={"rotate_client_token": True},
+        )
+    assert response.status_code == 200
+    assert response.json()["listen"] == "0.0.0.0:8099"
+    assert len(response.json()["client_token"]) >= 32
+    assert calls[0][0] == "sascha@10.5.85.5"
+    assert "butler-secret" not in calls[0][1]
 
 
 def test_sysctl_audit_reads_fixed_keys_from_inventory_host(monkeypatch):
@@ -108,6 +212,46 @@ def test_inventory_upsert_uses_base64_script(monkeypatch):
     assert "\\nname =" not in calls[0][1]
 
 
+def test_ansible_run_supports_safe_tune_action_and_syncs_approved_files(monkeypatch):
+    calls = []
+
+    def fake_ssh(host, command, timeout=600):
+        calls.append((host, command, timeout))
+        return 0, "changed=1 failed=0", ""
+
+    monkeypatch.setattr(app, "_ssh", fake_ssh)
+    with TestClient(app.app) as client:
+        response = client.post(
+            "/ansible/run",
+            headers={"Authorization": "Bearer test-token"},
+            json={"hostname": "emby-sascha", "action": "tune"},
+        )
+    assert response.status_code == 200
+    assert response.json()["action"] == "tune"
+    assert "git fetch origin master" in calls[0][1]
+    assert "git show origin/master:roles/sysctl/tasks/main.yml" in calls[0][1]
+    assert "git pull --ff-only" not in calls[0][1]
+    assert "bash pfannkuchen.sh tune emby-sascha" in calls[0][1]
+    assert len(calls) == 1
+
+
+def test_ansible_run_rejects_unknown_action_and_shell_metacharacters(monkeypatch):
+    monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not SSH")))
+    with TestClient(app.app) as client:
+        bad_action = client.post(
+            "/ansible/run",
+            headers={"Authorization": "Bearer test-token"},
+            json={"hostname": "emby-sascha", "action": "shell"},
+        )
+        bad_host = client.post(
+            "/ansible/run",
+            headers={"Authorization": "Bearer test-token"},
+            json={"hostname": "emby-sascha;id", "action": "tune"},
+        )
+    assert bad_action.status_code == 400
+    assert bad_host.status_code == 400
+
+
 def test_docker_inspect_returns_sanitized_summary(monkeypatch):
     raw = [{
         "Name": "/fileflows",

tests/test_bw_manager_deploy.py

@@ -0,0 +1,32 @@
+import importlib.util
+from pathlib import Path
+
+import pytest
+from fastapi.testclient import TestClient
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def load_app(monkeypatch):
+    monkeypatch.setenv("BUTLER_TOKEN", "test-token")
+    spec = importlib.util.spec_from_file_location("butler_bw_test", ROOT / "app.py")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_bw_manager_deploy_rejects_bundle_without_and_gate(monkeypatch):
+    app = load_app(monkeypatch)
+    files = {path: "placeholder" for path in app.BW_MANAGER_REPO_FILES}
+    files["compose.yaml"] = "services:\n  bw-manager:\n    build: ./src\n"
+    files["src/app.py"] = "def old_policy(): pass\n"
+
+    with pytest.raises(ValueError, match="AND gate"):
+        app._deploy_bw_manager_compose(files)
+
+
+def test_bw_manager_deploy_endpoint_requires_auth(monkeypatch):
+    app = load_app(monkeypatch)
+    response = TestClient(app.app).post("/vps/bw-manager/deploy")
+    assert response.status_code in (401, 403)