app.py

@@ -11,7 +11,7 @@ from fastapi.responses import JSONResponse, RedirectResponse
 from contextlib import asynccontextmanager
 
 log = logging.getLogger("butler")
-VERSION = "2.3.5"
+VERSION = "2.3.2"
 
 API_DIR = os.environ.get("API_KEY_DIR", "/data/api")
 VAULT_CACHE_DIR = os.environ.get("VAULT_CACHE_DIR", "/data/vault-cache")
@@ -278,7 +278,6 @@ async def root():
             "status": "GET /status - health of all backends",
             "overview": "GET /overview?details=false - deterministic homelab verdict for small models",
             "audit": "GET /audit - recent API calls",
-            "sysctl_audit": "GET /system/sysctl/{host} - read-only live and persistent network tuning",
         },
         "vault_items": len(_vault_cache),
     }
@@ -1050,7 +1049,6 @@ SPEEDTEST_REPO_FILES = (
     "streamscope/static/index.html",
     "streamscope/static/assets/app.css",
     "streamscope/static/assets/app.js",
-    "streamscope/static/assets/longterm-metrics.js",
 )
 
 
@@ -1126,7 +1124,7 @@ os.chmod(env, 0o600)
     if rc != 0:
         _ssh(VPS_SSH, rollback, timeout=180)
         raise RuntimeError(f"StreamScope build preflight failed: {err[-500:]}")
-    deploy = "cd /app-config/github/speedtest && (docker rm -f speedtest >/dev/null 2>&1 || true) && docker compose up -d --remove-orphans"
+    deploy = "cd /app-config/github/speedtest && docker compose up -d --remove-orphans"
     rc, out, err = _ssh(VPS_SSH, deploy, timeout=180)
     if rc != 0:
         _ssh(VPS_SSH, rollback, timeout=180)
@@ -1164,117 +1162,6 @@ async def vps_speedtest_deploy(req: SpeedtestDeployRequest, _=Depends(_verify)):
     return result
 
 
-BW_MANAGER_REPO_FILES = (
-    ".env.example", ".gitignore", "README.md", "compose.yaml",
-    "src/.dockerignore", "src/Dockerfile", "src/app.py",
-    "src/remote_policy.py", "src/requirements.txt",
-    "src/templates/base.html", "src/templates/history.html",
-    "src/templates/index.html", "src/templates/users.html",
-)
-
-
-async def _fetch_bw_manager_text(path: str) -> str:
-    if path not in BW_MANAGER_REPO_FILES:
-        raise ValueError("unsupported BW Manager file")
-    cfg = SERVICES.get("forgejo", {})
-    base_url, token = cfg.get("url"), _get_key(cfg)
-    if not base_url or not token:
-        raise RuntimeError("Forgejo service configuration is unavailable")
-    url = f"{base_url}/api/v1/repos/sascha/bw-manager/contents/{path}"
-    async with httpx.AsyncClient(timeout=30) as client:
-        response = await client.get(
-            url, params={"ref": "main"},
-            headers={"Authorization": f"token {token}"},
-        )
-        response.raise_for_status()
-    return base64.b64decode(response.json()["content"]).decode()
-
-
-def _deploy_bw_manager_compose(files: dict[str, str]) -> dict:
-    if set(files) != set(BW_MANAGER_REPO_FILES):
-        raise ValueError("BW Manager source bundle is incomplete")
-    if "build: ./src" not in files["compose.yaml"]:
-        raise ValueError("BW Manager compose contract is invalid")
-    if "build_gated_targets" not in files["src/app.py"]:
-        raise ValueError("BW Manager candidate lacks the user/network AND gate")
-    rc, working_dir, err = _ssh(
-        VPS_SSH,
-        "docker inspect -f '{{ index .Config.Labels \"com.docker.compose.project.working_dir\" }}' bw-manager",
-        timeout=30,
-    )
-    working_dir = working_dir.strip()
-    if rc != 0 or not re.fullmatch(r"/app-config/[A-Za-z0-9_./-]+", working_dir):
-        raise RuntimeError(f"cannot determine safe BW Manager working directory: {(err or working_dir)[-300:]}")
-    timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
-    candidate = f"/app-config/deployment-candidates/bw-manager-{timestamp}"
-    backup = f"/app-config/deployment-backups/bw-manager-{timestamp}"
-    image = "bw-manager-bw-manager"
-    rollback_image = f"{image}:rollback-{timestamp}"
-    init_script = f"""from pathlib import Path
-import shutil
-candidate = Path({candidate!r})
-if candidate.exists(): shutil.rmtree(candidate)
-candidate.mkdir(parents=True)
-live_env = Path({working_dir!r}) / '.env'
-if live_env.exists(): shutil.copy2(live_env, candidate / '.env')
-"""
-    rc, _out, err = _remote_python(init_script)
-    if rc != 0:
-        raise RuntimeError(f"BW Manager candidate initialization failed: {err[-300:]}")
-    # Stage one file per SSH call. Sending the complete repository in one
-    # command exceeds Linux's argv limit once app.py and templates are encoded.
-    for relative, content in files.items():
-        file_script = f"""from pathlib import Path
-target = Path({candidate!r}) / {relative!r}
-target.parent.mkdir(parents=True, exist_ok=True)
-target.write_text({content!r})
-"""
-        rc, _out, err = _remote_python(file_script)
-        if rc != 0:
-            raise RuntimeError(f"BW Manager staging failed for {relative}: {err[-300:]}")
-    rc, _out, err = _ssh(VPS_SSH, f"cd {candidate} && docker compose config -q && docker compose build --pull", timeout=600)
-    if rc != 0:
-        raise RuntimeError(f"BW Manager candidate build failed: {err[-500:]}")
-    deploy_script = f"""from pathlib import Path
-import shutil
-live, backup, candidate = Path({working_dir!r}), Path({backup!r}), Path({candidate!r})
-backup.parent.mkdir(parents=True, exist_ok=True)
-if backup.exists(): shutil.rmtree(backup)
-shutil.copytree(live, backup)
-for relative in {BW_MANAGER_REPO_FILES!r}:
-    source, target = candidate / relative, live / relative
-    target.parent.mkdir(parents=True, exist_ok=True)
-    shutil.copy2(source, target)
-"""
-    rc, _out, err = _remote_python(deploy_script)
-    if rc != 0:
-        raise RuntimeError(f"BW Manager live file switch failed: {err[-300:]}")
-    _ssh(VPS_SSH, f"docker image tag {image} {rollback_image}", timeout=60)
-    rollback = (
-        f"rm -rf {working_dir} && cp -a {backup} {working_dir} && "
-        f"docker image tag {rollback_image} {image} && cd {working_dir} && "
-        "docker compose up -d --no-build"
-    )
-    rc, out, err = _ssh(VPS_SSH, f"cd {working_dir} && docker compose up -d --build --remove-orphans", timeout=600)
-    if rc != 0:
-        _ssh(VPS_SSH, rollback, timeout=180)
-        raise RuntimeError(f"BW Manager deployment failed: {(err or out)[-500:]}")
-    health = "for i in $(seq 1 45); do curl -fsS --max-time 3 http://127.0.0.1:8870/api/status >/dev/null && exit 0; sleep 2; done; exit 1"
-    rc, _out, err = _ssh(VPS_SSH, health, timeout=105)
-    if rc != 0:
-        _ssh(VPS_SSH, rollback, timeout=180)
-        raise RuntimeError(f"BW Manager health failed; rollback attempted: {err[-300:]}")
-    return {"status": "deployed", "health": "ok", "working_dir": working_dir, "backup": backup}
-
-
-@app.post("/vps/bw-manager/deploy")
-async def vps_bw_manager_deploy(_=Depends(_verify)):
-    contents = await asyncio.gather(*(_fetch_bw_manager_text(path) for path in BW_MANAGER_REPO_FILES))
-    result = await asyncio.to_thread(_deploy_bw_manager_compose, dict(zip(BW_MANAGER_REPO_FILES, contents)))
-    _audit("/vps/bw-manager/deploy", "POST", 200, "Git-managed BW Manager deployment")
-    return result
-
-
 # --- VM Lifecycle Endpoints ---
 import subprocess as _sp
 
@@ -1298,76 +1185,6 @@ def _ssh(host, cmd, timeout=600):
     except _sp.TimeoutExpired:
         return 124, "", f"SSH command timed out after {timeout} seconds"
 
-
-SYSCTL_AUDIT_KEYS = (
-    "net.core.default_qdisc",
-    "net.core.rmem_default",
-    "net.core.rmem_max",
-    "net.core.wmem_default",
-    "net.core.wmem_max",
-    "net.core.netdev_max_backlog",
-    "net.core.somaxconn",
-    "net.ipv4.ip_forward",
-    "net.ipv4.tcp_congestion_control",
-    "net.ipv4.tcp_fastopen",
-    "net.ipv4.tcp_mtu_probing",
-    "net.ipv4.tcp_no_metrics_save",
-    "net.ipv4.tcp_rmem",
-    "net.ipv4.tcp_slow_start_after_idle",
-    "net.ipv4.tcp_window_scaling",
-    "net.ipv4.tcp_wmem",
-)
-
-
-def _sysctl_audit_command() -> str:
-    script = f'''import glob, json
-from pathlib import Path
-keys = {SYSCTL_AUDIT_KEYS!r}
-live, errors = {{}}, {{}}
-for key in keys:
-    try:
-        live[key] = Path("/proc/sys/" + key.replace(".", "/")).read_text().strip()
-    except OSError as exc:
-        errors[key] = str(exc)[:160]
-persistent = {{}}
-for path in ["/etc/sysctl.conf", *sorted(glob.glob("/etc/sysctl.d/*.conf"))]:
-    try:
-        with open(path, encoding="utf-8", errors="replace") as handle:
-            for raw in handle:
-                line = raw.split("#", 1)[0].strip()
-                if "=" not in line:
-                    continue
-                key, value = (part.strip() for part in line.split("=", 1))
-                if key in keys:
-                    persistent.setdefault(key, []).append({{"file": path, "value": value}})
-    except (FileNotFoundError, PermissionError):
-        pass
-print(json.dumps({{"live": live, "persistent": persistent, "errors": errors}}))
-'''
-    encoded = base64.b64encode(script.encode()).decode()
-    return f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"'
-
-
-@app.get("/system/sysctl/{host}")
-async def system_sysctl_audit(host: str, _=Depends(_verify)):
-    if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,62}", host):
-        raise HTTPException(400, "Invalid host name")
-    if host == "vps":
-        target = VPS_SSH
-    else:
-        inventory = await asyncio.to_thread(_find_inventory_host, host)
-        if not inventory:
-            raise HTTPException(404, f"Host {host} not found")
-        target = f'{inventory["user"]}@{inventory["ip"]}'
-    rc, out, err = await asyncio.to_thread(_ssh, target, _sysctl_audit_command(), 30)
-    if rc != 0:
-        raise HTTPException(502, (err or out).strip()[-500:] or "sysctl audit failed")
-    try:
-        result = json.loads(out)
-    except json.JSONDecodeError as exc:
-        raise HTTPException(502, "sysctl audit returned invalid JSON") from exc
-    return {"host": host, **result}
-
 def _pve_auth():
     pv = _parse_kv("proxmox")
     return f"PVEAPIToken={pv.get('tokenid','')}={pv.get('secret','')}"
@@ -1678,47 +1495,10 @@ print("updated" if updated else "added")'''
 async def ansible_run(request: Request, _=Depends(_verify)):
     body = await request.json()
     hostname = body.get("limit", body.get("hostname", ""))
+    template_id = body.get("template_id", 10)
     if not hostname:
         return JSONResponse({"error": "limit/hostname required"}, status_code=400)
-    action = body.get("action", "setup")
+    rc, out, err = _ssh(AUTOMATION1, f"cd /app-config/ansible && bash pfannkuchen.sh setup {hostname}", timeout=600)
-    if action not in {"setup", "tune", "pvetune"}:
-        return JSONResponse({"error": "action must be setup, tune or pvetune"}, status_code=400)
-    if not re.fullmatch(r"[a-zA-Z0-9_.:-]+", hostname):
-        return JSONResponse({"error": "invalid hostname/limit"}, status_code=400)
-    if action in {"tune", "pvetune"}:
-        approved_files = (
-            "roles/sysctl/defaults/main.yml",
-            "roles/sysctl/tasks/main.yml",
-            "group_vars/vps/sysctl.yml",
-            "sysctl-proxmox.yaml",
-            "roles/sysctl_proxmox/tasks/main.yml",
-        )
-        file_sync = " && ".join(
-            f"git show origin/master:{path} > {path}" for path in approved_files
-        )
-        command = (
-            "cd /app-config/ansible && "
-            "git fetch origin master && "
-            f"{file_sync} && "
-            f"bash pfannkuchen.sh {action} {hostname}"
-        )
-    else:
-        command = (
-            "cd /app-config/ansible && "
-            "git pull --ff-only origin master && "
-            f"bash pfannkuchen.sh {action} {hostname}"
-        )
-    rc, out, err = _ssh(AUTOMATION1, command, timeout=600)
-    _audit("/ansible/run", "POST", 200 if rc == 0 else 502, f"{action} {hostname}")
-    if action != "setup":
-        return {
-            "status": "ok" if rc == 0 else "error",
-            "action": action,
-            "hostname": hostname,
-            "rc": rc,
-            "output": out[-4000:],
-            "error": err[-1000:] if rc != 0 else "",
-        }
     
     # After successful ansible run: sync Hawser token to Dockhand
     if rc == 0:

tests/test_app.py

@@ -42,32 +42,7 @@ def test_health_exposes_current_version():
     with TestClient(app.app) as client:
         response = client.get("/health")
     assert response.status_code == 200
-    assert response.json()["version"] == app.VERSION == "2.3.5"
+    assert response.json()["version"] == app.VERSION == "2.3.2"
-
-
-def test_sysctl_audit_reads_fixed_keys_from_inventory_host(monkeypatch):
-    payload = {
-        "live": {"net.ipv4.tcp_congestion_control": "bbr"},
-        "persistent": {"net.ipv4.tcp_congestion_control": [{"file": "/etc/sysctl.d/99-net-tuning.conf", "value": "bbr"}]},
-        "errors": {},
-    }
-    calls = []
-    monkeypatch.setattr(app, "_find_inventory_host", lambda name: {"name": name, "user": "root", "ip": "10.5.85.16"})
-    monkeypatch.setattr(app, "_ssh", lambda host, command, timeout=600: (calls.append((host, command, timeout)) or (0, __import__("json").dumps(payload), "")))
-    with TestClient(app.app) as client:
-        response = client.get("/system/sysctl/node6", headers={"Authorization": "Bearer test-token"})
-    assert response.status_code == 200
-    assert response.json()["live"]["net.ipv4.tcp_congestion_control"] == "bbr"
-    assert calls[0][0] == "root@10.5.85.16"
-    assert "base64.b64decode" in calls[0][1]
-
-
-def test_sysctl_audit_rejects_unknown_host_without_ssh(monkeypatch):
-    monkeypatch.setattr(app, "_find_inventory_host", lambda _name: None)
-    monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not SSH")))
-    with TestClient(app.app) as client:
-        response = client.get("/system/sysctl/not-there", headers={"Authorization": "Bearer test-token"})
-    assert response.status_code == 404
 
 
 def test_invalid_log_target_is_rejected_before_ssh():
@@ -108,46 +83,6 @@ def test_inventory_upsert_uses_base64_script(monkeypatch):
     assert "\\nname =" not in calls[0][1]
 
 
-def test_ansible_run_supports_safe_tune_action_and_syncs_approved_files(monkeypatch):
-    calls = []
-
-    def fake_ssh(host, command, timeout=600):
-        calls.append((host, command, timeout))
-        return 0, "changed=1 failed=0", ""
-
-    monkeypatch.setattr(app, "_ssh", fake_ssh)
-    with TestClient(app.app) as client:
-        response = client.post(
-            "/ansible/run",
-            headers={"Authorization": "Bearer test-token"},
-            json={"hostname": "emby-sascha", "action": "tune"},
-        )
-    assert response.status_code == 200
-    assert response.json()["action"] == "tune"
-    assert "git fetch origin master" in calls[0][1]
-    assert "git show origin/master:roles/sysctl/tasks/main.yml" in calls[0][1]
-    assert "git pull --ff-only" not in calls[0][1]
-    assert "bash pfannkuchen.sh tune emby-sascha" in calls[0][1]
-    assert len(calls) == 1
-
-
-def test_ansible_run_rejects_unknown_action_and_shell_metacharacters(monkeypatch):
-    monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not SSH")))
-    with TestClient(app.app) as client:
-        bad_action = client.post(
-            "/ansible/run",
-            headers={"Authorization": "Bearer test-token"},
-            json={"hostname": "emby-sascha", "action": "shell"},
-        )
-        bad_host = client.post(
-            "/ansible/run",
-            headers={"Authorization": "Bearer test-token"},
-            json={"hostname": "emby-sascha;id", "action": "tune"},
-        )
-    assert bad_action.status_code == 400
-    assert bad_host.status_code == 400
-
-
 def test_docker_inspect_returns_sanitized_summary(monkeypatch):
     raw = [{
         "Name": "/fileflows",

tests/test_bw_manager_deploy.py

@@ -1,32 +0,0 @@
-import importlib.util
-from pathlib import Path
-
-import pytest
-from fastapi.testclient import TestClient
-
-
-ROOT = Path(__file__).resolve().parents[1]
-
-
-def load_app(monkeypatch):
-    monkeypatch.setenv("BUTLER_TOKEN", "test-token")
-    spec = importlib.util.spec_from_file_location("butler_bw_test", ROOT / "app.py")
-    module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)
-    return module
-
-
-def test_bw_manager_deploy_rejects_bundle_without_and_gate(monkeypatch):
-    app = load_app(monkeypatch)
-    files = {path: "placeholder" for path in app.BW_MANAGER_REPO_FILES}
-    files["compose.yaml"] = "services:\n  bw-manager:\n    build: ./src\n"
-    files["src/app.py"] = "def old_policy(): pass\n"
-
-    with pytest.raises(ValueError, match="AND gate"):
-        app._deploy_bw_manager_compose(files)
-
-
-def test_bw_manager_deploy_endpoint_requires_auth(monkeypatch):
-    app = load_app(monkeypatch)
-    response = TestClient(app.app).post("/vps/bw-manager/deploy")
-    assert response.status_code in (401, 403)

tests/test_streamscope_deploy.py

@@ -1,38 +0,0 @@
-import app
-
-
-def test_streamscope_deploy_replaces_legacy_container_before_compose(monkeypatch):
-    assert "streamscope/static/assets/longterm-metrics.js" in app.SPEEDTEST_REPO_FILES
-    files = {path: "placeholder" for path in app.SPEEDTEST_REPO_FILES}
-    files["compose.yaml"] = """services:
-  streamscope:
-    build: .
-    ports:
-      - '127.0.0.1:8080:8080'
-    environment:
-      ADMIN_PASSWORD: "${ADMIN_PASSWORD:?required}"
-      SESSION_SECRET: "${SESSION_SECRET:?required}"
-    volumes:
-      - /app-config/speedtest/data:/data
-    cap_add:
-      - NET_RAW
-"""
-    files["Dockerfile"] = "FROM python:3.13-slim\nRUN apt-get install -y mtr-tiny\n"
-    commands = []
-
-    monkeypatch.setattr(app, "_remote_python", lambda script: (0, "", ""))
-
-    def fake_ssh(host, command, timeout=600):
-        commands.append(command)
-        return 0, "ok", ""
-
-    monkeypatch.setattr(app, "_ssh", fake_ssh)
-    result = app._deploy_speedtest_compose(
-        files,
-        "correct-horse-battery-staple",
-        "streamscope-session-secret-with-entropy",
-    )
-
-    deploy = next(command for command in commands if "compose up -d --remove-orphans" in command)
-    assert "docker rm -f speedtest" in deploy
-    assert result["application"] == "streamscope"