Merge pull request 'Kontrollierte Bereinigung des fehlgeschlagenen Qwen-Deployments' (#28) from fix/qwen-cleanup-endpoints-20260814 into main

app.py

@@ -1325,6 +1325,139 @@ async def system_forensics(host: str, since_hours: int = Query(48, ge=1, le=168)
     return {"host": host, "since_hours": since_hours, "checks": result}
 
 
+def _docker_residue_cleanup_command(dry_run: bool) -> str:
+    script = f'''import json, os, shlex, shutil, subprocess
+
+def run(args):
+    proc = subprocess.run(args, text=True, capture_output=True, timeout=30)
+    return {{"rc": proc.returncode, "stdout": proc.stdout.strip(), "stderr": proc.stderr.strip()}}
+
+def docker_rule_count():
+    proc = run(["iptables-save"])
+    return sum(1 for line in proc["stdout"].splitlines() if "docker" in line.lower())
+
+result = {{"dry_run": {str(dry_run)}, "before_rule_count": docker_rule_count(), "removed_rules": [], "removed_chains": [], "removed_links": [], "removed_paths": [], "errors": []}}
+docker_binary = shutil.which("docker")
+unit_state = run(["systemctl", "is-active", "docker", "containerd"])["stdout"].splitlines()
+if docker_binary or any(state == "active" for state in unit_state):
+    result["error"] = "Docker or containerd is still installed/active; refusing residue cleanup"
+    print(json.dumps(result)); raise SystemExit(2)
+if result["dry_run"]:
+    result["would_remove_paths"] = [path for path in ("/var/lib/docker", "/var/lib/containerd", "/etc/docker") if os.path.exists(path)]
+    print(json.dumps(result)); raise SystemExit(0)
+for table in ("filter", "nat"):
+    saved = run(["iptables-save", "-t", table])
+    rules = []
+    for line in saved["stdout"].splitlines():
+        if line.startswith("-A ") and "docker" in line.lower():
+            rules.append(line)
+    for line in rules:
+        args = ["iptables", "-t", table] + shlex.split(line)
+        args[3] = "-D"
+        removed = run(args)
+        if removed["rc"] == 0:
+            result["removed_rules"].append(table + ":" + line)
+        else:
+            result["errors"].append(table + ":" + line + ":" + removed["stderr"])
+for table, chains in (("filter", ("DOCKER-USER", "DOCKER-FORWARD", "DOCKER-BRIDGE", "DOCKER-CT", "DOCKER-INTERNAL", "DOCKER")), ("nat", ("DOCKER",))):
+    for chain in chains:
+        run(["iptables", "-t", table, "-F", chain])
+        deleted = run(["iptables", "-t", table, "-X", chain])
+        if deleted["rc"] == 0:
+            result["removed_chains"].append(table + ":" + chain)
+for link in ("docker0", "docker_gwbridge"):
+    exists = run(["ip", "link", "show", link])
+    if exists["rc"] == 0:
+        deleted = run(["ip", "link", "delete", link])
+        if deleted["rc"] == 0: result["removed_links"].append(link)
+        else: result["errors"].append(link + ":" + deleted["stderr"])
+for path in ("/var/lib/docker", "/var/lib/containerd", "/etc/docker"):
+    if os.path.exists(path):
+        shutil.rmtree(path)
+        result["removed_paths"].append(path)
+result["after_rule_count"] = docker_rule_count()
+result["forward_rules"] = run(["iptables", "-S", "FORWARD"])["stdout"].splitlines()
+print(json.dumps(result))
+if result["errors"] or result["after_rule_count"] != 0: raise SystemExit(1)
+'''
+    encoded = base64.b64encode(script.encode()).decode()
+    return f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"'
+
+
+@app.post("/system/cleanup/docker-residue/{host}")
+async def cleanup_docker_residue(host: str, dry_run: bool = Query(True), _=Depends(_verify)):
+    """Remove only stale Docker firewall/data residue after Docker itself is absent."""
+    if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,62}", host):
+        raise HTTPException(400, "Invalid host name")
+    inventory = await asyncio.to_thread(_find_inventory_host, host)
+    if not inventory:
+        raise HTTPException(404, f"Host {host} not found")
+    target = f'{inventory["user"]}@{inventory["ip"]}'
+    rc, out, err = await asyncio.to_thread(_ssh, target, _docker_residue_cleanup_command(dry_run), 90)
+    try:
+        result = json.loads(out)
+    except json.JSONDecodeError as exc:
+        raise HTTPException(502, (err or out).strip()[-500:] or "cleanup returned invalid JSON") from exc
+    if rc != 0:
+        raise HTTPException(409 if result.get("error") else 502, result)
+    _audit(f"/system/cleanup/docker-residue/{host}", "POST", 200, f"dry_run={dry_run}", dry_run=dry_run)
+    return {"host": host, **result}
+
+
+def _iso_builder_restore_command(dry_run: bool) -> str:
+    script = f'''import glob, hashlib, json, os, subprocess, tempfile
+repo = "/app-config/ansible"
+paths = ("iso-builder/build-iso.sh", "iso-builder/preseed.cfg.tpl")
+result = {{"dry_run": {str(dry_run)}, "restored": [], "removed_outputs": [], "validation": {{}}}}
+def run(args):
+    proc = subprocess.run(args, cwd=repo, text=True, capture_output=True, timeout=60)
+    return {{"rc": proc.returncode, "stdout": proc.stdout.strip(), "stderr": proc.stderr.strip()}}
+fetch = run(["git", "fetch", "origin", "master"])
+if fetch["rc"] != 0:
+    result["error"] = "git fetch failed"; result["detail"] = fetch["stderr"][-500:]; print(json.dumps(result)); raise SystemExit(1)
+outputs = sorted(glob.glob(os.path.join(repo, "iso-builder/output/debian-13-minecraft*.iso")))
+result["would_remove_outputs"] = outputs
+for path in paths:
+    blob = subprocess.run(["git", "show", "origin/master:" + path], cwd=repo, capture_output=True, timeout=30)
+    if blob.returncode != 0:
+        result["error"] = "missing canonical file " + path; print(json.dumps(result)); raise SystemExit(1)
+    current = open(os.path.join(repo, path), "rb").read() if os.path.exists(os.path.join(repo, path)) else b""
+    result.setdefault("hashes", {{}})[path] = {{"live_before": hashlib.sha256(current).hexdigest(), "canonical": hashlib.sha256(blob.stdout).hexdigest()}}
+    if not result["dry_run"]:
+        destination = os.path.join(repo, path)
+        fd, temporary = tempfile.mkstemp(dir=os.path.dirname(destination))
+        with os.fdopen(fd, "wb") as handle: handle.write(blob.stdout)
+        os.chmod(temporary, 0o755 if path.endswith(".sh") else 0o644)
+        os.replace(temporary, destination)
+        result["restored"].append(path)
+if not result["dry_run"]:
+    for output in outputs:
+        os.remove(output); result["removed_outputs"].append(output)
+    syntax = run(["bash", "-n", "iso-builder/build-iso.sh"])
+    diff = run(["git", "diff", "--quiet", "origin/master", "--", *paths])
+    result["validation"] = {{"bash_syntax_rc": syntax["rc"], "canonical_diff_rc": diff["rc"]}}
+    if syntax["rc"] != 0 or diff["rc"] != 0:
+        result["error"] = "post-restore validation failed"; print(json.dumps(result)); raise SystemExit(1)
+print(json.dumps(result))
+'''
+    encoded = base64.b64encode(script.encode()).decode()
+    return f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"'
+
+
+@app.post("/system/restore/iso-builder")
+async def restore_iso_builder(dry_run: bool = Query(True), _=Depends(_verify)):
+    """Restore only the canonical ISO-builder files and remove generated Minecraft ISOs."""
+    rc, out, err = await asyncio.to_thread(_ssh, AUTOMATION1, _iso_builder_restore_command(dry_run), 120)
+    try:
+        result = json.loads(out)
+    except json.JSONDecodeError as exc:
+        raise HTTPException(502, (err or out).strip()[-500:] or "restore returned invalid JSON") from exc
+    if rc != 0:
+        raise HTTPException(502, result)
+    _audit("/system/restore/iso-builder", "POST", 200, f"dry_run={dry_run}", dry_run=dry_run)
+    return result
+
+
 def _pve_auth():
     pv = _parse_kv("proxmox")
     return f"PVEAPIToken={pv.get('tokenid','')}={pv.get('secret','')}"

tests/test_app.py

@@ -201,6 +201,45 @@ def test_host_forensics_rejects_unknown_host_and_invalid_window(monkeypatch):
     assert bad_window.status_code == 422
 
 
+def test_docker_residue_cleanup_defaults_to_dry_run(monkeypatch):
+    payload = {"dry_run": True, "before_rule_count": 25, "removed_rules": [], "removed_chains": [], "removed_links": [], "removed_paths": [], "errors": []}
+    calls = []
+    monkeypatch.setattr(app, "_find_inventory_host", lambda name: {"name": name, "user": "root", "ip": "10.5.85.13"})
+    monkeypatch.setattr(app, "_ssh", lambda host, command, timeout=600: (calls.append((host, command, timeout)) or (0, __import__("json").dumps(payload), "")))
+    with TestClient(app.app) as client:
+        response = client.post("/system/cleanup/docker-residue/node3", headers={"Authorization": "Bearer test-token"})
+    assert response.status_code == 200
+    assert response.json()["dry_run"] is True
+    assert calls[0][0] == "root@10.5.85.13"
+    assert calls[0][2] == 90
+
+
+def test_docker_residue_cleanup_refuses_active_docker(monkeypatch):
+    payload = {"dry_run": False, "error": "Docker or containerd is still installed/active; refusing residue cleanup"}
+    monkeypatch.setattr(app, "_find_inventory_host", lambda name: {"name": name, "user": "root", "ip": "10.5.85.13"})
+    monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (2, __import__("json").dumps(payload), ""))
+    with TestClient(app.app) as client:
+        response = client.post("/system/cleanup/docker-residue/node3?dry_run=false", headers={"Authorization": "Bearer test-token"})
+    assert response.status_code == 409
+
+
+def test_iso_builder_restore_defaults_to_dry_run_and_has_no_free_target(monkeypatch):
+    payload = {"dry_run": True, "restored": [], "removed_outputs": [], "validation": {}}
+    calls = []
+    monkeypatch.setattr(app, "_ssh", lambda host, command, timeout=600: (calls.append((host, command, timeout)) or (0, __import__("json").dumps(payload), "")))
+    with TestClient(app.app) as client:
+        response = client.post("/system/restore/iso-builder", headers={"Authorization": "Bearer test-token"})
+    assert response.status_code == 200
+    assert response.json()["dry_run"] is True
+    assert calls[0][0] == app.AUTOMATION1
+    assert calls[0][2] == 120
+    command = app._iso_builder_restore_command(True)
+    encoded = command.split("base64.b64decode('", 1)[1].split("')", 1)[0]
+    decoded = __import__("base64").b64decode(encoded).decode()
+    assert "origin/master" in decoded
+    assert "/app-config/ansible" in decoded
+
+
 def test_invalid_log_target_is_rejected_before_ssh():
     with TestClient(app.app) as client:
         response = client.get(