Kontrollierte Bereinigung des fehlgeschlagenen Qwen-Deployments #28
1 changed files with 133 additions and 0 deletions
133
app.py
133
app.py
|
|
@ -1325,6 +1325,139 @@ async def system_forensics(host: str, since_hours: int = Query(48, ge=1, le=168)
|
||||||
return {"host": host, "since_hours": since_hours, "checks": result}
|
return {"host": host, "since_hours": since_hours, "checks": result}
|
||||||
|
|
||||||
|
|
||||||
|
def _docker_residue_cleanup_command(dry_run: bool) -> str:
|
||||||
|
script = f'''import json, os, shlex, shutil, subprocess
|
||||||
|
|
||||||
|
def run(args):
|
||||||
|
proc = subprocess.run(args, text=True, capture_output=True, timeout=30)
|
||||||
|
return {{"rc": proc.returncode, "stdout": proc.stdout.strip(), "stderr": proc.stderr.strip()}}
|
||||||
|
|
||||||
|
def docker_rule_count():
|
||||||
|
proc = run(["iptables-save"])
|
||||||
|
return sum(1 for line in proc["stdout"].splitlines() if "docker" in line.lower())
|
||||||
|
|
||||||
|
result = {{"dry_run": {str(dry_run)}, "before_rule_count": docker_rule_count(), "removed_rules": [], "removed_chains": [], "removed_links": [], "removed_paths": [], "errors": []}}
|
||||||
|
docker_binary = shutil.which("docker")
|
||||||
|
unit_state = run(["systemctl", "is-active", "docker", "containerd"])["stdout"].splitlines()
|
||||||
|
if docker_binary or any(state == "active" for state in unit_state):
|
||||||
|
result["error"] = "Docker or containerd is still installed/active; refusing residue cleanup"
|
||||||
|
print(json.dumps(result)); raise SystemExit(2)
|
||||||
|
if result["dry_run"]:
|
||||||
|
result["would_remove_paths"] = [path for path in ("/var/lib/docker", "/var/lib/containerd", "/etc/docker") if os.path.exists(path)]
|
||||||
|
print(json.dumps(result)); raise SystemExit(0)
|
||||||
|
for table in ("filter", "nat"):
|
||||||
|
saved = run(["iptables-save", "-t", table])
|
||||||
|
rules = []
|
||||||
|
for line in saved["stdout"].splitlines():
|
||||||
|
if line.startswith("-A ") and "docker" in line.lower():
|
||||||
|
rules.append(line)
|
||||||
|
for line in rules:
|
||||||
|
args = ["iptables", "-t", table] + shlex.split(line)
|
||||||
|
args[3] = "-D"
|
||||||
|
removed = run(args)
|
||||||
|
if removed["rc"] == 0:
|
||||||
|
result["removed_rules"].append(table + ":" + line)
|
||||||
|
else:
|
||||||
|
result["errors"].append(table + ":" + line + ":" + removed["stderr"])
|
||||||
|
for table, chains in (("filter", ("DOCKER-USER", "DOCKER-FORWARD", "DOCKER-BRIDGE", "DOCKER-CT", "DOCKER-INTERNAL", "DOCKER")), ("nat", ("DOCKER",))):
|
||||||
|
for chain in chains:
|
||||||
|
run(["iptables", "-t", table, "-F", chain])
|
||||||
|
deleted = run(["iptables", "-t", table, "-X", chain])
|
||||||
|
if deleted["rc"] == 0:
|
||||||
|
result["removed_chains"].append(table + ":" + chain)
|
||||||
|
for link in ("docker0", "docker_gwbridge"):
|
||||||
|
exists = run(["ip", "link", "show", link])
|
||||||
|
if exists["rc"] == 0:
|
||||||
|
deleted = run(["ip", "link", "delete", link])
|
||||||
|
if deleted["rc"] == 0: result["removed_links"].append(link)
|
||||||
|
else: result["errors"].append(link + ":" + deleted["stderr"])
|
||||||
|
for path in ("/var/lib/docker", "/var/lib/containerd", "/etc/docker"):
|
||||||
|
if os.path.exists(path):
|
||||||
|
shutil.rmtree(path)
|
||||||
|
result["removed_paths"].append(path)
|
||||||
|
result["after_rule_count"] = docker_rule_count()
|
||||||
|
result["forward_rules"] = run(["iptables", "-S", "FORWARD"])["stdout"].splitlines()
|
||||||
|
print(json.dumps(result))
|
||||||
|
if result["errors"] or result["after_rule_count"] != 0: raise SystemExit(1)
|
||||||
|
'''
|
||||||
|
encoded = base64.b64encode(script.encode()).decode()
|
||||||
|
return f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"'
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/system/cleanup/docker-residue/{host}")
|
||||||
|
async def cleanup_docker_residue(host: str, dry_run: bool = Query(True), _=Depends(_verify)):
|
||||||
|
"""Remove only stale Docker firewall/data residue after Docker itself is absent."""
|
||||||
|
if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,62}", host):
|
||||||
|
raise HTTPException(400, "Invalid host name")
|
||||||
|
inventory = await asyncio.to_thread(_find_inventory_host, host)
|
||||||
|
if not inventory:
|
||||||
|
raise HTTPException(404, f"Host {host} not found")
|
||||||
|
target = f'{inventory["user"]}@{inventory["ip"]}'
|
||||||
|
rc, out, err = await asyncio.to_thread(_ssh, target, _docker_residue_cleanup_command(dry_run), 90)
|
||||||
|
try:
|
||||||
|
result = json.loads(out)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise HTTPException(502, (err or out).strip()[-500:] or "cleanup returned invalid JSON") from exc
|
||||||
|
if rc != 0:
|
||||||
|
raise HTTPException(409 if result.get("error") else 502, result)
|
||||||
|
_audit(f"/system/cleanup/docker-residue/{host}", "POST", 200, f"dry_run={dry_run}", dry_run=dry_run)
|
||||||
|
return {"host": host, **result}
|
||||||
|
|
||||||
|
|
||||||
|
def _iso_builder_restore_command(dry_run: bool) -> str:
|
||||||
|
script = f'''import glob, hashlib, json, os, subprocess, tempfile
|
||||||
|
repo = "/app-config/ansible"
|
||||||
|
paths = ("iso-builder/build-iso.sh", "iso-builder/preseed.cfg.tpl")
|
||||||
|
result = {{"dry_run": {str(dry_run)}, "restored": [], "removed_outputs": [], "validation": {{}}}}
|
||||||
|
def run(args):
|
||||||
|
proc = subprocess.run(args, cwd=repo, text=True, capture_output=True, timeout=60)
|
||||||
|
return {{"rc": proc.returncode, "stdout": proc.stdout.strip(), "stderr": proc.stderr.strip()}}
|
||||||
|
fetch = run(["git", "fetch", "origin", "master"])
|
||||||
|
if fetch["rc"] != 0:
|
||||||
|
result["error"] = "git fetch failed"; result["detail"] = fetch["stderr"][-500:]; print(json.dumps(result)); raise SystemExit(1)
|
||||||
|
outputs = sorted(glob.glob(os.path.join(repo, "iso-builder/output/debian-13-minecraft*.iso")))
|
||||||
|
result["would_remove_outputs"] = outputs
|
||||||
|
for path in paths:
|
||||||
|
blob = subprocess.run(["git", "show", "origin/master:" + path], cwd=repo, capture_output=True, timeout=30)
|
||||||
|
if blob.returncode != 0:
|
||||||
|
result["error"] = "missing canonical file " + path; print(json.dumps(result)); raise SystemExit(1)
|
||||||
|
current = open(os.path.join(repo, path), "rb").read() if os.path.exists(os.path.join(repo, path)) else b""
|
||||||
|
result.setdefault("hashes", {{}})[path] = {{"live_before": hashlib.sha256(current).hexdigest(), "canonical": hashlib.sha256(blob.stdout).hexdigest()}}
|
||||||
|
if not result["dry_run"]:
|
||||||
|
destination = os.path.join(repo, path)
|
||||||
|
fd, temporary = tempfile.mkstemp(dir=os.path.dirname(destination))
|
||||||
|
with os.fdopen(fd, "wb") as handle: handle.write(blob.stdout)
|
||||||
|
os.chmod(temporary, 0o755 if path.endswith(".sh") else 0o644)
|
||||||
|
os.replace(temporary, destination)
|
||||||
|
result["restored"].append(path)
|
||||||
|
if not result["dry_run"]:
|
||||||
|
for output in outputs:
|
||||||
|
os.remove(output); result["removed_outputs"].append(output)
|
||||||
|
syntax = run(["bash", "-n", "iso-builder/build-iso.sh"])
|
||||||
|
diff = run(["git", "diff", "--quiet", "origin/master", "--", *paths])
|
||||||
|
result["validation"] = {{"bash_syntax_rc": syntax["rc"], "canonical_diff_rc": diff["rc"]}}
|
||||||
|
if syntax["rc"] != 0 or diff["rc"] != 0:
|
||||||
|
result["error"] = "post-restore validation failed"; print(json.dumps(result)); raise SystemExit(1)
|
||||||
|
print(json.dumps(result))
|
||||||
|
'''
|
||||||
|
encoded = base64.b64encode(script.encode()).decode()
|
||||||
|
return f'python3 -c "import base64;exec(base64.b64decode(\'{encoded}\'))"'
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/system/restore/iso-builder")
|
||||||
|
async def restore_iso_builder(dry_run: bool = Query(True), _=Depends(_verify)):
|
||||||
|
"""Restore only the canonical ISO-builder files and remove generated Minecraft ISOs."""
|
||||||
|
rc, out, err = await asyncio.to_thread(_ssh, AUTOMATION1, _iso_builder_restore_command(dry_run), 120)
|
||||||
|
try:
|
||||||
|
result = json.loads(out)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise HTTPException(502, (err or out).strip()[-500:] or "restore returned invalid JSON") from exc
|
||||||
|
if rc != 0:
|
||||||
|
raise HTTPException(502, result)
|
||||||
|
_audit("/system/restore/iso-builder", "POST", 200, f"dry_run={dry_run}", dry_run=dry_run)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def _pve_auth():
|
def _pve_auth():
|
||||||
pv = _parse_kv("proxmox")
|
pv = _parse_kv("proxmox")
|
||||||
return f"PVEAPIToken={pv.get('tokenid','')}={pv.get('secret','')}"
|
return f"PVEAPIToken={pv.get('tokenid','')}={pv.get('secret','')}"
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue