diff --git a/README.md b/README.md index aa96e1d..d0a464f 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,6 @@ Known secret response fields such as Dockhand's `hawserToken` and `webhookSecret | `/docker/inspect/{host}/{container}` | GET | Sanitized image, runtime, resources, mounts and state | | `/docker/restart/{host}/{container}` | POST | Restart container; supports `dry_run=true` | | `/config/reload` | POST | Reload YAML configuration and credential cache | -| `/media/verify` | POST | ffprobe duration check on a NAS media file to catch truncated Sonarr/Radarr imports; flags `suspect` if deviation from `expected_minutes` exceeds `tolerance_pct` | ## VM lifecycle and inventory @@ -98,14 +97,6 @@ Integration Compose definition: `tests/compose.integration.yaml` (binds only to ## Changelog -### 2.4.8 — 17.09.2026 - -- Fixed `POST /media/verify` false-positive rejection: filenames containing a legitimate apostrophe (e.g. "La'An" in a Star Trek episode title) were blocked as "unsafe characters". Replaced the naive single-quote wrapping + apostrophe blocklist with proper `shlex.quote()` escaping for the remote ffprobe command; only newline/NUL byte injection is still rejected. - -### 2.4.7 — 17.09.2026 - -- Added `POST /media/verify`: probes a media file's real duration via `ffprobe` (running inside the existing `bazarrUHD` container on `arrapps`, which already mounts `/data` and ships ffmpeg — no new service needed) and flags it as `suspect` when it deviates from an `expected_minutes` value beyond `tolerance_pct`. Catches truncated Sonarr/Radarr imports (e.g. a re-grab that lands as 1.8 GB instead of the expected 8 GB for a UHD episode) after the file is already on the NAS. - ### 2.3.2 — 22.07.2026 - Make Vaultwarden refresh durable: persistent named cache volume, protected runtime credentials, automatic API-key re-login and atomic cache writes. diff --git a/app.py b/app.py index 7ddb58a..b950c4d 100644 --- a/app.py +++ b/app.py @@ -1,7 +1,7 @@ """Homelab Butler v2.1 – Unified API proxy for Pfannkuchen homelab. Reads service config from butler.yaml, credentials from Vaultwarden cache with flat-file fallback.""" -import os, json, asyncio, logging, time, base64, re, subprocess, ipaddress, secrets, sqlite3, math, hashlib, shlex +import os, json, asyncio, logging, time, base64, re, subprocess, ipaddress, secrets, sqlite3, math, hashlib from datetime import datetime, timezone import httpx, yaml from typing import Literal @@ -12,7 +12,7 @@ from contextlib import asynccontextmanager from contextvars import ContextVar log = logging.getLogger("butler") -VERSION = "2.4.9" +VERSION = "2.4.6" API_DIR = os.environ.get("API_KEY_DIR", "/data/api") VAULT_CACHE_DIR = os.environ.get("VAULT_CACHE_DIR", "/data/vault-cache") @@ -1546,102 +1546,6 @@ async def vault_reload(_=Depends(_verify)): return {"reloaded": True, "items": len(_vault_cache)} -# --- Media file integrity verification --- - -MEDIA_VERIFY_PROBES = { - "arrapps": ("bazarrUHD", "sascha"), - "arr-chris": (None, "chris"), - "arr-chris-live": (None, "chris"), -} - - -class MediaVerifyRequest(BaseModel): - host: str = Field(..., max_length=64) - path: str = Field(..., max_length=1000) - expected_minutes: float | None = Field(None, ge=0, le=1440) - tolerance_pct: float = Field(20.0, gt=0, le=100) - - -@app.post("/media/verify") -async def media_verify(payload: MediaVerifyRequest, _=Depends(_verify)): - """Probe a media file's real duration via ffprobe to catch truncated imports. - - Runs `ffprobe` inside an existing container (bazarrUHD on arrapps, which already - mounts /data and ships ffmpeg) so no new service is required. Compares the - measured duration against an expected runtime (minutes) supplied by the caller - (e.g. Sonarr/Radarr's runtime field) within tolerance_pct. - """ - if not re.fullmatch(r"[A-Za-z0-9_.-]+", payload.host): - raise HTTPException(400, "Invalid host name") - if payload.host not in MEDIA_VERIFY_PROBES: - raise HTTPException(400, f"No ffprobe container configured for host {payload.host}") - if not re.fullmatch(r"/data/[^\x00]+\.(mkv|mp4|avi|m4v|ts)", payload.path): - raise HTTPException(400, "Path must be an absolute /data media file") - if ".." in payload.path or "\n" in payload.path or "\x00" in payload.path: - raise HTTPException(400, "Path contains unsafe characters") - - container, _default_user = MEDIA_VERIFY_PROBES[payload.host] - if not container: - raise HTTPException(400, f"Host {payload.host} has no configured ffprobe container yet") - - target = _find_inventory_host(payload.host) - if not target: - raise HTTPException(404, f"Host {payload.host} not found in inventory") - - # expected_minutes == 0 or None: no reference runtime available, skip verification gracefully - if not payload.expected_minutes or payload.expected_minutes <= 0: - return { - "host": payload.host, - "path": payload.path, - "duration_seconds": None, - "duration_minutes": None, - "expected_minutes": payload.expected_minutes, - "verified": False, - "suspect": False, - "skipped": True, - "skip_reason": "no_reference_runtime", - "deviation_pct": None, - } - - probe_cmd = ( - f"docker exec {container} ffprobe -v error " - f"-show_entries format=duration -of default=noprint_wrappers=1:nokey=1 {shlex.quote(payload.path)}" - ) - rc, out, err = await asyncio.to_thread( - _ssh, f'{target["user"]}@{target["ip"]}', f"sudo -n {probe_cmd} || {probe_cmd}", 30 - ) - if rc != 0: - _audit("/media/verify", "POST", 502, f"host={payload.host} path={payload.path}") - raise HTTPException(502, (err or out).strip()[:500] or "ffprobe failed") - - raw_duration = out.strip() - try: - duration_seconds = float(raw_duration) - except ValueError: - _audit("/media/verify", "POST", 502, f"host={payload.host} unparsable duration") - raise HTTPException(502, "ffprobe returned no parsable duration; file is likely corrupt") - - duration_minutes = duration_seconds / 60 - result = { - "host": payload.host, - "path": payload.path, - "duration_seconds": round(duration_seconds, 1), - "duration_minutes": round(duration_minutes, 2), - "expected_minutes": payload.expected_minutes, - "verified": True, - "suspect": False, - } - if payload.expected_minutes: - deviation_pct = abs(duration_minutes - payload.expected_minutes) / payload.expected_minutes * 100 - result["deviation_pct"] = round(deviation_pct, 1) - result["suspect"] = deviation_pct > payload.tolerance_pct - _audit( - "/media/verify", "POST", 200, - f"host={payload.host} dur={duration_minutes:.1f}m suspect={result['suspect']}", - ) - return result - - # --- VPS reverse-proxy and DNS management --- VPS_SSH = "root@46.225.230.72" @@ -4217,11 +4121,7 @@ async def media_handoff(payload: MediaHandoffPayload, request: Request): _audit("/media/handoff", "POST", 403, f"caller={caller} action={payload.action}") raise HTTPException(403, "Media handoff caller is not allowed") - # 16.09.2026: serienen/videoen ergaenzt. Die englischen Arr-Instanzen - # sonarrEN (Port 8991, Root /data/FHD/serienen) und radarrEN (Port 7880, - # Root /data/FHD/videoen) nutzen eigene SAB-Kategorien. Ohne sie brach der - # Handoff mit 422 ab und Releases blieben in /usenet/complete liegen. - allowed_categories = {"serien4k", "serien", "serienen", "video4k", "video", "videoen"} + allowed_categories = {"serien4k", "serien", "video4k", "video"} if payload.action == "start": if payload.category not in allowed_categories: raise HTTPException(422, "Unsupported media category") diff --git a/tests/test_app.py b/tests/test_app.py index 45011c2..3178eb8 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,7 +1,6 @@ import os import asyncio import json -import shlex import time from datetime import datetime, timedelta, timezone @@ -118,99 +117,6 @@ def test_media_handoff_rejects_wrong_category_path(monkeypatch): assert response.status_code == 422 -def test_media_handoff_accepts_english_arr_categories(monkeypatch): - """serienen/videoen muessen durchgehen (sonarrEN 8991 / radarrEN 7880). - - 16.09.2026: fehlten in allowed_categories -> HTTP 422 -> movetdarr.sh - brach mit "n8n-Handoff konnte nicht registriert werden" ab und liess - Mutiny.2026 x2 tagelang in /usenet/complete/videoen liegen. - """ - seen = [] - - class FakeResponse: - status_code = 200 - - def json(self): - return {"ok": True, "jobId": "test-job-5678", "state": "registered"} - - class FakeClient: - def __init__(self, **_kwargs): - pass - - async def __aenter__(self): - return self - - async def __aexit__(self, *_args): - return None - - async def post(self, url, json, headers): - seen.append(json) - return FakeResponse() - - monkeypatch.setattr(app.httpx, "AsyncClient", FakeClient) - with TestClient(app.app) as client: - monkeypatch.setattr(app, "SERVICES", {"n8n": {"url": "http://n8n:5678", "auth": "n8n"}}) - for category in ("serienen", "videoen"): - response = client.post( - "/media/handoff", - headers={"Authorization": "Bearer test-token"}, - json={ - "action": "start", - "category": category, - "directory": f"/usenet/complete/{category}/Release.2026", - "release": "Release.2026-GRP", - "cleanName": "Release 2026", - "expectedFiles": 1, - }, - ) - assert response.status_code == 200, (category, response.text) - assert response.json()["state"] == "registered", category - - assert [item["category"] for item in seen] == ["serienen", "videoen"] - - -def test_media_handoff_still_rejects_unknown_category(monkeypatch): - """Fail-closed bleibt: eine frei erfundene Kategorie wird nicht geproxyt.""" - monkeypatch.setattr( - app.httpx, - "AsyncClient", - lambda **_kwargs: (_ for _ in ()).throw(AssertionError("must not proxy")), - ) - with TestClient(app.app) as client: - response = client.post( - "/media/handoff", - headers={"Authorization": "Bearer test-token"}, - json={ - "action": "start", - "category": "hoerbuecher", - "directory": "/usenet/complete/hoerbuecher/Buch", - "release": "Buch", - }, - ) - assert response.status_code == 422 - - -def test_media_handoff_english_category_path_must_match(monkeypatch): - """Pfadpruefung gilt auch fuer die neuen Kategorien.""" - monkeypatch.setattr( - app.httpx, - "AsyncClient", - lambda **_kwargs: (_ for _ in ()).throw(AssertionError("must not proxy")), - ) - with TestClient(app.app) as client: - response = client.post( - "/media/handoff", - headers={"Authorization": "Bearer test-token"}, - json={ - "action": "start", - "category": "videoen", - "directory": "/usenet/complete/video4k/Wrong", - "release": "Wrong", - }, - ) - assert response.status_code == 422 - - def test_paperless_import_queues_pdf_through_butler(monkeypatch): captured = {} @@ -902,115 +808,6 @@ def test_uptime_monitor_remove_rejects_invalid_expected_name_before_ssh(monkeypa assert response.status_code == 400 -def test_media_verify_returns_duration_and_not_suspect_when_within_tolerance(monkeypatch): - monkeypatch.setattr(app, "_find_inventory_host", lambda host: {"user": "sascha", "ip": "10.2.1.100"}) - calls = [] - - def fake_ssh(host, command, timeout=30): - calls.append((host, command, timeout)) - return 0, "2967.355000\n", "" - - monkeypatch.setattr(app, "_ssh", fake_ssh) - with TestClient(app.app) as client: - response = client.post( - "/media/verify", - headers={"Authorization": "Bearer test-token"}, - json={"host": "arrapps", "path": "/data/UHD/serien/Show/ep.mkv", "expected_minutes": 49.5}, - ) - assert response.status_code == 200 - body = response.json() - assert body["duration_minutes"] == 49.46 - assert body["suspect"] is False - assert calls[0][0] == "sascha@10.2.1.100" - assert "bazarrUHD" in calls[0][1] - assert "ffprobe" in calls[0][1] - - -def test_media_verify_flags_truncated_file_as_suspect(monkeypatch): - monkeypatch.setattr(app, "_find_inventory_host", lambda host: {"user": "sascha", "ip": "10.2.1.100"}) - monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (0, "1836.0\n", "")) - with TestClient(app.app) as client: - response = client.post( - "/media/verify", - headers={"Authorization": "Bearer test-token"}, - json={"host": "arrapps", "path": "/data/UHD/serien/Show/ep.mkv", "expected_minutes": 49.5}, - ) - assert response.status_code == 200 - body = response.json() - assert body["suspect"] is True - assert body["deviation_pct"] > 20 - - -def test_media_verify_rejects_path_outside_data_before_ssh(monkeypatch): - monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not SSH"))) - with TestClient(app.app) as client: - response = client.post( - "/media/verify", - headers={"Authorization": "Bearer test-token"}, - json={"host": "arrapps", "path": "/etc/passwd"}, - ) - assert response.status_code == 400 - - -def test_media_verify_allows_apostrophe_in_filename_and_quotes_it_safely(monkeypatch): - monkeypatch.setattr(app, "_find_inventory_host", lambda host: {"user": "sascha", "ip": "10.2.1.100"}) - calls = [] - - def fake_ssh(host, command, timeout=30): - calls.append((host, command, timeout)) - return 0, "2967.0\n", "" - - monkeypatch.setattr(app, "_ssh", fake_ssh) - path = "/data/FHD/serien/Star Trek - Strange New Worlds (2022)/Season 04/Once La'An a Time.mkv" - with TestClient(app.app) as client: - response = client.post( - "/media/verify", - headers={"Authorization": "Bearer test-token"}, - json={"host": "arrapps", "path": path, "expected_minutes": 49.5}, - ) - assert response.status_code == 200 - # shlex.quote must produce a command the remote shell parses as ONE argument, - # i.e. no unescaped apostrophe breaks out of quoting. - executed_cmd = calls[0][1] - quoted = shlex.quote(path) - assert quoted in executed_cmd - assert shlex.split(executed_cmd)[-1] == path - - -def test_media_verify_rejects_shell_metacharacters_before_ssh(monkeypatch): - monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not SSH"))) - with TestClient(app.app) as client: - response = client.post( - "/media/verify", - headers={"Authorization": "Bearer test-token"}, - json={"host": "arrapps", "path": "/data/UHD/serien/a\x00.mkv"}, - ) - assert response.status_code == 400 - - -def test_media_verify_rejects_unknown_host_before_ssh(monkeypatch): - monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("must not SSH"))) - with TestClient(app.app) as client: - response = client.post( - "/media/verify", - headers={"Authorization": "Bearer test-token"}, - json={"host": "unknown-host", "path": "/data/UHD/serien/ep.mkv"}, - ) - assert response.status_code == 400 - - -def test_media_verify_returns_502_on_unparsable_ffprobe_output(monkeypatch): - monkeypatch.setattr(app, "_find_inventory_host", lambda host: {"user": "sascha", "ip": "10.2.1.100"}) - monkeypatch.setattr(app, "_ssh", lambda *_args, **_kwargs: (0, "N/A\n", "")) - with TestClient(app.app) as client: - response = client.post( - "/media/verify", - headers={"Authorization": "Bearer test-token"}, - json={"host": "arrapps", "path": "/data/UHD/serien/ep.mkv", "expected_minutes": 49.5}, - ) - assert response.status_code == 502 - - def test_invalid_log_target_is_rejected_before_ssh(): with TestClient(app.app) as client: response = client.get(