sascha
619193bddd
Control-plane health semantics, concurrent backup checks and lightweight-model overview.
245 lines
9.2 KiB
Python
245 lines
9.2 KiB
Python
import os
|
|
import asyncio
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
os.environ.setdefault("BUTLER_TOKEN", "test-token")
|
|
|
|
from fastapi.testclient import TestClient
|
|
|
|
import app
|
|
|
|
|
|
def test_redact_response_recursively():
|
|
source = {
|
|
"id": 6,
|
|
"hawserToken": "secret-value",
|
|
"nested": [{"webhookSecret": "also-secret", "name": "tdarr"}],
|
|
}
|
|
result = app._redact_response(source)
|
|
assert result["hawserToken"] == "[REDACTED]"
|
|
assert result["nested"][0]["webhookSecret"] == "[REDACTED]"
|
|
assert result["nested"][0]["name"] == "tdarr"
|
|
|
|
|
|
def test_inventory_defaults_node7_and_explicit_user():
|
|
inventory = """
|
|
[node]
|
|
node2 ansible_host=10.5.85.12
|
|
[apps]
|
|
tdarr ansible_host=10.2.1.104
|
|
emby-chris ansible_host=10.7.1.106
|
|
special ansible_host=10.7.1.200 ansible_user=operator
|
|
"""
|
|
hosts = {item["name"]: item for item in app._inventory_hosts(inventory)}
|
|
assert hosts["node2"]["user"] == "root"
|
|
assert hosts["tdarr"]["user"] == "sascha"
|
|
assert hosts["emby-chris"]["user"] == "chris"
|
|
assert hosts["special"]["user"] == "operator"
|
|
|
|
|
|
def test_health_exposes_current_version():
|
|
with TestClient(app.app) as client:
|
|
response = client.get("/health")
|
|
assert response.status_code == 200
|
|
assert response.json()["version"] == app.VERSION == "2.3.0"
|
|
|
|
|
|
def test_invalid_log_target_is_rejected_before_ssh():
|
|
with TestClient(app.app) as client:
|
|
response = client.get(
|
|
"/logs/tdarr/fileflows;rm",
|
|
headers={"Authorization": "Bearer test-token"},
|
|
)
|
|
assert response.status_code == 400
|
|
|
|
|
|
def test_inventory_rejects_invalid_ip_before_ssh():
|
|
with TestClient(app.app) as client:
|
|
response = client.post(
|
|
"/inventory/host",
|
|
headers={"Authorization": "Bearer test-token"},
|
|
json={"name": "bad-host", "ip": "not-an-ip", "group": "auto"},
|
|
)
|
|
assert response.status_code == 400
|
|
|
|
|
|
def test_inventory_upsert_uses_base64_script(monkeypatch):
|
|
calls = []
|
|
|
|
def fake_ssh(host, command, timeout=600):
|
|
calls.append((host, command, timeout))
|
|
return 0, "updated", ""
|
|
|
|
monkeypatch.setattr(app, "_ssh", fake_ssh)
|
|
with TestClient(app.app) as client:
|
|
response = client.post(
|
|
"/inventory/host",
|
|
headers={"Authorization": "Bearer test-token"},
|
|
json={"name": "pfannkuchen", "ip": "46.225.230.72", "group": "vps", "user": "root"},
|
|
)
|
|
assert response.status_code == 200
|
|
assert "base64.b64decode" in calls[0][1]
|
|
assert "\\nname =" not in calls[0][1]
|
|
|
|
|
|
def test_docker_inspect_returns_sanitized_summary(monkeypatch):
|
|
raw = [{
|
|
"Name": "/fileflows",
|
|
"Config": {"Image": "revenz/fileflows:26.06", "Env": ["API_TOKEN=secret", "TZ=Europe/Berlin"]},
|
|
"State": {"Status": "running", "Running": True, "OOMKilled": False, "ExitCode": 0},
|
|
"RestartCount": 0,
|
|
"HostConfig": {"Runtime": "nvidia", "Memory": 0, "MemoryReservation": 0, "NanoCpus": 0,
|
|
"DeviceRequests": [], "RestartPolicy": {"Name": "always"}, "LogConfig": {"Type": "json-file"}},
|
|
"Mounts": [{"Type": "bind", "Source": "/app-config/fileflows-data", "Destination": "/app/Data", "RW": True}],
|
|
}]
|
|
monkeypatch.setattr(app, "_find_inventory_host", lambda _name: {"user": "sascha", "ip": "10.2.1.104"})
|
|
monkeypatch.setattr(app, "_ssh", lambda *args, **kwargs: (0, __import__("json").dumps(raw), ""))
|
|
with TestClient(app.app) as client:
|
|
response = client.get(
|
|
"/docker/inspect/tdarr/fileflows",
|
|
headers={"Authorization": "Bearer test-token"},
|
|
)
|
|
assert response.status_code == 200
|
|
result = response.json()
|
|
assert result["runtime"] == "nvidia"
|
|
assert result["environment_keys"] == ["API_TOKEN", "TZ"]
|
|
assert "secret" not in response.text
|
|
|
|
|
|
def test_classify_http_status_distinguishes_auth_and_route_errors():
|
|
assert app._classify_http_status(200, {200}) == "healthy"
|
|
assert app._classify_http_status(401, {200}) == "auth_failed"
|
|
assert app._classify_http_status(403, {200}) == "auth_failed"
|
|
assert app._classify_http_status(404, {200}) == "misconfigured"
|
|
assert app._classify_http_status(429, {200}) == "degraded"
|
|
assert app._classify_http_status(503, {200}) == "degraded"
|
|
|
|
|
|
def test_service_auth_headers_support_lightweight_health_probes(monkeypatch):
|
|
monkeypatch.setattr(app, "_get_key", lambda _cfg: "test-secret")
|
|
assert app._service_auth({"auth": "bearer"})["headers"] == {
|
|
"Authorization": "Bearer test-secret"
|
|
}
|
|
assert app._service_auth({"auth": "apikey"})["headers"] == {
|
|
"X-Api-Key": "test-secret"
|
|
}
|
|
assert app._service_auth({"auth": "n8n"})["headers"] == {
|
|
"X-N8N-API-KEY": "test-secret"
|
|
}
|
|
|
|
|
|
def _archive_json(start, name="archive"):
|
|
return __import__("json").dumps([{"archives": [{"start": start, "name": name}]}])
|
|
|
|
|
|
def test_backup_item_reports_age_and_severity():
|
|
now = datetime.now(timezone.utc)
|
|
recent = (now - timedelta(hours=4)).isoformat()
|
|
warning = (now - timedelta(hours=36)).isoformat()
|
|
critical = (now - timedelta(hours=60)).isoformat()
|
|
|
|
assert app._backup_item(0, _archive_json(recent), "")["state"] == "healthy"
|
|
assert app._backup_item(0, _archive_json(warning), "")["state"] == "warning"
|
|
assert app._backup_item(0, _archive_json(critical), "")["state"] == "critical"
|
|
assert app._backup_item(1, "", "timeout")["state"] == "unknown"
|
|
|
|
|
|
def test_ssh_timeout_is_normalized_instead_of_crashing_collection(monkeypatch):
|
|
def timeout(*_args, **_kwargs):
|
|
raise app._sp.TimeoutExpired(cmd=["ssh"], timeout=1)
|
|
|
|
monkeypatch.setattr(app._sp, "run", timeout)
|
|
rc, out, err = app._ssh("sascha@example", "true", timeout=1)
|
|
assert (rc, out) == (124, "")
|
|
assert "timed out" in err.lower()
|
|
|
|
|
|
def test_service_login_failure_is_isolated(monkeypatch):
|
|
monkeypatch.setattr(app, "SERVICES", {
|
|
"dockhand": {"url": "http://dockhand.invalid", "auth": "session", "health_path": "/api/health"}
|
|
})
|
|
|
|
async def failed_login(_client):
|
|
raise RuntimeError("login failed")
|
|
|
|
monkeypatch.setattr(app, "_dockhand_login", failed_login)
|
|
result = asyncio.run(app._collect_service_status())
|
|
assert result["dockhand"]["status"] == "offline"
|
|
assert result["dockhand"]["reachable"] is False
|
|
|
|
|
|
def test_backup_collection_runs_hosts_concurrently(monkeypatch):
|
|
active = 0
|
|
max_active = 0
|
|
|
|
monkeypatch.setattr(app, "_get_inventory_hosts", lambda: [
|
|
{"name": f"vm-{index}", "user": "sascha", "ip": f"10.1.1.{index}"}
|
|
for index in range(1, 5)
|
|
])
|
|
|
|
def fake_ssh(*_args, **_kwargs):
|
|
nonlocal active, max_active
|
|
active += 1
|
|
max_active = max(max_active, active)
|
|
time.sleep(0.05)
|
|
active -= 1
|
|
return 0, _archive_json(datetime.now(timezone.utc).isoformat()), ""
|
|
|
|
monkeypatch.setattr(app, "_ssh", fake_ssh)
|
|
result = asyncio.run(app._collect_backup_status(concurrency=4))
|
|
|
|
assert max_active > 1
|
|
assert result["summary"] == {
|
|
"total": 4, "healthy": 4, "warning": 0, "critical": 0, "unknown": 0
|
|
}
|
|
|
|
|
|
def test_overview_openapi_has_stable_enums_and_schema():
|
|
schema = app.app.openapi()
|
|
response_schema = schema["paths"]["/overview"]["get"]["responses"]["200"]["content"]["application/json"]["schema"]
|
|
assert response_schema["$ref"].endswith("/OverviewResponse")
|
|
overview_schema = schema["components"]["schemas"]["OverviewResponse"]
|
|
assert overview_schema["properties"]["overall_state"]["enum"] == ["healthy", "warning", "critical"]
|
|
finding_schema = schema["components"]["schemas"]["OverviewFinding"]
|
|
assert finding_schema["properties"]["severity"]["enum"] == ["healthy", "warning", "critical"]
|
|
|
|
|
|
def test_overview_is_compact_deterministic_and_light_model_friendly(monkeypatch):
|
|
async def service_data():
|
|
return {
|
|
"ok": {"status": "healthy", "reachable": True, "http": 200},
|
|
"bad-auth": {"status": "auth_failed", "reachable": True, "http": 401},
|
|
}
|
|
|
|
async def host_data():
|
|
return {
|
|
"vm1": {"reachable": True, "containers": ["app: Up 1 hour (healthy)"]},
|
|
"vm2": {"reachable": False, "containers": [], "error": "timeout"},
|
|
}
|
|
|
|
async def backup_data():
|
|
return {
|
|
"summary": {"total": 1, "healthy": 0, "warning": 1, "critical": 0, "unknown": 0},
|
|
"hosts": {"vm1": {"state": "warning", "age_hours": 36}},
|
|
}
|
|
|
|
async def disk_data():
|
|
return {"vm1": {"pct": "85%"}}
|
|
|
|
monkeypatch.setattr(app, "_collect_service_status", service_data)
|
|
monkeypatch.setattr(app, "_collect_health_all", host_data)
|
|
monkeypatch.setattr(app, "_collect_backup_status", backup_data)
|
|
monkeypatch.setattr(app, "_collect_disk_usage", disk_data)
|
|
|
|
with TestClient(app.app) as client:
|
|
response = client.get("/overview", headers={"Authorization": "Bearer test-token"})
|
|
|
|
assert response.status_code == 200
|
|
result = response.json()
|
|
assert result["schema_version"] == 1
|
|
assert result["overall_state"] == "critical"
|
|
assert result["summary"] == {"critical": 2, "warning": 2, "healthy": 2}
|
|
assert [item["code"] for item in result["findings"]] == [
|
|
"host_unreachable", "service_auth_failed", "backup_warning", "disk_high"
|
|
]
|