From 5c35a1ed3625d12b222434ad4868e4c08c1dad11 Mon Sep 17 00:00:00 2001 From: feldjaeger Date: Mon, 13 Apr 2026 09:27:14 +0200 Subject: [PATCH] split: monitoring in 3 Stacks aufgeteilt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - monitoring: Prometheus, Exporters, InfluxDB (owns monitoring_network) - teslamate/: TeslaMate + Grafana + Postgres + Mosquitto - backup-monitor/: Backup-Monitor + MongoDB - Jeder Stack unabhängig steuerbar, kein gegenseitiges Risiko --- .env.enc | 15 ++ backup-monitor/Dockerfile | 7 + backup-monitor/app.py | 396 ++++++++++++++++++++++++++++ backup-monitor/compose.yaml | 27 ++ backup-monitor/requirements.txt | 4 + backup-monitor/static/css/style.css | 1 + backup-monitor/static/js/app.js | 360 +++++++++++++++++++++++++ backup-monitor/templates/index.html | 295 +++++++++++++++++++++ compose.yaml | 94 ------- teslamate/compose.yaml | 87 ++++++ 10 files changed, 1192 insertions(+), 94 deletions(-) create mode 100644 .env.enc create mode 100644 backup-monitor/Dockerfile create mode 100644 backup-monitor/app.py create mode 100644 backup-monitor/compose.yaml create mode 100644 backup-monitor/requirements.txt create mode 100644 backup-monitor/static/css/style.css create mode 100644 backup-monitor/static/js/app.js create mode 100644 backup-monitor/templates/index.html create mode 100644 teslamate/compose.yaml diff --git a/.env.enc b/.env.enc new file mode 100644 index 0000000..8b39254 --- /dev/null +++ b/.env.enc @@ -0,0 +1,15 @@ +TM_DB_USER=ENC[AES256_GCM,data:oDO0wGr3X8We,iv:0QHLprRYHGprqYssXr6XAxeK/+Pks4Tx7UIgddU6q/g=,tag:OFDAtVpbUdyu2MPSBAiGqw==,type:str] +TM_DB_PASS=ENC[AES256_GCM,data:8QJIAhpYVg==,iv:b51ZXnk+UhtjbczyQeFVeftwYMDwH3H0R4k6JA4HMrU=,tag:fXsa1CD85UsEh5qvVtNqWA==,type:str] +TM_DB_NAME=ENC[AES256_GCM,data:i12zyQchoiAa,iv:6t1sWpzh3JXTMkcn1iXXn6ZB65gidl1qREvCN+I0ddw=,tag:0u+TOfVmyaqQ6BZHmCMwgg==,type:str] +GRAFANA_USER=ENC[AES256_GCM,data:Y59OifM=,iv:k6IBKx7NbP/EsURrRDsg3ktDj1eVaPBTUfYVvV4wWQA=,tag:bUds1/urN8Ld6B/T6tulGw==,type:str] +GRAFANA_PW=ENC[AES256_GCM,data:yy3/Y9V4,iv:4eYOuqNZsPAp25Z+N5JU4338NkDlBVyVNsx1WJIgAEc=,tag:PxgcOslxsAtX4a7H2kK/Jw==,type:str] +FQDN_TM=ENC[AES256_GCM,data:tH0Zb7lPF+35hjKFzT7m4hjJUoY=,iv:eop9OU3wvc3yHP6OxlKgXfUMUPD2z+8oSDr4ZQ47BJw=,tag:1gGP1355cOQFqz3pdZV/pQ==,type:str] +TM_TZ=ENC[AES256_GCM,data:Q7Ved1xMYTeYCCdFkA==,iv:ab8lIJVCqDVyltYCEXyqyZ+2iOzMgwvYUie1Vq72tr8=,tag:vQw2Oh+c/5C4LydLGIhW1A==,type:str] +LETSENCRYPT_EMAIL=ENC[AES256_GCM,data:SqjA551w6MCKmGwQ2NNm4F3QyAMr,iv:/TlDZlauRW1coRfA2osoCGiSnEWgVqxjlgKtDmklDUw=,tag:t+s9RO3Tdjm+ta+80j02fA==,type:str] +ENCRYPTION_KEY=ENC[AES256_GCM,data:XnnT0HJxjfFtwCulZAKQ4Ocz0wPUipmoXo/JniDl5Cl6jSEPfqsbOwCQcnnSG0yiJg==,iv:HRSx2B1jUi2pd50lvSiawQz8XiJpPBg3pIIVwwbjcV0=,tag:42ZOXqffvdmyX0Cz8bL0MQ==,type:str] +sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVMVAweGNuWlRBK2U2aUs1\nRVo5VENnc2ZUK2pIbG5kSmtndXFVbWJPQkJRCldZaksyOS9JS0lkR0lyUEFqTGtB\nMmdjVDg0YWo5VkhTaHVwdnVXRTIyRkkKLS0tIGtHNFkxbDZ3WGI5SEJFNHkrVkdH\nSkFqM3BnWWJaMmQrMTI4aDl0aktNV0UKgtBtdUerHifZmAvZy8+4v3YJF/WHHysh\nWSqhxFQmGLt9/GvWr6wVpzQ9RzxVb/a4klzo/MhbmJvrLWOP7MESOA==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_0__map_recipient=age1z8gak2l4h0vpcnhtcdxmem2u9h2n54vuksk8ys82609qtzampuvqh50wdr +sops_lastmodified=2026-04-04T07:42:39Z +sops_mac=ENC[AES256_GCM,data:UUHJL94dfwfPEHrQxWB1fkzbejgikrR0mlDPkcvJrrpLtHTjKqso+wCCs61dy2ChbE/Kp2Vfy0Xl/wZOGCP6sZr5SEAyXvKfUspw8sEsp2UscPgQ9oDaLvk7mKACYHzSzUmDzy7Ic35kyBE+PcKckrt14X6Ki/dp6fK6QxiGUqQ=,iv:BEcLF6VwqO7T8uP59iuN6OKPA1L07zMC07AiVa9KLzs=,tag:miPyabX8wfMXMboz0KuChg==,type:str] +sops_unencrypted_suffix=_unencrypted +sops_version=3.12.2 diff --git a/backup-monitor/Dockerfile b/backup-monitor/Dockerfile new file mode 100644 index 0000000..217c6c7 --- /dev/null +++ b/backup-monitor/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.12-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . +EXPOSE 9999 +CMD ["gunicorn", "-b", "0.0.0.0:9999", "-w", "2", "--timeout", "30", "app:app"] diff --git a/backup-monitor/app.py b/backup-monitor/app.py new file mode 100644 index 0000000..71e18f4 --- /dev/null +++ b/backup-monitor/app.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python3 +""" +Backup Monitor – Backend +MongoDB-backed backup monitoring with Web UI, Uptime Kuma, Prometheus & Webhook integration. +""" +from flask import Flask, request, jsonify, render_template, Response +from pymongo import MongoClient, DESCENDING +from datetime import datetime, timedelta +from functools import wraps +import os, time, requests, logging, threading, secrets as _secrets + +app = Flask(__name__) +logging.basicConfig(level=logging.INFO) +log = logging.getLogger("backup-monitor") + +MONGO_URI = os.environ.get("MONGO_URI", "mongodb://mongo:27017") +KUMA_URL = os.environ.get("KUMA_URL", "") +KUMA_TOKEN = os.environ.get("KUMA_TOKEN", "") +STALE_HOURS = int(os.environ.get("STALE_HOURS", "26")) + +# API Key Auth – set API_KEY to enable, leave empty to disable (open access) +API_KEY = os.environ.get("API_KEY", "") + +db = MongoClient(MONGO_URI).backup_monitor + + +# ── Auth Decorator ───────────────────────────────────────────────────────── + +def require_api_key(f): + """Protect write endpoints. Checks X-API-Key header or ?api_key= query param.""" + @wraps(f) + def decorated(*args, **kwargs): + if not API_KEY: + return f(*args, **kwargs) + key = request.headers.get("X-API-Key") or request.args.get("api_key") + if not key or key != API_KEY: + return jsonify({"error": "Unauthorized – invalid or missing API key"}), 401 + return f(*args, **kwargs) + return decorated +db.hosts.create_index("name", unique=True) +db.history.create_index([("host", 1), ("timestamp", -1)]) +db.history.create_index("timestamp", expireAfterSeconds=90 * 86400) # 90 Tage TTL + + +# ── API: Push (called by borgmatic after_backup hook) ────────────────────── + +@app.route("/api/push", methods=["POST", "GET"]) +@require_api_key +def push(): + host = request.args.get("host") or request.json.get("host", "") if request.is_json else request.args.get("host") + if not host: + return jsonify({"error": "host required"}), 400 + + data = request.json if request.is_json else {} + now = datetime.utcnow() + + entry = { + "host": host, + "timestamp": now, + "status": data.get("status", request.args.get("status", "ok")), + "duration_sec": data.get("duration_sec", 0), + "original_size": data.get("original_size", 0), + "deduplicated_size": data.get("deduplicated_size", 0), + "compressed_size": data.get("compressed_size", 0), + "nfiles_new": data.get("nfiles_new", 0), + "nfiles_changed": data.get("nfiles_changed", 0), + "message": data.get("message", request.args.get("msg", "")), + } + db.history.insert_one(entry) + + # Update host record + db.hosts.update_one( + {"name": host}, + {"$set": {"last_backup": now, "last_status": entry["status"], "last_message": entry["message"]}, + "$setOnInsert": {"name": host, "enabled": True, "created": now, "kuma_push_url": ""}}, + upsert=True + ) + + # Uptime Kuma push + h = db.hosts.find_one({"name": host}) + if h and h.get("kuma_push_url"): + try: + status_param = "up" if entry["status"] == "ok" else "down" + msg = f"Backup OK – {_fmt_bytes(entry['original_size'])}" if entry["status"] == "ok" else entry["message"] + requests.get(h["kuma_push_url"], params={"status": status_param, "msg": msg}, timeout=5) + except Exception as e: + log.warning(f"Kuma push failed for {host}: {e}") + + # Webhooks + if entry["status"] == "error": + _send_webhooks("error", host, entry.get("message", "Backup fehlgeschlagen")) + + # Check for stale hosts + _check_stale_hosts() + + return jsonify({"ok": True, "host": host}) + + +# ── API: Hosts CRUD ──────────────────────────────────────────────────────── + +@app.route("/api/hosts", methods=["GET"]) +def list_hosts(): + hosts = [] + now = datetime.utcnow() + for h in db.hosts.find().sort("name", 1): + age_h = (now - h.get("last_backup", now)).total_seconds() / 3600 if h.get("last_backup") else 999 + if not h.get("enabled", True): + status = "disabled" + elif age_h > STALE_HOURS: + status = "stale" + elif h.get("last_status") == "error": + status = "error" + else: + status = "ok" + + # Last 7 days summary + week_ago = now - timedelta(days=7) + recent = list(db.history.find({"host": h["name"], "timestamp": {"$gte": week_ago}}).sort("timestamp", -1)) + + hosts.append({ + "name": h["name"], + "enabled": h.get("enabled", True), + "status": status, + "last_backup": h.get("last_backup", "").isoformat() + "Z" if h.get("last_backup") else None, + "last_status": h.get("last_status", "unknown"), + "last_message": h.get("last_message", ""), + "age_hours": round(age_h, 1), + "kuma_push_url": h.get("kuma_push_url", ""), + "backup_count_7d": len(recent), + "total_size_7d": sum(r.get("original_size", 0) for r in recent), + "avg_duration_7d": round(sum(r.get("duration_sec", 0) for r in recent) / max(len(recent), 1)), + }) + return jsonify(hosts) + + +@app.route("/api/hosts", methods=["POST"]) +@require_api_key +def add_host(): + data = request.json + name = data.get("name", "").strip() + if not name: + return jsonify({"error": "name required"}), 400 + db.hosts.update_one( + {"name": name}, + {"$setOnInsert": {"name": name, "enabled": True, "created": datetime.utcnow(), + "kuma_push_url": data.get("kuma_push_url", "")}}, + upsert=True + ) + return jsonify({"ok": True, "name": name}) + + +@app.route("/api/hosts/", methods=["PUT"]) +@require_api_key +def update_host(name): + data = request.json + update = {} + if "enabled" in data: + update["enabled"] = data["enabled"] + if "kuma_push_url" in data: + update["kuma_push_url"] = data["kuma_push_url"] + if update: + db.hosts.update_one({"name": name}, {"$set": update}) + return jsonify({"ok": True}) + + +@app.route("/api/hosts/", methods=["DELETE"]) +@require_api_key +def delete_host(name): + db.hosts.delete_one({"name": name}) + db.history.delete_many({"host": name}) + return jsonify({"ok": True}) + + +# ── API: History ─────────────────────────────────────────────────────────── + +@app.route("/api/history/") +def host_history(host): + days = int(request.args.get("days", 30)) + since = datetime.utcnow() - timedelta(days=days) + entries = [] + for e in db.history.find({"host": host, "timestamp": {"$gte": since}}).sort("timestamp", DESCENDING): + entries.append({ + "timestamp": e["timestamp"].isoformat() + "Z", + "status": e.get("status", "ok"), + "duration_sec": e.get("duration_sec", 0), + "original_size": e.get("original_size", 0), + "deduplicated_size": e.get("deduplicated_size", 0), + "compressed_size": e.get("compressed_size", 0), + "nfiles_new": e.get("nfiles_new", 0), + "nfiles_changed": e.get("nfiles_changed", 0), + "message": e.get("message", ""), + }) + return jsonify(entries) + + +@app.route("/api/calendar/") +def host_calendar(host): + """30-day calendar heatmap data.""" + days = int(request.args.get("days", 30)) + since = datetime.utcnow() - timedelta(days=days) + pipeline = [ + {"$match": {"host": host, "timestamp": {"$gte": since}}}, + {"$group": { + "_id": {"$dateToString": {"format": "%Y-%m-%d", "date": "$timestamp"}}, + "count": {"$sum": 1}, + "total_size": {"$sum": "$original_size"}, + "has_error": {"$max": {"$cond": [{"$eq": ["$status", "error"]}, 1, 0]}}, + "avg_duration": {"$avg": "$duration_sec"}, + }}, + {"$sort": {"_id": 1}} + ] + result = {} + for day in db.history.aggregate(pipeline): + result[day["_id"]] = { + "count": day["count"], + "total_size": day["total_size"], + "has_error": bool(day["has_error"]), + "avg_duration": round(day.get("avg_duration", 0)), + } + return jsonify(result) + + +# ── API: Dashboard summary ───────────────────────────────────────────────── + +@app.route("/api/summary") +def summary(): + now = datetime.utcnow() + hosts = list(db.hosts.find({"enabled": True})) + total = len(hosts) + ok = stale = error = 0 + for h in hosts: + age_h = (now - h.get("last_backup", now)).total_seconds() / 3600 if h.get("last_backup") else 999 + if age_h > STALE_HOURS: + stale += 1 + elif h.get("last_status") == "error": + error += 1 + else: + ok += 1 + + today = now.replace(hour=0, minute=0, second=0, microsecond=0) + today_backups = db.history.count_documents({"timestamp": {"$gte": today}}) + today_size = sum(e.get("original_size", 0) for e in db.history.find({"timestamp": {"$gte": today}})) + + return jsonify({ + "total_hosts": total, "ok": ok, "stale": stale, "error": error, + "today_backups": today_backups, "today_size": today_size, + }) + + +# ── Homepage Widget API ──────────────────────────────────────────────────── + +@app.route("/api/homepage") +def homepage_widget(): + """Returns data in Homepage custom API widget format.""" + now = datetime.utcnow() + hosts = list(db.hosts.find({"enabled": True})) + total = len(hosts) + ok = stale = error = 0 + for h in hosts: + age_h = (now - h["last_backup"]).total_seconds() / 3600 if h.get("last_backup") else 999 + if age_h > STALE_HOURS: stale += 1 + elif h.get("last_status") == "error": error += 1 + else: ok += 1 + + today = now.replace(hour=0, minute=0, second=0, microsecond=0) + today_size = sum(e.get("original_size", 0) for e in db.history.find({"timestamp": {"$gte": today}})) + + return jsonify({ + "hosts_ok": f"{ok}/{total}", + "errors": error, + "stale": stale, + "today": _fmt_bytes(today_size), + }) + + +# ── Prometheus Metrics ────────────────────────────────────────────────────── + +@app.route("/metrics") +def prometheus_metrics(): + now = datetime.utcnow() + hosts = list(db.hosts.find()) + lines = [ + "# HELP backup_hosts_total Total number of monitored hosts", + "# TYPE backup_hosts_total gauge", + f"backup_hosts_total {len([h for h in hosts if h.get('enabled', True)])}", + "# HELP backup_host_last_seconds Seconds since last backup", + "# TYPE backup_host_last_seconds gauge", + "# HELP backup_host_status Backup status (1=ok, 0=error, -1=stale, -2=disabled)", + "# TYPE backup_host_status gauge", + "# HELP backup_host_duration_seconds Duration of last backup", + "# TYPE backup_host_duration_seconds gauge", + "# HELP backup_host_size_bytes Original size of last backup", + "# TYPE backup_host_size_bytes gauge", + "# HELP backup_host_dedup_bytes Deduplicated size of last backup", + "# TYPE backup_host_dedup_bytes gauge", + "# HELP backup_host_files_new New files in last backup", + "# TYPE backup_host_files_new gauge", + ] + for h in hosts: + name = h["name"] + labels = f'host="{name}"' + age = (now - h["last_backup"]).total_seconds() if h.get("last_backup") else 999999 + + if not h.get("enabled", True): + status_val = -2 + elif age > STALE_HOURS * 3600: + status_val = -1 + elif h.get("last_status") == "error": + status_val = 0 + else: + status_val = 1 + + lines.append(f"backup_host_last_seconds{{{labels}}} {int(age)}") + lines.append(f"backup_host_status{{{labels}}} {status_val}") + + last = db.history.find_one({"host": name}, sort=[("timestamp", DESCENDING)]) + if last: + lines.append(f'backup_host_duration_seconds{{{labels}}} {last.get("duration_sec", 0)}') + lines.append(f'backup_host_size_bytes{{{labels}}} {last.get("original_size", 0)}') + lines.append(f'backup_host_dedup_bytes{{{labels}}} {last.get("deduplicated_size", 0)}') + lines.append(f'backup_host_files_new{{{labels}}} {last.get("nfiles_new", 0)}') + + today = now.replace(hour=0, minute=0, second=0, microsecond=0) + today_count = db.history.count_documents({"timestamp": {"$gte": today}}) + today_size = sum(e.get("original_size", 0) for e in db.history.find({"timestamp": {"$gte": today}})) + lines += [ + "# HELP backup_today_total Backups completed today", + "# TYPE backup_today_total gauge", + f"backup_today_total {today_count}", + "# HELP backup_today_bytes Total bytes backed up today", + "# TYPE backup_today_bytes gauge", + f"backup_today_bytes {today_size}", + ] + return Response("\n".join(lines) + "\n", mimetype="text/plain; version=0.0.4") + + +# ── Webhooks (Notifications) ────────────────────────────────────────────── + +WEBHOOK_URLS = [u.strip() for u in os.environ.get("WEBHOOK_URLS", "").split(",") if u.strip()] +WEBHOOK_EVENTS = os.environ.get("WEBHOOK_EVENTS", "error,stale").split(",") + + +def _send_webhooks(event, host, message): + """Fire webhooks in background thread.""" + if event not in WEBHOOK_EVENTS or not WEBHOOK_URLS: + return + payload = { + "event": event, + "host": host, + "message": message, + "timestamp": datetime.utcnow().isoformat() + "Z", + } + def _fire(): + for url in WEBHOOK_URLS: + try: + requests.post(url, json=payload, timeout=10) + except Exception as e: + log.warning(f"Webhook failed ({url}): {e}") + threading.Thread(target=_fire, daemon=True).start() + + +# ── Stale Check (runs after each push) ──────────────────────────────────── + +def _check_stale_hosts(): + """Check all hosts for stale status and fire webhooks.""" + now = datetime.utcnow() + for h in db.hosts.find({"enabled": True}): + if not h.get("last_backup"): + continue + age_h = (now - h["last_backup"]).total_seconds() / 3600 + if age_h > STALE_HOURS and not h.get("_stale_notified"): + _send_webhooks("stale", h["name"], f"Kein Backup seit {int(age_h)}h") + db.hosts.update_one({"name": h["name"]}, {"$set": {"_stale_notified": True}}) + elif age_h <= STALE_HOURS and h.get("_stale_notified"): + db.hosts.update_one({"name": h["name"]}, {"$unset": {"_stale_notified": ""}}) + + +# ── Web UI ───────────────────────────────────────────────────────────────── + +@app.route("/") +def index(): + return render_template("index.html", api_key_required=bool(API_KEY)) + + +# ── Helpers ──────────────────────────────────────────────────────────────── + +def _fmt_bytes(b): + for unit in ["B", "KB", "MB", "GB", "TB"]: + if b < 1024: + return f"{b:.1f} {unit}" + b /= 1024 + return f"{b:.1f} PB" + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=9999, debug=False) diff --git a/backup-monitor/compose.yaml b/backup-monitor/compose.yaml new file mode 100644 index 0000000..3d170fa --- /dev/null +++ b/backup-monitor/compose.yaml @@ -0,0 +1,27 @@ +networks: + monitoring_network: + external: true + +services: + backup-monitor: + build: ../backup-monitor + container_name: backup-monitor + restart: always + ports: + - "9999:9999" + environment: + - MONGO_URI=mongodb://backup-mongo:27017 + - STALE_HOURS=26 + depends_on: + - backup-mongo + networks: + - monitoring_network + + backup-mongo: + image: mongo:4.4 + container_name: backup-mongo + restart: always + volumes: + - /app-config/backup_mongo_data:/data/db + networks: + - monitoring_network diff --git a/backup-monitor/requirements.txt b/backup-monitor/requirements.txt new file mode 100644 index 0000000..ae30eb6 --- /dev/null +++ b/backup-monitor/requirements.txt @@ -0,0 +1,4 @@ +Flask==3.1.* +pymongo==4.12.* +requests==2.32.* +gunicorn==23.* diff --git a/backup-monitor/static/css/style.css b/backup-monitor/static/css/style.css new file mode 100644 index 0000000..9452692 --- /dev/null +++ b/backup-monitor/static/css/style.css @@ -0,0 +1 @@ +/* Sentinel Design System – all styles via Tailwind CSS */ diff --git a/backup-monitor/static/js/app.js b/backup-monitor/static/js/app.js new file mode 100644 index 0000000..73ccfd8 --- /dev/null +++ b/backup-monitor/static/js/app.js @@ -0,0 +1,360 @@ +/* ── The Sentinel – Backup Monitor Frontend ────────────────── */ + +const API = ''; +let apiKey = localStorage.getItem('bm_api_key') || ''; +let allHosts = []; +let currentPage = 'dashboard'; + +// ── Init ────────────────────────────────────────────────── +document.addEventListener('DOMContentLoaded', () => { + loadAll(); + setInterval(loadAll, 30000); +}); + +function authHeaders() { + const h = {'Content-Type': 'application/json'}; + if (apiKey) h['X-API-Key'] = apiKey; + return h; +} + +async function apiFetch(url, opts = {}) { + if (!opts.headers) opts.headers = {}; + if (apiKey) opts.headers['X-API-Key'] = apiKey; + const r = await fetch(url, opts); + if (r.status === 401) { + const key = prompt('🔑 API-Key eingeben:'); + if (key) { apiKey = key; localStorage.setItem('bm_api_key', key); opts.headers['X-API-Key'] = key; return fetch(url, opts); } + } + return r; +} + +async function loadAll() { + const [sumR, hostsR] = await Promise.all([fetch(`${API}/api/summary`), fetch(`${API}/api/hosts`)]); + const sum = await sumR.json(); + allHosts = await hostsR.json(); + renderDashboard(sum); + renderAlerts(); + renderHostGrid(); + document.getElementById('lastScan').textContent = new Date().toLocaleTimeString('de-DE', {hour:'2-digit',minute:'2-digit'}); + // System status indicator + const ss = document.getElementById('sysStatus'); + if (sum.error > 0) { ss.innerHTML = 'errorErrors Active'; } + else if (sum.stale > 0) { ss.innerHTML = 'warningStale Hosts'; } + else { ss.innerHTML = 'cloud_doneAll Systems OK'; } +} + +// ── Dashboard ───────────────────────────────────────────── +function renderDashboard(sum) { + document.getElementById('mOk').textContent = `${sum.ok}/${sum.total_hosts}`; + document.getElementById('mSize').textContent = fmtBytes(sum.today_size); + document.getElementById('mWarn').textContent = sum.error + sum.stale; + const wc = document.getElementById('mWarnCard'); + wc.className = 'bg-surface-container-low p-6 rounded-xl' + ((sum.error + sum.stale > 0) ? ' border border-error/20' : ''); + + // Latest backup + const sorted = [...allHosts].filter(h => h.last_backup).sort((a,b) => new Date(b.last_backup) - new Date(a.last_backup)); + if (sorted.length) { + document.getElementById('mLatest').textContent = sorted[0].last_status === 'ok' ? 'Success' : 'Error'; + document.getElementById('mLatestHost').textContent = sorted[0].name; + } + + // Cluster list + const cl = document.getElementById('clusterList'); + const groups = { ok: [], stale: [], error: [], disabled: [] }; + allHosts.forEach(h => groups[h.status]?.push(h)); + let html = ''; + if (groups.error.length) { html += clusterGroup('ERRORS', groups.error, 'error'); } + if (groups.stale.length) { html += clusterGroup('STALE', groups.stale, 'tertiary'); } + if (groups.ok.length) { html += clusterGroup('OPERATIONAL', groups.ok, 'secondary'); } + if (groups.disabled.length) { html += clusterGroup('DISABLED', groups.disabled, 'outline'); } + cl.innerHTML = html; + + // Live stream + renderLiveStream(); + loadVolumeChart(); +} + +function clusterGroup(label, hosts, color) { + return ` +
+
${label}
+ ${hosts.map(h => ` +
+
+
${h.name}
+
schedule ${h.last_backup ? timeAgo(h.last_backup) : 'Never'}
+
+ ${h.status.toUpperCase()} +
+ `).join('')} +
`; +} + +function renderLiveStream() { + const sorted = [...allHosts].filter(h => h.last_backup).sort((a,b) => new Date(b.last_backup) - new Date(a.last_backup)).slice(0, 8); + const ls = document.getElementById('liveStream'); + ls.innerHTML = sorted.map(h => { + const t = new Date(h.last_backup).toLocaleTimeString('de-DE', {hour:'2-digit',minute:'2-digit',second:'2-digit'}); + const isErr = h.last_status !== 'ok'; + return ` +
+ ${t} +
+ ${isErr ? 'ERROR: ' : ''}Backup for ${h.name} ${isErr ? 'failed' : 'completed successfully'}. + ${h.last_message ? `${h.last_message}` : ''} +
`; + }).join(''); +} + +async function loadVolumeChart() { + // Aggregate daily totals from all hosts + const days = []; + for (let i = 29; i >= 0; i--) { const d = new Date(); d.setDate(d.getDate() - i); days.push(d.toISOString().split('T')[0]); } + const dailyTotals = {}; + days.forEach(d => dailyTotals[d] = 0); + + // Fetch calendar for top hosts (limit to avoid too many requests) + const topHosts = allHosts.slice(0, 10); + const cals = await Promise.all(topHosts.map(h => fetch(`${API}/api/calendar/${h.name}?days=30`).then(r => r.json()))); + cals.forEach(cal => { Object.entries(cal).forEach(([day, data]) => { if (dailyTotals[day] !== undefined) dailyTotals[day] += data.total_size; }); }); + + const values = days.map(d => dailyTotals[d]); + const max = Math.max(...values, 1); + const points = values.map((v, i) => `${(i / (values.length - 1)) * 100},${100 - (v / max) * 80}`); + const pathD = 'M' + points.join(' L'); + const fillD = pathD + ` L100,100 L0,100 Z`; + + document.getElementById('chartSvg').innerHTML = ` + + ${[20,40,60,80].map(y => ``).join('')} + + + `; + document.getElementById('chartSvg').setAttribute('viewBox', '0 0 100 100'); + document.getElementById('chartSvg').setAttribute('preserveAspectRatio', 'none'); +} + +// ── Alerts ──────────────────────────────────────────────── +function renderAlerts() { + const issues = allHosts.filter(h => h.status === 'error' || h.status === 'stale'); + document.getElementById('aCrit').textContent = String(allHosts.filter(h => h.status === 'error').length).padStart(2, '0'); + document.getElementById('aStale').textContent = String(allHosts.filter(h => h.status === 'stale').length).padStart(2, '0'); + + const al = document.getElementById('alertList'); + if (!issues.length) { al.innerHTML = '
No active alerts – all systems operational ✓
'; return; } + + al.innerHTML = issues.map(h => { + const isCrit = h.status === 'error'; + const color = isCrit ? 'error' : 'tertiary'; + const icon = isCrit ? 'error' : 'warning'; + const label = isCrit ? 'CRITICAL' : 'STALE'; + return ` +
+
+
+ ${icon} +
+
+
+

${isCrit ? 'Backup Failed' : 'Backup Overdue'} – ${h.name}

+ ${label} +
+
+ dns ${h.name} + schedule ${h.last_backup ? timeAgo(h.last_backup) : 'Never'} + ${h.last_message ? `${h.last_message}` : `${Math.round(h.age_hours)}h without backup`} +
+
+
+ +
+
+
`; + }).join(''); +} + +// ── Host Grid ───────────────────────────────────────────── +function renderHostGrid() { + const grid = document.getElementById('hostGrid'); + grid.innerHTML = allHosts.map(h => ` +
+
+
+
${h.name}
+ ${h.status.toUpperCase()} +
+
+
Last Backup${h.last_backup ? timeAgo(h.last_backup) : 'Never'}
+
7d Backups${h.backup_count_7d}
+
Avg Duration${fmtDuration(h.avg_duration_7d)}
+
7d Volume${fmtBytes(h.total_size_7d)}
+
+
+ `).join(''); +} + +// ── Host Detail Drawer ──────────────────────────────────── +async function openHost(name) { + document.getElementById('drawerTitle').textContent = name; + document.getElementById('drawerBg').classList.remove('opacity-0','pointer-events-none'); + document.getElementById('drawer').classList.remove('translate-x-full'); + const body = document.getElementById('drawerBody'); + body.innerHTML = '
Loading...
'; + + const [histR, calR] = await Promise.all([fetch(`${API}/api/history/${name}?days=30`), fetch(`${API}/api/calendar/${name}?days=30`)]); + const history = await histR.json(); + const calendar = await calR.json(); + const host = allHosts.find(h => h.name === name) || {}; + + const totalSize = history.reduce((s,e) => s + e.original_size, 0); + const avgDur = history.length ? Math.round(history.reduce((s,e) => s + e.duration_sec, 0) / history.length) : 0; + const rate = history.length ? Math.round(history.filter(e => e.status === 'ok').length / history.length * 100) : 0; + + body.innerHTML = ` + +
+
${history.length}
Backups
+
${rate}%
Success
+
${fmtDuration(avgDur)}
Avg Duration
+
+ + +

30-Day Calendar

+
${buildCalendar(calendar)}
+ + +

Data Volume

+
${buildSizeChart(history)}
+ + +

Recent Backups

+
+ ${history.slice(0, 15).map(e => ` +
+ ${new Date(e.timestamp).toLocaleString('de-DE',{day:'2-digit',month:'2-digit',hour:'2-digit',minute:'2-digit'})} + + ${fmtDuration(e.duration_sec)} + ${fmtBytes(e.original_size)} + ${e.nfiles_new ? `+${e.nfiles_new}` : ''} +
+ `).join('')} +
+ + +
+ + +
+ `; +} + +function buildCalendar(cal) { + const days = []; + for (let i = 29; i >= 0; i--) { + const d = new Date(); d.setDate(d.getDate() - i); + const key = d.toISOString().split('T')[0]; + const data = cal[key]; + const num = d.getDate(); + if (!data) { days.push(`
${num}
`); } + else { + const cls = data.has_error ? 'bg-error/20 text-error' : 'bg-secondary/20 text-secondary'; + days.push(`
${num}
`); + } + } + return days.join(''); +} + +function buildSizeChart(history) { + const byDay = {}; + history.forEach(e => { const d = e.timestamp.split('T')[0]; byDay[d] = (byDay[d]||0) + e.original_size; }); + const days = []; + for (let i = 29; i >= 0; i--) { const d = new Date(); d.setDate(d.getDate()-i); days.push({key:d.toISOString().split('T')[0], size: byDay[d.toISOString().split('T')[0]]||0}); } + const max = Math.max(...days.map(d=>d.size), 1); + return days.map(d => { + const h = d.size ? Math.max(6, (d.size/max)*100) : 4; + return `
`; + }).join(''); +} + +function closeDrawer() { + document.getElementById('drawerBg').classList.add('opacity-0','pointer-events-none'); + document.getElementById('drawer').classList.add('translate-x-full'); +} + +// ── Modal ───────────────────────────────────────────────── +function openAddHost() { + document.getElementById('modalTitle').textContent = 'Add Host'; + document.getElementById('formMode').value = 'add'; + document.getElementById('formName').value = ''; document.getElementById('formName').disabled = false; + document.getElementById('formKumaUrl').value = ''; + document.getElementById('formEnabled').checked = true; + openModal(); +} + +async function openEditHost(name) { + closeDrawer(); + const h = allHosts.find(x => x.name === name); if (!h) return; + document.getElementById('modalTitle').textContent = `Edit: ${name}`; + document.getElementById('formMode').value = 'edit'; + document.getElementById('formName').value = h.name; document.getElementById('formName').disabled = true; + document.getElementById('formKumaUrl').value = h.kuma_push_url || ''; + document.getElementById('formEnabled').checked = h.enabled; + openModal(); +} + +async function saveHost(e) { + e.preventDefault(); + const mode = document.getElementById('formMode').value; + const name = document.getElementById('formName').value.trim(); + const kuma = document.getElementById('formKumaUrl').value.trim(); + const enabled = document.getElementById('formEnabled').checked; + if (mode === 'add') { + await apiFetch(`${API}/api/hosts`, { method:'POST', headers:authHeaders(), body:JSON.stringify({name, kuma_push_url:kuma}) }); + toast(`${name} added`); + } else { + await apiFetch(`${API}/api/hosts/${name}`, { method:'PUT', headers:authHeaders(), body:JSON.stringify({kuma_push_url:kuma, enabled}) }); + toast(`${name} updated`); + } + closeModal(); loadAll(); +} + +async function confirmDelete(name) { + if (!confirm(`Delete "${name}" and all history?`)) return; + await apiFetch(`${API}/api/hosts/${name}`, { method:'DELETE', headers:authHeaders() }); + toast(`${name} deleted`); closeDrawer(); loadAll(); +} + +function openModal() { document.getElementById('modalBg').classList.remove('opacity-0','pointer-events-none'); const m = document.getElementById('modal'); m.classList.remove('scale-95','opacity-0','pointer-events-none'); } +function closeModal() { document.getElementById('modalBg').classList.add('opacity-0','pointer-events-none'); const m = document.getElementById('modal'); m.classList.add('scale-95','opacity-0','pointer-events-none'); } + +// ── Navigation ──────────────────────────────────────────── +function showPage(page) { + currentPage = page; + ['dashboard','alerts','hosts','config'].forEach(p => { + document.getElementById(`page-${p}`).classList.toggle('hidden', p !== page); + // Nav highlights + const nav = document.getElementById(`nav-${p}`); + const side = document.getElementById(`side-${p}`); + if (nav) { nav.className = p === page ? 'text-sm font-bold tracking-tight text-blue-400 px-3 py-1 rounded-lg font-headline' : 'text-sm font-medium tracking-tight text-slate-400 hover:bg-slate-800/50 px-3 py-1 rounded-lg transition-colors font-headline'; } + if (side) { side.className = p === page ? 'flex items-center gap-3 px-4 py-3 rounded-xl bg-blue-600/10 text-blue-400 border-r-2 border-blue-500 font-headline text-sm font-semibold' : 'flex items-center gap-3 px-4 py-3 rounded-xl text-slate-500 hover:text-slate-300 hover:bg-slate-900/80 transition-all font-headline text-sm font-semibold'; } + }); +} + +// ── Toast ───────────────────────────────────────────────── +function toast(msg) { + const t = document.createElement('div'); + t.className = 'glass px-5 py-3 rounded-xl text-sm font-medium text-white shadow-2xl flex items-center gap-2 animate-[slideIn_0.3s_ease-out]'; + t.innerHTML = `check_circle ${msg}`; + document.getElementById('toasts').appendChild(t); + setTimeout(() => t.remove(), 4000); +} + +// ── Helpers ─────────────────────────────────────────────── +function fmtBytes(b) { if (!b) return '0 B'; const u = ['B','KB','MB','GB','TB']; const i = Math.floor(Math.log(b)/Math.log(1024)); return (b/Math.pow(1024,i)).toFixed(i>0?1:0)+' '+u[i]; } +function fmtDuration(s) { if (!s) return '–'; if (s<60) return s+'s'; if (s<3600) return Math.floor(s/60)+'m'; return Math.floor(s/3600)+'h '+Math.floor((s%3600)/60)+'m'; } +function timeAgo(iso) { const d=(Date.now()-new Date(iso).getTime())/1000; if(d<60) return 'just now'; if(d<3600) return Math.floor(d/60)+'m ago'; if(d<86400) return Math.floor(d/3600)+'h ago'; return Math.floor(d/86400)+'d ago'; } +function statusChipClass(s) { return { ok:'bg-secondary/10 text-secondary border border-secondary/20', error:'bg-error/10 text-error border border-error/20', stale:'bg-tertiary/10 text-tertiary border border-tertiary/20', disabled:'bg-outline/10 text-outline border border-outline/20' }[s] || ''; } diff --git a/backup-monitor/templates/index.html b/backup-monitor/templates/index.html new file mode 100644 index 0000000..6eb279d --- /dev/null +++ b/backup-monitor/templates/index.html @@ -0,0 +1,295 @@ + + + + + +The Sentinel – Backup Monitor + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+

Vault Overview

+

Operational command for Borgmatic backup infrastructure.

+
+
+
+ Last Scan: + – +
+ +
+
+ + +
+
+
+
+
check_circle
+
+
–
+
Hosts OK
+
+
+
+
database
+
+
–
+
Today Backed Up
+
+
+
+
bolt
+
LIVE
+
+
–
+
Latest Backup
+
–
+
+
+
+
report_problem
+
+
0
+
Issues
+ +
+
+ + +
+ +
+
+
+

Backup Volume Trends

+

Storage growth across all hosts (Last 30 Days)

+
+
+
+ +
+
+ +
+

Host Status

+
+
+
+ + +
+
+

Live Backup Stream

+ Auto-refresh: 30s +
+
+
+
+ + + + + + + + + + +
+ + +
+ + + +
+
+
+

Add Host

+ +
+
+ +
+ + +
+
+ + +
+ +
+ + +
+
+
+ + +
+ + + + diff --git a/compose.yaml b/compose.yaml index 3d66cd4..38d8fc8 100644 --- a/compose.yaml +++ b/compose.yaml @@ -3,89 +3,6 @@ networks: name: monitoring_network services: - teslamate: - image: teslamate/teslamate:latest - container_name: teslamate - restart: always - depends_on: - - teslamate_database - environment: - - DATABASE_USER=${TM_DB_USER} - - DATABASE_PASS=${TM_DB_PASS} - - DATABASE_NAME=${TM_DB_NAME} - - DATABASE_HOST=teslamate_database - - MQTT_HOST=mosquitto - - VIRTUAL_HOST=${FQDN_TM} - - CHECK_ORIGIN=true - - TZ=${TM_TZ} - - ENCRYPTION_KEY=${ENCRYPTION_KEY} - volumes: - - /app-config/teslamate_config:/opt/app/import - cap_drop: - - all - ports: - - "4000:4000" - networks: - - monitoring_network - - teslamate_database: - image: postgres:17 - container_name: teslamate_database - restart: always - environment: - - POSTGRES_USER=${TM_DB_USER} - - POSTGRES_PASSWORD=${TM_DB_PASS} - - POSTGRES_DB=${TM_DB_NAME} - volumes: - - /app-config/teslamate_database:/var/lib/postgresql/data - networks: - - monitoring_network - - grafana: - image: teslamate/grafana:latest - container_name: grafana - restart: always - environment: - - DATABASE_USER=${TM_DB_USER} - - DATABASE_PASS=${TM_DB_PASS} - - DATABASE_NAME=${TM_DB_NAME} - - DATABASE_HOST=teslamate_database - - GRAFANA_PASSWD=${GRAFANA_PW} - - GF_SECURITY_ADMIN_USER=${GRAFANA_USER} - - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PW} - - GF_AUTH_ANONYMOUS_ENABLED=false - - GF_SERVER_DOMAIN=grafana.sascha-lutz.de - - GF_SERVER_ROOT_URL=https://grafana.sascha-lutz.de - - GF_SERVER_SERVE_FROM_SUB_PATH=false - - GF_AUTH_GENERIC_OAUTH_ENABLED=true - - GF_AUTH_GENERIC_OAUTH_NAME=Authentik - - GF_AUTH_GENERIC_OAUTH_CLIENT_ID=${GF_OAUTH_CLIENT_ID} - - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${GF_OAUTH_CLIENT_SECRET} - - GF_AUTH_GENERIC_OAUTH_SCOPES=openid profile email - - GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://auth.sascha-lutz.de/application/o/authorize/ - - GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://auth.sascha-lutz.de/application/o/token/ - - GF_AUTH_GENERIC_OAUTH_API_URL=https://auth.sascha-lutz.de/application/o/userinfo/ - - GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=contains(groups[*], 'authentik Admins') && 'Admin' || 'Viewer' - - GF_AUTH_SIGNOUT_REDIRECT_URL=https://auth.sascha-lutz.de/application/o/grafana/end-session/ - - GF_AUTH_GENERIC_OAUTH_AUTO_LOGIN=false - volumes: - - /app-config/grafana_data:/var/lib/grafana - ports: - - "3000:3000" - networks: - - monitoring_network - - mosquitto: - image: eclipse-mosquitto:2 - container_name: mosquitto - command: mosquitto -c /mosquitto-no-auth.conf - restart: always - volumes: - - /app-config/mosquitto_config:/mosquitto/config - - /app-config/mosquitto_data:/mosquitto/data - networks: - - monitoring_network - prometheus: image: prom/prometheus container_name: prometheus @@ -176,14 +93,3 @@ services: - '--path.sysfs=/host/sys' - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($|/)' restart: unless-stopped - - backup-status: - build: ./backup-status - container_name: backup-status - restart: always - ports: - - "9999:9999" - volumes: - - /app-config/backup_status_data:/data - networks: - - monitoring_network diff --git a/teslamate/compose.yaml b/teslamate/compose.yaml new file mode 100644 index 0000000..fcded3c --- /dev/null +++ b/teslamate/compose.yaml @@ -0,0 +1,87 @@ +networks: + monitoring_network: + external: true + +services: + teslamate: + image: teslamate/teslamate:latest + container_name: teslamate + restart: always + depends_on: + - teslamate_database + environment: + - DATABASE_USER=${TM_DB_USER} + - DATABASE_PASS=${TM_DB_PASS} + - DATABASE_NAME=${TM_DB_NAME} + - DATABASE_HOST=teslamate_database + - MQTT_HOST=mosquitto + - VIRTUAL_HOST=${FQDN_TM} + - CHECK_ORIGIN=true + - TZ=${TM_TZ} + - ENCRYPTION_KEY=${ENCRYPTION_KEY} + volumes: + - /app-config/teslamate_config:/opt/app/import + cap_drop: + - all + ports: + - "4000:4000" + networks: + - monitoring_network + + teslamate_database: + image: postgres:17 + container_name: teslamate_database + restart: always + environment: + - POSTGRES_USER=${TM_DB_USER} + - POSTGRES_PASSWORD=${TM_DB_PASS} + - POSTGRES_DB=${TM_DB_NAME} + volumes: + - /app-config/teslamate_database:/var/lib/postgresql/data + networks: + - monitoring_network + + grafana: + image: teslamate/grafana:latest + container_name: grafana + restart: always + environment: + - DATABASE_USER=${TM_DB_USER} + - DATABASE_PASS=${TM_DB_PASS} + - DATABASE_NAME=${TM_DB_NAME} + - DATABASE_HOST=teslamate_database + - GRAFANA_PASSWD=${GRAFANA_PW} + - GF_SECURITY_ADMIN_USER=${GRAFANA_USER} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PW} + - GF_AUTH_ANONYMOUS_ENABLED=false + - GF_SERVER_DOMAIN=grafana.sascha-lutz.de + - GF_SERVER_ROOT_URL=https://grafana.sascha-lutz.de + - GF_SERVER_SERVE_FROM_SUB_PATH=false + - GF_AUTH_GENERIC_OAUTH_ENABLED=true + - GF_AUTH_GENERIC_OAUTH_NAME=Authentik + - GF_AUTH_GENERIC_OAUTH_CLIENT_ID=${GF_OAUTH_CLIENT_ID} + - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${GF_OAUTH_CLIENT_SECRET} + - GF_AUTH_GENERIC_OAUTH_SCOPES=openid profile email + - GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://auth.sascha-lutz.de/application/o/authorize/ + - GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://auth.sascha-lutz.de/application/o/token/ + - GF_AUTH_GENERIC_OAUTH_API_URL=https://auth.sascha-lutz.de/application/o/userinfo/ + - GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=contains(groups[*], 'authentik Admins') && 'Admin' || 'Viewer' + - GF_AUTH_SIGNOUT_REDIRECT_URL=https://auth.sascha-lutz.de/application/o/grafana/end-session/ + - GF_AUTH_GENERIC_OAUTH_AUTO_LOGIN=false + volumes: + - /app-config/grafana_data:/var/lib/grafana + ports: + - "3000:3000" + networks: + - monitoring_network + + mosquitto: + image: eclipse-mosquitto:2 + container_name: mosquitto + command: mosquitto -c /mosquitto-no-auth.conf + restart: always + volumes: + - /app-config/mosquitto_config:/mosquitto/config + - /app-config/mosquitto_data:/mosquitto/data + networks: + - monitoring_network