diff --git a/nfs-stability.yml b/nfs-stability.yml new file mode 100644 index 0000000..c5c3014 --- /dev/null +++ b/nfs-stability.yml @@ -0,0 +1,6 @@ +--- +- name: NFS and mergerfs stability + hosts: emby-sascha:emby-chris:arrapps + become: yes + roles: + - nfs_stability diff --git a/pfannkuchen.ini b/pfannkuchen.ini index 9d72c0b..943d6b7 100644 --- a/pfannkuchen.ini +++ b/pfannkuchen.ini @@ -40,7 +40,7 @@ n8n ansible_host=10.4.1.113 hermes ansible_host=10.4.1.100 monitoring ansible_host=10.1.1.111 automation1 ansible_host=10.5.85.5 -outline ansible_host=10.1.1.100 + satisfactory ansible_host=10.3.1.120 [communication] diff --git a/pfannkuchen.sh b/pfannkuchen.sh index 6e91824..7f0955b 100755 --- a/pfannkuchen.sh +++ b/pfannkuchen.sh @@ -23,6 +23,7 @@ usage() { echo -e " ${C}wstunnel${N} [host] wstunnel + WireGuard deployen" echo -e " ${C}sshfs${N} [host] SSHFS Mounts einrichten" echo -e " ${C}tune${N} Sysctl Netzwerk-Tuning" + echo -e " ${C}nfs${N} NFS-/mergerfs-Stabilisierung ausrollen" echo -e " ${C}pvetune${N} [host] Proxmox Host Tuning (sysctl, resolv, hosts)" echo -e " ${C}tc${N} [host] tc per-flow Rate-Limit (50 Mbit/s pro Connection)" echo -e " ${C}watchdog${N} Network Watchdog deployen (Whitelist aus Ansible)" @@ -103,6 +104,10 @@ case "$CMD" in [ -z "$HOST" ] && echo -e "${R}Fehler: Host angeben${N}" && exit 1 run sysctl.yaml -l "$HOST" ;; + nfs) + [ -z "$HOST" ] && echo -e "${R}Fehler: Host angeben${N}" && exit 1 + run nfs-stability.yml -l "$HOST" + ;; pvetune) if [ -n "$HOST" ]; then run sysctl-proxmox.yaml -l "$HOST" diff --git a/roles/nfs_stability/tasks/main.yml b/roles/nfs_stability/tasks/main.yml new file mode 100644 index 0000000..8073ba6 --- /dev/null +++ b/roles/nfs_stability/tasks/main.yml @@ -0,0 +1,134 @@ +--- +- name: Ensure this role only targets the three mergerfs media hosts + ansible.builtin.assert: + that: + - inventory_hostname in ['emby-sascha', 'emby-chris', 'arrapps'] + fail_msg: "nfs_stability must not run on {{ inventory_hostname }}" + +- name: Stop and disable the destructive legacy retry timer first + ansible.builtin.systemd: + name: remount-nfs.timer + state: stopped + enabled: false + failed_when: false + +- name: Stop a currently running destructive legacy retry service + ansible.builtin.systemd: + name: remount-nfs.service + state: stopped + failed_when: false + +- name: Mark mergerfs as a network-dependent mount without remounting it live + ansible.builtin.replace: + path: /etc/systemd/system/mnt-media.mount + regexp: '^Options=(?![^\n]*_netdev)(.*)$' + replace: 'Options=_netdev,\1' + backup: true + register: mergerfs_unit + +- name: Install non-destructive one-shot NFS recovery helper + ansible.builtin.copy: + dest: /usr/local/sbin/remount-nfs-safe + owner: root + group: root + mode: '0755' + content: | + #!/bin/bash + set -u + is_nfs() { + findmnt -rn --target "$1" -t nfs,nfs4 -o TARGET | grep -Fxq "$1" + } + is_media() { + findmnt -rn --target /mnt/media -t fuse.mergerfs -o TARGET | grep -Fxq /mnt/media + } + for attempt in 1 2 3; do + missing=0 + for entry in 'mnt-nas.mount:/mnt/nas' 'mnt-qnap1.mount:/mnt/qnap1' 'mnt-qnap2.mount:/mnt/qnap2'; do + unit="${entry%%:*}" + target="${entry#*:}" + if ! is_nfs "$target"; then + systemctl start "$unit" || true + fi + is_nfs "$target" || missing=1 + done + if [ "$missing" -eq 0 ]; then + if ! is_media; then + systemctl start mnt-media.mount || true + fi + is_media && exit 0 + fi + sleep 10 + done + echo 'NFS recovery failed without restarting any active mount' >&2 + exit 1 + +- name: Install safe one-shot NFS recovery service + ansible.builtin.copy: + dest: /etc/systemd/system/remount-nfs.service + owner: root + group: root + mode: '0644' + backup: true + content: | + [Unit] + Description=Safely start missing NFS mounts once after boot + After=network-online.target + Wants=network-online.target + + [Service] + Type=oneshot + ExecStart=/usr/local/sbin/remount-nfs-safe + +- name: Install once-per-boot NFS recovery timer + ansible.builtin.copy: + dest: /etc/systemd/system/remount-nfs.timer + owner: root + group: root + mode: '0644' + backup: true + content: | + [Unit] + Description=One-shot NFS mount validation after boot + + [Timer] + OnBootSec=45 + AccuracySec=5 + Persistent=false + Unit=remount-nfs.service + + [Install] + WantedBy=timers.target + +- name: Reload systemd and enable the one-shot boot timer + ansible.builtin.systemd: + daemon_reload: true + name: remount-nfs.timer + enabled: true + state: started + +- name: Run the safe helper once to recover mounts left missing by the legacy loop + ansible.builtin.command: + argv: + - /usr/local/sbin/remount-nfs-safe + changed_when: false + +- name: Verify systemd unit dependency graph + ansible.builtin.command: + argv: + - systemd-analyze + - verify + - /etc/systemd/system/mnt-media.mount + - /etc/systemd/system/remount-nfs.service + - /etc/systemd/system/remount-nfs.timer + changed_when: false + +- name: Verify all active storage mounts without restarting them + ansible.builtin.shell: | + set -e + for target in /mnt/nas /mnt/qnap1 /mnt/qnap2; do + findmnt -rn --target "$target" -t nfs,nfs4 -o TARGET | grep -Fxq "$target" + done + findmnt -rn --target /mnt/media -t fuse.mergerfs -o TARGET | grep -Fxq /mnt/media + args: + executable: /bin/bash + changed_when: false diff --git a/site.yml b/site.yml index f359ccd..6152040 100644 --- a/site.yml +++ b/site.yml @@ -1,4 +1,11 @@ --- +# Stabilize NFS/mergerfs only on hosts that use the shared media pool. +- name: NFS and mergerfs stability + hosts: emby-sascha:emby-chris:arrapps + become: yes + roles: + - nfs_stability + # Neue VM komplett einrichten - name: VM Setup hosts: all diff --git a/tests/test_nfs_stability_role.py b/tests/test_nfs_stability_role.py new file mode 100644 index 0000000..820ddfd --- /dev/null +++ b/tests/test_nfs_stability_role.py @@ -0,0 +1,27 @@ +from pathlib import Path + +ROLE = Path("roles/nfs_stability/tasks/main.yml").read_text() +SITE = Path("site.yml").read_text() + + +def test_recovery_is_non_destructive(): + assert "systemctl restart" not in ROLE + assert "OnUnitActiveSec" not in ROLE + assert "systemctl start" in ROLE + + +def test_recovery_validates_real_nfs_and_mergerfs_mounts(): + assert "-t nfs,nfs4" in ROLE + assert "-t fuse.mergerfs" in ROLE + assert all(path in ROLE for path in ("/mnt/nas", "/mnt/qnap1", "/mnt/qnap2", "/mnt/media")) + + +def test_legacy_loop_is_stopped_before_replacement(): + stop_at = ROLE.index("Stop and disable the destructive legacy retry timer first") + install_at = ROLE.index("Install once-per-boot NFS recovery timer") + assert stop_at < install_at + + +def test_role_is_scoped_to_media_mount_hosts(): + assert "hosts: emby-sascha:emby-chris:arrapps" in SITE + assert "- nfs_stability" in SITE