diff options
| author | schererleander <leander@schererleander.de> | 2026-05-25 01:56:22 +0200 |
|---|---|---|
| committer | schererleander <leander@schererleander.de> | 2026-05-25 02:11:52 +0200 |
| commit | 033f25f6840bf41ab07c6f0c3cc2880b794b5b57 (patch) | |
| tree | 3fb61b7d6714e1bdd6d2dc0a82b71d99dc3b7f25 | |
| parent | 523a00457a79b35ada98ac10172be557e3eea235 (diff) | |
feat(borg-backup): add failure notifications and log stats
| -rw-r--r-- | modules/services/git.nix | 5 | ||||
| -rw-r--r-- | modules/services/mail.nix | 60 | ||||
| -rw-r--r-- | modules/services/nextcloud.nix | 7 |
3 files changed, 67 insertions, 5 deletions
diff --git a/modules/services/git.nix b/modules/services/git.nix index 4249bd0..12ef804 100644 --- a/modules/services/git.nix +++ b/modules/services/git.nix @@ -82,14 +82,15 @@ config.sops.secrets."borgbase_ssh_key".path } -o StrictHostKeyChecking=accept-new"; }; + extraCreateArgs = [ "--info" "--stats" ]; compression = "auto,lzma"; startAt = "daily"; preHook = '' set -euo pipefail - - # Exporting the specific Git repo secret export BORG_REPO="$(cat ${config.sops.secrets."borg_git_repo".path})" ''; }; + + systemd.services."borgbackup-job-git".unitConfig.OnFailure = [ "notify-backup-failure@%n.service" ]; }; } diff --git a/modules/services/mail.nix b/modules/services/mail.nix index 0df1909..385d912 100644 --- a/modules/services/mail.nix +++ b/modules/services/mail.nix @@ -32,6 +32,66 @@ }; }; + systemd.services."notify-backup-failure@" = { + description = "Notify backup failure for %i"; + serviceConfig.Type = "oneshot"; + script = '' + UNIT_NAME="%i" + HOSTNAME=$(${pkgs.coreutils}/bin/cat /etc/hostname) + TIMESTAMP=$(${pkgs.coreutils}/bin/date "+%Y-%m-%d %H:%M:%S %Z") + + # Get logs + LOGS=$(${pkgs.systemd}/bin/journalctl -u "$UNIT_NAME" -n 50 --no-pager) + + ( + ${pkgs.coreutils}/bin/echo "To: leander@schererleander.de" + ${pkgs.coreutils}/bin/echo "From: root@sachiel.schererleander.de" + ${pkgs.coreutils}/bin/echo "Subject: Backup Failure: $UNIT_NAME" + ${pkgs.coreutils}/bin/echo "Content-Type: text/html; charset=UTF-8" + ${pkgs.coreutils}/bin/echo "" + ${pkgs.coreutils}/bin/cat <<EOF + <!DOCTYPE html> + <html> + <head> + <meta name="color-scheme" content="light dark"> + <style> + :root { color-scheme: light dark; } + body { font-family: sans-serif; line-height: 1.5; color: #000; background: #fff; max-width: 800px; margin: 0 auto; padding: 20px; } + h1 { border-bottom: 2px solid #000; color: #c00; } + pre, .crit { background: #f0f0f0; padding: 10px; font-family: monospace; font-size: 13px; } + .crit { border-left: 4px solid #c00; } + table { width: 100%; border-collapse: collapse; margin-bottom: 15px; font-size: 13px; } + th, td { text-align: left; padding: 6px; border-bottom: 1px solid #ddd; } + @media (prefers-color-scheme: dark) { + body { background: #121212; color: #eee; } + h1, th { border-color: #555; } + h1 { color: #ff6666; } + pre, .crit { background: #1e1e1e; border-color: #eee; } + .crit { border-left-color: #ff6666; } + th, td { border-color: #333; } + } + </style> + </head> + <body> + <h1>Backup Failure Alert</h1> + + <table> + <tr><th>Unit</th><td>$UNIT_NAME</td></tr> + <tr><th>Host</th><td>$HOSTNAME</td></tr> + <tr><th>Time</th><td>$TIMESTAMP</td></tr> + </table> + + <p><strong>Last 50 log lines:</strong></p> + <div class="crit"> + <pre>$LOGS</pre> + </div> + </body> + </html> +EOF + ) | /run/wrappers/bin/sendmail -f root@sachiel.schererleander.de leander@schererleander.de + ''; + }; + services.rspamd = { enable = true; locals."dkim_signing.conf".text = '' diff --git a/modules/services/nextcloud.nix b/modules/services/nextcloud.nix index ef6f791..38b6145 100644 --- a/modules/services/nextcloud.nix +++ b/modules/services/nextcloud.nix @@ -85,6 +85,7 @@ } -o StrictHostKeyChecking=accept-new"; TMPDIR = "/var/tmp"; }; + extraCreateArgs = [ "--info" "--stats" ]; compression = "auto,lzma"; startAt = "daily"; readWritePaths = [ @@ -93,10 +94,8 @@ ]; preHook = '' set -euo pipefail - - # Exporting the specific Nextcloud repo secret export BORG_REPO="$(cat ${config.sops.secrets."borg_nextcloud_repo".path})" - + INSTALL="${pkgs.coreutils}/bin/install" FIND="${pkgs.findutils}/bin/find" MYSQLDUMP="${pkgs.mariadb.client}/bin/mariadb-dump" @@ -123,6 +122,8 @@ ''; }; + systemd.services."borgbackup-job-nextcloud".unitConfig.OnFailure = [ "notify-backup-failure@%n.service" ]; + services.fail2ban = { enable = true; bantime = lib.mkDefault "1h"; |