From 033f25f6840bf41ab07c6f0c3cc2880b794b5b57 Mon Sep 17 00:00:00 2001 From: schererleander Date: Mon, 25 May 2026 01:56:22 +0200 Subject: feat(borg-backup): add failure notifications and log stats --- modules/services/git.nix | 5 ++-- modules/services/mail.nix | 60 ++++++++++++++++++++++++++++++++++++++++++ modules/services/nextcloud.nix | 7 ++--- 3 files changed, 67 insertions(+), 5 deletions(-) (limited to 'modules/services') diff --git a/modules/services/git.nix b/modules/services/git.nix index 4249bd0..12ef804 100644 --- a/modules/services/git.nix +++ b/modules/services/git.nix @@ -82,14 +82,15 @@ config.sops.secrets."borgbase_ssh_key".path } -o StrictHostKeyChecking=accept-new"; }; + extraCreateArgs = [ "--info" "--stats" ]; compression = "auto,lzma"; startAt = "daily"; preHook = '' set -euo pipefail - - # Exporting the specific Git repo secret export BORG_REPO="$(cat ${config.sops.secrets."borg_git_repo".path})" ''; }; + + systemd.services."borgbackup-job-git".unitConfig.OnFailure = [ "notify-backup-failure@%n.service" ]; }; } diff --git a/modules/services/mail.nix b/modules/services/mail.nix index 0df1909..385d912 100644 --- a/modules/services/mail.nix +++ b/modules/services/mail.nix @@ -32,6 +32,66 @@ }; }; + systemd.services."notify-backup-failure@" = { + description = "Notify backup failure for %i"; + serviceConfig.Type = "oneshot"; + script = '' + UNIT_NAME="%i" + HOSTNAME=$(${pkgs.coreutils}/bin/cat /etc/hostname) + TIMESTAMP=$(${pkgs.coreutils}/bin/date "+%Y-%m-%d %H:%M:%S %Z") + + # Get logs + LOGS=$(${pkgs.systemd}/bin/journalctl -u "$UNIT_NAME" -n 50 --no-pager) + + ( + ${pkgs.coreutils}/bin/echo "To: leander@schererleander.de" + ${pkgs.coreutils}/bin/echo "From: root@sachiel.schererleander.de" + ${pkgs.coreutils}/bin/echo "Subject: Backup Failure: $UNIT_NAME" + ${pkgs.coreutils}/bin/echo "Content-Type: text/html; charset=UTF-8" + ${pkgs.coreutils}/bin/echo "" + ${pkgs.coreutils}/bin/cat < + + + + + + +

Backup Failure Alert

+ + + + + +
Unit$UNIT_NAME
Host$HOSTNAME
Time$TIMESTAMP
+ +

Last 50 log lines:

+
+ 
$LOGS
+
+ + +EOF + ) | /run/wrappers/bin/sendmail -f root@sachiel.schererleander.de leander@schererleander.de + ''; + }; + services.rspamd = { enable = true; locals."dkim_signing.conf".text = '' diff --git a/modules/services/nextcloud.nix b/modules/services/nextcloud.nix index ef6f791..38b6145 100644 --- a/modules/services/nextcloud.nix +++ b/modules/services/nextcloud.nix @@ -85,6 +85,7 @@ } -o StrictHostKeyChecking=accept-new"; TMPDIR = "/var/tmp"; }; + extraCreateArgs = [ "--info" "--stats" ]; compression = "auto,lzma"; startAt = "daily"; readWritePaths = [ @@ -93,10 +94,8 @@ ]; preHook = '' set -euo pipefail - - # Exporting the specific Nextcloud repo secret export BORG_REPO="$(cat ${config.sops.secrets."borg_nextcloud_repo".path})" - + INSTALL="${pkgs.coreutils}/bin/install" FIND="${pkgs.findutils}/bin/find" MYSQLDUMP="${pkgs.mariadb.client}/bin/mariadb-dump" @@ -123,6 +122,8 @@ ''; }; + systemd.services."borgbackup-job-nextcloud".unitConfig.OnFailure = [ "notify-backup-failure@%n.service" ]; + services.fail2ban = { enable = true; bantime = lib.mkDefault "1h"; -- cgit v1.3.1