feat: 10min incremental snapshots/backups

flake.lock

@@ -48,11 +48,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1769526729,
+        "lastModified": 1769949729,
-        "narHash": "sha256-/ew2cd+lz9xOkTLsYZLqYKfnOoxqlAhpRR7jAqOUljc=",
+        "narHash": "sha256-6JCFnE4qtV/nbwqbAT/ePE6tSNmLYGW31w9xp06X2G8=",
         "ref": "refs/heads/main",
-        "rev": "59180721d3b95f28c17eff9e3a18e54b0a1fa588",
+        "rev": "0ec88131ff6faf182c2bd6a886b91dbff64290ac",
-        "revCount": 23,
+        "revCount": 24,
         "type": "git",
         "url": "ssh://forgejo@git.lukadeka.com:6968/LukaDeka/HexName-Backend.git"
       },
@@ -101,11 +101,11 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1769598131,
+        "lastModified": 1769741972,
-        "narHash": "sha256-e7VO/kGLgRMbWtpBqdWl0uFg8Y2XWFMdz0uUJvlML8o=",
+        "narHash": "sha256-RxSg1EioTWNpoLaykiT1UQKTo/K0PPdLqCyQgNjNqWs=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "fa83fd837f3098e3e678e6cf017b2b36102c7211",
+        "rev": "63590ac958a8af30ebd52c7a0309d8c52a94dd77",
         "type": "github"
       },
       "original": {
@@ -117,11 +117,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1769461804,
+        "lastModified": 1769789167,
-        "narHash": "sha256-msG8SU5WsBUfVVa/9RPLaymvi5bI8edTavbIq3vRlhI=",
+        "narHash": "sha256-kKB3bqYJU5nzYeIROI82Ef9VtTbu4uA3YydSk/Bioa8=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "bfc1b8a4574108ceef22f02bafcf6611380c100d",
+        "rev": "62c8382960464ceb98ea593cb8321a2cf8f9e3e5",
         "type": "github"
       },
       "original": {

flake.nix

@@ -26,24 +26,27 @@
           ./pkgs/powerdns.nix
           ./pkgs/postgres.nix
 
+          ./pkgs/restic.nix # Defines `restic-backups-postgres.service`
+
           ######## Networking ########
           ./pkgs/ssh.nix
           ./pkgs/tailscale.nix
           ./pkgs/virtualisation.nix
-          ./pkgs/stalwart.nix
+          ./pkgs/stalwart.nix # Mailserver
 
           ######## Sysadmin ########
           ./pkgs/neovim.nix
           ./pkgs/common-packages.nix
           ./pkgs/aliases.nix
 
+          ######## Services ########
+          ./services/backup-postgres.nix # Runs `pg_dump` every 10 mins
+          ./services/update-containers.nix # Runs podman pull weekly
+          ./services/restart-powerdns.nix
+
           ######## etc. ########
           ./pkgs/extra.nix
           ./pkgs/zfs.nix
-
-          ######## Scripts ########
-          ./services/update-containers.nix # Runs podman pull weekly
-          ./services/restart-powerdns.nix
         ];
       };
     };

pkgs/backend.nix

@@ -30,7 +30,7 @@
       Group = "hexname";
       Type = "simple";
       Restart = "always";
-      ExecStart = "${pkgs.hexname-backend}/bin/dns-backend";
+      ExecStart = "${pkgs.hexname-backend}/bin/hexname-backend";
     };
   };
 }

pkgs/postgres.nix

@@ -14,43 +14,24 @@
       listen_addresses = lib.mkForce "127.0.0.1,100.123.91.36";
     };
 
-    # Allow root to log in as postgres in the DB (for the PowerDNS container)
+    # authentication = ''
-    # identMap = ''
+    #   host all replication_user 100.112.93.9/32 scram-sha-256
-    #   postgres root postgres
     # '';
 
-    authentication = ''
+    authentication = lib.mkForce ''
-      host all replication_user 100.112.93.9/32 scram-sha-256
+      # TYPE DATABASE        USER             ADDRESS         AUTH-METHOD
+
+      # Allow the replication subscriber to connect from Tailscale IP
+      host   all             replication_user 100.112.93.9/32 scram-sha-256
+
+      host   hexname-backend hexname-backend  127.0.0.1/32    scram-sha-256
+      host   powerdns        powerdns         127.0.0.1/32    scram-sha-256
+      local  all             all                              peer
+
+      # local  all             postgres                         peer
     '';
 
-    # authentication = lib.mkForce ''
-    #   # TYPE DATABASE        USER            ADDRESS       AUTH-METHOD   [auth-options]
-    #   # host   hexname-backend hexname-backend 127.0.0.1/24  scram-sha-256
-    #   # host   postgres        postgres        127.0.0.1/24  scram-sha-256
-    #   host   all        all        127.0.0.1/24  scram-sha-256
-    #   # host   all             powerdns-user   127.0.0.1/24  scram-sha-256
-    #   # local  all             root                          trust
-    # '';
-
-    # This password is only the initial one - don't get too excited
     initialScript = "/etc/env/hexname/init-db-dns.sql";
-    # initialScript = pkgs.writeText "set-initial-password-script" ''
-      # CREATE USER powerdns WITH PASSWORD 'shuaze-gagyof';
-      # CREATE USER hexname_backend WITH PASSWORD 'shuaze-gagyof2' CREATEDB;
-      # CREATE USER replication_user SUPERUSER WITH PASSWORD 'shuaze-gagyof3';
-
-      # CREATE DATABASE powerdns OWNER "powerdns";
-      # CREATE DATABASE hexname_backend OWNER "hexname-backend";
-      # CREATE DATABASE hexname_backend OWNER "hexname-backend";
-
-      # \c powerdns;
-      # CREATE PUBLICATION powerdns_pub FOR ALL TABLES;
-
-      # CREATE SUBSCRIPTION hexname_ns2
-      #   CONNECTION 'host=publisher_ip_address port=5432 user=replication_user password=' ' dbname=powerdns'
-      #   PUBLICATION powerdns_pub;
-    # '';
-      # ALTER USER "postgres" WITH PASSWORD 'shuaze-gagyof';
   };
 }
 

pkgs/restic.nix

@@ -0,0 +1,38 @@
+{ config, ... }:
+
+{
+  # users.groups.hexname-postgres-backup = {};
+  # users.users = {
+  #   hexname-postgres-backup = {
+  #     group = "hexname-postgres-backup";
+  #     isSystemUser = true;
+  #     createHome = true;
+  #     home = "/var/lib/hexname/postgres";
+  #   };
+  # };
+
+  services.restic.backups = {
+    postgres = {
+      paths = [
+        # This is where I store the output of `pg_dump`, not the DB itself
+        "/var/lib/hexname/postgres"
+      ];
+      repository = "sftp:hexname-postgres-backup@ns2.hexname.com:/etc/backups/ns1";
+      extraOptions = [
+        "sftp.command='ssh -p 6968 hexname-postgres-backup@ns2.hexname.com -i /etc/env/restic/id_ed25519 -s sftp'"
+      ];
+      passwordFile = "/etc/env/restic/gateway-password";
+      initialize = true;
+
+      timerConfig = null; # Explicitly invoked by backup-postgres.service
+      pruneOpts = [
+        "--keep-last 36" # 6 hours' worth
+        "--keep-daily 7"
+        "--keep-weekly 2"
+        "--keep-monthly 3"
+        "--keep-yearly 1"
+      ];
+    };
+  };
+}
+

pkgs/zfs.nix

@@ -0,0 +1,19 @@
+{ ... }:
+
+{
+  services.zfs = {
+    autoSnapshot = {
+      enable = true;
+      flags = "-k -p -u";
+
+      monthly = 6;
+      weekly = 2;
+      daily = 7;
+      hourly = 24;
+      frequent = 8;
+    };
+
+    autoScrub.enable = true;
+  };
+}
+

scripts/pdns-init-config-dns.sh

@@ -71,11 +71,11 @@ add_record "A" 3600 "mta-sts.hexname.com." "188.245.239.209"
 add_record "CNAME" 3600 "mail.hexname.com." "mx.hexname.com."
 add_record "MX" 3600 "hexname.com." "10 mx.hexname.com."
 
+add_record "TXT" 3600 "hexname.com." "\"google-site-verification=yOG8oyjZY9hKDxSNiD730PD9HO-zsQz0xEke49U6lrM\""
+add_record "TXT" 3600 "hexname.com." "\"ahrefs-site-verification_eadd521586ff26f292e19c7a99f7feb5c22420c290ce62b452ce381b2f1b9c16\""
 add_record "TXT" 3600 "_mta-sts.hexname.com." "\"v=STSv1; id=20260127182600Z;\""
-
 add_record "TXT" 3600 "202601e._domainkey.hexname.com." "\"v=DKIM1; k=ed25519; h=sha256; p=C30gZd1CbkUpIGInw/wZgZQD0pmUEnwTp+svCLm1oCk=\""
 add_record "TXT" 3600 "202601r._domainkey.hexname.com." "\"v=DKIM1; k=rsa; h=sha256; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnyQRHisJtMpgRCAKAE5mfq63n1hvguiNheRrGWcLjEziA9r3M8oaxM71gDNeDEZj19yXlYBWlZZiPsdkMNsumFaElTt3E810JjZxvWslvRgCQ9qMK6lE4ytJZHXJD1a+g82/j4Pfu3C0iz0GfMvngXf03pDl5jWeScwfSFgvKx/0tRdzCAzwkSZfZaSKCh5bcvVwoxzXqHjz3zxwxDJGlUPoERymd18/7NkdRRfJZoqAo/aHdsh5JsYa8APtNIHjSjp2vUBPQnNrtx9+lI0qRnwdyrim8v8CRKin+QUW0sstWGuyqZxgxOGXO2Ek2fqTrpzVu2fu6pzGqJdbTVf5BQIDAQAB\""
-
 add_record "TXT" 3600 "mx.hexname.com." "\"v=spf1 a ra=spf-reports -all\""
 add_record "TXT" 3600 "hexname.com." "\"v=spf1 mx ra=spf-reports -all\""
 add_record "TXT" 3600 "_dmarc.hexname.com." "\"v=DMARC1; p=reject; rua=mailto:dmarc-reports@hexname.com; ruf=mailto:dmarc-reports@hexname.com\""

services/backup-postgres.nix

@@ -0,0 +1,47 @@
+{ config, pkgs, ... }:
+
+let
+  serviceName = "backup-postgres";
+in
+{
+  environment.systemPackages = [
+    pkgs.restic
+  ];
+
+  systemd.timers."${serviceName}" = {
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnBootSec = "10m";
+      OnUnitActiveSec = "10m";
+      Unit = "${serviceName}.service";
+    };
+  };
+  systemd.services."${serviceName}"= {
+    script = ''
+      set -euo pipefail
+
+      cd /var/lib/hexname/postgres
+      ${config.services.postgresql.package}/bin/pg_dumpall \
+        --exclude-database=roundcube \
+        --exclude-database=template0 \
+        --exclude-database=template1 \
+        --exclude-database=postgres \
+        --clean \
+        --file ./postgres.out
+    '';
+    wants = [ "restic-backups-postgres.service" ];
+    serviceConfig = {
+      Type = "oneshot";
+      User = "postgres";
+      Group = "postgres";
+      RemainAfterExit = true; # Don't start the service on every rebuild
+    };
+  };
+
+  # Overwrite the restic backup service so that it's triggered after `pg_dump`
+  systemd.services.restic-backups-postgres = {
+    after = [ "backup-postgres.service" ];
+    # requires = [ "backup-postgres.service" ];
+  };
+}
+

services/update-containers.nix

@@ -1,4 +1,4 @@
-{ pkgs, lib, ... }:
+{ lib, pkgs, ... }:
 
 {
   systemd.timers.update-containers = {