Merge pull request #49809 from griff/rspamd-workers

nixos/rspamd: Multiple workers, extraConfig priority & postfix integration

nixos/modules/services/mail/rspamd.nix

@@ -6,6 +6,7 @@ let
 
   cfg = config.services.rspamd;
   opts = options.services.rspamd;
+  postfixCfg = config.services.postfix;
 
   bindSocketOpts = {options, config, ... }: {
     options = {
@@ -58,7 +59,7 @@ let
       };
       type = mkOption {
         type = types.nullOr (types.enum [
-          "normal" "controller" "fuzzy_storage" "proxy" "lua"
+          "normal" "controller" "fuzzy_storage" "rspamd_proxy" "lua"
         ]);
         description = "The type of this worker";
       };
@@ -99,19 +100,21 @@ let
         description = "Additional entries to put verbatim into worker section of rspamd config file.";
       };
     };
-    config = mkIf (name == "normal" || name == "controller" || name == "fuzzy") {
+    config = mkIf (name == "normal" || name == "controller" || name == "fuzzy" || name == "rspamd_proxy") {
       type = mkDefault name;
-      includes = mkDefault [ "$CONFDIR/worker-${name}.inc" ];
+      includes = mkDefault [ "$CONFDIR/worker-${if name == "rspamd_proxy" then "proxy" else name}.inc" ];
-      bindSockets = mkDefault (if name == "normal"
+      bindSockets =
-        then [{
+        let
-              socket = "/run/rspamd/rspamd.sock";
+          unixSocket = name: {
-              mode = "0660";
+            mode = "0660";
-              owner = cfg.user;
+            socket = "/run/rspamd/${name}.sock";
-              group = cfg.group;
+            owner = cfg.user;
-            }]
+            group = cfg.group;
-        else if name == "controller"
+          };
-        then [ "localhost:11334" ]
+        in mkDefault (if name == "normal" then [(unixSocket "rspamd")]
-        else [] );
+          else if name == "controller" then [ "localhost:11334" ]
+          else if name == "rspamd_proxy" then [ (unixSocket "proxy") ]
+          else [] );
     };
   };
 
@@ -138,19 +141,25 @@ let
         .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/logging.inc"
       }
 
-      ${concatStringsSep "\n" (mapAttrsToList (name: value: ''
+      ${concatStringsSep "\n" (mapAttrsToList (name: value: let
-        worker ${optionalString (value.name != "normal" && value.name != "controller") "${value.name}"} {
+          includeName = if name == "rspamd_proxy" then "proxy" else name;
+          tryOverride = if value.extraConfig == "" then "true" else "false";
+        in ''
+        worker "${value.type}" {
           type = "${value.type}";
           ${optionalString (value.enable != null)
             "enabled = ${if value.enable != false then "yes" else "no"};"}
           ${mkBindSockets value.enable value.bindSockets}
           ${optionalString (value.count != null) "count = ${toString value.count};"}
           ${concatStringsSep "\n  " (map (each: ".include \"${each}\"") value.includes)}
-          ${value.extraConfig}
+          .include(try=true; priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/worker-${includeName}.inc"
+          .include(try=${tryOverride}; priority=10) "$LOCAL_CONFDIR/override.d/worker-${includeName}.inc"
         }
       '') cfg.workers)}
 
-      ${cfg.extraConfig}
+      ${optionalString (cfg.extraConfig != "") ''
+        .include(priority=10) "$LOCAL_CONFDIR/override.d/extra-config.inc"
+      ''}
    '';
 
   filterFiles = files: filterAttrs (n: v: v.enable) files;
@@ -189,6 +198,15 @@ let
         in mkDefault (pkgs.writeText name' config.text));
     };
   };
+
+  configOverrides =
+    (mapAttrs' (n: v: nameValuePair "worker-${if n == "rspamd_proxy" then "proxy" else n}.inc" {
+      text = v.extraConfig;
+    })
+    (filterAttrs (n: v: v.extraConfig != "") cfg.workers))
+    // (if cfg.extraConfig == "" then {} else {
+      "extra-config.inc".text = cfg.extraConfig;
+    });
 in
 
 {
@@ -285,7 +303,7 @@ in
         description = ''
           User to use when no root privileges are required.
         '';
-       };
+      };
 
       group = mkOption {
         type = types.string;
@@ -293,7 +311,30 @@ in
         description = ''
           Group to use when no root privileges are required.
         '';
-       };
+      };
+
+      postfix = {
+        enable = mkOption {
+          type = types.bool;
+          default = false;
+          description = "Add rspamd milter to postfix main.conf";
+        };
+
+        config = mkOption {
+          type = with types; attrsOf (either bool (either str (listOf str)));
+          description = ''
+            Addon to postfix configuration
+          '';
+          default = {
+            smtpd_milters = ["unix:/run/rspamd/rspamd-milter.sock"];
+            non_smtpd_milters = ["unix:/run/rspamd/rspamd-milter.sock"];
+          };
+          example = {
+            smtpd_milters = ["unix:/run/rspamd/rspamd-milter.sock"];
+            non_smtpd_milters = ["unix:/run/rspamd/rspamd-milter.sock"];
+          };
+        };
+      };
     };
   };
 
@@ -301,6 +342,25 @@ in
   ###### implementation
 
   config = mkIf cfg.enable {
+    services.rspamd.overrides = configOverrides;
+    services.rspamd.workers = mkIf cfg.postfix.enable {
+      controller = {};
+      rspamd_proxy = {
+        bindSockets = [ {
+          mode = "0660";
+          socket = "/run/rspamd/rspamd-milter.sock";
+          owner = cfg.user;
+          group = postfixCfg.group;
+        } ];
+        extraConfig = ''
+          upstream "local" {
+            default = yes; # Self-scan upstreams are always default
+            self_scan = yes; # Enable self-scan
+          }
+        '';
+      };
+    };
+    services.postfix.config = mkIf cfg.postfix.enable cfg.postfix.config;
 
     # Allow users to run 'rspamc' and 'rspamadm'.
     environment.systemPackages = [ pkgs.rspamd ];

nixos/tests/rspamd.nix

@@ -28,6 +28,8 @@ let
       ${checkSocket "/run/rspamd/rspamd.sock" "rspamd" "rspamd" "660" }
       sleep 10;
       $machine->log($machine->succeed("cat /etc/rspamd/rspamd.conf"));
+      $machine->log($machine->succeed("grep 'CONFDIR/worker-controller.inc' /etc/rspamd/rspamd.conf"));
+      $machine->log($machine->succeed("grep 'CONFDIR/worker-normal.inc' /etc/rspamd/rspamd.conf"));
       $machine->log($machine->succeed("systemctl cat rspamd.service"));
       $machine->log($machine->succeed("curl http://localhost:11334/auth"));
       $machine->log($machine->succeed("curl http://127.0.0.1:11334/auth"));
@@ -56,6 +58,8 @@ in
       ${checkSocket "/run/rspamd.sock" "root" "root" "600" }
       ${checkSocket "/run/rspamd-worker.sock" "root" "root" "666" }
       $machine->log($machine->succeed("cat /etc/rspamd/rspamd.conf"));
+      $machine->log($machine->succeed("grep 'CONFDIR/worker-controller.inc' /etc/rspamd/rspamd.conf"));
+      $machine->log($machine->succeed("grep 'CONFDIR/worker-normal.inc' /etc/rspamd/rspamd.conf"));
       $machine->log($machine->succeed("rspamc -h /run/rspamd-worker.sock stat"));
       $machine->log($machine->succeed("curl --unix-socket /run/rspamd-worker.sock http://localhost/ping"));
     '';
@@ -78,6 +82,15 @@ in
           owner = "root";
           group = "root";
         }];
+        workers.controller2 = {
+          type = "controller";
+          bindSockets = [ "0.0.0.0:11335" ];
+          extraConfig = ''
+            static_dir = "''${WWWDIR}";
+            secure_ip = null;
+            password = "verysecretpassword";
+          '';
+        };
       };
     };
 
@@ -87,8 +100,14 @@ in
       ${checkSocket "/run/rspamd.sock" "root" "root" "600" }
       ${checkSocket "/run/rspamd-worker.sock" "root" "root" "666" }
       $machine->log($machine->succeed("cat /etc/rspamd/rspamd.conf"));
+      $machine->log($machine->succeed("grep 'CONFDIR/worker-controller.inc' /etc/rspamd/rspamd.conf"));
+      $machine->log($machine->succeed("grep 'CONFDIR/worker-normal.inc' /etc/rspamd/rspamd.conf"));
+      $machine->log($machine->succeed("grep 'LOCAL_CONFDIR/override.d/worker-controller2.inc' /etc/rspamd/rspamd.conf"));
+      $machine->log($machine->succeed("grep 'verysecretpassword' /etc/rspamd/override.d/worker-controller2.inc"));
+      $machine->waitUntilSucceeds("journalctl -u rspamd | grep -i 'starting controller process' >&2");
       $machine->log($machine->succeed("rspamc -h /run/rspamd-worker.sock stat"));
       $machine->log($machine->succeed("curl --unix-socket /run/rspamd-worker.sock http://localhost/ping"));
+      $machine->log($machine->succeed("curl http://localhost:11335/ping"));
     '';
   };
   customLuaRules = makeTest {
@@ -183,4 +202,48 @@ in
       $machine->log($machine->succeed("cat /etc/tests/muh.eml | rspamc -h 127.0.0.1:11334 symbols | grep NO_MUH"));
     '';
   };
+  postfixIntegration = makeTest {
+    name = "rspamd-postfix-integration";
+    machine = {
+      environment.systemPackages = with pkgs; [ msmtp ];
+      environment.etc."tests/gtube.eml".text = ''
+        From: Sheep1<bah@example.com>
+        To: Sheep2<tester@example.com>
+        Subject: Evil cows
+
+        I find cows to be evil don't you?
+
+        XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
+      '';
+      environment.etc."tests/example.eml".text = ''
+        From: Sheep1<bah@example.com>
+        To: Sheep2<tester@example.com>
+        Subject: Evil cows
+
+        I find cows to be evil don't you?
+      '';
+      users.users.tester.password = "test";
+      services.postfix = {
+        enable = true;
+        destination = ["example.com"];
+      };
+      services.rspamd = {
+        enable = true;
+        postfix.enable = true;
+      };
+    };
+    testScript = ''
+      ${initMachine}
+      $machine->waitForOpenPort(11334);
+      $machine->waitForOpenPort(25);
+      ${checkSocket "/run/rspamd/rspamd-milter.sock" "rspamd" "postfix" "660" }
+      $machine->log($machine->succeed("rspamc -h 127.0.0.1:11334 stat"));
+      $machine->log($machine->succeed("msmtp --host=localhost -t --read-envelope-from < /etc/tests/example.eml"));
+      $machine->log($machine->fail("msmtp --host=localhost -t --read-envelope-from < /etc/tests/gtube.eml"));
+
+      $machine->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]');
+      $machine->fail("journalctl -u postfix | grep -i error >&2");
+      $machine->fail("journalctl -u postfix | grep -i warning >&2");
+    '';
+  };
 }