From dade94cdb99525c42d075e10773a362404fc88c2 Mon Sep 17 00:00:00 2001 From: Ben Harper Date: Sat, 23 Nov 2019 21:30:48 +1100 Subject: [PATCH] nixos/awstats: refactor module --- nixos/doc/manual/release-notes/rl-2003.xml | 12 + nixos/modules/services/logging/awstats.nix | 299 +++++++++++++++------ 2 files changed, 231 insertions(+), 80 deletions(-) diff --git a/nixos/doc/manual/release-notes/rl-2003.xml b/nixos/doc/manual/release-notes/rl-2003.xml index 5744de96b74b..10ba1c180f6f 100644 --- a/nixos/doc/manual/release-notes/rl-2003.xml +++ b/nixos/doc/manual/release-notes/rl-2003.xml @@ -315,6 +315,18 @@ services.xserver.displayManager.defaultSession = "xfce+icewm"; + + + The awstats module has been rewritten + to serve stats via static html pages, updated on a timer, over nginx, + instead of dynamic cgi pages over apache. + + + Minor changes will be required to migrate existing configurations. Details of the + required changes can seen by looking through the awstats + module. + + diff --git a/nixos/modules/services/logging/awstats.nix b/nixos/modules/services/logging/awstats.nix index a92ff3bee490..d51c9a5cffab 100644 --- a/nixos/modules/services/logging/awstats.nix +++ b/nixos/modules/services/logging/awstats.nix @@ -4,31 +4,116 @@ with lib; let cfg = config.services.awstats; - httpd = config.services.httpd; package = pkgs.awstats; -in + configOpts = {name, config, ...}: { + options = { + type = mkOption{ + type = types.enum [ "mail" "web" ]; + default = "web"; + example = "mail"; + description = '' + The type of log being collected. + ''; + }; + domain = mkOption { + type = types.str; + default = name; + description = "The domain name to collect stats for."; + example = "example.com"; + }; -{ - options.services.awstats = { - enable = mkOption { - type = types.bool; - default = cfg.service.enable; - description = '' - Enable the awstats program (but not service). - Currently only simple httpd (Apache) configs are supported, - and awstats plugins may not work correctly. - ''; + logFile = mkOption { + type = types.str; + example = "/var/spool/nginx/logs/access.log"; + description = '' + The log file to be scanned. + + For mail, set this to + + journalctl $OLD_CURSOR -u postfix.service | ''${pkgs.perl}/bin/perl ''${pkgs.awstats.out}/share/awstats/tools/maillogconvert.pl standard | + + ''; + }; + + logFormat = mkOption { + type = types.str; + default = "1"; + description = '' + The log format being used. + + For mail, set this to + + %time2 %email %email_r %host %host_r %method %url %code %bytesd + + ''; + }; + + hostAliases = mkOption { + type = types.listOf types.str; + default = []; + example = "[ \"www.example.org\" ]"; + description = '' + List of aliases the site has. + ''; + }; + + extraConfig = mkOption { + type = types.attrsOf types.str; + default = {}; + example = literalExample '' + { + "ValidHTTPCodes" = "404"; + } + ''; + }; + + webService = { + enable = mkEnableOption "awstats web service"; + + hostname = mkOption { + type = types.str; + default = config.domain; + description = "The hostname the web service appears under."; + }; + + urlPrefix = mkOption { + type = types.str; + default = "/awstats"; + description = "The URL prefix under which the awstats pages appear."; + }; + }; }; - vardir = mkOption { + }; + webServices = filterAttrs (name: value: value.webService.enable) cfg.configs; +in +{ + imports = [ + (mkRemovedOptionModule [ "services" "awstats" "service" "enable" ] "Please enable per domain with `services.awstats.configs..webService.enable`") + (mkRemovedOptionModule [ "services" "awstats" "service" "urlPrefix" ] "Please set per domain with `services.awstats.configs..webService.urlPrefix`") + (mkRenamedOptionModule [ "services" "awstats" "vardir" ] [ "services" "awstats" "dataDir" ]) + ]; + + options.services.awstats = { + enable = mkEnableOption "awstats"; + + dataDir = mkOption { type = types.path; default = "/var/lib/awstats"; - description = "The directory where variable awstats data will be stored."; + description = "The directory where awstats data will be stored."; }; - extraConfig = mkOption { - type = types.lines; - default = ""; - description = "Extra configuration to be appendend to awstats.conf."; + configs = mkOption { + type = types.attrsOf (types.submodule configOpts); + default = {}; + example = literalExample '' + { + "mysite" = { + domain = "example.com"; + logFile = "/var/spool/nginx/logs/access.log"; + }; + } + ''; + description = "Attribute set of domains to collect stats for."; }; updateAt = mkOption { @@ -42,75 +127,129 @@ in 7) ''; }; - - service = { - enable = mkOption { - type = types.bool; - default = false; - description = ''Enable the awstats web service. This switches on httpd.''; - }; - urlPrefix = mkOption { - type = types.str; - default = "/awstats"; - description = "The URL prefix under which the awstats service appears."; - }; - }; }; config = mkIf cfg.enable { environment.systemPackages = [ package.bin ]; - /* TODO: - - heed config.services.httpd.logPerVirtualHost, etc. - - Can't AllowToUpdateStatsFromBrowser, as CGI scripts don't have permission - to read the logs, and our httpd config apparently doesn't an option for that. - */ - environment.etc."awstats/awstats.conf".source = pkgs.runCommand "awstats.conf" + + environment.etc = mapAttrs' (name: opts: + nameValuePair "awstats/awstats.${name}.conf" { + source = pkgs.runCommand "awstats.${name}.conf" { preferLocalBuild = true; } - ( let - logFormat = - if httpd.logFormat == "combined" then "1" else - if httpd.logFormat == "common" then "4" else - throw "awstats service doesn't support Apache log format `${httpd.logFormat}`"; - in + ('' + sed \ + '' + # set up mail stats + + optionalString (opts.type == "mail") + '' + -e 's|^\(LogType\)=.*$|\1=M|' \ + -e 's|^\(LevelForBrowsersDetection\)=.*$|\1=0|' \ + -e 's|^\(LevelForOSDetection\)=.*$|\1=0|' \ + -e 's|^\(LevelForRefererAnalyze\)=.*$|\1=0|' \ + -e 's|^\(LevelForRobotsDetection\)=.*$|\1=0|' \ + -e 's|^\(LevelForSearchEnginesDetection\)=.*$|\1=0|' \ + -e 's|^\(LevelForFileTypesDetection\)=.*$|\1=0|' \ + -e 's|^\(LevelForWormsDetection\)=.*$|\1=0|' \ + -e 's|^\(ShowMenu\)=.*$|\1=1|' \ + -e 's|^\(ShowSummary\)=.*$|\1=HB|' \ + -e 's|^\(ShowMonthStats\)=.*$|\1=HB|' \ + -e 's|^\(ShowDaysOfMonthStats\)=.*$|\1=HB|' \ + -e 's|^\(ShowDaysOfWeekStats\)=.*$|\1=HB|' \ + -e 's|^\(ShowHoursStats\)=.*$|\1=HB|' \ + -e 's|^\(ShowDomainsStats\)=.*$|\1=0|' \ + -e 's|^\(ShowHostsStats\)=.*$|\1=HB|' \ + -e 's|^\(ShowAuthenticatedUsers\)=.*$|\1=0|' \ + -e 's|^\(ShowRobotsStats\)=.*$|\1=0|' \ + -e 's|^\(ShowEMailSenders\)=.*$|\1=HBML|' \ + -e 's|^\(ShowEMailReceivers\)=.*$|\1=HBML|' \ + -e 's|^\(ShowSessionsStats\)=.*$|\1=0|' \ + -e 's|^\(ShowPagesStats\)=.*$|\1=0|' \ + -e 's|^\(ShowFileTypesStats\)=.*$|\1=0|' \ + -e 's|^\(ShowFileSizesStats\)=.*$|\1=0|' \ + -e 's|^\(ShowBrowsersStats\)=.*$|\1=0|' \ + -e 's|^\(ShowOSStats\)=.*$|\1=0|' \ + -e 's|^\(ShowOriginStats\)=.*$|\1=0|' \ + -e 's|^\(ShowKeyphrasesStats\)=.*$|\1=0|' \ + -e 's|^\(ShowKeywordsStats\)=.*$|\1=0|' \ + -e 's|^\(ShowMiscStats\)=.*$|\1=0|' \ + -e 's|^\(ShowHTTPErrorsStats\)=.*$|\1=0|' \ + -e 's|^\(ShowSMTPErrorsStats\)=.*$|\1=1|' \ + '' + + + # common options + '' + -e 's|^\(DirData\)=.*$|\1="${cfg.dataDir}/${name}"|' \ + -e 's|^\(DirIcons\)=.*$|\1="icons"|' \ + -e 's|^\(CreateDirDataIfNotExists\)=.*$|\1=1|' \ + -e 's|^\(SiteDomain\)=.*$|\1="${name}"|' \ + -e 's|^\(LogFile\)=.*$|\1="${opts.logFile}"|' \ + -e 's|^\(LogFormat\)=.*$|\1="${opts.logFormat}"|' \ + '' + + + # extra config + concatStringsSep "\n" (mapAttrsToList (n: v: '' + -e 's|^\(${n}\)=.*$|\1="${v}"|' \ + '') opts.extraConfig) + + + '' + < '${package.out}/wwwroot/cgi-bin/awstats.model.conf' > "$out" + ''); + }) cfg.configs; + + # create data directory with the correct permissions + systemd.tmpfiles.rules = + [ "d '${cfg.dataDir}' 755 root root - -" ] ++ + mapAttrsToList (name: opts: "d '${cfg.dataDir}/${name}' 755 root root - -") cfg.configs ++ + [ "Z '${cfg.dataDir}' 755 root root - -" ]; + + # nginx options + services.nginx.virtualHosts = mapAttrs'(name: opts: { + name = opts.webService.hostname; + value = { + locations = { + "${opts.webService.urlPrefix}/css/" = { + alias = "${package.out}/wwwroot/css/"; + }; + "${opts.webService.urlPrefix}/icons/" = { + alias = "${package.out}/wwwroot/icon/"; + }; + "${opts.webService.urlPrefix}/" = { + alias = "${cfg.dataDir}/${name}/"; + extraConfig = '' + autoindex on; + ''; + }; + }; + }; + }) webServices; + + # update awstats + systemd.services = mkIf (cfg.updateAt != null) (mapAttrs' (name: opts: + nameValuePair "awstats-${name}-update" { + description = "update awstats for ${name}"; + script = optionalString (opts.type == "mail") '' - sed \ - -e 's|^\(DirData\)=.*$|\1="${cfg.vardir}"|' \ - -e 's|^\(DirIcons\)=.*$|\1="icons"|' \ - -e 's|^\(CreateDirDataIfNotExists\)=.*$|\1=1|' \ - -e 's|^\(SiteDomain\)=.*$|\1="${httpd.hostName}"|' \ - -e 's|^\(LogFile\)=.*$|\1="${httpd.logDir}/access_log"|' \ - -e 's|^\(LogFormat\)=.*$|\1=${logFormat}|' \ - < '${package.out}/wwwroot/cgi-bin/awstats.model.conf' > "$out" - echo '${cfg.extraConfig}' >> "$out" - ''); - - systemd.tmpfiles.rules = optionals cfg.service.enable [ - "d '${cfg.vardir}' - ${httpd.user} ${httpd.group} - -" - "Z '${cfg.vardir}' - ${httpd.user} ${httpd.group} - -" - ]; - - # The httpd sub-service showing awstats. - services.httpd = optionalAttrs cfg.service.enable { - enable = true; - extraConfig = '' - Alias ${cfg.service.urlPrefix}/classes "${package.out}/wwwroot/classes/" - Alias ${cfg.service.urlPrefix}/css "${package.out}/wwwroot/css/" - Alias ${cfg.service.urlPrefix}/icons "${package.out}/wwwroot/icon/" - ScriptAlias ${cfg.service.urlPrefix}/ "${package.out}/wwwroot/cgi-bin/" - - - Options None - Require all granted - - ''; - }; - - systemd.services.awstats-update = mkIf (cfg.updateAt != null) { - description = "awstats log collector"; - script = "exec '${package.bin}/bin/awstats' -update -config=awstats.conf"; - startAt = cfg.updateAt; - }; + if [[ -f "${cfg.dataDir}/${name}-cursor" ]]; then + CURSOR="$(cat "${cfg.dataDir}/${name}-cursor" | tr -d '\n')" + if [[ -n "$CURSOR" ]]; then + echo "Using cursor: $CURSOR" + export OLD_CURSOR="--cursor $CURSOR" + fi + fi + NEW_CURSOR="$(journalctl $OLD_CURSOR -u postfix.service --show-cursor | tail -n 1 | tr -d '\n' | sed -e 's#^-- cursor: \(.*\)#\1#')" + echo "New cursor: $NEW_CURSOR" + ${package.bin}/bin/awstats -update -config=${name} + if [ -n "$NEW_CURSOR" ]; then + echo -n "$NEW_CURSOR" > ${cfg.dataDir}/${name}-cursor + fi + '' + '' + ${package.out}/share/awstats/tools/awstats_buildstaticpages.pl \ + -config=${name} -update -dir=${cfg.dataDir}/${name} \ + -awstatsprog=${package.bin}/bin/awstats + ''; + startAt = cfg.updateAt; + }) cfg.configs); }; }