beegfs: init at 6.17

package, kernel module, nixos module, and nixos test

nixos/modules/module-list.nix

@@ -404,6 +404,7 @@
   ./services/monitoring/vnstat.nix
   ./services/monitoring/zabbix-agent.nix
   ./services/monitoring/zabbix-server.nix
+  ./services/network-filesystems/beegfs.nix
   ./services/network-filesystems/cachefilesd.nix
   ./services/network-filesystems/davfs2.nix
   ./services/network-filesystems/drbd.nix

nixos/modules/services/network-filesystems/beegfs.nix

@@ -0,0 +1,343 @@
+{ config, lib, pkgs, ...} :
+
+with lib;
+
+let
+  cfg = config.services.beegfs;
+
+  # functions for the generations of config files
+
+  configMgmtd = name: cfg: pkgs.writeText "mgmt-${name}.conf" ''
+    storeMgmtdDirectory = ${cfg.mgmtd.storeDir}
+    storeAllowFirstRunInit = false
+    connAuthFile = ${cfg.connAuthFile}
+    connPortShift = ${toString cfg.connPortShift}
+
+    ${cfg.mgmtd.extraConfig}
+  '';
+
+  configAdmon = name: cfg: pkgs.writeText "admon-${name}.conf" ''
+    sysMgmtdHost = ${cfg.mgmtdHost}
+    connAuthFile = ${cfg.connAuthFile}
+    connPortShift = ${toString cfg.connPortShift}
+
+    ${cfg.admon.extraConfig}
+  '';
+
+  configMeta = name: cfg: pkgs.writeText "meta-${name}.conf" ''
+    storeMetaDirectory = ${cfg.meta.storeDir}
+    sysMgmtdHost = ${cfg.mgmtdHost}
+    connAuthFile = ${cfg.connAuthFile}
+    connPortShift = ${toString cfg.connPortShift}
+    storeAllowFirstRunInit = false
+
+    ${cfg.mgmtd.extraConfig}
+  '';
+
+  configStorage = name: cfg: pkgs.writeText "storage-${name}.conf" ''
+    storeStorageDirectory = ${cfg.storage.storeDir}
+    sysMgmtdHost = ${cfg.mgmtdHost}
+    connAuthFile = ${cfg.connAuthFile}
+    connPortShift = ${toString cfg.connPortShift}
+    storeAllowFirstRunInit = false
+
+    ${cfg.storage.extraConfig}
+  '';
+
+  configHelperd = name: cfg: pkgs.writeText "helperd-${name}.conf" ''
+    connAuthFile = ${cfg.connAuthFile}
+    ${cfg.helperd.extraConfig}
+  '';
+
+  configClientFilename = name : "/etc/beegfs/client-${name}.conf";
+
+  configClient = name: cfg: ''
+    sysMgmtdHost = ${cfg.mgmtdHost}
+    connAuthFile = ${cfg.connAuthFile}
+    connPortShift = ${toString cfg.connPortShift}
+
+    ${cfg.client.extraConfig}
+  '';
+
+  serviceList = [
+    { service = "admon"; cfgFile = configAdmon; }
+    { service = "meta"; cfgFile = configMeta; }
+    { service = "mgmtd"; cfgFile = configMgmtd; }
+    { service = "storage"; cfgFile = configStorage; }
+  ];
+
+  # functions to generate systemd.service entries
+
+  systemdEntry = service: cfgFile: (mapAttrs' ( name: cfg:
+    (nameValuePair "beegfs-${service}-${name}" (mkIf cfg."${service}".enable {
+    wantedBy = [ "multi-user.target" ];
+    requires = [ "network-online.target" ];
+    after = [ "network-online.target" ];
+    serviceConfig = rec {
+      ExecStart = ''
+        ${pkgs.beegfs}/bin/beegfs-${service} \
+          cfgFile=${cfgFile name cfg} \
+          pidFile=${PIDFile}
+      '';
+      PIDFile = "/run/beegfs-${service}-${name}.pid";
+      TimeoutStopSec = "300";
+    };
+  }))) cfg);
+
+  systemdHelperd =  mapAttrs' ( name: cfg:
+    (nameValuePair "beegfs-helperd-${name}" (mkIf cfg.client.enable {
+    wantedBy = [ "multi-user.target" ];
+    requires = [ "network-online.target" ];
+    after = [ "network-online.target" ];
+    serviceConfig = rec {
+      ExecStart = ''
+        ${pkgs.beegfs}/bin/beegfs-helperd \
+          cfgFile=${configHelperd name cfg} \
+          pidFile=${PIDFile}
+      '';
+      PIDFile = "/run/beegfs-helperd-${name}.pid";
+      TimeoutStopSec = "300";
+    };
+   }))) cfg;
+
+  # wrappers to beegfs tools. Avoid typing path of config files
+  utilWrappers = mapAttrsToList ( name: cfg:
+      ( pkgs.runCommand "beegfs-utils-${name}" { nativeBuildInputs = [ pkgs.makeWrapper ]; } ''
+        mkdir -p $out/bin
+
+        makeWrapper ${pkgs.beegfs}/bin/beegfs-check-servers \
+                    $out/bin/beegfs-check-servers-${name} \
+                    --add-flags "-c ${configClientFilename name}" \
+                    --prefix PATH : ${lib.makeBinPath [ pkgs.beegfs ]}
+
+        makeWrapper ${pkgs.beegfs}/bin/beegfs-ctl \
+                    $out/bin/beegfs-ctl-${name} \
+                    --add-flags "--cfgFile=${configClientFilename name}"
+
+        makeWrapper ${pkgs.beegfs}/bin/beegfs-ctl \
+                    $out/bin/beegfs-df-${name} \
+                    --add-flags "--cfgFile=${configClientFilename name}" \
+                    --add-flags --listtargets  \
+                    --add-flags --hidenodeid \
+                    --add-flags --pools \
+                    --add-flags --spaceinfo
+
+        makeWrapper ${pkgs.beegfs}/bin/beegfs-fsck \
+                    $out/bin/beegfs-fsck-${name} \
+                    --add-flags "--cfgFile=${configClientFilename name}"
+      ''
+     )) cfg;
+in
+{
+  ###### interface
+
+  options = {
+    services.beegfsEnable = mkEnableOption "BeeGFS";
+
+    services.beegfs = mkOption {
+      default = {};
+      description = ''
+        BeeGFS configurations. Every mount point requires a separate configuration.
+      '';
+      type = with types; attrsOf (submodule ({ config, ... } : {
+        options = {
+          mgmtdHost = mkOption {
+            type = types.str;
+            default = null;
+            example = "master";
+            description = ''Hostname of managament host.'';
+          };
+
+          connAuthFile = mkOption {
+            type = types.str;
+            default = "";
+            example = "/etc/my.key";
+            description = "File containing shared secret authentication.";
+          };
+
+          connPortShift = mkOption {
+            type = types.int;
+            default = 0;
+            example = 5;
+            description = ''
+              For each additional beegfs configuration shift all
+              service TCP/UDP ports by at least 5.
+            '';
+          };
+
+          client = {
+            enable = mkEnableOption "BeeGFS client";
+
+            mount = mkOption {
+              type = types.bool;
+              default = true;
+              description = "Create fstab entry automatically";
+            };
+
+            mountPoint = mkOption {
+              type = types.str;
+              default = "/run/beegfs";
+              description = ''
+                Mount point under which the beegfs filesytem should be mounted.
+                If mounted manually the mount option specifing the config file is needed:
+                cfgFile=/etc/beegfs/beegfs-client-<name>.conf
+              '';
+            };
+
+            extraConfig = mkOption {
+              type = types.lines;
+              default = "";
+              description = ''
+                Additional lines for beegfs-client.conf.
+                See documentation for further details.
+             '';
+            };
+          };
+
+          helperd = {
+            extraConfig = mkOption {
+              type = types.lines;
+              default = "";
+              description = ''
+                Additional lines for beegfs-helperd.conf. See documentation
+                for further details.
+              '';
+            };
+          };
+
+          mgmtd = {
+            enable = mkEnableOption "BeeGFS mgmtd daemon";
+
+            storeDir = mkOption {
+              type = types.path;
+              default = null;
+              example = "/data/beegfs-mgmtd";
+              description = ''
+                Data directory for mgmtd.
+                Must not be shared with other beegfs daemons.
+                This directory must exist and it must be initialized
+                with beegfs-setup-mgmtd, e.g. "beegfs-setup-mgmtd -C -p <storeDir>"
+              '';
+            };
+
+            extraConfig = mkOption {
+              type = types.lines;
+              default = "";
+              description = ''
+                Additional lines for beegfs-mgmtd.conf. See documentation
+                for further details.
+              '';
+            };
+          };
+
+          admon = {
+            enable = mkEnableOption "BeeGFS admon daemon";
+
+            extraConfig = mkOption {
+              type = types.lines;
+              default = "";
+              description = ''
+                Additional lines for beegfs-admon.conf. See documentation
+                for further details.
+              '';
+            };
+          };
+
+          meta = {
+            enable = mkEnableOption "BeeGFS meta data daemon";
+
+            storeDir = mkOption {
+              type = types.path;
+              default = null;
+              example = "/data/beegfs-meta";
+              description = ''
+                Data directory for meta data service.
+                Must not be shared with other beegfs daemons.
+                The underlying filesystem must be mounted with xattr turned on.
+                This directory must exist and it must be initialized
+                with beegfs-setup-meta, e.g.
+                "beegfs-setup-meta -C -s <serviceID> -p <storeDir>"
+              '';
+            };
+
+            extraConfig = mkOption {
+              type = types.str;
+              default = "";
+              description = ''
+                Additional lines for beegfs-meta.conf. See documentation
+                for further details.
+              '';
+            };
+          };
+
+          storage = {
+            enable = mkEnableOption "BeeGFS storage daemon";
+
+            storeDir = mkOption {
+              type = types.path;
+              default = null;
+              example = "/data/beegfs-storage";
+              description = ''
+                Data directories for storage service.
+                Must not be shared with other beegfs daemons.
+                The underlying filesystem must be mounted with xattr turned on.
+                This directory must exist and it must be initialized
+                with beegfs-setup-storage, e.g.
+                "beegfs-setup-storage -C -s <serviceID> -i <storageTargetID> -p <storeDir>"
+              '';
+            };
+
+            extraConfig = mkOption {
+              type = types.str;
+              default = "";
+              description = ''
+                Addional lines for beegfs-storage.conf. See documentation
+                for further details.
+              '';
+            };
+          };
+        };
+      }));
+    };
+  };
+
+  ###### implementation
+
+  config =
+    mkIf config.services.beegfsEnable {
+
+    environment.systemPackages = utilWrappers;
+
+    # Put the client.conf files in /etc since they are needed
+    # by the commandline tools
+    environment.etc = mapAttrs' ( name: cfg:
+      (nameValuePair "beegfs/client-${name}.conf" (mkIf (cfg.client.enable)
+    {
+      enable = true;
+      text = configClient name cfg;
+    }))) cfg;
+
+    # Kernel module, we need it only once per host.
+    boot = mkIf (
+      foldr (a: b: a || b) false
+        (map (x: x.client.enable) (collect (x: x ? client) cfg)))
+    {
+      kernelModules = [ "beegfs" ];
+      extraModulePackages = [ pkgs.linuxPackages.beegfs-module ];
+    };
+
+    # generate fstab entries
+    fileSystems = mapAttrs' (name: cfg:
+      (nameValuePair cfg.client.mountPoint (optionalAttrs cfg.client.mount (mkIf cfg.client.enable {
+      device = "beegfs_nodev";
+      fsType = "beegfs";
+      mountPoint = cfg.client.mountPoint;
+      options = [ "cfgFile=${configClientFilename name}" "_netdev" ];
+    })))) cfg;
+
+    # generate systemd services
+    systemd.services = systemdHelperd //
+      foldr (a: b: a // b) {}
+        (map (x: systemdEntry x.service x.cfgFile) serviceList);
+  };
+}

nixos/release.nix

@@ -232,6 +232,7 @@ in rec {
   tests.atd = callTest tests/atd.nix {};
   tests.acme = callTest tests/acme.nix {};
   tests.avahi = callTest tests/avahi.nix {};
+  tests.beegfs = callTest tests/beegfs.nix {};
   tests.bittorrent = callTest tests/bittorrent.nix {};
   tests.blivet = callTest tests/blivet.nix {};
   tests.boot = callSubTests tests/boot.nix {};

nixos/tests/beegfs.nix

@@ -0,0 +1,115 @@
+import ./make-test.nix ({ pkgs, ... } :
+
+let
+  connAuthFile="beegfs/auth-def.key";
+
+  client = { config, pkgs, lib, ... } : {
+    networking.firewall.enable = false;
+    services.beegfsEnable = true;
+    services.beegfs.default = {
+      mgmtdHost = "mgmt";
+      connAuthFile = "/etc/${connAuthFile}";
+      client = {
+        mount = false;
+        enable = true;
+      };
+    };
+
+    fileSystems = pkgs.lib.mkVMOverride # FIXME: this should be creatd by the module
+      [ { mountPoint = "/beegfs";
+          device = "default";
+          fsType = "beegfs";
+          options = [ "cfgFile=/etc/beegfs/client-default.conf" "_netdev" ];
+        }
+      ];
+
+    environment.etc."${connAuthFile}" = {
+      enable = true;
+      text = "ThisIsALousySecret";
+      mode = "0600";
+    };
+  };
+
+
+  server = service : { config, pkgs, lib, ... } : {
+    networking.firewall.enable = false;
+    boot.initrd.postDeviceCommands = ''
+      ${pkgs.e2fsprogs}/bin/mkfs.ext4 -L data /dev/vdb
+    '';
+
+    virtualisation.emptyDiskImages = [ 4096 ];
+
+    fileSystems = pkgs.lib.mkVMOverride
+      [ { mountPoint = "/data";
+          device = "/dev/disk/by-label/data";
+          fsType = "ext4";
+        }
+      ];
+
+    environment.systemPackages = with pkgs; [ beegfs ];
+    environment.etc."${connAuthFile}" = {
+      enable = true;
+      text = "ThisIsALousySecret";
+      mode = "0600";
+    };
+
+    services.beegfsEnable = true;
+    services.beegfs.default = {
+      mgmtdHost = "mgmt";
+      connAuthFile = "/etc/${connAuthFile}";
+      "${service}" = {
+        enable = true;
+        storeDir = "/data";
+      };
+    };
+  };
+
+in
+{
+  name = "beegfs";
+
+  nodes = {
+    meta = server "meta";
+    mgmt = server "mgmtd";
+    storage1 = server "storage";
+    storage2 = server "storage";
+    client1 = client;
+    client2 = client;
+  };
+
+  testScript = ''
+    # Initalize the data directories
+    $mgmt->waitForUnit("default.target");
+    $mgmt->succeed("beegfs-setup-mgmtd -C -f -p /data");
+    $mgmt->succeed("systemctl start beegfs-mgmtd-default");
+
+    $meta->waitForUnit("default.target");
+    $meta->succeed("beegfs-setup-meta -C -f -s 1 -p /data");
+    $meta->succeed("systemctl start beegfs-meta-default");
+
+    $storage1->waitForUnit("default.target");
+    $storage1->succeed("beegfs-setup-storage -C -f -s 1 -i 1 -p /data");
+    $storage1->succeed("systemctl start beegfs-storage-default");
+
+    $storage2->waitForUnit("default.target");
+    $storage2->succeed("beegfs-setup-storage -C -f -s 2 -i 2 -p /data");
+    $storage2->succeed("systemctl start beegfs-storage-default");
+
+    #
+
+    # Basic test
+    $client1->waitForUnit("beegfs.mount");
+    $client1->succeed("beegfs-check-servers-default");
+    $client1->succeed("echo test > /beegfs/test");
+    $client2->waitForUnit("beegfs.mount");
+    $client2->succeed("test -e /beegfs/test");
+    $client2->succeed("cat /beegfs/test | grep test");
+
+    # test raid0/stripping
+    $client1->succeed("dd if=/dev/urandom bs=1M count=10 of=/beegfs/striped");
+    $client2->succeed("cat /beegfs/striped > /dev/null");
+
+    # check if fs is still healthy
+    $client1->succeed("beegfs-fsck-default --checkfs");
+  '';
+})

pkgs/os-specific/linux/beegfs/default.nix

@@ -0,0 +1,122 @@
+{ stdenv, fetchurl, pkgconfig, unzip, which
+, libuuid, attr, xfsprogs, cppunit
+, zlib, openssl, sqlite, jre, openjdk, ant
+} :
+
+let
+  version = "6.17";
+
+  subdirs = [
+    "beegfs_thirdparty/build"
+    "beegfs_opentk_lib/build"
+    "beegfs_common/build"
+    "beegfs_admon/build"
+    "beegfs_java_lib/build"
+    "beegfs_ctl/build"
+    "beegfs_fsck/build"
+    "beegfs_helperd/build"
+    "beegfs_meta/build"
+    "beegfs_mgmtd/build"
+    "beegfs_online_cfg/build"
+    "beegfs_storage/build"
+    "beegfs_utils/build"
+  ];
+
+in stdenv.mkDerivation rec {
+  name = "beegfs-${version}";
+
+  src = fetchurl {
+    url = "https://git.beegfs.com/pub/v6/repository/archive.tar.bz2?ref=${version}";
+    sha256 = "10xs7gzdmlg23k6zn1b7jij3lljn7rr1j6h476hq4lbg981qk3n3";
+  };
+
+  nativeBuildInputs = [ which unzip pkgconfig cppunit openjdk ant];
+  buildInputs = [ libuuid attr xfsprogs zlib openssl sqlite jre ];
+
+  postPatch = ''
+    patchShebangs ./
+    find -type f -name Makefile -exec sed -i "s:/bin/bash:${stdenv.shell}:" \{} \;
+    find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \;
+    find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \;
+  '';
+
+  buildPhase = ''
+    for i in ${toString subdirs}; do
+      make -C $i
+    done
+    make -C beegfs_admon/build admon_gui
+  '';
+
+  installPhase = ''
+    binDir=$out/bin
+    docDir=$out/share/doc/beegfs
+    includeDir=$out/include/beegfs
+    libDir=$out/lib
+    libDirPkg=$out/lib/beegfs
+
+    mkdir -p $binDir $libDir $libDirPkg $docDir $includeDir
+
+    cp beegfs_admon/build/beegfs-admon $binDir
+    cp beegfs_admon/build/dist/usr/bin/beegfs-admon-gui $binDir
+    cp beegfs_admon_gui/dist/beegfs-admon-gui.jar $libDirPkg
+    cp beegfs_admon/build/dist/etc/beegfs-admon.conf $docDir
+
+    cp beegfs_java_lib/build/jbeegfs.jar $libDirPkg
+    cp beegfs_java_lib/build/libjbeegfs.so $libDir
+
+    cp beegfs_ctl/build/beegfs-ctl $binDir
+    cp beegfs_fsck/build/beegfs-fsck $binDir
+
+    cp beegfs_utils/scripts/beegfs-check-servers $binDir
+    cp beegfs_utils/scripts/beegfs-df $binDir
+    cp beegfs_utils/scripts/beegfs-net $binDir
+
+    cp beegfs_helperd/build/beegfs-helperd $binDir
+    cp beegfs_helperd/build/dist/etc/beegfs-helperd.conf $docDir
+
+    cp beegfs_client_module/build/dist/sbin/beegfs-setup-client $binDir
+    cp beegfs_client_module/build/dist/etc/beegfs-client.conf $docDir
+
+    cp beegfs_meta/build/beegfs-meta $binDir
+    cp beegfs_meta/build/dist/sbin/beegfs-setup-meta $binDir
+    cp beegfs_meta/build/dist/etc/beegfs-meta.conf $docDir
+
+    cp beegfs_mgmtd/build/beegfs-mgmtd $binDir
+    cp beegfs_mgmtd/build/dist/sbin/beegfs-setup-mgmtd $binDir
+    cp beegfs_mgmtd/build/dist/etc/beegfs-mgmtd.conf $docDir
+
+    cp beegfs_storage/build/beegfs-storage $binDir
+    cp beegfs_storage/build/dist/sbin/beegfs-setup-storage $binDir
+    cp beegfs_storage/build/dist/etc/beegfs-storage.conf $docDir
+
+    cp beegfs_opentk_lib/build/libbeegfs-opentk.so $libDir
+
+    cp beegfs_client_devel/build/dist/usr/share/doc/beegfs-client-devel/examples/* $docDir
+    cp -r beegfs_client_devel/include/* $includeDir
+  '';
+
+  postFixup = ''
+    substituteInPlace $out/bin/beegfs-admon-gui \
+      --replace " java " " ${jre}/bin/java " \
+      --replace "/opt/beegfs/beegfs-admon-gui/beegfs-admon-gui.jar" \
+                "$libDirPkg/beegfs-admon-gui.jar"
+  '';
+
+  doCheck = true;
+
+  checkPhase = ''
+    beegfs_common/build/test-runner --text
+  '';
+
+  meta = with stdenv.lib; {
+    description = "High performance distributed filesystem with RDMA support";
+    homepage = "https://www.beegfs.io";
+    platforms = [ "i686-linux" "x86_64-linux" ];
+    license = {
+      fullName = "BeeGFS_EULA";
+      url = "https://www.beegfs.io/docs/BeeGFS_EULA.txt";
+      free = false;
+    };
+    maintainers = with maintainers; [ markuskowa ];
+  };
+}

pkgs/os-specific/linux/beegfs/kernel-module.nix

@@ -0,0 +1,43 @@
+{ stdenv, fetchurl, which
+, kmod, kernel
+} :
+
+let
+  version = "6.17";
+in stdenv.mkDerivation {
+  name = "beegfs-module-${version}-${kernel.version}";
+
+  src = fetchurl {
+    url = "https://git.beegfs.com/pub/v6/repository/archive.tar.bz2?ref=${version}";
+    sha256 = "10xs7gzdmlg23k6zn1b7jij3lljn7rr1j6h476hq4lbg981qk3n3";
+  };
+
+  hardeningDisable = [ "fortify" "pic" "stackprotector" ];
+
+  nativeBuildInputs = [ which kmod ];
+
+  makeFlags = [ "KDIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build/" ];
+
+  postPatch = ''
+    patchShebangs ./
+    find -type f -name Makefile -exec sed -i "s:/bin/bash:${stdenv.shell}:" \{} \;
+    find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \;
+    find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \;
+  '';
+
+  preBuild = "cd beegfs_client_module/build";
+
+  installPhase = ''
+    instdir=$out/lib/modules/${kernel.modDirVersion}/extras/fs/beegfs
+    mkdir -p $instdir
+    cp beegfs.ko $instdir
+  '';
+
+  meta = with stdenv.lib; {
+    description = "High performance distributed filesystem with RDMA support";
+    homepage = "https://www.beegfs.io";
+    platforms = [ "i686-linux" "x86_64-linux" ];
+    license = licenses.gpl2;
+    maintainers = with maintainers; [ markuskowa ];
+  };
+}

pkgs/top-level/all-packages.nix

@@ -1338,6 +1338,8 @@ with pkgs;
 
   beanstalkd = callPackage ../servers/beanstalkd { };
 
+  beegfs = callPackage ../os-specific/linux/beegfs { };
+
   beets = callPackage ../tools/audio/beets {
     pythonPackages = python2Packages;
   };
@@ -12821,6 +12823,8 @@ with pkgs;
 
     bbswitch = callPackage ../os-specific/linux/bbswitch {};
 
+    beegfs-module = callPackage ../os-specific/linux/beegfs/kernel-module.nix { };
+
     ati_drivers_x11 = callPackage ../os-specific/linux/ati-drivers { };
 
     blcr = callPackage ../os-specific/linux/blcr { };