Created
March 29, 2024 08:31
-
-
Save juselius/7c1584c75530bc85cec31c3d13156a75 to your computer and use it in GitHub Desktop.
NixOS BeeGFS 7.4.0
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ pkgs, kernel ? pkgs.linux, ... } : | |
with pkgs; | |
let | |
version = "7.4.0"; | |
in stdenvNoCC.mkDerivation { | |
pname = "beegfs"; | |
inherit version; | |
src = fetchurl { | |
name = "beegfs-archive-${version}.tar.bz2"; | |
# url = "https://git.beegfs.com/pub/v7/repository/archive.tar.bz2?ref=${version}"; | |
url = "https://git.beegfs.io/pub/v7/-/archive/${version}/v7-${version}.tar.bz2"; | |
sha256 = "sha256-VwD3z3lZIs5aOIBbwUvEkOxkFggTCv8OWuJMCga2ooo="; | |
}; | |
nativeBuildInputs = [ which unzip pkg-config cppunit perl makeWrapper ]; | |
buildInputs = [ | |
gcc12 | |
libuuid | |
attr | |
xfsprogs | |
zlib | |
openssl | |
sqlite | |
rdma-core | |
openssh | |
gfortran | |
influxdb | |
curl | |
rdma-core | |
pahole | |
]; | |
hardeningDisable = [ "format" ]; # required for building beeond | |
postPatch = '' | |
patchShebangs ./ | |
find -type f -name Makefile -exec sed -i "s:/bin/bash:${stdenv.shell}:" \{} \; | |
find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \; | |
find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \; | |
''; | |
buildPhase = '' | |
make BEEGFS_OPENTK_IBVERBS=1 \ | |
KDIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build \ | |
''${enableParallelBuilding:+-j''${NIX_BUILD_CORES} \ | |
-l''${NIX_BUILD_CORES}} | |
''; | |
enableParallelBuilding = true; | |
installPhase = '' | |
binDir=$out/bin | |
docDir=$out/share/doc/beegfs | |
includeDir=$out/include/beegfs | |
libDir=$out/lib | |
libDirPkg=$out/lib/beegfs | |
mkdir -p $binDir $libDir $libDirPkg $docDir $includeDir | |
cp common/build/libbeegfs_ib.so $libDir | |
cp ctl/build/beegfs-ctl $binDir | |
cp fsck/build/beegfs-fsck $binDir | |
cp utils/scripts/beegfs-check-servers $binDir | |
cp utils/scripts/beegfs-df $binDir | |
cp utils/scripts/beegfs-net $binDir | |
cp helperd/build/beegfs-helperd $binDir | |
cp helperd/build/dist/etc/beegfs-helperd.conf $docDir | |
cp client_module/build/dist/sbin/beegfs-setup-client $binDir | |
cp client_module/build/dist/etc/beegfs-client.conf $docDir | |
cp meta/build/beegfs-meta $binDir | |
cp meta/build/dist/sbin/beegfs-setup-meta $binDir | |
cp meta/build/dist/etc/beegfs-meta.conf $docDir | |
cp mgmtd/build/beegfs-mgmtd $binDir | |
cp mgmtd/build/dist/sbin/beegfs-setup-mgmtd $binDir | |
cp mgmtd/build/dist/etc/beegfs-mgmtd.conf $docDir | |
cp storage/build/beegfs-storage $binDir | |
cp storage/build/dist/sbin/beegfs-setup-storage $binDir | |
cp storage/build/dist/etc/beegfs-storage.conf $docDir | |
cp client_devel/build/dist/usr/share/doc/beegfs-client-devel/examples/* $docDir | |
cp -r client_devel/include/* $includeDir | |
''; | |
# postFixup = '' | |
# for i in $(find $out/bin -type f -executable); do | |
# wrapProgram "$i" --prefix LD_LIBRARY_PATH : $out/lib | |
# done | |
# ''; | |
doCheck = true; | |
# checkPhase = '' | |
# LD_LIBRARY_PATH=$LD_LIBRARY_PATH:`pwd`/opentk_lib/build/ \ | |
# common/build/test-runner --text | |
# ''; | |
meta = with lib; { | |
description = "High performance distributed filesystem with RDMA support"; | |
homepage = "https://www.beegfs.io"; | |
platforms = [ "i686-linux" "x86_64-linux" ]; | |
license = { | |
fullName = "BeeGFS_EULA"; | |
url = "https://www.beegfs.io/docs/BeeGFS_EULA.txt"; | |
free = true; | |
}; | |
maintainers = with maintainers; [ "juselius" ]; | |
}; | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Heavily based on the great work by @markuskowa | |
{ config, lib, pkgs, ...} : | |
with lib; | |
let | |
cfg = config.features.hpc.beegfs.beegfs; | |
# kernel = pkgs.linuxPackages_5_4.kernel; | |
kernel = config.boot.kernelPackages.kernel; | |
beegfs = pkgs.callPackage ./beegfs.nix { | |
inherit kernel; | |
}; | |
beegfs-module = pkgs.callPackage ./kernel-module.nix { | |
inherit kernel; | |
}; | |
# functions for the generations of config files | |
configMgmtd = name: cfg: pkgs.writeText "mgmt-${name}.conf" '' | |
storeMgmtdDirectory = ${cfg.mgmtd.storeDir} | |
storeAllowFirstRunInit = false | |
connAuthFile = ${cfg.connAuthFile} | |
connPortShift = ${toString cfg.connPortShift} | |
${cfg.mgmtd.extraConfig} | |
''; | |
configMeta = name: cfg: pkgs.writeText "meta-${name}.conf" '' | |
storeMetaDirectory = ${cfg.meta.storeDir} | |
sysMgmtdHost = ${cfg.mgmtdHost} | |
connAuthFile = ${cfg.connAuthFile} | |
connPortShift = ${toString cfg.connPortShift} | |
storeAllowFirstRunInit = false | |
${cfg.meta.extraConfig} | |
''; | |
configStorage = name: cfg: pkgs.writeText "storage-${name}.conf" '' | |
storeStorageDirectory = ${cfg.storage.storeDir} | |
sysMgmtdHost = ${cfg.mgmtdHost} | |
connAuthFile = ${cfg.connAuthFile} | |
connPortShift = ${toString cfg.connPortShift} | |
storeAllowFirstRunInit = false | |
${cfg.storage.extraConfig} | |
''; | |
configHelperd = name: cfg: pkgs.writeText "helperd-${name}.conf" '' | |
connAuthFile = ${cfg.connAuthFile} | |
${cfg.helperd.extraConfig} | |
''; | |
configClientFilename = name : "/etc/beegfs/client-${name}.conf"; | |
configClient = name: cfg: '' | |
sysMgmtdHost = ${cfg.mgmtdHost} | |
connAuthFile = ${cfg.connAuthFile} | |
connPortShift = ${toString cfg.connPortShift} | |
${cfg.client.extraConfig} | |
''; | |
serviceList = [ | |
{ service = "meta"; cfgFile = configMeta; } | |
{ service = "mgmtd"; cfgFile = configMgmtd; } | |
{ service = "storage"; cfgFile = configStorage; } | |
]; | |
# functions to generate systemd.service entries | |
systemdEntry = service: cfgFile: (mapAttrs' ( name: cfg: | |
(nameValuePair "beegfs-${service}-${name}" (mkIf cfg.${service}.enable { | |
wantedBy = [ "multi-user.target" ]; | |
requires = [ "network-online.target" ]; | |
after = [ "network-online.target" ]; | |
serviceConfig = rec { | |
ExecStart = '' | |
${beegfs}/bin/beegfs-${service} \ | |
cfgFile=${cfgFile name cfg} \ | |
pidFile=${PIDFile} | |
''; | |
PIDFile = "/run/beegfs-${service}-${name}.pid"; | |
TimeoutStopSec = "300"; | |
}; | |
}))) cfg); | |
systemdHelperd = mapAttrs' ( name: cfg: | |
(nameValuePair "beegfs-helperd-${name}" (mkIf cfg.client.enable { | |
wantedBy = [ "multi-user.target" ]; | |
requires = [ "network-online.target" ]; | |
after = [ "network-online.target" ]; | |
environment = { | |
LD_LIBRARY_PATH = "${beegfs}/lib"; | |
}; | |
serviceConfig = rec { | |
ExecStart = '' | |
${beegfs}/bin/beegfs-helperd \ | |
cfgFile=${configHelperd name cfg} \ | |
pidFile=${PIDFile} | |
''; | |
PIDFile = "/run/beegfs-helperd-${name}.pid"; | |
TimeoutStopSec = "300"; | |
}; | |
}))) cfg; | |
# wrappers to beegfs tools. Avoid typing path of config files | |
utilWrappers = mapAttrsToList ( name: cfg: | |
( pkgs.runCommand "beegfs-utils-${name}" { | |
nativeBuildInputs = [ pkgs.makeWrapper ]; | |
preferLocalBuild = true; | |
} '' | |
mkdir -p $out/bin | |
makeWrapper ${beegfs}/bin/beegfs-check-servers \ | |
$out/bin/beegfs-check-servers-${name} \ | |
--add-flags "-c ${configClientFilename name}" \ | |
--prefix PATH : ${lib.makeBinPath [ beegfs ]} | |
makeWrapper ${beegfs}/bin/beegfs-ctl \ | |
$out/bin/beegfs-ctl-${name} \ | |
--add-flags "--cfgFile=${configClientFilename name}" | |
makeWrapper ${beegfs}/bin/beegfs-ctl \ | |
$out/bin/beegfs-df-${name} \ | |
--add-flags "--cfgFile=${configClientFilename name}" \ | |
--add-flags --listtargets \ | |
--add-flags --hidenodeid \ | |
--add-flags --pools \ | |
--add-flags --spaceinfo | |
makeWrapper ${beegfs}/bin/beegfs-fsck \ | |
$out/bin/beegfs-fsck-${name} \ | |
--add-flags "--cfgFile=${configClientFilename name}" | |
'' | |
)) cfg; | |
beegfsOptions = { | |
options = { | |
mgmtdHost = mkOption { | |
type = types.str; | |
default = null; | |
example = "master"; | |
description = ''Hostname of managament host.''; | |
}; | |
connAuthFile = mkOption { | |
type = types.str; | |
default = ""; | |
example = "/etc/my.key"; | |
description = "File containing shared secret authentication."; | |
}; | |
connPortShift = mkOption { | |
type = types.int; | |
default = 0; | |
example = 5; | |
description = '' | |
For each additional beegfs configuration shift all | |
service TCP/UDP ports by at least 5. | |
''; | |
}; | |
client = { | |
enable = mkEnableOption "BeeGFS client"; | |
mount = mkOption { | |
type = types.bool; | |
default = true; | |
description = "Create fstab entry automatically"; | |
}; | |
mountPoint = mkOption { | |
type = types.str; | |
default = "/run/beegfs"; | |
description = '' | |
Mount point under which the beegfs filesytem should be mounted. | |
If mounted manually the mount option specifing the config file is needed: | |
cfgFile=/etc/beegfs/beegfs-client-<name>.conf | |
''; | |
}; | |
extraConfig = mkOption { | |
type = types.lines; | |
default = ""; | |
description = '' | |
Additional lines for beegfs-client.conf. | |
See documentation for further details. | |
''; | |
}; | |
}; | |
helperd = { | |
enable = mkOption { | |
type = types.bool; | |
default = true; | |
description = '' | |
Enable the BeeGFS helperd. | |
The helpered is need for logging purposes on the client. | |
Disabling <literal>helperd</literal> allows for runing the client | |
with <literal>allowUnfree = false</literal>. | |
''; | |
}; | |
extraConfig = mkOption { | |
type = types.lines; | |
default = ""; | |
description = '' | |
Additional lines for beegfs-helperd.conf. See documentation | |
for further details. | |
''; | |
}; | |
}; | |
mgmtd = { | |
enable = mkEnableOption "BeeGFS mgmtd daemon"; | |
storeDir = mkOption { | |
type = types.path; | |
default = null; | |
example = "/data/beegfs-mgmtd"; | |
description = '' | |
Data directory for mgmtd. | |
Must not be shared with other beegfs daemons. | |
This directory must exist and it must be initialized | |
with beegfs-setup-mgmtd, e.g. "beegfs-setup-mgmtd -C -p <storeDir>" | |
''; | |
}; | |
extraConfig = mkOption { | |
type = types.lines; | |
default = ""; | |
description = '' | |
Additional lines for beegfs-mgmtd.conf. See documentation | |
for further details. | |
''; | |
}; | |
}; | |
meta = { | |
enable = mkEnableOption "BeeGFS meta data daemon"; | |
storeDir = mkOption { | |
type = types.path; | |
default = null; | |
example = "/data/beegfs-meta"; | |
description = '' | |
Data directory for meta data service. | |
Must not be shared with other beegfs daemons. | |
The underlying filesystem must be mounted with xattr turned on. | |
This directory must exist and it must be initialized | |
with beegfs-setup-meta, e.g. | |
"beegfs-setup-meta -C -s <serviceID> -p <storeDir>" | |
''; | |
}; | |
extraConfig = mkOption { | |
type = types.str; | |
default = ""; | |
description = '' | |
Additional lines for beegfs-meta.conf. See documentation | |
for further details. | |
''; | |
}; | |
}; | |
storage = { | |
enable = mkEnableOption "BeeGFS storage daemon"; | |
storeDir = mkOption { | |
type = types.path; | |
default = null; | |
example = "/data/beegfs-storage"; | |
description = '' | |
Data directories for storage service. | |
Must not be shared with other beegfs daemons. | |
The underlying filesystem must be mounted with xattr turned on. | |
This directory must exist and it must be initialized | |
with beegfs-setup-storage, e.g. | |
"beegfs-setup-storage -C -s <serviceID> -i <storageTargetID> -p <storeDir>" | |
''; | |
}; | |
extraConfig = mkOption { | |
type = types.str; | |
default = ""; | |
description = '' | |
Addional lines for beegfs-storage.conf. See documentation | |
for further details. | |
''; | |
}; | |
}; | |
}; | |
}; | |
in | |
{ | |
###### interface | |
options.features.hpc.beegfs = { | |
enable = mkEnableOption "BeeGFS"; | |
beegfs = mkOption { | |
type = with types; attrsOf (submodule ({ ... } : beegfsOptions )); | |
default = {}; | |
description = '' | |
BeeGFS configurations. Every mount point requires a separate configuration. | |
''; | |
}; | |
}; | |
###### implementation | |
config = mkIf config.features.hpc.beegfs.enable { | |
environment.systemPackages = utilWrappers; | |
# Put the client.conf files in /etc since they are needed | |
# by the commandline tools | |
environment.etc = mapAttrs' ( name: cfg: | |
(nameValuePair "beegfs/client-${name}.conf" (mkIf (cfg.client.enable) | |
{ | |
enable = true; | |
text = configClient name cfg; | |
}))) cfg; | |
# Kernel module, we need it only once per host. | |
boot = mkIf ( | |
foldr (a: b: a || b) false | |
(map (x: x.client.enable) (collect (x: x ? client) cfg))) | |
{ | |
kernelModules = [ "beegfs" ]; | |
extraModulePackages = [ beegfs-module ]; | |
}; | |
# generate fstab entries | |
fileSystems = mapAttrs' (name: cfg: | |
(nameValuePair cfg.client.mountPoint (optionalAttrs cfg.client.mount (mkIf cfg.client.enable { | |
device = "beegfs_nodev"; | |
fsType = "beegfs"; | |
mountPoint = cfg.client.mountPoint; | |
options = [ "cfgFile=${configClientFilename name}" "_netdev" ]; | |
})))) cfg; | |
# generate systemd services | |
systemd.services = systemdHelperd // | |
foldr (a: b: a // b) {} | |
(map (x: systemdEntry x.service x.cfgFile) serviceList); | |
}; | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ pkgs, kernel ? pkgs.linux, ... } : | |
with pkgs; | |
let | |
version = "7.4.0"; | |
beegfs = pkgs.callPackage ./beegfs.nix { inherit kernel; }; | |
in stdenvNoCC.mkDerivation { | |
name = "beegfs-module-${version}-${kernel.version}"; | |
src = fetchurl { | |
name = "beegfs-archive-${version}.tar.bz2"; | |
# url = "https://git.beegfs.com/pub/v7/repository/archive.tar.bz2?ref=${version}"; | |
url = "https://git.beegfs.io/pub/v7/-/archive/${version}/v7-${version}.tar.bz2"; | |
sha256 = "sha256-VwD3z3lZIs5aOIBbwUvEkOxkFggTCv8OWuJMCga2ooo="; | |
}; | |
hardeningDisable = [ "fortify" "pic" "stackprotector" ]; | |
nativeBuildInputs = [ gcc12 which kmod pahole ]; | |
buildInputs = kernel.moduleBuildDependencies; | |
makeFlags = [ "KDIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build/" ]; | |
postPatch = '' | |
patchShebangs ./ | |
find -type f -name Makefile -exec sed -i "s:/bin/bash:${stdenv.shell}:" \{} \; | |
find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \; | |
find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \; | |
find -type f -name "configure" -exec sed -i "s:/bin/:/usr/bin/env :" \{} \; | |
find -type f -name "configure" -exec sed -i "s:/usr/bin/:/usr/bin/env :" \{} \; | |
sed -i 's,libbeegfs_ib.so,${beegfs}/lib/&,' common/source/common/net/sock/RDMASocket.cpp | |
''; | |
preBuild = "cd client_module/build"; | |
installPhase = '' | |
instdir=$out/lib/modules/${kernel.modDirVersion}/extras/fs/beegfs | |
mkdir -p $instdir | |
cp beegfs.ko $instdir | |
''; | |
meta = with lib; { | |
description = "High performance distributed filesystem with RDMA support"; | |
homepage = "https://www.beegfs.io"; | |
platforms = [ "i686-linux" "x86_64-linux" ]; | |
license = licenses.gpl2; | |
maintainers = with maintainers; [ "juselius" ]; | |
# broken = stdenv.lib.versionAtLeast kernel.version "4.18"; | |
}; | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment