From fb22d67fa7e4719af2af286f8596108af5e623f7 Mon Sep 17 00:00:00 2001 From: Johan Thomsen Date: Fri, 14 Jun 2019 12:46:07 +0200 Subject: [PATCH] ceph: 13.2.4 -> 14.2.1 * remove kinetic * release note * add johanot as maintainer nixos/ceph: create option for mgr_module_path - since the upstream default is no longer correct in v14 * fix module, default location for libexec has changed * ceph: fix test --- nixos/doc/manual/release-notes/rl-1909.xml | 7 +++ .../services/network-filesystems/ceph.nix | 10 +++- nixos/tests/ceph.nix | 36 +++++++------ .../ceph/0000-ceph-volume-allow-loop.patch | 35 ------------- .../0000-dont-check-cherrypy-version.patch | 51 ++++++++++++++++--- pkgs/tools/filesystems/ceph/default.nix | 51 +++++++++++-------- pkgs/top-level/all-packages.nix | 2 +- 7 files changed, 108 insertions(+), 84 deletions(-) delete mode 100644 pkgs/tools/filesystems/ceph/0000-ceph-volume-allow-loop.patch diff --git a/nixos/doc/manual/release-notes/rl-1909.xml b/nixos/doc/manual/release-notes/rl-1909.xml index c0b7cc48a461..78e911fa5d93 100644 --- a/nixos/doc/manual/release-notes/rl-1909.xml +++ b/nixos/doc/manual/release-notes/rl-1909.xml @@ -710,6 +710,13 @@ nix-shell -p altcoins.dogecoin, etc. + + Ceph has been upgraded to v14.2.1. + See the release notes for details. + The mgr dashboard as well as osds backed by loop-devices is no longer explicitly supported by the package and module. + Note: There's been some issues with python-cherrypy, which is used by the dashboard + and prometheus mgr modules (and possibly others), hence 0000-dont-check-cherrypy-version.patch. + diff --git a/nixos/modules/services/network-filesystems/ceph.nix b/nixos/modules/services/network-filesystems/ceph.nix index 58922897a181..aef284c31a5a 100644 --- a/nixos/modules/services/network-filesystems/ceph.nix +++ b/nixos/modules/services/network-filesystems/ceph.nix @@ -38,7 +38,7 @@ let -f --cluster ${clusterName} --id ${daemonId} --setuser ceph \ --setgroup ${if daemonType == "osd" then "disk" else "ceph"}''; } // extraServiceConfig - // optionalAttrs (daemonType == "osd") { ExecStartPre = ''${ceph.out}/libexec/ceph/ceph-osd-prestart.sh \ + // optionalAttrs (daemonType == "osd") { ExecStartPre = ''${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh \ --id ${daemonId} --cluster ${clusterName}''; }; } // optionalAttrs (builtins.elem daemonType [ "mds" "mon" "rgw" "mgr" ]) { preStart = '' @@ -88,6 +88,14 @@ in ''; }; + mgrModulePath = mkOption { + type = types.path; + default = "${pkgs.ceph.lib}/lib/ceph/mgr"; + description = '' + Path at which to find ceph-mgr modules. + ''; + }; + monInitialMembers = mkOption { type = with types; nullOr commas; default = null; diff --git a/nixos/tests/ceph.nix b/nixos/tests/ceph.nix index a80f63da6655..7a6e238d9157 100644 --- a/nixos/tests/ceph.nix +++ b/nixos/tests/ceph.nix @@ -7,6 +7,7 @@ import ./make-test.nix ({pkgs, lib, ...}: rec { nodes = { aio = { pkgs, ... }: { virtualisation = { + memorySize = 1536; emptyDiskImages = [ 20480 20480 ]; vlans = [ 1 ]; }; @@ -24,9 +25,6 @@ import ./make-test.nix ({pkgs, lib, ...}: rec { ceph xfsprogs ]; - nixpkgs.config.packageOverrides = super: { - ceph = super.ceph.override({ nss = super.nss; libxfs = super.libxfs; libaio = super.libaio; jemalloc = super.jemalloc; }); - }; boot.kernelModules = [ "xfs" ]; @@ -40,9 +38,6 @@ import ./make-test.nix ({pkgs, lib, ...}: rec { services.ceph.mon = { enable = true; daemons = [ "aio" ]; - extraConfig = { - "mgr initial modules" = "dashboard"; - }; }; services.ceph.mgr = { @@ -67,23 +62,26 @@ import ./make-test.nix ({pkgs, lib, ...}: rec { # Create the ceph-related directories $aio->mustSucceed( - "mkdir -p /var/lib/ceph/mgr/ceph-aio/", - "mkdir -p /var/lib/ceph/mon/ceph-aio/", - "mkdir -p /var/lib/ceph/osd/ceph-{0..1}/", - "chown ceph:ceph -R /var/lib/ceph/" + "mkdir -p /var/lib/ceph/mgr/ceph-aio", + "mkdir -p /var/lib/ceph/mon/ceph-aio", + "mkdir -p /var/lib/ceph/osd/ceph-{0,1}", + "chown ceph:ceph -R /var/lib/ceph/", + "mkdir -p /etc/ceph", + "chown ceph:ceph -R /etc/ceph" ); # Bootstrap ceph-mon daemon $aio->mustSucceed( "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'", - "ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'", - "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring", - "monmaptool --create --add aio 192.168.1.1 --fsid 066ae264-2a5d-4729-8001-6ad265f50b03 /tmp/monmap", + "sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'", + "sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring", + "monmaptool --create --add aio 192.168.1.1 --fsid 066ae264-2a5d-4729-8001-6ad265f50b03 /tmp/monmap", "sudo -u ceph ceph-mon --mkfs -i aio --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring", - "touch /var/lib/ceph/mon/ceph-aio/done", + "sudo -u ceph touch /var/lib/ceph/mon/ceph-aio/done", "systemctl start ceph-mon-aio" ); $aio->waitForUnit("ceph-mon-aio"); + $aio->mustSucceed("ceph mon enable-msgr2"); # Can't check ceph status until a mon is up $aio->succeed("ceph -s | grep 'mon: 1 daemons'"); @@ -95,7 +93,7 @@ import ./make-test.nix ({pkgs, lib, ...}: rec { ); $aio->waitForUnit("ceph-mgr-aio"); $aio->waitUntilSucceeds("ceph -s | grep 'quorum aio'"); - $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active)'"); + $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active,'"); # Bootstrap both OSDs $aio->mustSucceed( @@ -118,8 +116,8 @@ import ./make-test.nix ({pkgs, lib, ...}: rec { "systemctl start ceph-osd-1" ); - $aio->waitUntilSucceeds("ceph osd stat | grep '2 osds: 2 up, 2 in'"); - $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active)'"); + $aio->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'"); + $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active,'"); $aio->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'"); $aio->mustSucceed( @@ -156,8 +154,8 @@ import ./make-test.nix ({pkgs, lib, ...}: rec { $aio->waitForUnit("ceph-osd-1"); $aio->succeed("ceph -s | grep 'mon: 1 daemons'"); $aio->waitUntilSucceeds("ceph -s | grep 'quorum aio'"); - $aio->waitUntilSucceeds("ceph osd stat | grep '2 osds: 2 up, 2 in'"); - $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active)'"); + $aio->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'"); + $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active,'"); $aio->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'"); ''; }) diff --git a/pkgs/tools/filesystems/ceph/0000-ceph-volume-allow-loop.patch b/pkgs/tools/filesystems/ceph/0000-ceph-volume-allow-loop.patch deleted file mode 100644 index d103c42b5189..000000000000 --- a/pkgs/tools/filesystems/ceph/0000-ceph-volume-allow-loop.patch +++ /dev/null @@ -1,35 +0,0 @@ -diff --git a/src/ceph-disk/ceph_disk/main.py b/src/ceph-disk/ceph_disk/main.py -index f8ef35503e..2cc8a75d2b 100644 ---- a/src/ceph-disk/ceph_disk/main.py -+++ b/src/ceph-disk/ceph_disk/main.py -@@ -881,13 +881,14 @@ def is_partition(dev): - if not stmode_is_diskdevice(st.st_mode): - raise Error('not a block device', dev) - -+ major = os.major(st.st_rdev) -+ minor = os.minor(st.st_rdev) -+ - name = get_dev_name(dev) -- if os.path.exists(os.path.join(BLOCKDIR, name)): -+ if os.path.exists(os.path.join(BLOCKDIR, name)) or os.path.exists(os.path.join(BLOCKDIR, "loop%d" % minor)): - return False - - # make sure it is a partition of something else -- major = os.major(st.st_rdev) -- minor = os.minor(st.st_rdev) - if os.path.exists('/sys/dev/block/%d:%d/partition' % (major, minor)): - return True - -diff --git a/src/ceph-volume/ceph_volume/util/disk.py b/src/ceph-volume/ceph_volume/util/disk.py -index c3a7915400..9a708b05f2 100644 ---- a/src/ceph-volume/ceph_volume/util/disk.py -+++ b/src/ceph-volume/ceph_volume/util/disk.py -@@ -203,7 +203,7 @@ def is_device(dev): - # use lsblk first, fall back to using stat - TYPE = lsblk(dev).get('TYPE') - if TYPE: -- return TYPE == 'disk' -+ return TYPE == 'disk' or TYPE == 'loop' - - # fallback to stat - return _stat_is_device(os.lstat(dev).st_mode) diff --git a/pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch b/pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch index 154e90d91467..9604da87fb50 100644 --- a/pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch +++ b/pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch @@ -1,12 +1,11 @@ diff --git a/src/pybind/mgr/dashboard/module.py b/src/pybind/mgr/dashboard/module.py -index 0b53743d74..2e17ce1168 100644 +index a8a3ec07c1..bcc9b86c37 100644 --- a/src/pybind/mgr/dashboard/module.py +++ b/src/pybind/mgr/dashboard/module.py -@@ -28,28 +28,6 @@ except ImportError: - # To be picked up and reported by .can_run() - cherrypy = None +@@ -25,40 +25,6 @@ except ImportError: + + from .services.sso import load_sso_db -- -# The SSL code in CherryPy 3.5.0 is buggy. It was fixed long ago, -# but 3.5.0 is still shipping in major linux distributions -# (Fedora 27, Ubuntu Xenial), so we must monkey patch it to get SSL working. @@ -14,7 +13,7 @@ index 0b53743d74..2e17ce1168 100644 - v = StrictVersion(cherrypy.__version__) - # It was fixed in 3.7.0. Exact lower bound version is probably earlier, - # but 3.5.0 is what this monkey patch is tested on. -- if v >= StrictVersion("3.5.0") and v < StrictVersion("3.7.0"): +- if StrictVersion("3.5.0") <= v < StrictVersion("3.7.0"): - from cherrypy.wsgiserver.wsgiserver2 import HTTPConnection,\ - CP_fileobject - @@ -27,7 +26,45 @@ index 0b53743d74..2e17ce1168 100644 - - HTTPConnection.__init__ = fixed_init - +-# When the CherryPy server in 3.2.2 (and later) starts it attempts to verify +-# that the ports its listening on are in fact bound. When using the any address +-# "::" it tries both ipv4 and ipv6, and in some environments (e.g. kubernetes) +-# ipv6 isn't yet configured / supported and CherryPy throws an uncaught +-# exception. +-if cherrypy is not None: +- v = StrictVersion(cherrypy.__version__) +- # the issue was fixed in 3.2.3. it's present in 3.2.2 (current version on +- # centos:7) and back to at least 3.0.0. +- if StrictVersion("3.1.2") <= v < StrictVersion("3.2.3"): +- # https://github.com/cherrypy/cherrypy/issues/1100 +- from cherrypy.process import servers +- servers.wait_for_occupied_port = lambda host, port: None - if 'COVERAGE_ENABLED' in os.environ: import coverage - _cov = coverage.Coverage(config_file="{}/.coveragerc".format(os.path.dirname(__file__))) + __cov = coverage.Coverage(config_file="{}/.coveragerc".format(os.path.dirname(__file__)), +diff --git a/src/pybind/mgr/prometheus/module.py b/src/pybind/mgr/prometheus/module.py +index b7fecf8d85..dfd4160591 100644 +--- a/src/pybind/mgr/prometheus/module.py ++++ b/src/pybind/mgr/prometheus/module.py +@@ -18,20 +18,6 @@ from rbd import RBD + DEFAULT_ADDR = '::' + DEFAULT_PORT = 9283 + +-# When the CherryPy server in 3.2.2 (and later) starts it attempts to verify +-# that the ports its listening on are in fact bound. When using the any address +-# "::" it tries both ipv4 and ipv6, and in some environments (e.g. kubernetes) +-# ipv6 isn't yet configured / supported and CherryPy throws an uncaught +-# exception. +-if cherrypy is not None: +- v = StrictVersion(cherrypy.__version__) +- # the issue was fixed in 3.2.3. it's present in 3.2.2 (current version on +- # centos:7) and back to at least 3.0.0. +- if StrictVersion("3.1.2") <= v < StrictVersion("3.2.3"): +- # https://github.com/cherrypy/cherrypy/issues/1100 +- from cherrypy.process import servers +- servers.wait_for_occupied_port = lambda host, port: None +- + # cherrypy likes to sys.exit on error. don't let it take us down too! + def os_exit_noop(*args, **kwargs): + pass diff --git a/pkgs/tools/filesystems/ceph/default.nix b/pkgs/tools/filesystems/ceph/default.nix index b9c71c49e807..efb292687ab4 100644 --- a/pkgs/tools/filesystems/ceph/default.nix +++ b/pkgs/tools/filesystems/ceph/default.nix @@ -2,18 +2,18 @@ , ensureNewerSourcesHook , cmake, pkgconfig , which, git -, boost, python2Packages +, boost, python3Packages , libxml2, zlib, lz4 , openldap, lttng-ust , babeltrace, gperf , cunit, snappy , rocksdb, makeWrapper -, leveldb, oathToolkit +, leveldb, oathToolkit, removeReferencesTo # Optional Dependencies , yasm ? null, fcgi ? null, expat ? null , curl ? null, fuse ? null -, libedit ? null, libatomic_ops ? null, kinetic-cpp-client ? null +, libedit ? null, libatomic_ops ? null , libs3 ? null # Mallocs @@ -24,7 +24,7 @@ , nss ? null, nspr ? null # Linux Only Dependencies -, linuxHeaders, utillinux, libuuid, udev, keyutils +, linuxHeaders, utillinux, libuuid, udev, keyutils, rdma-core, rabbitmq-c , libaio ? null, libxfs ? null, zfs ? null , ... }: @@ -43,7 +43,6 @@ let optFuse = shouldUsePkg fuse; optLibedit = shouldUsePkg libedit; optLibatomic_ops = shouldUsePkg libatomic_ops; - optKinetic-cpp-client = shouldUsePkg kinetic-cpp-client; optLibs3 = shouldUsePkg libs3; optJemalloc = shouldUsePkg jemalloc; @@ -73,7 +72,7 @@ let none = [ ]; }; - ceph-python-env = python2Packages.python.withPackages (ps: [ + ceph-python-env = python3Packages.python.withPackages (ps: [ ps.sphinx ps.flask ps.cython @@ -84,30 +83,30 @@ let ps.cherrypy ps.pecan ps.prettytable + ps.pyjwt ps.webob ps.bcrypt + ps.six ]); - version = "13.2.4"; + version = "14.2.1"; in rec { ceph = stdenv.mkDerivation { name="ceph-${version}"; src = fetchurl { url = "http://download.ceph.com/tarballs/ceph-${version}.tar.gz"; - sha256 = "0g2mc6rp84ia44vz8kl449820m9hmfavzfmwn8fy6li14xr8a00w"; + sha256 = "0qa9p8xp26d45h3jfj1rbwhmqv44f9n1mvccmpzaf2i05v42kmzb"; }; patches = [ ./0000-fix-SPDK-build-env.patch - # TODO: remove when https://github.com/ceph/ceph/pull/21289 is merged - ./0000-ceph-volume-allow-loop.patch ./0000-dont-check-cherrypy-version.patch ]; nativeBuildInputs = [ cmake - pkgconfig which git python2Packages.wrapPython makeWrapper + pkgconfig which git python3Packages.wrapPython makeWrapper (ensureNewerSourcesHook { year = "1980"; }) ]; @@ -115,9 +114,11 @@ in rec { boost ceph-python-env libxml2 optYasm optLibatomic_ops optLibs3 malloc zlib openldap lttng-ust babeltrace gperf cunit snappy rocksdb lz4 oathToolkit leveldb - optKinetic-cpp-client + removeReferencesTo ] ++ optionals stdenv.isLinux [ linuxHeaders utillinux libuuid udev keyutils optLibaio optLibxfs optZfs + # ceph 14 + rdma-core rabbitmq-c ] ++ optionals hasRadosgw [ optFcgi optExpat optCurl optFuse optLibedit ]; @@ -126,20 +127,21 @@ in rec { substituteInPlace src/common/module.c --replace "/sbin/modinfo" "modinfo" substituteInPlace src/common/module.c --replace "/sbin/modprobe" "modprobe" # Since Boost 1.67 this seems to have changed - substituteInPlace CMakeLists.txt --replace "list(APPEND BOOST_COMPONENTS python)" "list(APPEND BOOST_COMPONENTS python27)" - substituteInPlace src/CMakeLists.txt --replace "Boost::python " "Boost::python27 " + substituteInPlace CMakeLists.txt --replace "list(APPEND BOOST_COMPONENTS python)" "list(APPEND BOOST_COMPONENTS python37)" + substituteInPlace src/CMakeLists.txt --replace "Boost::python " "Boost::python37 " # for pybind/rgw to find internal dep export LD_LIBRARY_PATH="$PWD/build/lib:$LD_LIBRARY_PATH" # install target needs to be in PYTHONPATH for "*.pth support" check to succeed - export PYTHONPATH=$lib/lib/python2.7/site-packages/:$out/lib/python2.7/site-packages/ + export PYTHONPATH=${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/python3.7/site-packages/:$out/lib/python3.7/site-packages/ patchShebangs src/spdk ''; cmakeFlags = [ - "-DWITH_SYSTEM_ROCKSDB=ON" - "-DROCKSDB_INCLUDE_DIR=${rocksdb}/include/rocksdb" + "-DWITH_PYTHON3=ON" + "-DWITH_SYSTEM_ROCKSDB=OFF" + "-DWITH_SYSTEM_BOOST=ON" "-DWITH_SYSTEMD=OFF" "-DWITH_TESTS=OFF" @@ -147,9 +149,16 @@ in rec { "-DWITH_MGR_DASHBOARD_FRONTEND=OFF" ]; + preFixup = '' + find $lib -type f -exec remove-references-to -t $out '{}' + + mv $out/share/ceph/mgr $lib/lib/ceph/ + ''; + postFixup = '' + export PYTHONPATH="${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/ceph/mgr:$out/lib/python3.7/site-packages/" wrapPythonPrograms - wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "$lib/lib/ceph/mgr:$out/lib/python2.7/site-packages/" + wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/ceph/mgr:$out/lib/python3.7/site-packages/" + wrapProgram $out/bin/ceph-volume --prefix PYTHONPATH ":" "${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/ceph/mgr:$out/lib/python3.7/site-packages/" ''; enableParallelBuilding = true; @@ -160,7 +169,7 @@ in rec { homepage = https://ceph.com/; description = "Distributed storage system"; license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ]; - maintainers = with maintainers; [ adev ak krav ]; + maintainers = with maintainers; [ adev ak krav johanot ]; platforms = platforms.unix; }; @@ -176,11 +185,11 @@ in rec { platforms = platforms.unix; }; } '' - mkdir -p $out/{bin,etc,lib/python2.7/site-packages} + mkdir -p $out/{bin,etc,lib/python3.7/site-packages} cp -r ${ceph}/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $out/bin cp -r ${ceph}/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $out/bin cp -r ${ceph}/bin/rbd-replay* $out/bin - cp -r ${ceph}/lib/python2.7/site-packages $out/lib/python2.7/ + cp -r ${ceph}/lib/python3.7/site-packages $out/lib/python3.7/ cp -r ${ceph}/etc/bash_completion.d $out/etc # wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths substituteInPlace $out/bin/ceph --replace ${ceph} $out diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 85f7a786a6c3..21f5f14a2dcb 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -2194,7 +2194,7 @@ in libceph = ceph.lib; inherit (callPackages ../tools/filesystems/ceph { - boost = boost167.override { enablePython = true; }; + boost = boost167.override { enablePython = true; python = python37; }; }) ceph ceph-client;