From 34b5c9a4de92120d2c302e084f0139bc3cd8f155 Mon Sep 17 00:00:00 2001 From: Fernando J Pando Date: Mon, 30 Jan 2017 12:44:14 -0500 Subject: [PATCH 1/3] buildbot: 0.9.0.post1 -> 0.9.3 - Fixes unneeded patching - Adds worker to build inputs now needed for tests - Replaces enableworker option with worker configuration module - Openssh required for tests - Fixes worker hardcoded paths - Tested on Nixos Unstable --- nixos/modules/module-list.nix | 1 + .../buildbot/master.nix | 30 ++-- .../buildbot/worker.nix | 128 ++++++++++++++++++ .../tools/build-managers/buildbot/default.nix | 42 ++---- .../tools/build-managers/buildbot/plugins.nix | 25 ++-- .../tools/build-managers/buildbot/worker.nix | 10 +- pkgs/top-level/all-packages.nix | 1 - 7 files changed, 176 insertions(+), 61 deletions(-) create mode 100644 nixos/modules/services/continuous-integration/buildbot/worker.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index bd427bc679c7..b8c8775fce1c 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -141,6 +141,7 @@ ./services/computing/torque/mom.nix ./services/computing/slurm/slurm.nix ./services/continuous-integration/buildbot/master.nix + ./services/continuous-integration/buildbot/worker.nix ./services/continuous-integration/buildkite-agent.nix ./services/continuous-integration/hydra/default.nix ./services/continuous-integration/gitlab-runner.nix diff --git a/nixos/modules/services/continuous-integration/buildbot/master.nix b/nixos/modules/services/continuous-integration/buildbot/master.nix index a40be4f546ea..512e09eb8041 100644 --- a/nixos/modules/services/continuous-integration/buildbot/master.nix +++ b/nixos/modules/services/continuous-integration/buildbot/master.nix @@ -7,7 +7,7 @@ with lib; let cfg = config.services.buildbot-master; escapeStr = s: escape ["'"] s; - masterCfg = pkgs.writeText "master.cfg" '' + masterCfg = if cfg.masterCfg == null then pkgs.writeText "master.cfg" '' from buildbot.plugins import * factory = util.BuildFactory() c = BuildmasterConfig = dict( @@ -27,9 +27,8 @@ let factory.addStep(step) ${cfg.extraConfig} - ''; - - configFile = if cfg.masterCfg == null then masterCfg else cfg.masterCfg; + '' + else pkgs.writeText "master.cfg" cfg.masterCfg; in { options = { @@ -67,15 +66,13 @@ in { }; masterCfg = mkOption { - type = with types; nullOr path; + type = types.str; description = '' - Optionally pass path to raw master.cfg file. + Optionally pass raw master.cfg file as string. Other options in this configuration will be ignored. ''; default = null; - example = literalExample '' - pkgs.writeText "master.cfg" "BuildmasterConfig = c = {}" - ''; + example = "BuildmasterConfig = c = {}"; }; schedulers = mkOption { @@ -99,9 +96,9 @@ in { type = types.listOf types.str; description = "List of Workers."; default = [ - "worker.Worker('default-worker', 'password')" + "worker.Worker('example-worker', 'pass')" ]; - example = [ "worker.LocalWorker('default-worker')" ]; + example = [ "worker.LocalWorker('example-worker')" ]; }; status = mkOption { @@ -209,7 +206,7 @@ in { users.extraUsers = optional (cfg.user == "buildbot") { name = "buildbot"; - description = "buildbot user"; + description = "Buildbot User."; isNormalUser = true; createHome = true; home = cfg.home; @@ -219,7 +216,7 @@ in { }; systemd.services.buildbot-master = { - description = "Buildbot Continuous Integration Server"; + description = "Buildbot Continuous Integration Server."; after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; path = cfg.packages; @@ -233,9 +230,8 @@ in { }; preStart = '' - mkdir -vp ${cfg.buildbotDir} - chown -c ${cfg.user}:${cfg.group} ${cfg.buildbotDir} - ln -sf ${configFile} ${cfg.buildbotDir}/master.cfg + ${pkgs.coreutils}/bin/mkdir -vp ${cfg.buildbotDir} + ${pkgs.coreutils}/bin/ln -sfv ${masterCfg} ${cfg.buildbotDir}/master.cfg ${cfg.package}/bin/buildbot create-master ${cfg.buildbotDir} ''; @@ -247,4 +243,6 @@ in { }; }; + meta.maintainers = with lib.maintainers; [ nand0p Mic92 ]; + } diff --git a/nixos/modules/services/continuous-integration/buildbot/worker.nix b/nixos/modules/services/continuous-integration/buildbot/worker.nix new file mode 100644 index 000000000000..430fd4e53f1c --- /dev/null +++ b/nixos/modules/services/continuous-integration/buildbot/worker.nix @@ -0,0 +1,128 @@ +# NixOS module for Buildbot Worker. + +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.buildbot-worker; + +in { + options = { + services.buildbot-worker = { + + enable = mkOption { + type = types.bool; + default = false; + description = "Whether to enable the Buildbot Worker."; + }; + + user = mkOption { + default = "bbworker"; + type = types.str; + description = "User the buildbot Worker should execute under."; + }; + + group = mkOption { + default = "bbworker"; + type = types.str; + description = "Primary group of buildbot Worker user."; + }; + + extraGroups = mkOption { + type = types.listOf types.str; + default = [ "nixbld" ]; + description = "List of extra groups that the Buildbot Worker user should be a part of."; + }; + + home = mkOption { + default = "/home/bbworker"; + type = types.path; + description = "Buildbot home directory."; + }; + + buildbotDir = mkOption { + default = "${cfg.home}/worker"; + type = types.path; + description = "Specifies the Buildbot directory."; + }; + + workerUser = mkOption { + default = "example-worker"; + type = types.str; + description = "Specifies the Buildbot Worker user."; + }; + + workerPass = mkOption { + default = "pass"; + type = types.str; + description = "Specifies the Buildbot Worker password."; + }; + + masterUrl = mkOption { + default = "localhost:9989"; + type = types.str; + description = "Specifies the Buildbot Worker connection string."; + }; + + package = mkOption { + type = types.package; + default = pkgs.buildbot-worker; + description = "Package to use for buildbot worker."; + example = pkgs.buildbot-worker; + }; + + packages = mkOption { + default = [ ]; + example = [ pkgs.git ]; + type = types.listOf types.package; + description = "Packages to add to PATH for the buildbot process."; + }; + + }; + }; + + config = mkIf cfg.enable { + users.extraGroups = optional (cfg.group == "bbworker") { + name = "bbworker"; + }; + + users.extraUsers = optional (cfg.user == "bbworker") { + name = "bbworker"; + description = "Buildbot Worker User."; + isNormalUser = true; + createHome = true; + home = cfg.home; + group = cfg.group; + extraGroups = cfg.extraGroups; + useDefaultShell = true; + }; + + systemd.services.buildbot-worker = { + description = "Buildbot Worker."; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + wants = [ "buildbot-master.service" ]; + path = cfg.packages; + + preStart = '' + # NOTE: ensure master has time to start in case running on localhost + ${pkgs.coreutils}/bin/sleep 4 + ${pkgs.coreutils}/bin/mkdir -vp ${cfg.buildbotDir} + ${cfg.package}/bin/buildbot-worker create-worker ${cfg.buildbotDir} ${cfg.masterUrl} ${cfg.workerUser} ${cfg.workerPass} + ''; + + serviceConfig = { + Type = "forking"; + User = cfg.user; + Group = cfg.group; + WorkingDirectory = cfg.home; + ExecStart = "${cfg.package}/bin/buildbot-worker start ${cfg.buildbotDir}"; + }; + + }; + }; + + meta.maintainers = with lib.maintainers; [ nand0p ]; + +} diff --git a/pkgs/development/tools/build-managers/buildbot/default.nix b/pkgs/development/tools/build-managers/buildbot/default.nix index 8e85c645e2e0..427ff7058405 100644 --- a/pkgs/development/tools/build-managers/buildbot/default.nix +++ b/pkgs/development/tools/build-managers/buildbot/default.nix @@ -1,21 +1,13 @@ -{ stdenv, - lib, - pythonPackages, - fetchurl, - coreutils, - openssh, - buildbot-worker, - plugins ? [], - enableLocalWorker ? false -}: +{ stdenv, lib, fetchurl, coreutils, openssh, buildbot-worker, makeWrapper, + pythonPackages, gnused, plugins ? [] }: pythonPackages.buildPythonApplication (rec { name = "${pname}-${version}"; pname = "buildbot"; - version = "0.9.0.post1"; + version = "0.9.3"; src = fetchurl { url = "mirror://pypi/b/${pname}/${name}.tar.gz"; - sha256 = "18rnsp691cnmbymlch6czx3mrcmifmf6dk97h9nslgfkkyf25n5g"; + sha256 = "1yw7knk5dcvwms14vqwlp89flhjf8567l17s9cq7vydh760nmg62"; }; buildInputs = with pythonPackages; [ @@ -31,7 +23,11 @@ pythonPackages.buildPythonApplication (rec { pylint astroid pyflakes - ] ++ lib.optionals (enableLocalWorker) [openssh]; + openssh + buildbot-worker + makeWrapper + treq + ]; propagatedBuildInputs = with pythonPackages; [ @@ -39,7 +35,6 @@ pythonPackages.buildPythonApplication (rec { twisted jinja2 zope_interface - future sqlalchemy sqlalchemy_migrate future @@ -61,32 +56,21 @@ pythonPackages.buildPythonApplication (rec { ramlfications sphinx-jinja - ] ++ plugins ++ - lib.optionals (enableLocalWorker) [buildbot-worker]; - - preInstall = '' - # writes out a file that can't be read properly - sed -i.bak -e '69,84d' buildbot/test/unit/test_www_config.py - ''; + ] ++ plugins; postPatch = '' - # re-hardcode path to tail - sed -i 's|/usr/bin/tail|${coreutils}/bin/tail|' buildbot/scripts/logwatcher.py + ${gnused}/bin/sed -i 's|/usr/bin/tail|${coreutils}/bin/tail|' buildbot/scripts/logwatcher.py ''; postFixup = '' - mv -v $out/bin/buildbot $out/bin/.wrapped-buildbot - echo "#!/bin/sh" > $out/bin/buildbot - echo "export PYTHONPATH=$PYTHONPATH" >> $out/bin/buildbot - echo "exec $out/bin/.wrapped-buildbot \"\$@\"" >> $out/bin/buildbot - chmod -c 555 $out/bin/buildbot + makeWrapper $out/bin/.buildbot-wrapped $out/bin/buildbot --set PYTHONPATH "$PYTHONPATH" ''; meta = with stdenv.lib; { homepage = http://buildbot.net/; description = "Continuous integration system that automates the build/test cycle"; maintainers = with maintainers; [ nand0p ryansydnor ]; - platforms = platforms.all; + platforms = platforms.linux; license = licenses.gpl2; }; }) diff --git a/pkgs/development/tools/build-managers/buildbot/plugins.nix b/pkgs/development/tools/build-managers/buildbot/plugins.nix index 2875f6942a9e..f2fdd1535bfd 100644 --- a/pkgs/development/tools/build-managers/buildbot/plugins.nix +++ b/pkgs/development/tools/build-managers/buildbot/plugins.nix @@ -4,11 +4,11 @@ let buildbot-pkg = pythonPackages.buildPythonPackage rec { name = "${pname}-${version}"; pname = "buildbot-pkg"; - version = "0.9.0.post1"; + version = "0.9.3"; src = fetchurl { url = "mirror://pypi/b/${pname}/${name}.tar.gz"; - sha256 = "0frmnc73dsyc9mjnrnpm4vdrwb7c63gc6maq6xvlp486v7sdhjbi"; + sha256 = "02949cvmghyh313i1hmplwxp3nzq789kk85xjx2ir82cpr1d6h6j"; }; propagatedBuildInputs = with pythonPackages; [ setuptools ]; @@ -26,14 +26,15 @@ in { www = pythonPackages.buildPythonPackage rec { name = "${pname}-${version}"; pname = "buildbot_www"; - version = "0.9.0.post1"; + version = "0.9.3"; # NOTE: wheel is used due to buildbot circular dependency format = "wheel"; - src = fetchurl { - url = "https://pypi.python.org/packages/02/d0/fc56ee27a09498638a47dcc5637ee5412ab7a67bfb4b3ff47e041f3d7b66/${name}-py2-none-any.whl"; - sha256 = "14ghch67k6090736n89l401swz7r9hnk2zlmdb59niq8lg7dyg9q"; + src = pythonPackages.fetchPypi { + inherit pname version format; + python = "py2"; + sha256 = "0yggg6mcykcnv41srl2sp2zwx2r38vb6a8jgxh1a4825mspm2jf7"; }; meta = with stdenv.lib; { @@ -48,14 +49,14 @@ in { console-view = pythonPackages.buildPythonPackage rec { name = "${pname}-${version}"; pname = "buildbot-console-view"; - version = "0.9.0.post1"; + version = "0.9.3"; src = fetchurl { url = "mirror://pypi/b/${pname}/${name}.tar.gz"; - sha256 = "0dc7rb7mrpva5gj7l57i96a78d6yj28pkkj9hfim1955z9dgn58l"; + sha256 = "1rkzakm05x72nvdivc5bc3gab3nyasdfvlwnwril90jj9q1b92dk"; }; - propagatedBuildInputs = [ buildbot-pkg ]; + propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ]; meta = with stdenv.lib; { homepage = http://buildbot.net/; @@ -69,14 +70,14 @@ in { waterfall-view = pythonPackages.buildPythonPackage rec { name = "${pname}-${version}"; pname = "buildbot-waterfall-view"; - version = "0.9.0.post1"; + version = "0.9.3"; src = fetchurl { url = "mirror://pypi/b/${pname}/${name}.tar.gz"; - sha256 = "0x9vvw15zzgj4w3qcxh8r10rb36ni0qh1215y7wbawh5lggnjm0g"; + sha256 = "033x2cs0znhk1j0lw067nmjw2m7yy1fdq5qch0sx50jnpjiq6g6g"; }; - propagatedBuildInputs = [ buildbot-pkg ]; + propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ]; meta = with stdenv.lib; { homepage = http://buildbot.net/; diff --git a/pkgs/development/tools/build-managers/buildbot/worker.nix b/pkgs/development/tools/build-managers/buildbot/worker.nix index 7d7ecc1c52d3..861ed647c5df 100644 --- a/pkgs/development/tools/build-managers/buildbot/worker.nix +++ b/pkgs/development/tools/build-managers/buildbot/worker.nix @@ -1,18 +1,22 @@ -{ stdenv, fetchurl, pythonPackages }: +{ stdenv, fetchurl, gnused, coreutils, pythonPackages }: pythonPackages.buildPythonApplication (rec { name = "${pname}-${version}"; pname = "buildbot-worker"; - version = "0.9.0.post1"; + version = "0.9.3"; src = fetchurl { url = "mirror://pypi/b/${pname}/${name}.tar.gz"; - sha256 = "1f8ij3y62r9z7qv92x21rg9h9whhakkwv59rgniq09j64ggjz8lx"; + sha256 = "176kp04g4c7gj15f73wppraqrirbfclyx214gcz966019niikcsp"; }; buildInputs = with pythonPackages; [ setuptoolsTrial mock ]; propagatedBuildInputs = with pythonPackages; [ twisted future ]; + postPatch = '' + ${gnused}/bin/sed -i 's|/usr/bin/tail|${coreutils}/bin/tail|' buildbot_worker/scripts/logwatcher.py + ''; + meta = with stdenv.lib; { homepage = http://buildbot.net/; description = "Buildbot Worker Daemon"; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index e9e20f795fcf..1de24c26a2b9 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -6137,7 +6137,6 @@ with pkgs; }; buildbot-full = self.buildbot.override { plugins = with self.buildbot-plugins; [ www console-view waterfall-view ]; - enableLocalWorker = true; }; buildkite-agent = callPackage ../development/tools/continuous-integration/buildkite-agent { }; From 487229654ed2dde979ffdd8c19580d47c425a1a3 Mon Sep 17 00:00:00 2001 From: Fernando J Pando Date: Thu, 9 Feb 2017 10:08:32 -0500 Subject: [PATCH 2/3] incremental: init 16.10.1 Tested on NixOS unstable --- .../python-modules/incremental/default.nix | 19 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 ++ 2 files changed, 21 insertions(+) create mode 100644 pkgs/development/python-modules/incremental/default.nix diff --git a/pkgs/development/python-modules/incremental/default.nix b/pkgs/development/python-modules/incremental/default.nix new file mode 100644 index 000000000000..b8565a8b7587 --- /dev/null +++ b/pkgs/development/python-modules/incremental/default.nix @@ -0,0 +1,19 @@ +{ stdenv, buildPythonPackage, fetchurl }: + +buildPythonPackage rec { + name = "${pname}-${version}"; + pname = "incremental"; + version = "16.10.1"; + + src = fetchurl { + url = "mirror://pypi/i/${pname}/${name}.tar.gz"; + sha256 = "0hh382gsj5lfl3fsabblk2djngl4n5yy90xakinasyn41rr6pb8l"; + }; + + meta = with stdenv.lib; { + homepage = http://github.com/twisted/treq; + description = "Incremental is a small library that versions your Python projects"; + license = licenses.mit; + maintainers = with maintainers; [ nand0p ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 84971210b6c2..40fda8e7a8bb 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -31906,6 +31906,8 @@ EOF }; }; + incremental = callPackage ../development/python-modules/incremental { }; + }); in fix' (extends overrides packages) From 6fc34fd48a01840fb6b11f6927145509736ec716 Mon Sep 17 00:00:00 2001 From: Fernando J Pando Date: Mon, 6 Feb 2017 15:27:24 -0500 Subject: [PATCH 3/3] treq: init 16.12.1 Tested on NixOS unstable --- .../python-modules/treq/default.nix | 51 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 53 insertions(+) create mode 100644 pkgs/development/python-modules/treq/default.nix diff --git a/pkgs/development/python-modules/treq/default.nix b/pkgs/development/python-modules/treq/default.nix new file mode 100644 index 000000000000..37f9e3324c7e --- /dev/null +++ b/pkgs/development/python-modules/treq/default.nix @@ -0,0 +1,51 @@ +{ stdenv, fetchurl, buildPythonPackage, service-identity, requests2, + six, mock, twisted, incremental, coreutils, gnumake, pep8, sphinx, + openssl, pyopenssl }: + +buildPythonPackage rec { + name = "${pname}-${version}"; + pname = "treq"; + version = "16.12.0"; + + src = fetchurl { + url = "mirror://pypi/t/${pname}/${name}.tar.gz"; + sha256 = "1aci3f3rmb5mdf4s6s4k4kghmnyy784cxgi3pz99m5jp274fs25h"; + }; + + buildInputs = [ + pep8 + mock + ]; + + propagatedBuildInputs = [ + service-identity + requests2 + twisted + incremental + sphinx + six + openssl + pyopenssl + ]; + + checkPhase = '' + ${pep8}/bin/pep8 --ignore=E902 treq + trial treq + ''; + + doCheck = false; + # Failure: twisted.web._newclient.RequestTransmissionFailed: [] + + postBuild = '' + ${coreutils}/bin/mkdir -pv treq + ${coreutils}/bin/echo "${version}" | ${coreutils}/bin/tee treq/_version + cd docs && ${gnumake}/bin/make html && cd .. + ''; + + meta = with stdenv.lib; { + homepage = http://github.com/twisted/treq; + description = "A requests-like API built on top of twisted.web's Agent"; + license = licenses.mit; + maintainers = with maintainers; [ nand0p ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 40fda8e7a8bb..4e0d235caa5f 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -31908,6 +31908,8 @@ EOF incremental = callPackage ../development/python-modules/incremental { }; + treq = callPackage ../development/python-modules/treq { }; + }); in fix' (extends overrides packages)