JakeHillion/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #24131 from nand0p/buildbot-0.9.5

Browse Source 
buildbot: 0.9.4 -> 0.9.5
This commit is contained in:
Frederik Rietdijk 2017-05-03 07:56:29 +02:00 committed by GitHub
commit 9e48fc3268
6 changed files with 141 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
nixos/modules/services/continuous-integration/buildbot/master.nix
View File

@ -12,7 +12,7 @@ let
factory = util.BuildFactory()
c = BuildmasterConfig = dict(
workers = [${concatStringsSep "," cfg.workers}],
protocols = { 'pb': {'port': ${cfg.bpPort} } },
protocols = { 'pb': {'port': ${toString cfg.bpPort} } },
title = '${escapeStr cfg.title}',
titleURL = '${escapeStr cfg.titleUrl}',
buildbotURL = '${escapeStr cfg.buildbotUrl}',
@ -62,7 +62,7 @@ in {
extraConfig = mkOption {
type = types.str;
description = "Extra configuration to append to master.cfg";
default = "";
default = "c['buildbotNetUsageData'] = None";
};
masterCfg = mkOption {
@ -92,10 +92,7 @@ in {
workers = mkOption {
type = types.listOf types.str;
description = "List of Workers.";
default = [
"worker.Worker('example-worker', 'pass')"
];
example = [ "worker.LocalWorker('example-worker')" ];
default = [ "worker.Worker('example-worker', 'pass')" ];
};
status = mkOption {
@ -135,9 +132,8 @@ in {
};
bpPort = mkOption {
default = "9989";
type = types.string;
example = "tcp:10000:interface=127.0.0.1";
default = 9989;
type = types.int;
description = "Port where the master will listen to Buildbot Worker.";
};
@ -179,14 +175,14 @@ in {
package = mkOption {
type = types.package;
default = pkgs.buildbot-ui;
defaultText = "pkgs.buildbot-ui";
default = pkgs.buildbot-full;
defaultText = "pkgs.buildbot-full";
description = "Package to use for buildbot.";
example = literalExample "pkgs.buildbot-full";
};
packages = mkOption {
default = [ ];
default = with pkgs; [ python27Packages.twisted git ];
example = literalExample "[ pkgs.git ]";
type = types.listOf types.package;
description = "Packages to add to PATH for the buildbot process.";
@ -212,29 +208,30 @@ in {
systemd.services.buildbot-master = {
description = "Buildbot Continuous Integration Server.";
after = [ "network.target" ];
after = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
path = cfg.packages;
preStart = ''
env > envvars
mkdir -vp ${cfg.buildbotDir}
ln -sfv ${masterCfg} ${cfg.buildbotDir}/master.cfg
rm -fv $cfg.buildbotDir}/buildbot.tac
${cfg.package}/bin/buildbot create-master ${cfg.buildbotDir}
'';
serviceConfig = {
Type = "simple";
User = cfg.user;
Group = cfg.group;
WorkingDirectory = cfg.home;
ExecStart = "${cfg.package}/bin/buildbot start --nodaemon ${cfg.buildbotDir}";
Environment = "PYTHONPATH=${cfg.package}/lib/python2.7/site-packages:${pkgs.buildbot-plugins.www}/lib/python2.7/site-packages:${pkgs.buildbot-plugins.waterfall-view}/lib/python2.7/site-packages:${pkgs.buildbot-plugins.console-view}/lib/python2.7/site-packages:${pkgs.python27Packages.future}/lib/python2.7/site-packages:${pkgs.python27Packages.dateutil}/lib/python2.7/site-packages:${pkgs.python27Packages.six}/lib/python2.7/site-packages:${pkgs.python27Packages.sqlalchemy}/lib/python2.7/site-packages:${pkgs.python27Packages.jinja2}/lib/python2.7/site-packages:${pkgs.python27Packages.markupsafe}/lib/python2.7/site-packages:${pkgs.python27Packages.sqlalchemy_migrate}/lib/python2.7/site-packages:${pkgs.python27Packages.tempita}/lib/python2.7/site-packages:${pkgs.python27Packages.decorator}/lib/python2.7/site-packages:${pkgs.python27Packages.sqlparse}/lib/python2.7/site-packages:${pkgs.python27Packages.txaio}/lib/python2.7/site-packages:${pkgs.python27Packages.autobahn}/lib/python2.7/site-packages:${pkgs.python27Packages.pyjwt}/lib/python2.7/site-packages:${pkgs.python27Packages.distro}/lib/python2.7/site-packages:${pkgs.python27Packages.pbr}/lib/python2.7/site-packages:${pkgs.python27Packages.urllib3}/lib/python2.7/site-packages";
# NOTE: call twistd directly with stdout logging for systemd
#ExecStart = "${cfg.package}/bin/buildbot start --nodaemon ${cfg.buildbotDir}";
ExecStart = "${pkgs.python27Packages.twisted}/bin/twistd -n -l - -y ${cfg.buildbotDir}/buildbot.tac";
};
preStart = ''
${pkgs.coreutils}/bin/mkdir -vp ${cfg.buildbotDir}
${pkgs.coreutils}/bin/ln -sfv ${masterCfg} ${cfg.buildbotDir}/master.cfg
${cfg.package}/bin/buildbot create-master ${cfg.buildbotDir}
'';
postStart = ''
until [[ $(${pkgs.curl}/bin/curl -s --head -w '\n%{http_code}' http://localhost:${toString cfg.port} | tail -n1) =~ ^(200|403)$ ]]; do
sleep 1
done
'';
};
};

11
nixos/modules/services/continuous-integration/buildbot/worker.nix
View File

@ -74,7 +74,7 @@ in {
};
packages = mkOption {
default = [ ];
default = with pkgs; [ python27Packages.twisted git ];
example = literalExample "[ pkgs.git ]";
type = types.listOf types.package;
description = "Packages to add to PATH for the buildbot process.";
@ -106,7 +106,8 @@ in {
path = cfg.packages;
preStart = ''
${pkgs.coreutils}/bin/mkdir -vp ${cfg.buildbotDir}
mkdir -vp ${cfg.buildbotDir}
rm -fv $cfg.buildbotDir}/buildbot.tac
${cfg.package}/bin/buildbot-worker create-worker ${cfg.buildbotDir} ${cfg.masterUrl} ${cfg.workerUser} ${cfg.workerPass}
'';
@ -115,7 +116,11 @@ in {
User = cfg.user;
Group = cfg.group;
WorkingDirectory = cfg.home;
ExecStart = "${cfg.package}/bin/buildbot-worker start --nodaemon ${cfg.buildbotDir}";
Environment = "PYTHONPATH=${cfg.package}/lib/python2.7/site-packages:${pkgs.python27Packages.future}/lib/python2.7/site-packages";
# NOTE: call twistd directly with stdout logging for systemd
#ExecStart = "${cfg.package}/bin/buildbot-worker start --nodaemon ${cfg.buildbotDir}";
ExecStart = "${pkgs.python27Packages.twisted}/bin/twistd -n -l - -y ${cfg.buildbotDir}/buildbot.tac";
};
};

95
nixos/tests/buildbot.nix
View File

@ -4,18 +4,22 @@ import ./make-test.nix ({ pkgs, ... } : {
name = "buildbot";
nodes = {
bbmaster = { config, pkgs, nodes, ... }: {
bbmaster = { config, pkgs, ... }: {
services.buildbot-master = {
enable = true;
package = pkgs.buildbot-full;
# NOTE: use fake repo due to no internet in hydra ci
factorySteps = [
"steps.Git(repourl='git://github.com/buildbot/pyflakes.git', mode='incremental')"
"steps.ShellCommand(command=['trial', 'pyflakes'])"
"steps.Git(repourl='git://gitrepo/fakerepo.git', mode='incremental')"
"steps.ShellCommand(command=['bash', 'fakerepo.sh'])"
];
changeSource = [
"changes.GitPoller('git://github.com/buildbot/pyflakes.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)"
"changes.GitPoller('git://gitrepo/fakerepo.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)"
];
};
networking.firewall.allowedTCPPorts = [ 8010 9989 ];
networking.firewall.allowedTCPPorts = [ 8010 8011 9989 ];
environment.systemPackages = with pkgs; [ git buildbot-full ];
};
bbworker = { config, pkgs, ... }: {
@ -23,24 +27,85 @@ import ./make-test.nix ({ pkgs, ... } : {
enable = true;
masterUrl = "bbmaster:9989";
};
environment.systemPackages = with pkgs; [ git buildbot-worker ];
};
gitrepo = { config, pkgs, ... }: {
services.openssh.enable = true;
networking.firewall.allowedTCPPorts = [ 22 9418 ];
environment.systemPackages = with pkgs; [ git ];
};
};
testScript = ''
#Start up and populate fake repo
$gitrepo->waitForUnit("multi-user.target");
print($gitrepo->execute(" \
git config --global user.name 'Nobody Fakeuser' && \
git config --global user.email 'nobody\@fakerepo.com' && \
rm -rvf /srv/repos/fakerepo.git /tmp/fakerepo && \
mkdir -pv /srv/repos/fakerepo ~/.ssh && \
ssh-keyscan -H gitrepo > ~/.ssh/known_hosts && \
cat ~/.ssh/known_hosts && \
cd /srv/repos/fakerepo && \
git init && \
echo -e '#!/bin/sh\necho fakerepo' > fakerepo.sh && \
cat fakerepo.sh && \
touch .git/git-daemon-export-ok && \
git add fakerepo.sh .git/git-daemon-export-ok && \
git commit -m fakerepo && \
git daemon --verbose --export-all --base-path=/srv/repos --reuseaddr & \
"));
$bbmaster->waitForUnit("network.target");
$bbworker->waitForUnit("network.target");
# Test gitrepo
$bbmaster->waitForUnit("network-online.target");
#$bbmaster->execute("nc -z gitrepo 9418");
print($bbmaster->execute(" \
rm -rfv /tmp/fakerepo && \
git clone git://gitrepo/fakerepo /tmp/fakerepo && \
pwd && \
ls -la && \
ls -la /tmp/fakerepo \
"));
# Additional tests to be added
#$bbmaster->waitForUnit("buildbot-master.service");
#$bbmaster->waitUntilSucceeds("curl -s --head http://bbmaster:8010") =~ /200 OK/ or die;
#$bbworker->waitForUnit("buildbot-worker.service");
#$bbworker->waitUntilSucceeds("tail -10 /home/bbworker/worker/twistd.log") =~ /success/ or die;
# Test start master and connect worker
$bbmaster->waitForUnit("buildbot-master.service");
$bbmaster->waitUntilSucceeds("curl -s --head http://bbmaster:8010") =~ /200 OK/;
$bbworker->waitForUnit("network-online.target");
$bbworker->execute("nc -z bbmaster 8010");
$bbworker->execute("nc -z bbmaster 9989");
$bbworker->waitForUnit("buildbot-worker.service");
print($bbworker->execute("ls -la /home/bbworker/worker"));
# Test stop buildbot master and worker
print($bbmaster->execute(" \
systemctl -l --no-pager status buildbot-master && \
systemctl stop buildbot-master \
"));
$bbworker->fail("nc -z bbmaster 8010");
$bbworker->fail("nc -z bbmaster 9989");
print($bbworker->execute(" \
systemctl -l --no-pager status buildbot-worker && \
systemctl stop buildbot-worker && \
ls -la /home/bbworker/worker \
"));
# Test buildbot daemon mode
# NOTE: daemon mode tests disabled due to broken PYTHONPATH child inheritence
#
#$bbmaster->execute("buildbot create-master /tmp");
#$bbmaster->execute("mv -fv /tmp/master.cfg.sample /tmp/master.cfg");
#$bbmaster->execute("sed -i 's/8010/8011/' /tmp/master.cfg");
#$bbmaster->execute("buildbot start /tmp");
#$bbworker->execute("nc -z bbmaster 8011");
#$bbworker->waitUntilSucceeds("curl -s --head http://bbmaster:8011") =~ /200 OK/;
#$bbmaster->execute("buildbot stop /tmp");
#$bbworker->fail("nc -z bbmaster 8011");
'';
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ nand0p ];
};
meta.maintainers = with pkgs.stdenv.lib.maintainers; [ nand0p ];
})

17
pkgs/development/tools/build-managers/buildbot/default.nix
View File

@ -5,17 +5,19 @@ let
buildInputs = [ makeWrapper ] ++ plugins;
passthru.withPlugins = moarPlugins: withPlugins (moarPlugins ++ plugins);
} ''
makeWrapper ${package}/bin/buildbot $out/bin/buildbot --prefix PYTHONPATH : $PYTHONPATH
makeWrapper ${package}/bin/buildbot $out/bin/buildbot \
--prefix PYTHONPATH : "${package}/lib/python2.7/site-packages:$PYTHONPATH"
ln -sfv ${package}/lib $out/lib
'';
package = pythonPackages.buildPythonApplication (rec {
name = "${pname}-${version}";
pname = "buildbot";
version = "0.9.4";
version = "0.9.5";
src = pythonPackages.fetchPypi {
inherit pname version;
sha256 = "0wklrn4fszac9wi8zw3vbsznwyff6y57cz0i81zvh46skb6n3086";
sha256 = "11r553nmh87a9pm58wycimapk2pw9hnlc7hffn97xwbqprd8qh66";
};
buildInputs = with pythonPackages; [
@ -49,6 +51,7 @@ let
txaio
autobahn
pyjwt
distro
# tls
pyopenssl
@ -68,6 +71,14 @@ let
postPatch = ''
substituteInPlace buildbot/scripts/logwatcher.py --replace '/usr/bin/tail' "$(type -P tail)"
# NOTE: secrets management tests currently broken
rm -fv buildbot/test/integration/test_integration_secrets.py
rm -fv buildbot/test/integration/test_integration_secrets_with_vault.py
rm -fv buildbot/test/unit/test_fake_secrets_manager.py
rm -fv buildbot/test/unit/test_interpolate_secrets.py
rm -fv buildbot/test/unit/test_secret_in_file.py
rm -fv buildbot/test/unit/test_secret_in_vault.py
'';
passthru = { inherit withPlugins; };

31
pkgs/development/tools/build-managers/buildbot/plugins.nix
View File

@ -1,14 +1,14 @@
{ stdenv, fetchurl, pythonPackages }:
{ stdenv, pythonPackages }:
let
buildbot-pkg = pythonPackages.buildPythonPackage rec {
name = "${pname}-${version}";
pname = "buildbot-pkg";
version = "0.9.4";
version = "0.9.5";
src = fetchurl {
url = "mirror://pypi/b/${pname}/${name}.tar.gz";
sha256 = "09a3yvs5hhf8syrkyydznmymgg86dpvgrwy9rb3bryq00wpjb3wn";
src = pythonPackages.fetchPypi {
inherit pname version;
sha256 = "1xpi4w0lc6z97pmmms85dvdspacbzlvs8zi3kv1r4rypk3znwmi1";
};
propagatedBuildInputs = with pythonPackages; [ setuptools ];
@ -25,15 +25,14 @@ in {
www = pythonPackages.buildPythonPackage rec {
name = "${pname}-${version}";
pname = "buildbot_www";
version = "0.9.4";
version = "0.9.5";
# NOTE: wheel is used due to buildbot circular dependency
format = "wheel";
src = pythonPackages.fetchPypi {
inherit pname version format;
python = "py2";
sha256 = "08m4h2pf6hgi8igh2j0qzfq49izc2z0qqj6ddxk0di5l306jx4im";
sha256 = "1d7yjxka6slflm3wbdpq4sr1kagmgbqdv2zgx9bq77jvjh7ga0py";
};
meta = with stdenv.lib; {
@ -47,11 +46,11 @@ in {
console-view = pythonPackages.buildPythonPackage rec {
name = "${pname}-${version}";
pname = "buildbot-console-view";
version = "0.9.4";
version = "0.9.5";
src = fetchurl {
url = "mirror://pypi/b/${pname}/${name}.tar.gz";
sha256 = "1w2vv8iyzl7ak4161avp9n6mhh08adav2fl82bbm17a3064apl8n";
src = pythonPackages.fetchPypi {
inherit pname version;
sha256 = "1s6mvw955dsgk7hvb1xa32bbd7w2yma62py5s0vmi5shv8nwq3hb";
};
propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ];
@ -67,11 +66,11 @@ in {
waterfall-view = pythonPackages.buildPythonPackage rec {
name = "${pname}-${version}";
pname = "buildbot-waterfall-view";
version = "0.9.4";
version = "0.9.5";
src = fetchurl {
url = "mirror://pypi/b/${pname}/${name}.tar.gz";
sha256 = "17xn6vrr0k2xabw6hr9sdyy0ry3llyjfmc79qrpgp5bsly2qv3jf";
src = pythonPackages.fetchPypi {
inherit pname version;
sha256 = "116846d987wp1bz78f0h4lypqcns5073vzhb4vsqbf08sppgr67k";
};
propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ];

4
pkgs/development/tools/build-managers/buildbot/worker.nix
View File

@ -3,11 +3,11 @@
pythonPackages.buildPythonApplication (rec {
name = "${pname}-${version}";
pname = "buildbot-worker";
version = "0.9.4";
version = "0.9.5";
src = pythonPackages.fetchPypi {
inherit pname version;
sha256 = "0rdrr8x7sn2nxl51p6h9ad42s3c28lb6sys84zrg0d7fm4zhv7hj";
sha256 = "1al7jam351sf781axj4kfhj70cc0g21zv81ynk410kdccjyxp2dy";
};
buildInputs = with pythonPackages; [ setuptoolsTrial mock ];