diff --git a/nixos/modules/services/cluster/kubernetes.nix b/nixos/modules/services/cluster/kubernetes.nix index 7fd2d77aa825..d9ae0454ba5e 100644 --- a/nixos/modules/services/cluster/kubernetes.nix +++ b/nixos/modules/services/cluster/kubernetes.nix @@ -44,6 +44,12 @@ in { type = types.path; }; + dockerCfg = mkOption { + description = "Kubernetes contents of dockercfg file."; + default = ""; + type = types.lines; + }; + apiserver = { enable = mkOption { description = "Whether to enable kubernetes apiserver."; @@ -217,13 +223,13 @@ in { }; machines = mkOption { - description = "Kubernetes apiserver list of machines to schedule to schedule onto"; + description = "Kubernetes controller list of machines to schedule to schedule onto"; default = []; type = types.listOf types.str; }; extraOpts = mkOption { - description = "Kubernetes scheduler extra command line options."; + description = "Kubernetes controller extra command line options."; default = ""; type = types.str; }; @@ -260,6 +266,30 @@ in { type = types.bool; }; + apiServers = mkOption { + description = "Kubernetes kubelet list of Kubernetes API servers for publishing events, and reading pods and services."; + default = ["${cfg.apiserver.address}:${toString cfg.apiserver.port}"]; + type = types.listOf types.str; + }; + + cadvisorPort = mkOption { + description = "Kubernetes kubelet local cadvisor port."; + default = config.services.cadvisor.port; + type = types.int; + }; + + clusterDns = mkOption { + description = "Use alternative dns."; + default = ""; + type = types.str; + }; + + clusterDomain = mkOption { + description = "Use alternative domain."; + default = ""; + type = types.str; + }; + extraOpts = mkOption { description = "Kubernetes kubelet extra command line options."; default = ""; @@ -295,6 +325,7 @@ in { systemd.services.kubernetes-apiserver = { description = "Kubernetes Api Server"; wantedBy = [ "multi-user.target" ]; + requires = ["kubernetes-setup.service"]; after = [ "network-interfaces.target" "etcd.service" ]; serviceConfig = { ExecStart = let @@ -306,26 +337,25 @@ in { (concatImapStringsSep "\n" (i: v: v + "," + (toString i)) (mapAttrsToList (name: token: token + "," + name) cfg.apiserver.tokenAuth)); in ''${cfg.package}/bin/kube-apiserver \ - -etcd_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \ - -address=${cfg.apiserver.address} \ - -port=${toString cfg.apiserver.port} \ - -read_only_port=${toString cfg.apiserver.readOnlyPort} \ - -public_address_override=${cfg.apiserver.publicAddress} \ - -allow_privileged=${if cfg.apiserver.allowPrivileged then "true" else "false"} \ + --etcd_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \ + --address=${cfg.apiserver.address} \ + --port=${toString cfg.apiserver.port} \ + --read_only_port=${toString cfg.apiserver.readOnlyPort} \ + --public_address_override=${cfg.apiserver.publicAddress} \ + --allow_privileged=${if cfg.apiserver.allowPrivileged then "true" else "false"} \ ${optionalString (cfg.apiserver.tlsCertFile!="") - "-tls_cert_file=${cfg.apiserver.tlsCertFile}"} \ + "--tls_cert_file=${cfg.apiserver.tlsCertFile}"} \ ${optionalString (cfg.apiserver.tlsPrivateKeyFile!="") - "-tls_private_key_file=${cfg.apiserver.tlsPrivateKeyFile}"} \ + "--tls_private_key_file=${cfg.apiserver.tlsPrivateKeyFile}"} \ ${optionalString (cfg.apiserver.tokenAuth!=[]) - "-token_auth_file=${tokenAuthFile}"} \ - -authorization_mode=${cfg.apiserver.authorizationMode} \ + "--token_auth_file=${tokenAuthFile}"} \ + --authorization_mode=${cfg.apiserver.authorizationMode} \ ${optionalString (cfg.apiserver.authorizationMode == "ABAC") - "-authorization_policy_file=${authorizationPolicyFile}"} \ - ${optionalString (cfg.apiserver.tlsCertFile!="" && cfg.apiserver.tlsCertFile!="") - "-secure_port=${toString cfg.apiserver.securePort}"} \ - -portal_net=${cfg.apiserver.portalNet} \ - -logtostderr=true \ - ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \ + "--authorization_policy_file=${authorizationPolicyFile}"} \ + --secure_port=${toString cfg.apiserver.securePort} \ + --portal_net=${cfg.apiserver.portalNet} \ + --logtostderr=true \ + ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \ ${cfg.apiserver.extraOpts} ''; User = "kubernetes"; @@ -345,11 +375,11 @@ in { after = [ "network-interfaces.target" "kubernetes-apiserver.service" ]; serviceConfig = { ExecStart = ''${cfg.package}/bin/kube-scheduler \ - -address=${cfg.scheduler.address} \ - -port=${toString cfg.scheduler.port} \ - -master=${cfg.scheduler.master} \ - -logtostderr=true \ - ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \ + --address=${cfg.scheduler.address} \ + --port=${toString cfg.scheduler.port} \ + --master=${cfg.scheduler.master} \ + --logtostderr=true \ + ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \ ${cfg.scheduler.extraOpts} ''; User = "kubernetes"; @@ -364,13 +394,12 @@ in { after = [ "network-interfaces.target" "kubernetes-apiserver.service" ]; serviceConfig = { ExecStart = ''${cfg.package}/bin/kube-controller-manager \ - -address=${cfg.controllerManager.address} \ - -port=${toString cfg.controllerManager.port} \ - -master=${cfg.controllerManager.master} \ - ${optionalString (cfg.controllerManager.machines != []) - "-machines=${concatStringsSep "," cfg.controllerManager.machines}"} \ - -logtostderr=true \ - ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \ + --address=${cfg.controllerManager.address} \ + --port=${toString cfg.controllerManager.port} \ + --master=${cfg.controllerManager.master} \ + --machines=${concatStringsSep "," cfg.controllerManager.machines} \ + --logtostderr=true \ + ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \ ${cfg.controllerManager.extraOpts} ''; User = "kubernetes"; @@ -382,23 +411,28 @@ in { systemd.services.kubernetes-kubelet = { description = "Kubernetes Kubelet Service"; wantedBy = [ "multi-user.target" ]; + requires = ["kubernetes-setup.service"]; after = [ "network-interfaces.target" "etcd.service" "docker.service" ]; - serviceConfig = { - ExecStart = ''${cfg.package}/bin/kubelet \ - -etcd_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \ - -address=${cfg.kubelet.address} \ - -port=${toString cfg.kubelet.port} \ - -hostname_override=${cfg.kubelet.hostname} \ - -allow_privileged=${if cfg.kubelet.allowPrivileged then "true" else "false"} \ - -root_dir=${cfg.dataDir} \ - -logtostderr=true \ - ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \ + script = '' + export PATH="/bin:/sbin:/usr/bin:/usr/sbin:$PATH" + exec ${cfg.package}/bin/kubelet \ + --etcd_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \ + --api_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.kubelet.apiServers} \ + --address=${cfg.kubelet.address} \ + --port=${toString cfg.kubelet.port} \ + --hostname_override=${cfg.kubelet.hostname} \ + --allow_privileged=${if cfg.kubelet.allowPrivileged then "true" else "false"} \ + --root_dir=${cfg.dataDir} \ + --cadvisor_port=${toString cfg.kubelet.cadvisorPort} \ + ${optionalString (cfg.kubelet.clusterDns != "") + ''--cluster_dns=${cfg.kubelet.clusterDns}''} \ + ${optionalString (cfg.kubelet.clusterDomain != "") + ''--cluster_domain=${cfg.kubelet.clusterDomain}''} \ + --logtostderr=true \ + ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \ ${cfg.kubelet.extraOpts} ''; - User = "kubernetes"; - PermissionsStartOnly = true; - WorkingDirectory = cfg.dataDir; - }; + serviceConfig.WorkingDirectory = cfg.dataDir; }; }) @@ -409,10 +443,10 @@ in { after = [ "network-interfaces.target" "etcd.service" ]; serviceConfig = { ExecStart = ''${cfg.package}/bin/kube-proxy \ - -etcd_servers=${concatMapStringsSep "," (s: "http://${s}") cfg.etcdServers} \ - -bind_address=${cfg.proxy.address} \ - -logtostderr=true \ - ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \ + --etcd_servers=${concatMapStringsSep "," (s: "http://${s}") cfg.etcdServers} \ + --bind_address=${cfg.proxy.address} \ + --logtostderr=true \ + ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \ ${cfg.proxy.extraOpts} ''; }; @@ -427,6 +461,8 @@ in { (mkIf (any (el: el == "node") cfg.roles) { virtualisation.docker.enable = mkDefault true; + services.cadvisor.enable = mkDefault true; + services.cadvisor.port = mkDefault 4194; services.kubernetes.kubelet.enable = mkDefault true; services.kubernetes.proxy.enable = mkDefault true; }) @@ -442,6 +478,16 @@ in { cfg.kubelet.enable || cfg.proxy.enable ) { + systemd.services.kubernetes-setup = { + description = "Kubernetes setup."; + serviceConfig.Type = "oneshot"; + script = '' + mkdir -p /var/run/kubernetes + chown kubernetes /var/run/kubernetes + ln -fs ${pkgs.writeText "kubernetes-dockercfg" cfg.dockerCfg} /var/run/kubernetes/.dockercfg + ''; + }; + services.kubernetes.package = mkDefault pkgs.kubernetes; environment.systemPackages = [ cfg.package ]; diff --git a/nixos/tests/kubernetes.nix b/nixos/tests/kubernetes.nix index 8bc7c8d618ad..0a9eda930bad 100644 --- a/nixos/tests/kubernetes.nix +++ b/nixos/tests/kubernetes.nix @@ -13,7 +13,7 @@ import ./make-test.nix rec { id: redis-master-pod containers: - name: master - image: master:5000/scratch + image: master:5000/nix cpu: 100 ports: - name: redis-server @@ -50,8 +50,8 @@ import ./make-test.nix rec { virtualisation.memorySize = 768; services.kubernetes = { roles = ["master" "node"]; + dockerCfg = ''{"master:5000":{}}''; controllerManager.machines = ["master" "node"]; - kubelet.extraOpts = "-network_container_image=master:5000/pause"; apiserver.address = "0.0.0.0"; verbose = true; }; @@ -94,7 +94,8 @@ import ./make-test.nix rec { { services.kubernetes = { roles = ["node"]; - kubelet.extraOpts = "-network_container_image=master:5000/pause"; + dockerCfg = ''{"master:5000":{}}''; + kubelet.apiServers = ["master:8080"]; verbose = true; }; virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0 --insecure-registry master:5000"; @@ -155,14 +156,14 @@ import ./make-test.nix rec { $node->waitForUnit("kubernetes-kubelet.service"); $node->waitForUnit("kubernetes-proxy.service"); - $master->waitUntilSucceeds("kubecfg list minions | grep master"); - $master->waitUntilSucceeds("kubecfg list minions | grep node"); + $master->waitUntilSucceeds("kubectl get minions | grep master"); + $master->waitUntilSucceeds("kubectl get minions | grep node"); $client->waitForUnit("docker.service"); - $client->succeed("tar cv --files-from /dev/null | docker import - scratch"); - $client->succeed("docker tag scratch master:5000/scratch"); + $client->succeed("tar cv --files-from /dev/null | docker import - nix"); + $client->succeed("docker tag nix master:5000/nix"); $master->waitForUnit("docker-registry.service"); - $client->succeed("docker push master:5000/scratch"); + $client->succeed("docker push master:5000/nix"); $client->succeed("mkdir -p /root/pause"); $client->succeed("cp /etc/test/pause /root/pause/"); $client->succeed("cp /etc/test/Dockerfile /root/pause/"); diff --git a/pkgs/applications/networking/cluster/kubernetes/default.nix b/pkgs/applications/networking/cluster/kubernetes/default.nix index 7931da602d6e..c89ad0a51685 100644 --- a/pkgs/applications/networking/cluster/kubernetes/default.nix +++ b/pkgs/applications/networking/cluster/kubernetes/default.nix @@ -1,14 +1,14 @@ -{ stdenv, fetchFromGitHub, which, go, makeWrapper, iptables,rsync }: +{ stdenv, fetchFromGitHub, which, go, makeWrapper, iptables, rsync, utillinux, coreutils }: stdenv.mkDerivation rec { name = "kubernetes-${version}"; - version = "v0.5.4"; + version = "0.12.1"; src = fetchFromGitHub { owner = "GoogleCloudPlatform"; repo = "kubernetes"; - rev = version; - sha256 = "1pipcqpjz9zsi4kfsbdvbbbia642l4xg50pznjw5v061c5xk7vnk"; + rev = "v${version}"; + sha256 = "1891wpssfp04nkk1h4y3cdgn096b0kq16pc0m2fzilbh3daa6pml"; }; buildInputs = [ makeWrapper which go iptables rsync ]; @@ -24,7 +24,8 @@ stdenv.mkDerivation rec { ''; preFixup = '' - wrapProgram "$out/bin/kube-proxy" --set "PATH" "${iptables}/bin" + wrapProgram "$out/bin/kube-proxy" --prefix PATH : "${iptables}/bin" + wrapProgram "$out/bin/kubelet" --prefix PATH : "${utillinux}/bin" ''; meta = with stdenv.lib; {