Merge pull request #6874 from offlinehacker/pkgs/kubernetes/update

kubernetes: update to 0.12.1, fix module
2015-03-20 11:22:48 +01:00 · 2015-03-20 11:22:48 +01:00 · 89f41b90fa
commit 89f41b90fa
parent 3bb0e6df8a 2482b75eae
3 changed files with 109 additions and 61 deletions
--- a/nixos/modules/services/cluster/kubernetes.nix
+++ b/nixos/modules/services/cluster/kubernetes.nix
@ -44,6 +44,12 @@ in {
      type = types.path;
    };

+    dockerCfg = mkOption {
+      description = "Kubernetes contents of dockercfg file.";
+      default = "";
+      type = types.lines;
+    };
+
    apiserver = {
      enable = mkOption {
        description = "Whether to enable kubernetes apiserver.";
@ -217,13 +223,13 @@ in {
      };

      machines = mkOption {
-        description = "Kubernetes apiserver list of machines to schedule to schedule onto";
+        description = "Kubernetes controller list of machines to schedule to schedule onto";
        default = [];
        type = types.listOf types.str;
      };

      extraOpts = mkOption {
-        description = "Kubernetes scheduler extra command line options.";
+        description = "Kubernetes controller extra command line options.";
        default = "";
        type = types.str;
      };
@ -260,6 +266,30 @@ in {
        type = types.bool;
      };

+      apiServers = mkOption {
+        description = "Kubernetes kubelet list of Kubernetes API servers for publishing events, and reading pods and services.";
+        default = ["${cfg.apiserver.address}:${toString cfg.apiserver.port}"];
+        type = types.listOf types.str;
+      };
+
+      cadvisorPort = mkOption {
+        description = "Kubernetes kubelet local cadvisor port.";
+        default = config.services.cadvisor.port;
+        type = types.int;
+      };
+
+      clusterDns = mkOption {
+        description = "Use alternative dns.";
+        default = "";
+        type = types.str;
+      };
+
+      clusterDomain = mkOption {
+        description = "Use alternative domain.";
+        default = "";
+        type = types.str;
+      };
+
      extraOpts = mkOption {
        description = "Kubernetes kubelet extra command line options.";
        default = "";
@ -295,6 +325,7 @@ in {
      systemd.services.kubernetes-apiserver = {
        description = "Kubernetes Api Server";
        wantedBy = [ "multi-user.target" ];
+        requires = ["kubernetes-setup.service"];
        after = [ "network-interfaces.target" "etcd.service" ];
        serviceConfig = {
          ExecStart = let
@ -306,26 +337,25 @@ in {
                (concatImapStringsSep "\n" (i: v: v + "," + (toString i))
                    (mapAttrsToList (name: token: token + "," + name) cfg.apiserver.tokenAuth));
          in ''${cfg.package}/bin/kube-apiserver \
-            -etcd_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \
-            -address=${cfg.apiserver.address} \
-            -port=${toString cfg.apiserver.port} \
-            -read_only_port=${toString cfg.apiserver.readOnlyPort} \
-            -public_address_override=${cfg.apiserver.publicAddress} \
-            -allow_privileged=${if cfg.apiserver.allowPrivileged then "true" else "false"} \
+            --etcd_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \
+            --address=${cfg.apiserver.address} \
+            --port=${toString cfg.apiserver.port} \
+            --read_only_port=${toString cfg.apiserver.readOnlyPort} \
+            --public_address_override=${cfg.apiserver.publicAddress} \
+            --allow_privileged=${if cfg.apiserver.allowPrivileged then "true" else "false"} \
            ${optionalString (cfg.apiserver.tlsCertFile!="")
-              "-tls_cert_file=${cfg.apiserver.tlsCertFile}"} \
+              "--tls_cert_file=${cfg.apiserver.tlsCertFile}"} \
            ${optionalString (cfg.apiserver.tlsPrivateKeyFile!="")
-              "-tls_private_key_file=${cfg.apiserver.tlsPrivateKeyFile}"} \
+              "--tls_private_key_file=${cfg.apiserver.tlsPrivateKeyFile}"} \
            ${optionalString (cfg.apiserver.tokenAuth!=[])
-              "-token_auth_file=${tokenAuthFile}"} \
-            -authorization_mode=${cfg.apiserver.authorizationMode} \
+              "--token_auth_file=${tokenAuthFile}"} \
+            --authorization_mode=${cfg.apiserver.authorizationMode} \
            ${optionalString (cfg.apiserver.authorizationMode == "ABAC")
-              "-authorization_policy_file=${authorizationPolicyFile}"} \
-            ${optionalString (cfg.apiserver.tlsCertFile!="" && cfg.apiserver.tlsCertFile!="")
-              "-secure_port=${toString cfg.apiserver.securePort}"} \
-            -portal_net=${cfg.apiserver.portalNet} \
-            -logtostderr=true \
-            ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \
+              "--authorization_policy_file=${authorizationPolicyFile}"} \
+            --secure_port=${toString cfg.apiserver.securePort} \
+            --portal_net=${cfg.apiserver.portalNet} \
+            --logtostderr=true \
+            ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
            ${cfg.apiserver.extraOpts}
          '';
          User = "kubernetes";
@ -345,11 +375,11 @@ in {
        after = [ "network-interfaces.target" "kubernetes-apiserver.service" ];
        serviceConfig = {
          ExecStart = ''${cfg.package}/bin/kube-scheduler \
-            -address=${cfg.scheduler.address} \
-            -port=${toString cfg.scheduler.port} \
-            -master=${cfg.scheduler.master} \
-            -logtostderr=true \
-            ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \
+            --address=${cfg.scheduler.address} \
+            --port=${toString cfg.scheduler.port} \
+            --master=${cfg.scheduler.master} \
+            --logtostderr=true \
+            ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
            ${cfg.scheduler.extraOpts}
          '';
          User = "kubernetes";
@ -364,13 +394,12 @@ in {
        after = [ "network-interfaces.target" "kubernetes-apiserver.service" ];
        serviceConfig = {
          ExecStart = ''${cfg.package}/bin/kube-controller-manager \
-            -address=${cfg.controllerManager.address} \
-            -port=${toString cfg.controllerManager.port} \
-            -master=${cfg.controllerManager.master} \
-            ${optionalString (cfg.controllerManager.machines != [])
-                "-machines=${concatStringsSep "," cfg.controllerManager.machines}"} \
-            -logtostderr=true \
-            ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \
+            --address=${cfg.controllerManager.address} \
+            --port=${toString cfg.controllerManager.port} \
+            --master=${cfg.controllerManager.master} \
+            --machines=${concatStringsSep "," cfg.controllerManager.machines} \
+            --logtostderr=true \
+            ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
            ${cfg.controllerManager.extraOpts}
          '';
          User = "kubernetes";
@ -382,23 +411,28 @@ in {
      systemd.services.kubernetes-kubelet = {
        description = "Kubernetes Kubelet Service";
        wantedBy = [ "multi-user.target" ];
+        requires = ["kubernetes-setup.service"];
        after = [ "network-interfaces.target" "etcd.service" "docker.service" ];
-        serviceConfig = {
-          ExecStart = ''${cfg.package}/bin/kubelet \
-            -etcd_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \
-            -address=${cfg.kubelet.address} \
-            -port=${toString cfg.kubelet.port} \
-            -hostname_override=${cfg.kubelet.hostname} \
-            -allow_privileged=${if cfg.kubelet.allowPrivileged then "true" else "false"} \
-            -root_dir=${cfg.dataDir} \
-            -logtostderr=true \
-            ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \
+        script = ''
+          export PATH="/bin:/sbin:/usr/bin:/usr/sbin:$PATH"
+          exec ${cfg.package}/bin/kubelet \
+            --etcd_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \
+            --api_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.kubelet.apiServers}  \
+            --address=${cfg.kubelet.address} \
+            --port=${toString cfg.kubelet.port} \
+            --hostname_override=${cfg.kubelet.hostname} \
+            --allow_privileged=${if cfg.kubelet.allowPrivileged then "true" else "false"} \
+            --root_dir=${cfg.dataDir} \
+            --cadvisor_port=${toString cfg.kubelet.cadvisorPort} \
+            ${optionalString (cfg.kubelet.clusterDns != "")
+                ''--cluster_dns=${cfg.kubelet.clusterDns}''} \
+            ${optionalString (cfg.kubelet.clusterDomain != "")
+                ''--cluster_domain=${cfg.kubelet.clusterDomain}''} \
+            --logtostderr=true \
+            ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
            ${cfg.kubelet.extraOpts}
          '';
-          User = "kubernetes";
-          PermissionsStartOnly = true;
-          WorkingDirectory = cfg.dataDir;
-        };
+        serviceConfig.WorkingDirectory = cfg.dataDir;
      };
    })

@ -409,10 +443,10 @@ in {
        after = [ "network-interfaces.target" "etcd.service" ];
        serviceConfig = {
          ExecStart = ''${cfg.package}/bin/kube-proxy \
-            -etcd_servers=${concatMapStringsSep "," (s: "http://${s}") cfg.etcdServers} \
-            -bind_address=${cfg.proxy.address} \
-            -logtostderr=true \
-            ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \
+            --etcd_servers=${concatMapStringsSep "," (s: "http://${s}") cfg.etcdServers} \
+            --bind_address=${cfg.proxy.address} \
+            --logtostderr=true \
+            ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
            ${cfg.proxy.extraOpts}
          '';
        };
@ -427,6 +461,8 @@ in {

    (mkIf (any (el: el == "node") cfg.roles) {
      virtualisation.docker.enable = mkDefault true;
+      services.cadvisor.enable = mkDefault true;
+      services.cadvisor.port = mkDefault 4194;
      services.kubernetes.kubelet.enable = mkDefault true;
      services.kubernetes.proxy.enable = mkDefault true;
    })
@ -442,6 +478,16 @@ in {
        cfg.kubelet.enable ||
        cfg.proxy.enable
    ) {
+      systemd.services.kubernetes-setup = {
+        description = "Kubernetes setup.";
+        serviceConfig.Type = "oneshot";
+        script = ''
+          mkdir -p /var/run/kubernetes
+          chown kubernetes /var/run/kubernetes
+          ln -fs ${pkgs.writeText "kubernetes-dockercfg" cfg.dockerCfg} /var/run/kubernetes/.dockercfg
+        '';
+      };
+
      services.kubernetes.package = mkDefault pkgs.kubernetes;

      environment.systemPackages = [ cfg.package ];
--- a/nixos/tests/kubernetes.nix
+++ b/nixos/tests/kubernetes.nix
@ -13,7 +13,7 @@ import ./make-test.nix rec {
          id: redis-master-pod
          containers:
            - name: master
-              image: master:5000/scratch
+              image: master:5000/nix
              cpu: 100
              ports:
                - name: redis-server
@ -50,8 +50,8 @@ import ./make-test.nix rec {
          virtualisation.memorySize = 768;
          services.kubernetes = {
            roles = ["master" "node"];
+            dockerCfg = ''{"master:5000":{}}'';
            controllerManager.machines = ["master" "node"];
-            kubelet.extraOpts = "-network_container_image=master:5000/pause";
            apiserver.address = "0.0.0.0";
            verbose = true;
          };
@ -94,7 +94,8 @@ import ./make-test.nix rec {
        {
          services.kubernetes = {
            roles = ["node"];
-            kubelet.extraOpts = "-network_container_image=master:5000/pause";
+            dockerCfg = ''{"master:5000":{}}'';
+            kubelet.apiServers = ["master:8080"];
            verbose = true;
          };
          virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0 --insecure-registry master:5000";
@ -155,14 +156,14 @@ import ./make-test.nix rec {
    $node->waitForUnit("kubernetes-kubelet.service");
    $node->waitForUnit("kubernetes-proxy.service");

-    $master->waitUntilSucceeds("kubecfg list minions | grep master");
-    $master->waitUntilSucceeds("kubecfg list minions | grep node");
+    $master->waitUntilSucceeds("kubectl get minions | grep master");
+    $master->waitUntilSucceeds("kubectl get minions | grep node");

    $client->waitForUnit("docker.service");
-    $client->succeed("tar cv --files-from /dev/null | docker import - scratch");
-    $client->succeed("docker tag scratch master:5000/scratch");
+    $client->succeed("tar cv --files-from /dev/null | docker import - nix");
+    $client->succeed("docker tag nix master:5000/nix");
    $master->waitForUnit("docker-registry.service");
-    $client->succeed("docker push master:5000/scratch");
+    $client->succeed("docker push master:5000/nix");
    $client->succeed("mkdir -p /root/pause");
    $client->succeed("cp /etc/test/pause /root/pause/");
    $client->succeed("cp /etc/test/Dockerfile /root/pause/");
--- a/pkgs/applications/networking/cluster/kubernetes/default.nix
+++ b/pkgs/applications/networking/cluster/kubernetes/default.nix
@ -1,14 +1,14 @@
-{ stdenv, fetchFromGitHub, which, go, makeWrapper, iptables,rsync }:
+{ stdenv, fetchFromGitHub, which, go, makeWrapper, iptables, rsync, utillinux, coreutils }:

 stdenv.mkDerivation rec {
  name = "kubernetes-${version}";
-  version = "v0.5.4";
+  version = "0.12.1";

  src = fetchFromGitHub {
    owner = "GoogleCloudPlatform";
    repo = "kubernetes";
-    rev = version;
-    sha256 = "1pipcqpjz9zsi4kfsbdvbbbia642l4xg50pznjw5v061c5xk7vnk";
+    rev = "v${version}";
+    sha256 = "1891wpssfp04nkk1h4y3cdgn096b0kq16pc0m2fzilbh3daa6pml";
  };

  buildInputs = [ makeWrapper which go iptables rsync ];
@ -24,7 +24,8 @@ stdenv.mkDerivation rec {
  '';

  preFixup = ''
-    wrapProgram "$out/bin/kube-proxy" --set "PATH" "${iptables}/bin"
+    wrapProgram "$out/bin/kube-proxy" --prefix PATH : "${iptables}/bin"
+    wrapProgram "$out/bin/kubelet" --prefix PATH : "${utillinux}/bin"
  '';

  meta = with stdenv.lib; {