nixos: kubernetes fixes

* Fix reference CNI plugins
  * The plugins were split out of the upstream cni repo around version
    0.6.0

* Fix RBAC and DNS tests
  * Fix broken apiVersion fields
  * Change plugin linking to look in ${package}/bin rather than
    ${package.plugins}

* Initial work towards a working e2e test
  * Test still fails, but at least the expression evaluates now

Continues @srhb's work in #37199

Fixes #37199

nixos/modules/services/cluster/kubernetes/default.nix

@@ -766,7 +766,7 @@ in {
           rm /opt/cni/bin/* || true
           ${concatMapStrings (package: ''
             echo "Linking cni package: ${package}"
-            ln -fs ${package.plugins}/* /opt/cni/bin
+            ln -fs ${package}/bin/* /opt/cni/bin
           '') cfg.kubelet.cni.packages}
         '';
         serviceConfig = {
@@ -828,7 +828,7 @@ in {
       };
 
       # Allways include cni plugins
-      services.kubernetes.kubelet.cni.packages = [pkgs.cni];
+      services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins];
 
       boot.kernelModules = ["br_netfilter"];
 

nixos/release.nix

@@ -296,7 +296,10 @@ in rec {
   tests.kernel-copperhead = callTest tests/kernel-copperhead.nix {};
   tests.kernel-latest = callTest tests/kernel-latest.nix {};
   tests.kernel-lts = callTest tests/kernel-lts.nix {};
-  tests.kubernetes = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/kubernetes/default.nix {};
+  tests.kubernetes.dns = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/kubernetes/dns.nix {};
+  ## kubernetes.e2e should eventually replace kubernetes.rbac when it works
+  #tests.kubernetes.e2e = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/kubernetes/e2e.nix {};
+  tests.kubernetes.rbac = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/kubernetes/rbac.nix {};
   tests.latestKernel.login = callTest tests/login.nix { latestKernel = true; };
   tests.ldap = callTest tests/ldap.nix {};
   #tests.lightdm = callTest tests/lightdm.nix {};

nixos/tests/kubernetes/certs.nix

@@ -6,29 +6,62 @@
   kubelets
 }:
 let
-  runWithCFSSL = name: cmd:
-    builtins.fromJSON (builtins.readFile (
-      pkgs.runCommand "${name}-cfss.json" {
-        buildInputs = [ pkgs.cfssl ];
-      } "cfssl ${cmd} > $out"
-    ));
+   runWithCFSSL = name: cmd:
+     let secrets = pkgs.runCommand "${name}-cfss.json" {
+         buildInputs = [ pkgs.cfssl pkgs.jq ];
+         outputs = [ "out" "cert" "key" "csr" ];
+       }
+       ''
+         (
+           echo "${cmd}"
+           cfssl ${cmd} > tmp
+           cat tmp | jq -r .key > $key
+           cat tmp | jq -r .cert > $cert
+           cat tmp | jq -r .csr > $csr
 
-  writeCFSSL = content:
-    pkgs.runCommand content.name {
-      buildInputs = [ pkgs.cfssl ];
-    } ''
-      mkdir -p $out
-      cd $out
-      cat ${writeFile content} | cfssljson -bare ${content.name}
-    '';
+           touch $out
+         ) 2>&1 | fold -w 80 -s
+       '';
+     in {
+       key = secrets.key;
+       cert = secrets.cert;
+       csr = secrets.csr;
+     };
+
+   writeCFSSL = content:
+     pkgs.runCommand content.name {
+      buildInputs = [ pkgs.cfssl pkgs.jq ];
+     } ''
+       mkdir -p $out
+       cd $out
+
+       json=${pkgs.lib.escapeShellArg (builtins.toJSON content)}
+
+       # for a given $field in the $json, treat the associated value as a
+       # file path and substitute the contents thereof into the $json
+       # object.
+       expandFileField() {
+         local field=$1
+         if jq -e --arg field "$field" 'has($field)'; then
+           local path="$(echo "$json" | jq -r ".$field")"
+           json="$(echo "$json" | jq --arg val "$(cat "$path")" ".$field = \$val")"
+         fi
+       }
+
+       expandFileField key
+       expandFileField ca
+       expandFileField cert
+
+       echo "$json" | cfssljson -bare ${content.name}
+     '';
 
   noCSR = content: pkgs.lib.filterAttrs (n: v: n != "csr") content;
   noKey = content: pkgs.lib.filterAttrs (n: v: n != "key") content;
 
-  writeFile = content: pkgs.writeText "content" (
-    if pkgs.lib.isAttrs content then builtins.toJSON content
-    else toString content
-  );
+  writeFile = content:
+    if pkgs.lib.isDerivation content
+    then content
+    else pkgs.writeText "content" (builtins.toJSON content);
 
   createServingCertKey = { ca, cn, hosts? [], size ? 2048, name ? cn }:
     noCSR (

nixos/tests/kubernetes/e2e.nix

@@ -2,7 +2,7 @@
 with import ./base.nix { inherit system; };
 let
   domain = "my.zyx";
-  certs = import ./certs.nix { externalDomain = domain; };
+  certs = import ./certs.nix { externalDomain = domain; kubelets = ["machine1" "machine2"]; };
   kubeconfig = pkgs.writeText "kubeconfig.json" (builtins.toJSON {
     apiVersion = "v1";
     kind = "Config";

nixos/tests/kubernetes/rbac.nix

@@ -12,7 +12,7 @@ let
   });
 
   roRoleBinding = pkgs.writeText "ro-role-binding.json" (builtins.toJSON {
-    apiVersion = "rbac.authorization.k8s.io/v1beta1";
+    apiVersion = "rbac.authorization.k8s.io/v1";
     kind = "RoleBinding";
     metadata = {
       name = "read-pods";
@@ -31,7 +31,7 @@ let
   });
 
   roRole = pkgs.writeText "ro-role.json" (builtins.toJSON {
-    apiVersion = "rbac.authorization.k8s.io/v1beta1";
+    apiVersion = "rbac.authorization.k8s.io/v1";
     kind = "Role";
     metadata = {
       name = "pod-reader";

pkgs/applications/networking/cluster/cni/default.nix

@@ -13,17 +13,14 @@ stdenv.mkDerivation rec {
 
   buildInputs = [ go ];
 
-  outputs = ["out" "plugins"];
-
   buildPhase = ''
     patchShebangs build.sh
     ./build.sh
   '';
 
   installPhase = ''
-    mkdir -p $out/bin $plugins
+    mkdir -p $out/bin
     mv bin/cnitool $out/bin
-    mv bin/* $plugins/
   '';
 
   meta = with stdenv.lib; {

pkgs/applications/networking/cluster/cni/plugins.nix

@@ -0,0 +1,33 @@
+{ stdenv, lib, fetchFromGitHub, go }:
+
+stdenv.mkDerivation rec {
+  name = "cni-plugins-${version}";
+  version = "0.7.0";
+
+  src = fetchFromGitHub {
+    owner = "containernetworking";
+    repo = "plugins";
+    rev = "v${version}";
+    sha256 = "0m885v76azs7lrk6m6n53rwh0xadwvdcr90h0l3bxpdv87sj2mnf";
+  };
+
+  buildInputs = [ go ];
+
+  buildPhase = ''
+    patchShebangs build.sh
+    ./build.sh
+  '';
+
+  installPhase = ''
+    mkdir -p $out/bin
+    mv bin/* $out/bin
+  '';
+
+  meta = with lib; {
+    description = "Some standard networking plugins, maintained by the CNI team";
+    homepage = https://github.com/containernetworking/plugins;
+    license = licenses.asl20;
+    platforms = [ "x86_64-linux" ];
+    maintainers = with maintainers; [ cstrahan ];
+  };
+}

pkgs/top-level/all-packages.nix

@@ -15053,6 +15053,7 @@ with pkgs;
   };
 
   cni = callPackage ../applications/networking/cluster/cni {};
+  cni-plugins = callPackage ../applications/networking/cluster/cni/plugins.nix {};
 
   communi = libsForQt5.callPackage ../applications/networking/irc/communi { };