diff --git a/nixos/modules/services/cluster/kubernetes/addon-manager.nix b/nixos/modules/services/cluster/kubernetes/addon-manager.nix index 406b20b0d8d8..b9a56811d2bd 100644 --- a/nixos/modules/services/cluster/kubernetes/addon-manager.nix +++ b/nixos/modules/services/cluster/kubernetes/addon-manager.nix @@ -63,24 +63,48 @@ in }; enable = mkEnableOption "Whether to enable Kubernetes addon manager."; + + kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes addon manager"; + bootstrapAddonsKubeconfig = top.lib.mkKubeConfigOptions "Kubernetes addon manager bootstrap"; }; ###### implementation - config = mkIf cfg.enable { + config = let + + addonManagerPaths = filter (a: a != null) [ + cfg.kubeconfig.caFile + cfg.kubeconfig.certFile + cfg.kubeconfig.keyFile + ]; + bootstrapAddonsPaths = filter (a: a != null) [ + cfg.bootstrapAddonsKubeconfig.caFile + cfg.bootstrapAddonsKubeconfig.certFile + cfg.bootstrapAddonsKubeconfig.keyFile + ]; + + in mkIf cfg.enable { environment.etc."kubernetes/addons".source = "${addons}/"; + #TODO: Get rid of kube-addon-manager in the future for the following reasons + # - it is basically just a shell script wrapped around kubectl + # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount + # - it is designed to be used with k8s system components only + # - it would be better with a more Nix-oriented way of managing addons systemd.services.kube-addon-manager = { description = "Kubernetes addon manager"; - wantedBy = [ "kube-control-plane-online.target" ]; - after = [ "kube-addon-manager-bootstrap.service" ]; - before = [ "kube-control-plane-online.target" ]; - environment.ADDON_PATH = "/etc/kubernetes/addons/"; - path = [ pkgs.gawk ]; + wantedBy = [ "kubernetes.target" ]; + after = [ "kube-node-online.target" ]; + before = [ "kubernetes.target" ]; + environment = { + ADDON_PATH = "/etc/kubernetes/addons/"; + KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager" cfg.kubeconfig; + }; + path = with pkgs; [ gawk kubectl ]; preStart = '' - ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager; { - path = "/api/v1/namespaces/kube-system/serviceaccounts/default"; - cacert = top.caFile; - } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })} + until kubectl -n kube-system get serviceaccounts/default 2>/dev/null; do + echo kubectl -n kube-system get serviceaccounts/default: exit status $? + sleep 2 + done ''; serviceConfig = { Slice = "kubernetes.slice"; @@ -91,27 +115,52 @@ in Restart = "on-failure"; RestartSec = 10; }; + unitConfig.ConditionPathExists = addonManagerPaths; }; + systemd.paths.kube-addon-manager = { + wantedBy = [ "kube-addon-manager.service" ]; + pathConfig = { + PathExists = addonManagerPaths; + PathChanged = addonManagerPaths; + }; + }; + + services.kubernetes.addonManager.kubeconfig.server = mkDefault top.apiserverAddress; + systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) { wantedBy = [ "kube-control-plane-online.target" ]; after = [ "kube-apiserver.service" ]; before = [ "kube-control-plane-online.target" ]; path = [ pkgs.kubectl ]; + environment = { + KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager-bootstrap" cfg.bootstrapAddonsKubeconfig; + }; preStart = with pkgs; let files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v)) cfg.bootstrapAddons; in '' - ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager-bootstrap; { - path = "/api"; - cacert = top.caFile; - } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })} + until kubectl auth can-i '*' '*' -q 2>/dev/null; do + echo kubectl auth can-i '*' '*': exit status $? + sleep 2 + done kubectl apply -f ${concatStringsSep " \\\n -f " files} ''; script = "echo Ok"; + unitConfig.ConditionPathExists = bootstrapAddonsPaths; }; + systemd.paths.kube-addon-manager-bootstrap = { + wantedBy = [ "kube-addon-manager-bootstrap.service" ]; + pathConfig = { + PathExists = bootstrapAddonsPaths; + PathChanged = bootstrapAddonsPaths; + }; + }; + + services.kubernetes.addonManager.bootstrapAddonsKubeconfig.server = mkDefault top.apiserverAddress; + services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled (let name = system:kube-addon-manager; diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix index 8bacc07b0089..90b40dd4c1f6 100644 --- a/nixos/modules/services/cluster/kubernetes/pki.nix +++ b/nixos/modules/services/cluster/kubernetes/pki.nix @@ -27,12 +27,11 @@ let certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}"; cfsslAPITokenLength = 32; - clusterAdminKubeconfig = with cfg.certs.clusterAdmin; - top.lib.mkKubeConfig "cluster-admin" { - server = top.apiserverAddress; - certFile = cert; - keyFile = key; - }; + clusterAdminKubeconfig = with cfg.certs.clusterAdmin; { + server = top.apiserverAddress; + certFile = cert; + keyFile = key; + }; remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}"; in @@ -142,12 +141,6 @@ in config.services.etcd.keyFile config.services.etcd.trustedCaFile ]; - addonManagerPaths = mkIf top.addonManager.enable [ - cfg.certs.addonManager.cert - cfg.certs.addonManager.key - cfg.certs.clusterAdmin.cert - cfg.certs.clusterAdmin.key - ]; flannelPaths = [ cfg.certs.flannelClient.cert cfg.certs.flannelClient.key @@ -331,38 +324,6 @@ in }; }; - systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) { - environment = { - KUBECONFIG = clusterAdminKubeconfig; - inherit (cfg.certs.clusterAdmin) cert key; - }; - }; - - #TODO: Get rid of kube-addon-manager in the future for the following reasons - # - it is basically just a shell script wrapped around kubectl - # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount - # - it is designed to be used with k8s system components only - # - it would be better with a more Nix-oriented way of managing addons - systemd.services.kube-addon-manager = mkIf top.addonManager.enable { - environment = with cfg.certs.addonManager; { - KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager" { - server = top.apiserverAddress; - certFile = cert; - keyFile = key; - }; - inherit cert key; - }; - unitConfig.ConditionPathExists = addonManagerPaths; - }; - - systemd.paths.kube-addon-manager = mkIf top.addonManager.enable { - wantedBy = [ "kube-addon-manager.service" ]; - pathConfig = { - PathExists = addonManagerPaths; - PathChanged = addonManagerPaths; - }; - }; - systemd.services.kube-controller-manager = mkIf top.controllerManager.enable { environment = { inherit (cfg.certs.controllerManagerClient) cert key; }; unitConfig.ConditionPathExists = controllerManagerPaths; @@ -396,7 +357,7 @@ in }; environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig) - clusterAdminKubeconfig; + (top.lib.mkKubeConfig "cluster-admin" clusterAdminKubeconfig); environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [ (pkgs.writeScriptBin "nixos-kubernetes-node-join" '' @@ -538,6 +499,13 @@ in kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert; kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key; }); + addonManager = mkIf top.addonManager.enable { + kubeconfig = with cfg.certs.addonManager; { + certFile = mkDefault cert; + keyFile = mkDefault key; + }; + bootstrapAddonsKubeconfig = clusterAdminKubeconfig; + }; controllerManager = mkIf top.controllerManager.enable { serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key; rootCaFile = cfg.certs.controllerManagerClient.caCert;