401 lines
13 KiB
Nix
401 lines
13 KiB
Nix
{ config, lib, pkgs, ... }:
|
|
|
|
with lib;
|
|
|
|
let
|
|
top = config.services.kubernetes;
|
|
cfg = top.pki;
|
|
|
|
csrCA = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON {
|
|
key = {
|
|
algo = "rsa";
|
|
size = 2048;
|
|
};
|
|
names = singleton cfg.caSpec;
|
|
});
|
|
|
|
csrCfssl = pkgs.writeText "kube-pki-cfssl-csr.json" (builtins.toJSON {
|
|
key = {
|
|
algo = "rsa";
|
|
size = 2048;
|
|
};
|
|
CN = top.masterAddress;
|
|
hosts = cfg.cfsslAPIExtraSANs;
|
|
});
|
|
|
|
cfsslAPITokenBaseName = "apitoken.secret";
|
|
cfsslAPITokenPath = "${config.services.cfssl.dataDir}/${cfsslAPITokenBaseName}";
|
|
certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}";
|
|
cfsslAPITokenLength = 32;
|
|
|
|
clusterAdminKubeconfig = with cfg.certs.clusterAdmin;
|
|
top.lib.mkKubeConfig "cluster-admin" {
|
|
server = top.apiserverAddress;
|
|
certFile = cert;
|
|
keyFile = key;
|
|
};
|
|
|
|
remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}";
|
|
in
|
|
{
|
|
###### interface
|
|
options.services.kubernetes.pki = with lib.types; {
|
|
|
|
enable = mkEnableOption "easyCert issuer service";
|
|
|
|
certs = mkOption {
|
|
description = "List of certificate specs to feed to cert generator.";
|
|
default = {};
|
|
type = attrs;
|
|
};
|
|
|
|
genCfsslCACert = mkOption {
|
|
description = ''
|
|
Whether to automatically generate cfssl CA certificate and key,
|
|
if they don't exist.
|
|
'';
|
|
default = true;
|
|
type = bool;
|
|
};
|
|
|
|
genCfsslAPICerts = mkOption {
|
|
description = ''
|
|
Whether to automatically generate cfssl API webserver TLS cert and key,
|
|
if they don't exist.
|
|
'';
|
|
default = true;
|
|
type = bool;
|
|
};
|
|
|
|
cfsslAPIExtraSANs = mkOption {
|
|
description = ''
|
|
Extra x509 Subject Alternative Names to be added to the cfssl API webserver TLS cert.
|
|
'';
|
|
default = [];
|
|
example = [ "subdomain.example.com" ];
|
|
type = listOf str;
|
|
};
|
|
|
|
genCfsslAPIToken = mkOption {
|
|
description = ''
|
|
Whether to automatically generate cfssl API-token secret,
|
|
if they doesn't exist.
|
|
'';
|
|
default = true;
|
|
type = bool;
|
|
};
|
|
|
|
pkiTrustOnBootstrap = mkOption {
|
|
description = "Whether to always trust remote cfssl server upon initial PKI bootstrap.";
|
|
default = true;
|
|
type = bool;
|
|
};
|
|
|
|
caCertPathPrefix = mkOption {
|
|
description = ''
|
|
Path-prefrix for the CA-certificate to be used for cfssl signing.
|
|
Suffixes ".pem" and "-key.pem" will be automatically appended for
|
|
the public and private keys respectively.
|
|
'';
|
|
default = "${config.services.cfssl.dataDir}/ca";
|
|
type = str;
|
|
};
|
|
|
|
caSpec = mkOption {
|
|
description = "Certificate specification for the auto-generated CAcert.";
|
|
default = {
|
|
CN = "kubernetes-cluster-ca";
|
|
O = "NixOS";
|
|
OU = "services.kubernetes.pki.caSpec";
|
|
L = "auto-generated";
|
|
};
|
|
type = attrs;
|
|
};
|
|
|
|
etcClusterAdminKubeconfig = mkOption {
|
|
description = ''
|
|
Symlink a kubeconfig with cluster-admin privileges to environment path
|
|
(/etc/<path>).
|
|
'';
|
|
default = null;
|
|
type = nullOr str;
|
|
};
|
|
|
|
};
|
|
|
|
###### implementation
|
|
config = mkIf cfg.enable
|
|
(let
|
|
cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl";
|
|
cfsslCert = "${cfsslCertPathPrefix}.pem";
|
|
cfsslKey = "${cfsslCertPathPrefix}-key.pem";
|
|
in
|
|
{
|
|
|
|
services.cfssl = mkIf (top.apiserver.enable) {
|
|
enable = true;
|
|
address = "0.0.0.0";
|
|
tlsCert = cfsslCert;
|
|
tlsKey = cfsslKey;
|
|
configFile = toString (pkgs.writeText "cfssl-config.json" (builtins.toJSON {
|
|
signing = {
|
|
profiles = {
|
|
default = {
|
|
usages = ["digital signature"];
|
|
auth_key = "default";
|
|
expiry = "720h";
|
|
};
|
|
};
|
|
};
|
|
auth_keys = {
|
|
default = {
|
|
type = "standard";
|
|
key = "file:${cfsslAPITokenPath}";
|
|
};
|
|
};
|
|
}));
|
|
};
|
|
|
|
systemd.services.cfssl.preStart = with pkgs; with config.services.cfssl; mkIf (top.apiserver.enable)
|
|
(concatStringsSep "\n" [
|
|
"set -e"
|
|
(optionalString cfg.genCfsslCACert ''
|
|
if [ ! -f "${cfg.caCertPathPrefix}.pem" ]; then
|
|
${cfssl}/bin/cfssl genkey -initca ${csrCA} | \
|
|
${cfssl}/bin/cfssljson -bare ${cfg.caCertPathPrefix}
|
|
fi
|
|
'')
|
|
(optionalString cfg.genCfsslAPICerts ''
|
|
if [ ! -f "${dataDir}/cfssl.pem" ]; then
|
|
${cfssl}/bin/cfssl gencert -ca "${cfg.caCertPathPrefix}.pem" -ca-key "${cfg.caCertPathPrefix}-key.pem" ${csrCfssl} | \
|
|
${cfssl}/bin/cfssljson -bare ${cfsslCertPathPrefix}
|
|
fi
|
|
'')
|
|
(optionalString cfg.genCfsslAPIToken ''
|
|
if [ ! -f "${cfsslAPITokenPath}" ]; then
|
|
head -c ${toString (cfsslAPITokenLength / 2)} /dev/urandom | od -An -t x | tr -d ' ' >"${cfsslAPITokenPath}"
|
|
fi
|
|
chown cfssl "${cfsslAPITokenPath}" && chmod 400 "${cfsslAPITokenPath}"
|
|
'')]);
|
|
|
|
systemd.services.kube-certmgr-bootstrap = {
|
|
description = "Kubernetes certmgr bootstrapper";
|
|
wantedBy = [ "certmgr.service" ];
|
|
after = [ "cfssl.target" ];
|
|
script = concatStringsSep "\n" [''
|
|
set -e
|
|
|
|
# If there's a cfssl (cert issuer) running locally, then don't rely on user to
|
|
# manually paste it in place. Just symlink.
|
|
# otherwise, create the target file, ready for users to insert the token
|
|
|
|
if [ -f "${cfsslAPITokenPath}" ]; then
|
|
ln -fs "${cfsslAPITokenPath}" "${certmgrAPITokenPath}"
|
|
else
|
|
touch "${certmgrAPITokenPath}" && chmod 600 "${certmgrAPITokenPath}"
|
|
fi
|
|
''
|
|
(optionalString (cfg.pkiTrustOnBootstrap) ''
|
|
if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then
|
|
${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \
|
|
${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}
|
|
fi
|
|
'')
|
|
];
|
|
serviceConfig = {
|
|
RestartSec = "10s";
|
|
Restart = "on-failure";
|
|
};
|
|
};
|
|
|
|
services.certmgr = {
|
|
enable = true;
|
|
package = pkgs.certmgr-selfsigned;
|
|
svcManager = "command";
|
|
specs =
|
|
let
|
|
mkSpec = _: cert: {
|
|
inherit (cert) action;
|
|
authority = {
|
|
inherit remote;
|
|
file.path = cert.caCert;
|
|
root_ca = cert.caCert;
|
|
profile = "default";
|
|
auth_key_file = certmgrAPITokenPath;
|
|
};
|
|
certificate = {
|
|
path = cert.cert;
|
|
};
|
|
private_key = cert.privateKeyOptions;
|
|
request = {
|
|
inherit (cert) CN hosts;
|
|
key = {
|
|
algo = "rsa";
|
|
size = 2048;
|
|
};
|
|
names = [ cert.fields ];
|
|
};
|
|
};
|
|
in
|
|
mapAttrs mkSpec cfg.certs;
|
|
};
|
|
|
|
#TODO: Get rid of kube-addon-manager in the future for the following reasons
|
|
# - it is basically just a shell script wrapped around kubectl
|
|
# - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
|
|
# - it is designed to be used with k8s system components only
|
|
# - it would be better with a more Nix-oriented way of managing addons
|
|
systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{
|
|
environment.KUBECONFIG = with cfg.certs.addonManager;
|
|
top.lib.mkKubeConfig "addon-manager" {
|
|
server = top.apiserverAddress;
|
|
certFile = cert;
|
|
keyFile = key;
|
|
};
|
|
}
|
|
|
|
(optionalAttrs (top.addonManager.bootstrapAddons != {}) {
|
|
serviceConfig.PermissionsStartOnly = true;
|
|
preStart = with pkgs;
|
|
let
|
|
files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
|
|
top.addonManager.bootstrapAddons;
|
|
in
|
|
''
|
|
export KUBECONFIG=${clusterAdminKubeconfig}
|
|
${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
|
|
'';
|
|
})]);
|
|
|
|
environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig)
|
|
clusterAdminKubeconfig;
|
|
|
|
environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [
|
|
(pkgs.writeScriptBin "nixos-kubernetes-node-join" ''
|
|
set -e
|
|
exec 1>&2
|
|
|
|
if [ $# -gt 0 ]; then
|
|
echo "Usage: $(basename $0)"
|
|
echo ""
|
|
echo "No args. Apitoken must be provided on stdin."
|
|
echo "To get the apitoken, execute: 'sudo cat ${certmgrAPITokenPath}' on the master node."
|
|
exit 1
|
|
fi
|
|
|
|
if [ $(id -u) != 0 ]; then
|
|
echo "Run as root please."
|
|
exit 1
|
|
fi
|
|
|
|
read -r token
|
|
if [ ''${#token} != ${toString cfsslAPITokenLength} ]; then
|
|
echo "Token must be of length ${toString cfsslAPITokenLength}."
|
|
exit 1
|
|
fi
|
|
|
|
echo $token > ${certmgrAPITokenPath}
|
|
chmod 600 ${certmgrAPITokenPath}
|
|
|
|
echo "Restarting certmgr..." >&1
|
|
systemctl restart certmgr
|
|
|
|
echo "Waiting for certs to appear..." >&1
|
|
|
|
${optionalString top.kubelet.enable ''
|
|
while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done
|
|
echo "Restarting kubelet..." >&1
|
|
systemctl restart kubelet
|
|
''}
|
|
|
|
${optionalString top.proxy.enable ''
|
|
while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done
|
|
echo "Restarting kube-proxy..." >&1
|
|
systemctl restart kube-proxy
|
|
''}
|
|
|
|
${optionalString top.flannel.enable ''
|
|
while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done
|
|
echo "Restarting flannel..." >&1
|
|
systemctl restart flannel
|
|
''}
|
|
|
|
echo "Node joined succesfully"
|
|
'')];
|
|
|
|
# isolate etcd on loopback at the master node
|
|
# easyCerts doesn't support multimaster clusters anyway atm.
|
|
services.etcd = with cfg.certs.etcd; {
|
|
listenClientUrls = ["https://127.0.0.1:2379"];
|
|
listenPeerUrls = ["https://127.0.0.1:2380"];
|
|
advertiseClientUrls = ["https://etcd.local:2379"];
|
|
initialCluster = ["${top.masterAddress}=https://etcd.local:2380"];
|
|
initialAdvertisePeerUrls = ["https://etcd.local:2380"];
|
|
certFile = mkDefault cert;
|
|
keyFile = mkDefault key;
|
|
trustedCaFile = mkDefault caCert;
|
|
};
|
|
networking.extraHosts = mkIf (config.services.etcd.enable) ''
|
|
127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
|
|
'';
|
|
|
|
services.flannel = with cfg.certs.flannelClient; {
|
|
kubeconfig = top.lib.mkKubeConfig "flannel" {
|
|
server = top.apiserverAddress;
|
|
certFile = cert;
|
|
keyFile = key;
|
|
};
|
|
};
|
|
|
|
services.kubernetes = {
|
|
|
|
apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; {
|
|
etcd = with cfg.certs.apiserverEtcdClient; {
|
|
servers = ["https://etcd.local:2379"];
|
|
certFile = mkDefault cert;
|
|
keyFile = mkDefault key;
|
|
caFile = mkDefault caCert;
|
|
};
|
|
clientCaFile = mkDefault caCert;
|
|
tlsCertFile = mkDefault cert;
|
|
tlsKeyFile = mkDefault key;
|
|
serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert;
|
|
kubeletClientCaFile = mkDefault caCert;
|
|
kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert;
|
|
kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key;
|
|
proxyClientCertFile = mkDefault cfg.certs.apiserverProxyClient.cert;
|
|
proxyClientKeyFile = mkDefault cfg.certs.apiserverProxyClient.key;
|
|
});
|
|
controllerManager = mkIf top.controllerManager.enable {
|
|
serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key;
|
|
rootCaFile = cfg.certs.controllerManagerClient.caCert;
|
|
kubeconfig = with cfg.certs.controllerManagerClient; {
|
|
certFile = mkDefault cert;
|
|
keyFile = mkDefault key;
|
|
};
|
|
};
|
|
scheduler = mkIf top.scheduler.enable {
|
|
kubeconfig = with cfg.certs.schedulerClient; {
|
|
certFile = mkDefault cert;
|
|
keyFile = mkDefault key;
|
|
};
|
|
};
|
|
kubelet = mkIf top.kubelet.enable {
|
|
clientCaFile = mkDefault cfg.certs.kubelet.caCert;
|
|
tlsCertFile = mkDefault cfg.certs.kubelet.cert;
|
|
tlsKeyFile = mkDefault cfg.certs.kubelet.key;
|
|
kubeconfig = with cfg.certs.kubeletClient; {
|
|
certFile = mkDefault cert;
|
|
keyFile = mkDefault key;
|
|
};
|
|
};
|
|
proxy = mkIf top.proxy.enable {
|
|
kubeconfig = with cfg.certs.kubeProxyClient; {
|
|
certFile = mkDefault cert;
|
|
keyFile = mkDefault key;
|
|
};
|
|
};
|
|
};
|
|
});
|
|
}
|