kubernetes: fix tests
This commit is contained in:
Matej Cotman
Robin Gloster
parent
1febe07de7
commit
6ef8cad2a7
113
nixos/tests/kubernetes/base.nix
Normal file
113
nixos/tests/kubernetes/base.nix
Normal file
@ -0,0 +1,113 @@
|
||||
{ system ? builtins.currentSystem }:
|
||||
|
||||
with import ../../lib/testing.nix { inherit system; };
|
||||
with import ../../lib/qemu-flags.nix;
|
||||
with pkgs.lib;
|
||||
|
||||
let
|
||||
mkKubernetesBaseTest =
|
||||
{ name, domain ? "my.zyx", test, machines
|
||||
, pkgs ? import <nixpkgs> { inherit system; }
|
||||
, certs ? import ./certs.nix { inherit pkgs; externalDomain = domain; }
|
||||
, extraConfiguration ? null }:
|
||||
let
|
||||
masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
|
||||
master = machines.${masterName};
|
||||
extraHosts = ''
|
||||
${master.ip} etcd.${domain}
|
||||
${master.ip} api.${domain}
|
||||
${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip} ${machineName}.${domain}") (attrNames machines)}
|
||||
'';
|
||||
in makeTest {
|
||||
inherit name;
|
||||
|
||||
nodes = mapAttrs (machineName: machine:
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
mkMerge [
|
||||
{
|
||||
virtualisation.memorySize = mkDefault 768;
|
||||
virtualisation.diskSize = mkDefault 4096;
|
||||
networking = {
|
||||
inherit domain extraHosts;
|
||||
primaryIPAddress = mkForce machine.ip;
|
||||
|
||||
firewall = {
|
||||
allowedTCPPorts = [
|
||||
10250 # kubelet
|
||||
];
|
||||
trustedInterfaces = ["docker0"];
|
||||
|
||||
extraCommands = concatMapStrings (node: ''
|
||||
iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
|
||||
'') (attrValues nodes);
|
||||
};
|
||||
};
|
||||
programs.bash.enableCompletion = true;
|
||||
environment.variables = {
|
||||
ETCDCTL_CERT_FILE = "${certs.worker}/etcd-client.pem";
|
||||
ETCDCTL_KEY_FILE = "${certs.worker}/etcd-client-key.pem";
|
||||
ETCDCTL_CA_FILE = "${certs.worker}/ca.pem";
|
||||
ETCDCTL_PEERS = "https://etcd.${domain}:2379";
|
||||
};
|
||||
services.flannel.iface = "eth1";
|
||||
services.kubernetes.apiserver.advertiseAddress = master.ip;
|
||||
}
|
||||
(optionalAttrs (any (role: role == "master") machine.roles) {
|
||||
networking.firewall.allowedTCPPorts = [
|
||||
2379 2380 # etcd
|
||||
443 # kubernetes apiserver
|
||||
];
|
||||
services.etcd = {
|
||||
enable = true;
|
||||
certFile = "${certs.master}/etcd.pem";
|
||||
keyFile = "${certs.master}/etcd-key.pem";
|
||||
trustedCaFile = "${certs.master}/ca.pem";
|
||||
peerClientCertAuth = true;
|
||||
listenClientUrls = ["https://0.0.0.0:2379"];
|
||||
listenPeerUrls = ["https://0.0.0.0:2380"];
|
||||
advertiseClientUrls = ["https://etcd.${config.networking.domain}:2379"];
|
||||
initialCluster = ["${masterName}=https://etcd.${config.networking.domain}:2380"];
|
||||
initialAdvertisePeerUrls = ["https://etcd.${config.networking.domain}:2380"];
|
||||
};
|
||||
})
|
||||
(import ./kubernetes-common.nix { inherit (machine) roles; inherit pkgs config certs; })
|
||||
(optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; }))
|
||||
(optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))
|
||||
]
|
||||
) machines;
|
||||
|
||||
testScript = ''
|
||||
startAll;
|
||||
|
||||
${test}
|
||||
'';
|
||||
};
|
||||
|
||||
mkKubernetesMultiNodeTest = attrs: mkKubernetesBaseTest ({
|
||||
machines = {
|
||||
machine1 = {
|
||||
roles = ["master"];
|
||||
ip = "192.168.1.1";
|
||||
};
|
||||
machine2 = {
|
||||
roles = ["node"];
|
||||
ip = "192.168.1.2";
|
||||
};
|
||||
};
|
||||
} // attrs // {
|
||||
name = "kubernetes-${attrs.name}-multinode";
|
||||
});
|
||||
|
||||
mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({
|
||||
machines = {
|
||||
machine1 = {
|
||||
roles = ["master" "node"];
|
||||
ip = "192.168.1.1";
|
||||
};
|
||||
};
|
||||
} // attrs // {
|
||||
name = "kubernetes-${attrs.name}-singlenode";
|
||||
});
|
||||
in {
|
||||
inherit mkKubernetesBaseTest mkKubernetesSingleNodeTest mkKubernetesMultiNodeTest;
|
||||
}
|
||||
@ -1,229 +1,185 @@
|
||||
{
|
||||
pkgs ? import <nixpkgs> {},
|
||||
servers ? {test = "1.2.3.4";},
|
||||
internalDomain ? "cluster.local",
|
||||
externalDomain ? "nixos.xyz"
|
||||
internalDomain ? "cloud.yourdomain.net",
|
||||
externalDomain ? "myawesomecluster.cluster.yourdomain.net",
|
||||
serviceClusterIp ? "10.0.0.1"
|
||||
}:
|
||||
let
|
||||
mkAltNames = ipFrom: dnsFrom:
|
||||
pkgs.lib.concatImapStringsSep "\n" (i: v: "IP.${toString (i+ipFrom)} = ${v.ip}\nDNS.${toString (i+dnsFrom)} = ${v.name}.${externalDomain}") (pkgs.lib.mapAttrsToList (n: v: {name = n; ip = v;}) servers);
|
||||
runWithCFSSL = name: cmd:
|
||||
builtins.fromJSON (builtins.readFile (
|
||||
pkgs.runCommand "${name}-cfss.json" {
|
||||
buildInputs = [ pkgs.cfssl ];
|
||||
} "cfssl ${cmd} > $out"
|
||||
));
|
||||
|
||||
runWithOpenSSL = file: cmd: pkgs.runCommand file {
|
||||
buildInputs = [ pkgs.openssl ];
|
||||
passthru = { inherit file; };
|
||||
} cmd;
|
||||
|
||||
ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048";
|
||||
ca_pem = runWithOpenSSL "ca.pem" ''
|
||||
openssl req \
|
||||
-x509 -new -nodes -key ${ca_key} \
|
||||
-days 10000 -out $out -subj "/CN=etcd-ca"
|
||||
'';
|
||||
|
||||
etcd_cnf = pkgs.writeText "openssl.cnf" ''
|
||||
[req]
|
||||
req_extensions = v3_req
|
||||
distinguished_name = req_distinguished_name
|
||||
[req_distinguished_name]
|
||||
[ v3_req ]
|
||||
basicConstraints = CA:FALSE
|
||||
keyUsage = digitalSignature, keyEncipherment
|
||||
extendedKeyUsage = serverAuth
|
||||
subjectAltName = @alt_names
|
||||
[alt_names]
|
||||
DNS.1 = etcd.kubernetes.${externalDomain}
|
||||
IP.1 = 127.0.0.1
|
||||
'';
|
||||
etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048";
|
||||
etcd_csr = runWithOpenSSL "etcd.csr" ''
|
||||
openssl req \
|
||||
-new -key ${etcd_key} \
|
||||
-out $out -subj "/CN=etcd" \
|
||||
-config ${etcd_cnf}
|
||||
'';
|
||||
etcd_cert = runWithOpenSSL "etcd.pem" ''
|
||||
openssl x509 \
|
||||
-req -in ${etcd_csr} \
|
||||
-CA ${ca_pem} -CAkey ${ca_key} \
|
||||
-CAcreateserial -out $out \
|
||||
-days 3650 -extensions v3_req \
|
||||
-extfile ${etcd_cnf}
|
||||
'';
|
||||
|
||||
etcd_client_key = runWithOpenSSL "etcd-client-key.pem"
|
||||
"openssl genrsa -out $out 2048";
|
||||
|
||||
etcd_client_csr = runWithOpenSSL "etcd-client.csr" ''
|
||||
openssl req \
|
||||
-new -key ${etcd_client_key} \
|
||||
-out $out -subj "/CN=etcd-client" \
|
||||
-config ${client_openssl_cnf}
|
||||
'';
|
||||
|
||||
etcd_client_cert = runWithOpenSSL "etcd-client.crt" ''
|
||||
openssl x509 \
|
||||
-req -in ${etcd_client_csr} \
|
||||
-CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
|
||||
-out $out -days 3650 -extensions v3_req \
|
||||
-extfile ${client_openssl_cnf}
|
||||
'';
|
||||
|
||||
admin_key = runWithOpenSSL "admin-key.pem"
|
||||
"openssl genrsa -out $out 2048";
|
||||
|
||||
admin_csr = runWithOpenSSL "admin.csr" ''
|
||||
openssl req \
|
||||
-new -key ${admin_key} \
|
||||
-out $out -subj "/CN=admin/O=system:masters" \
|
||||
-config ${client_openssl_cnf}
|
||||
'';
|
||||
|
||||
admin_cert = runWithOpenSSL "admin.crt" ''
|
||||
openssl x509 \
|
||||
-req -in ${admin_csr} \
|
||||
-CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
|
||||
-out $out -days 3650 -extensions v3_req \
|
||||
-extfile ${client_openssl_cnf}
|
||||
'';
|
||||
|
||||
apiserver_key = runWithOpenSSL "apiserver-key.pem" "openssl genrsa -out $out 2048";
|
||||
|
||||
apiserver_csr = runWithOpenSSL "apiserver.csr" ''
|
||||
openssl req \
|
||||
-new -key ${apiserver_key} \
|
||||
-out $out -subj "/CN=kube-apiserver" \
|
||||
-config ${apiserver_cnf}
|
||||
'';
|
||||
|
||||
apiserver_cert = runWithOpenSSL "apiserver.pem" ''
|
||||
openssl x509 \
|
||||
-req -in ${apiserver_csr} \
|
||||
-CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
|
||||
-out $out -days 3650 -extensions v3_req \
|
||||
-extfile ${apiserver_cnf}
|
||||
'';
|
||||
|
||||
worker_key = runWithOpenSSL "worker-key.pem" "openssl genrsa -out $out 2048";
|
||||
|
||||
worker_csr = runWithOpenSSL "worker.csr" ''
|
||||
openssl req \
|
||||
-new -key ${worker_key} \
|
||||
-out $out -subj "/CN=kube-worker/O=system:authenticated" \
|
||||
-config ${worker_cnf}
|
||||
'';
|
||||
|
||||
worker_cert = runWithOpenSSL "worker.pem" ''
|
||||
openssl x509 \
|
||||
-req -in ${worker_csr} \
|
||||
-CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
|
||||
-out $out -days 3650 -extensions v3_req \
|
||||
-extfile ${worker_cnf}
|
||||
'';
|
||||
|
||||
openssl_cnf = pkgs.writeText "openssl.cnf" ''
|
||||
[req]
|
||||
req_extensions = v3_req
|
||||
distinguished_name = req_distinguished_name
|
||||
[req_distinguished_name]
|
||||
[ v3_req ]
|
||||
basicConstraints = CA:FALSE
|
||||
keyUsage = digitalSignature, keyEncipherment
|
||||
extendedKeyUsage = serverAuth
|
||||
subjectAltName = @alt_names
|
||||
[alt_names]
|
||||
DNS.1 = *.cluster.${externalDomain}
|
||||
IP.1 = 127.0.0.1
|
||||
${mkAltNames 1 1}
|
||||
'';
|
||||
|
||||
client_openssl_cnf = pkgs.writeText "client-openssl.cnf" ''
|
||||
[req]
|
||||
req_extensions = v3_req
|
||||
distinguished_name = req_distinguished_name
|
||||
[req_distinguished_name]
|
||||
[ v3_req ]
|
||||
basicConstraints = CA:FALSE
|
||||
keyUsage = digitalSignature, keyEncipherment
|
||||
extendedKeyUsage = clientAuth
|
||||
subjectAltName = @alt_names
|
||||
[alt_names]
|
||||
DNS.1 = kubernetes
|
||||
DNS.2 = kubernetes.default
|
||||
DNS.3 = kubernetes.default.svc
|
||||
DNS.4 = kubernetes.default.svc.${internalDomain}
|
||||
DNS.5 = kubernetes.${externalDomain}
|
||||
DNS.6 = *.cluster.${externalDomain}
|
||||
IP.1 = 10.1.10.1
|
||||
${mkAltNames 1 6}
|
||||
'';
|
||||
|
||||
apiserver_cnf = pkgs.writeText "apiserver-openssl.cnf" ''
|
||||
[req]
|
||||
req_extensions = v3_req
|
||||
distinguished_name = req_distinguished_name
|
||||
[req_distinguished_name]
|
||||
[ v3_req ]
|
||||
basicConstraints = CA:FALSE
|
||||
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
|
||||
extendedKeyUsage = serverAuth
|
||||
subjectAltName = @alt_names
|
||||
[alt_names]
|
||||
DNS.1 = kubernetes
|
||||
DNS.2 = kubernetes.default
|
||||
DNS.3 = kubernetes.default.svc
|
||||
DNS.4 = kubernetes.default.svc.${internalDomain}
|
||||
DNS.5 = kubernetes.${externalDomain}
|
||||
DNS.6 = *.cluster.${externalDomain}
|
||||
IP.1 = 10.1.10.1
|
||||
${mkAltNames 1 6}
|
||||
'';
|
||||
|
||||
worker_cnf = pkgs.writeText "worker-openssl.cnf" ''
|
||||
[req]
|
||||
req_extensions = v3_req
|
||||
distinguished_name = req_distinguished_name
|
||||
[req_distinguished_name]
|
||||
[ v3_req ]
|
||||
basicConstraints = CA:FALSE
|
||||
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
|
||||
subjectAltName = @alt_names
|
||||
[alt_names]
|
||||
DNS.1 = *.cluster.${externalDomain}
|
||||
IP.1 = 10.1.10.1
|
||||
${mkAltNames 1 1}
|
||||
'';
|
||||
|
||||
ln = cert: target: ''
|
||||
cp -v ${cert} ${target}/${cert.file}
|
||||
'';
|
||||
in
|
||||
pkgs.stdenv.mkDerivation rec {
|
||||
name = "kubernetes-certs";
|
||||
unpackPhase = "true";
|
||||
installPhase = ''
|
||||
set -xe
|
||||
writeCFSSL = content:
|
||||
pkgs.runCommand content.name {
|
||||
buildInputs = [ pkgs.cfssl ];
|
||||
} ''
|
||||
mkdir -p $out
|
||||
${ln ca_key "$out"}
|
||||
${ln ca_pem "$out"}
|
||||
|
||||
${ln etcd_key "$out"}
|
||||
${ln etcd_csr "$out"}
|
||||
${ln etcd_cert "$out"}
|
||||
|
||||
${ln etcd_client_key "$out"}
|
||||
${ln etcd_client_csr "$out"}
|
||||
${ln etcd_client_cert "$out"}
|
||||
|
||||
${ln apiserver_key "$out"}
|
||||
${ln apiserver_csr "$out"}
|
||||
${ln apiserver_cert "$out"}
|
||||
|
||||
${ln worker_key "$out"}
|
||||
${ln worker_csr "$out"}
|
||||
${ln worker_cert "$out"}
|
||||
|
||||
${ln admin_key "$out"}
|
||||
${ln admin_csr "$out"}
|
||||
${ln admin_cert "$out"}
|
||||
cd $out
|
||||
cat ${writeFile content} | cfssljson -bare ${content.name}
|
||||
'';
|
||||
}
|
||||
|
||||
noCSR = content: pkgs.lib.filterAttrs (n: v: n != "csr") content;
|
||||
noKey = content: pkgs.lib.filterAttrs (n: v: n != "key") content;
|
||||
|
||||
writeFile = content: pkgs.writeText "content" (
|
||||
if pkgs.lib.isAttrs content then builtins.toJSON content
|
||||
else toString content
|
||||
);
|
||||
|
||||
createServingCertKey = { ca, cn, hosts? [], size ? 2048, name ? cn }:
|
||||
noCSR (
|
||||
(runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=server -config=${writeFile ca.config} ${writeFile {
|
||||
CN = cn;
|
||||
hosts = hosts;
|
||||
key = { algo = "rsa"; inherit size; };
|
||||
}}") // { inherit name; }
|
||||
);
|
||||
|
||||
createClientCertKey = { ca, cn, groups ? [], size ? 2048, name ? cn }:
|
||||
noCSR (
|
||||
(runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=client -config=${writeFile ca.config} ${writeFile {
|
||||
CN = cn;
|
||||
names = map (group: {O = group;}) groups;
|
||||
hosts = [""];
|
||||
key = { algo = "rsa"; inherit size; };
|
||||
}}") // { inherit name; }
|
||||
);
|
||||
|
||||
createSigningCertKey = { C ? "xx", ST ? "x", L ? "x", O ? "x", OU ? "x", CN ? "ca", emailAddress ? "x", expiry ? "43800h", size ? 2048, name ? CN }:
|
||||
(noCSR (runWithCFSSL CN "genkey -initca ${writeFile {
|
||||
key = { algo = "rsa"; inherit size; };
|
||||
names = [{ inherit C ST L O OU CN emailAddress; }];
|
||||
}}")) // {
|
||||
inherit name;
|
||||
config.signing = {
|
||||
default.expiry = expiry;
|
||||
profiles = {
|
||||
server = {
|
||||
inherit expiry;
|
||||
usages = [
|
||||
"signing"
|
||||
"key encipherment"
|
||||
"server auth"
|
||||
];
|
||||
};
|
||||
client = {
|
||||
inherit expiry;
|
||||
usages = [
|
||||
"signing"
|
||||
"key encipherment"
|
||||
"client auth"
|
||||
];
|
||||
};
|
||||
peer = {
|
||||
inherit expiry;
|
||||
usages = [
|
||||
"signing"
|
||||
"key encipherment"
|
||||
"server auth"
|
||||
"client auth"
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
ca = createSigningCertKey {};
|
||||
|
||||
kube-apiserver = createServingCertKey {
|
||||
inherit ca;
|
||||
cn = "kube-apiserver";
|
||||
hosts = ["kubernetes.default" "kubernetes.default.svc" "localhost" "api.${externalDomain}" serviceClusterIp];
|
||||
};
|
||||
|
||||
kubelet = createServingCertKey {
|
||||
inherit ca;
|
||||
cn = "kubelet";
|
||||
hosts = ["*.${externalDomain}"];
|
||||
};
|
||||
|
||||
service-accounts = createServingCertKey {
|
||||
inherit ca;
|
||||
cn = "kube-service-accounts";
|
||||
};
|
||||
|
||||
etcd = createServingCertKey {
|
||||
inherit ca;
|
||||
cn = "etcd";
|
||||
hosts = ["etcd.${externalDomain}"];
|
||||
};
|
||||
|
||||
etcd-client = createClientCertKey {
|
||||
inherit ca;
|
||||
cn = "etcd-client";
|
||||
};
|
||||
|
||||
kubelet-client = createClientCertKey {
|
||||
inherit ca;
|
||||
cn = "kubelet-client";
|
||||
groups = ["system:masters"];
|
||||
};
|
||||
|
||||
apiserver-client = {
|
||||
kubelet = createClientCertKey {
|
||||
inherit ca;
|
||||
cn = "apiserver-client-kubelet";
|
||||
groups = ["system:nodes"];
|
||||
};
|
||||
|
||||
kube-proxy = createClientCertKey {
|
||||
inherit ca;
|
||||
name = "apiserver-client-kube-proxy";
|
||||
cn = "system:kube-proxy";
|
||||
groups = ["system:kube-proxy" "system:nodes"];
|
||||
};
|
||||
|
||||
kube-controller-manager = createClientCertKey {
|
||||
inherit ca;
|
||||
name = "apiserver-client-kube-controller-manager";
|
||||
cn = "system:kube-controller-manager";
|
||||
groups = ["system:masters"];
|
||||
};
|
||||
|
||||
kube-scheduler = createClientCertKey {
|
||||
inherit ca;
|
||||
name = "apiserver-client-kube-scheduler";
|
||||
cn = "system:kube-scheduler";
|
||||
groups = ["system:kube-scheduler"];
|
||||
};
|
||||
|
||||
admin = createClientCertKey {
|
||||
inherit ca;
|
||||
cn = "admin";
|
||||
groups = ["system:masters"];
|
||||
};
|
||||
};
|
||||
in {
|
||||
master = pkgs.buildEnv {
|
||||
name = "master-keys";
|
||||
paths = [
|
||||
(writeCFSSL (noKey ca))
|
||||
(writeCFSSL kube-apiserver)
|
||||
(writeCFSSL kubelet-client)
|
||||
(writeCFSSL apiserver-client.kube-controller-manager)
|
||||
(writeCFSSL apiserver-client.kube-scheduler)
|
||||
(writeCFSSL service-accounts)
|
||||
(writeCFSSL etcd)
|
||||
];
|
||||
};
|
||||
|
||||
worker = pkgs.buildEnv {
|
||||
name = "worker-keys";
|
||||
paths = [
|
||||
(writeCFSSL (noKey ca))
|
||||
(writeCFSSL kubelet)
|
||||
(writeCFSSL apiserver-client.kubelet)
|
||||
(writeCFSSL apiserver-client.kube-proxy)
|
||||
(writeCFSSL etcd-client)
|
||||
];
|
||||
};
|
||||
|
||||
admin = writeCFSSL apiserver-client.admin;
|
||||
}
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
{ system ? builtins.currentSystem }:
|
||||
{
|
||||
kubernetes-singlenode = import ./singlenode.nix { inherit system; };
|
||||
kubernetes-multinode-kubectl = import ./multinode-kubectl.nix { inherit system; };
|
||||
kubernetes-rbac = import ./rbac.nix { inherit system; };
|
||||
kubernetes-dns = import ./dns.nix { inherit system; };
|
||||
dns = import ./dns.nix { inherit system; };
|
||||
# e2e = import ./e2e.nix { inherit system; }; # TODO: make it pass
|
||||
# the following test(s) can be removed when e2e is working:
|
||||
rbac = import ./rbac.nix { inherit system; };
|
||||
}
|
||||
|
||||
@ -1,16 +1,11 @@
|
||||
{ system ? builtins.currentSystem }:
|
||||
|
||||
with import ../../lib/testing.nix { inherit system; };
|
||||
with import ../../lib/qemu-flags.nix;
|
||||
with pkgs.lib;
|
||||
|
||||
{ system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:
|
||||
with import ./base.nix { inherit system; };
|
||||
let
|
||||
servers.master = "192.168.1.1";
|
||||
servers.one = "192.168.1.10";
|
||||
domain = "my.zyx";
|
||||
|
||||
certs = import ./certs.nix { inherit servers; };
|
||||
certs = import ./certs.nix { externalDomain = domain; };
|
||||
|
||||
redisPod = pkgs.writeText "redis-master-pod.json" (builtins.toJSON {
|
||||
redisPod = pkgs.writeText "redis-pod.json" (builtins.toJSON {
|
||||
kind = "Pod";
|
||||
apiVersion = "v1";
|
||||
metadata.name = "redis";
|
||||
@ -40,64 +35,93 @@ let
|
||||
redisImage = pkgs.dockerTools.buildImage {
|
||||
name = "redis";
|
||||
tag = "latest";
|
||||
contents = [ pkgs.redis pkgs.bind.dnsutils pkgs.coreutils pkgs.inetutils pkgs.nmap ];
|
||||
contents = [ pkgs.redis pkgs.bind.host ];
|
||||
config.Entrypoint = "/bin/redis-server";
|
||||
};
|
||||
|
||||
test = ''
|
||||
$master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");
|
||||
$master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");
|
||||
probePod = pkgs.writeText "probe-pod.json" (builtins.toJSON {
|
||||
kind = "Pod";
|
||||
apiVersion = "v1";
|
||||
metadata.name = "probe";
|
||||
metadata.labels.name = "probe";
|
||||
spec.containers = [{
|
||||
name = "probe";
|
||||
image = "probe";
|
||||
args = [ "-f" ];
|
||||
tty = true;
|
||||
imagePullPolicy = "Never";
|
||||
}];
|
||||
});
|
||||
|
||||
$one->execute("docker load < ${redisImage}");
|
||||
|
||||
$master->waitUntilSucceeds("kubectl create -f ${redisPod} || kubectl apply -f ${redisPod}");
|
||||
$master->waitUntilSucceeds("kubectl create -f ${redisService} || kubectl apply -f ${redisService}");
|
||||
|
||||
$master->waitUntilSucceeds("kubectl get pod redis | grep Running");
|
||||
|
||||
$master->succeed("dig \@192.168.1.1 redis.default.svc.cluster.local");
|
||||
$one->succeed("dig \@192.168.1.10 redis.default.svc.cluster.local");
|
||||
|
||||
|
||||
$master->succeed("kubectl exec -ti redis -- cat /etc/resolv.conf | grep 'nameserver 192.168.1.10'");
|
||||
|
||||
$master->succeed("kubectl exec -ti redis -- dig \@192.168.1.10 redis.default.svc.cluster.local");
|
||||
'';
|
||||
|
||||
in makeTest {
|
||||
name = "kubernetes-dns";
|
||||
|
||||
nodes = {
|
||||
master =
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
mkMerge [
|
||||
{
|
||||
virtualisation.memorySize = 768;
|
||||
virtualisation.diskSize = 4096;
|
||||
networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}];
|
||||
networking.primaryIPAddress = mkForce servers.master;
|
||||
}
|
||||
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
|
||||
(import ./kubernetes-master.nix { inherit pkgs config certs; })
|
||||
];
|
||||
|
||||
one =
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
mkMerge [
|
||||
{
|
||||
virtualisation.memorySize = 768;
|
||||
virtualisation.diskSize = 4096;
|
||||
networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}];
|
||||
networking.primaryIPAddress = mkForce servers.one;
|
||||
services.kubernetes.roles = ["node"];
|
||||
}
|
||||
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
|
||||
];
|
||||
probeImage = pkgs.dockerTools.buildImage {
|
||||
name = "probe";
|
||||
tag = "latest";
|
||||
contents = [ pkgs.bind.host pkgs.busybox ];
|
||||
config.Entrypoint = "/bin/tail";
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
startAll;
|
||||
extraConfiguration = { config, pkgs, lib, nodes, ... }: {
|
||||
environment.systemPackages = [ pkgs.bind.host ];
|
||||
# virtualisation.docker.extraOptions = "--dns=${config.services.kubernetes.addons.dns.clusterIp}";
|
||||
services.dnsmasq.enable = true;
|
||||
services.dnsmasq.servers = [
|
||||
"/cluster.local/${config.services.kubernetes.addons.dns.clusterIp}#53"
|
||||
];
|
||||
};
|
||||
|
||||
${test}
|
||||
'';
|
||||
base = {
|
||||
name = "dns";
|
||||
inherit domain certs extraConfiguration;
|
||||
};
|
||||
|
||||
singleNodeTest = {
|
||||
test = ''
|
||||
# prepare machine1 for test
|
||||
$machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready");
|
||||
$machine1->execute("docker load < ${redisImage}");
|
||||
$machine1->waitUntilSucceeds("kubectl create -f ${redisPod}");
|
||||
$machine1->waitUntilSucceeds("kubectl create -f ${redisService}");
|
||||
$machine1->execute("docker load < ${probeImage}");
|
||||
$machine1->waitUntilSucceeds("kubectl create -f ${probePod}");
|
||||
|
||||
# check if pods are running
|
||||
$machine1->waitUntilSucceeds("kubectl get pod redis | grep Running");
|
||||
$machine1->waitUntilSucceeds("kubectl get pod probe | grep Running");
|
||||
$machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'kube-dns.*3/3'");
|
||||
|
||||
# check dns on host (dnsmasq)
|
||||
$machine1->succeed("host redis.default.svc.cluster.local");
|
||||
|
||||
# check dns inside the container
|
||||
$machine1->succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local");
|
||||
'';
|
||||
};
|
||||
|
||||
multiNodeTest = {
|
||||
test = ''
|
||||
# prepare machines for test
|
||||
$machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready");
|
||||
$machine1->waitUntilSucceeds("kubectl get node machine2.${domain} | grep -w Ready");
|
||||
$machine2->execute("docker load < ${redisImage}");
|
||||
$machine1->waitUntilSucceeds("kubectl create -f ${redisPod}");
|
||||
$machine1->waitUntilSucceeds("kubectl create -f ${redisService}");
|
||||
$machine2->execute("docker load < ${probeImage}");
|
||||
$machine1->waitUntilSucceeds("kubectl create -f ${probePod}");
|
||||
|
||||
# check if pods are running
|
||||
$machine1->waitUntilSucceeds("kubectl get pod redis | grep Running");
|
||||
$machine1->waitUntilSucceeds("kubectl get pod probe | grep Running");
|
||||
$machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'kube-dns.*3/3'");
|
||||
|
||||
# check dns on hosts (dnsmasq)
|
||||
$machine1->succeed("host redis.default.svc.cluster.local");
|
||||
$machine2->succeed("host redis.default.svc.cluster.local");
|
||||
|
||||
# check dns inside the container
|
||||
$machine1->succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local");
|
||||
'';
|
||||
};
|
||||
in {
|
||||
singlenode = mkKubernetesSingleNodeTest (base // singleNodeTest);
|
||||
multinode = mkKubernetesMultiNodeTest (base // multiNodeTest);
|
||||
}
|
||||
|
||||
40
nixos/tests/kubernetes/e2e.nix
Normal file
40
nixos/tests/kubernetes/e2e.nix
Normal file
@ -0,0 +1,40 @@
|
||||
{ system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:
|
||||
with import ./base.nix { inherit system; };
|
||||
let
|
||||
domain = "my.zyx";
|
||||
certs = import ./certs.nix { externalDomain = domain; };
|
||||
kubeconfig = pkgs.writeText "kubeconfig.json" (builtins.toJSON {
|
||||
apiVersion = "v1";
|
||||
kind = "Config";
|
||||
clusters = [{
|
||||
name = "local";
|
||||
cluster.certificate-authority = "${certs.master}/ca.pem";
|
||||
cluster.server = "https://api.${domain}";
|
||||
}];
|
||||
users = [{
|
||||
name = "kubelet";
|
||||
user = {
|
||||
client-certificate = "${certs.admin}/admin.pem";
|
||||
client-key = "${certs.admin}/admin-key.pem";
|
||||
};
|
||||
}];
|
||||
contexts = [{
|
||||
context = {
|
||||
cluster = "local";
|
||||
user = "kubelet";
|
||||
};
|
||||
current-context = "kubelet-context";
|
||||
}];
|
||||
});
|
||||
|
||||
base = {
|
||||
name = "e2e";
|
||||
inherit domain certs;
|
||||
test = ''
|
||||
$machine1->succeed("e2e.test -kubeconfig ${kubeconfig} -provider local -ginkgo.focus '\\[Conformance\\]' -ginkgo.skip '\\[Flaky\\]|\\[Serial\\]'");
|
||||
'';
|
||||
};
|
||||
in {
|
||||
singlenode = mkKubernetesSingleNodeTest base;
|
||||
multinode = mkKubernetesMultiNodeTest base;
|
||||
}
|
||||
@ -1,72 +1,59 @@
|
||||
{ config, pkgs, certs, servers }:
|
||||
|
||||
{ roles, config, pkgs, certs }:
|
||||
with pkgs.lib;
|
||||
let
|
||||
etcd_key = "${certs}/etcd-key.pem";
|
||||
etcd_cert = "${certs}/etcd.pem";
|
||||
ca_pem = "${certs}/ca.pem";
|
||||
etcd_client_cert = "${certs}/etcd-client.crt";
|
||||
etcd_client_key = "${certs}/etcd-client-key.pem";
|
||||
base = {
|
||||
inherit roles;
|
||||
featureGates = ["AllAlpha"];
|
||||
flannel.enable = true;
|
||||
addons.dashboard.enable = true;
|
||||
verbose = true;
|
||||
|
||||
worker_key = "${certs}/worker-key.pem";
|
||||
worker_cert = "${certs}/worker.pem";
|
||||
|
||||
rootCaFile = pkgs.writeScript "rootCaFile.pem" ''
|
||||
${pkgs.lib.readFile "${certs}/ca.pem"}
|
||||
|
||||
${pkgs.lib.readFile ("${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt")}
|
||||
'';
|
||||
|
||||
mkHosts =
|
||||
pkgs.lib.concatMapStringsSep "\n" (v: "${v.ip} ${v.name}.nixos.xyz") (pkgs.lib.mapAttrsToList (n: v: {name = n; ip = v;}) servers);
|
||||
|
||||
in
|
||||
{
|
||||
programs.bash.enableCompletion = true;
|
||||
environment.systemPackages = with pkgs; [ netcat bind etcd.bin ];
|
||||
|
||||
networking = {
|
||||
firewall.allowedTCPPorts = [
|
||||
10250 # kubelet
|
||||
];
|
||||
extraHosts = ''
|
||||
# register "external" domains
|
||||
${servers.master} etcd.kubernetes.nixos.xyz
|
||||
${servers.master} kubernetes.nixos.xyz
|
||||
${mkHosts}
|
||||
'';
|
||||
};
|
||||
services.flannel.iface = "eth1";
|
||||
environment.variables = {
|
||||
ETCDCTL_CERT_FILE = "${etcd_client_cert}";
|
||||
ETCDCTL_KEY_FILE = "${etcd_client_key}";
|
||||
ETCDCTL_CA_FILE = "${rootCaFile}";
|
||||
ETCDCTL_PEERS = "https://etcd.kubernetes.nixos.xyz:2379";
|
||||
};
|
||||
|
||||
services.kubernetes = {
|
||||
kubelet = {
|
||||
tlsKeyFile = worker_key;
|
||||
tlsCertFile = worker_cert;
|
||||
hostname = "${config.networking.hostName}.nixos.xyz";
|
||||
nodeIp = config.networking.primaryIPAddress;
|
||||
caFile = "${certs.master}/ca.pem";
|
||||
apiserver = {
|
||||
tlsCertFile = "${certs.master}/kube-apiserver.pem";
|
||||
tlsKeyFile = "${certs.master}/kube-apiserver-key.pem";
|
||||
kubeletClientCertFile = "${certs.master}/kubelet-client.pem";
|
||||
kubeletClientKeyFile = "${certs.master}/kubelet-client-key.pem";
|
||||
serviceAccountKeyFile = "${certs.master}/kube-service-accounts.pem";
|
||||
};
|
||||
etcd = {
|
||||
servers = ["https://etcd.kubernetes.nixos.xyz:2379"];
|
||||
keyFile = etcd_client_key;
|
||||
certFile = etcd_client_cert;
|
||||
caFile = ca_pem;
|
||||
servers = ["https://etcd.${config.networking.domain}:2379"];
|
||||
certFile = "${certs.worker}/etcd-client.pem";
|
||||
keyFile = "${certs.worker}/etcd-client-key.pem";
|
||||
};
|
||||
kubeconfig = {
|
||||
server = "https://kubernetes.nixos.xyz";
|
||||
caFile = rootCaFile;
|
||||
certFile = worker_cert;
|
||||
keyFile = worker_key;
|
||||
server = "https://api.${config.networking.domain}";
|
||||
};
|
||||
kubelet = {
|
||||
tlsCertFile = "${certs.worker}/kubelet.pem";
|
||||
tlsKeyFile = "${certs.worker}/kubelet-key.pem";
|
||||
hostname = "${config.networking.hostName}.${config.networking.domain}";
|
||||
kubeconfig = {
|
||||
certFile = "${certs.worker}/apiserver-client-kubelet.pem";
|
||||
keyFile = "${certs.worker}/apiserver-client-kubelet-key.pem";
|
||||
};
|
||||
};
|
||||
controllerManager = {
|
||||
serviceAccountKeyFile = "${certs.master}/kube-service-accounts-key.pem";
|
||||
kubeconfig = {
|
||||
certFile = "${certs.master}/apiserver-client-kube-controller-manager.pem";
|
||||
keyFile = "${certs.master}/apiserver-client-kube-controller-manager-key.pem";
|
||||
};
|
||||
};
|
||||
scheduler = {
|
||||
kubeconfig = {
|
||||
certFile = "${certs.master}/apiserver-client-kube-scheduler.pem";
|
||||
keyFile = "${certs.master}/apiserver-client-kube-scheduler-key.pem";
|
||||
};
|
||||
};
|
||||
proxy = {
|
||||
kubeconfig = {
|
||||
certFile = "${certs.worker}/apiserver-client-kube-proxy.pem";
|
||||
keyFile = "${certs.worker}//apiserver-client-kube-proxy-key.pem";
|
||||
};
|
||||
};
|
||||
flannel.enable = true;
|
||||
|
||||
dns.port = 4453;
|
||||
};
|
||||
|
||||
services.dnsmasq.enable = true;
|
||||
services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
|
||||
in {
|
||||
services.kubernetes = base;
|
||||
}
|
||||
|
||||
@ -1,63 +0,0 @@
|
||||
{ config, pkgs, certs }:
|
||||
let
|
||||
etcd_key = "${certs}/etcd-key.pem";
|
||||
etcd_cert = "${certs}/etcd.pem";
|
||||
ca_pem = "${certs}/ca.pem";
|
||||
etcd_client_cert = "${certs}/etcd-client.crt";
|
||||
etcd_client_key = "${certs}/etcd-client-key.pem";
|
||||
|
||||
apiserver_key = "${certs}/apiserver-key.pem";
|
||||
apiserver_cert = "${certs}/apiserver.pem";
|
||||
worker_key = "${certs}/worker-key.pem";
|
||||
worker_cert = "${certs}/worker.pem";
|
||||
|
||||
|
||||
rootCaFile = pkgs.writeScript "rootCaFile.pem" ''
|
||||
${pkgs.lib.readFile "${certs}/ca.pem"}
|
||||
|
||||
${pkgs.lib.readFile ("${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt")}
|
||||
'';
|
||||
in
|
||||
{
|
||||
networking = {
|
||||
firewall = {
|
||||
enable = true;
|
||||
allowPing = true;
|
||||
allowedTCPPorts = [
|
||||
2379 2380 # etcd
|
||||
443 # kubernetes apiserver
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
services.etcd = {
|
||||
enable = pkgs.lib.mkForce true;
|
||||
keyFile = etcd_key;
|
||||
certFile = etcd_cert;
|
||||
trustedCaFile = rootCaFile;
|
||||
peerClientCertAuth = true;
|
||||
listenClientUrls = ["https://0.0.0.0:2379"];
|
||||
listenPeerUrls = ["https://0.0.0.0:2380"];
|
||||
|
||||
advertiseClientUrls = ["https://etcd.kubernetes.nixos.xyz:2379"];
|
||||
initialCluster = ["master=https://etcd.kubernetes.nixos.xyz:2380"];
|
||||
initialAdvertisePeerUrls = ["https://etcd.kubernetes.nixos.xyz:2380"];
|
||||
};
|
||||
|
||||
services.kubernetes = {
|
||||
roles = ["master"];
|
||||
scheduler.leaderElect = true;
|
||||
controllerManager.rootCaFile = rootCaFile;
|
||||
controllerManager.serviceAccountKeyFile = apiserver_key;
|
||||
apiserver = {
|
||||
publicAddress = "192.168.1.1";
|
||||
advertiseAddress = "192.168.1.1";
|
||||
tlsKeyFile = apiserver_key;
|
||||
tlsCertFile = apiserver_cert;
|
||||
clientCaFile = rootCaFile;
|
||||
kubeletClientCaFile = rootCaFile;
|
||||
kubeletClientKeyFile = worker_key;
|
||||
kubeletClientCertFile = worker_cert;
|
||||
};
|
||||
};
|
||||
}
|
||||
@ -1,123 +0,0 @@
|
||||
{ system ? builtins.currentSystem }:
|
||||
|
||||
with import ../../lib/testing.nix { inherit system; };
|
||||
with import ../../lib/qemu-flags.nix;
|
||||
with pkgs.lib;
|
||||
|
||||
let
|
||||
servers.master = "192.168.1.1";
|
||||
servers.one = "192.168.1.10";
|
||||
servers.two = "192.168.1.20";
|
||||
|
||||
certs = import ./certs.nix { inherit servers; };
|
||||
|
||||
kubectlPod = pkgs.writeText "kubectl-pod.json" (builtins.toJSON {
|
||||
kind = "Pod";
|
||||
apiVersion = "v1";
|
||||
metadata.name = "kubectl";
|
||||
metadata.labels.name = "kubectl";
|
||||
spec.containers = [{
|
||||
name = "kubectl";
|
||||
image = "kubectl:latest";
|
||||
command = ["${pkgs.busybox}/bin/tail" "-f"];
|
||||
imagePullPolicy = "Never";
|
||||
tty = true;
|
||||
}];
|
||||
});
|
||||
|
||||
kubectlImage = pkgs.dockerTools.buildImage {
|
||||
name = "kubectl";
|
||||
tag = "latest";
|
||||
contents = [ pkgs.kubernetes pkgs.busybox certs kubeconfig ];
|
||||
config.Entrypoint = "${pkgs.busybox}/bin/sh";
|
||||
};
|
||||
|
||||
kubeconfig = pkgs.writeTextDir "kubeconfig.json" (builtins.toJSON {
|
||||
apiVersion = "v1";
|
||||
kind = "Config";
|
||||
clusters = [{
|
||||
name = "local";
|
||||
cluster.certificate-authority = "/ca.pem";
|
||||
cluster.server = "https://${servers.master}";
|
||||
}];
|
||||
users = [{
|
||||
name = "kubelet";
|
||||
user = {
|
||||
client-certificate = "/admin.crt";
|
||||
client-key = "/admin-key.pem";
|
||||
};
|
||||
}];
|
||||
contexts = [{
|
||||
context = {
|
||||
cluster = "local";
|
||||
user = "kubelet";
|
||||
};
|
||||
current-context = "kubelet-context";
|
||||
}];
|
||||
});
|
||||
|
||||
test = ''
|
||||
$master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");
|
||||
$master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");
|
||||
$master->waitUntilSucceeds("kubectl get node two.nixos.xyz | grep Ready");
|
||||
|
||||
$one->execute("docker load < ${kubectlImage}");
|
||||
$two->execute("docker load < ${kubectlImage}");
|
||||
|
||||
$master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}");
|
||||
|
||||
$master->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
|
||||
|
||||
$master->succeed("kubectl exec -ti kubectl -- kubectl --kubeconfig=/kubeconfig.json version");
|
||||
'';
|
||||
|
||||
in makeTest {
|
||||
name = "kubernetes-multinode-kubectl";
|
||||
|
||||
nodes = {
|
||||
master =
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
mkMerge [
|
||||
{
|
||||
virtualisation.memorySize = 768;
|
||||
virtualisation.diskSize = 4096;
|
||||
networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}];
|
||||
networking.primaryIPAddress = mkForce servers.master;
|
||||
}
|
||||
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
|
||||
(import ./kubernetes-master.nix { inherit pkgs config certs; })
|
||||
];
|
||||
|
||||
one =
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
mkMerge [
|
||||
{
|
||||
virtualisation.memorySize = 768;
|
||||
virtualisation.diskSize = 4096;
|
||||
networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}];
|
||||
networking.primaryIPAddress = mkForce servers.one;
|
||||
services.kubernetes.roles = ["node"];
|
||||
}
|
||||
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
|
||||
];
|
||||
|
||||
two =
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
mkMerge [
|
||||
{
|
||||
virtualisation.memorySize = 768;
|
||||
virtualisation.diskSize = 4096;
|
||||
networking.interfaces.eth1.ip4 = mkForce [{address = servers.two; prefixLength = 24;}];
|
||||
networking.primaryIPAddress = mkForce servers.two;
|
||||
services.kubernetes.roles = ["node"];
|
||||
}
|
||||
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
|
||||
];
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
startAll;
|
||||
|
||||
${test}
|
||||
'';
|
||||
}
|
||||
@ -1,14 +1,6 @@
|
||||
{ system ? builtins.currentSystem }:
|
||||
|
||||
with import ../../lib/testing.nix { inherit system; };
|
||||
with import ../../lib/qemu-flags.nix;
|
||||
with pkgs.lib;
|
||||
|
||||
{ system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:
|
||||
with import ./base.nix { inherit system; };
|
||||
let
|
||||
servers.master = "192.168.1.1";
|
||||
servers.one = "192.168.1.10";
|
||||
|
||||
certs = import ./certs.nix { inherit servers; };
|
||||
|
||||
roServiceAccount = pkgs.writeText "ro-service-account.json" (builtins.toJSON {
|
||||
kind = "ServiceAccount";
|
||||
@ -20,21 +12,21 @@ let
|
||||
});
|
||||
|
||||
roRoleBinding = pkgs.writeText "ro-role-binding.json" (builtins.toJSON {
|
||||
"apiVersion" = "rbac.authorization.k8s.io/v1beta1";
|
||||
"kind" = "RoleBinding";
|
||||
"metadata" = {
|
||||
"name" = "read-pods";
|
||||
"namespace" = "default";
|
||||
apiVersion = "rbac.authorization.k8s.io/v1beta1";
|
||||
kind = "RoleBinding";
|
||||
metadata = {
|
||||
name = "read-pods";
|
||||
namespace = "default";
|
||||
};
|
||||
"roleRef" = {
|
||||
"apiGroup" = "rbac.authorization.k8s.io";
|
||||
"kind" = "Role";
|
||||
"name" = "pod-reader";
|
||||
roleRef = {
|
||||
apiGroup = "rbac.authorization.k8s.io";
|
||||
kind = "Role";
|
||||
name = "pod-reader";
|
||||
};
|
||||
"subjects" = [{
|
||||
"kind" = "ServiceAccount";
|
||||
"name" = "read-only";
|
||||
"namespace" = "default";
|
||||
subjects = [{
|
||||
kind = "ServiceAccount";
|
||||
name = "read-only";
|
||||
namespace = "default";
|
||||
}];
|
||||
});
|
||||
|
||||
@ -62,7 +54,7 @@ let
|
||||
spec.containers = [{
|
||||
name = "kubectl";
|
||||
image = "kubectl:latest";
|
||||
command = ["${pkgs.busybox}/bin/tail" "-f"];
|
||||
command = ["/bin/tail" "-f"];
|
||||
imagePullPolicy = "Never";
|
||||
tty = true;
|
||||
}];
|
||||
@ -78,71 +70,68 @@ let
|
||||
spec.containers = [{
|
||||
name = "kubectl-2";
|
||||
image = "kubectl:latest";
|
||||
command = ["${pkgs.busybox}/bin/tail" "-f"];
|
||||
command = ["/bin/tail" "-f"];
|
||||
imagePullPolicy = "Never";
|
||||
tty = true;
|
||||
}];
|
||||
});
|
||||
|
||||
kubectl = pkgs.runCommand "copy-kubectl" { buildInputs = [ pkgs.kubernetes ]; } ''
|
||||
mkdir -p $out/bin
|
||||
cp ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl
|
||||
'';
|
||||
|
||||
kubectlImage = pkgs.dockerTools.buildImage {
|
||||
name = "kubectl";
|
||||
tag = "latest";
|
||||
contents = [ pkgs.kubernetes pkgs.busybox kubectlPod2 ]; # certs kubeconfig
|
||||
config.Entrypoint = "${pkgs.busybox}/bin/sh";
|
||||
contents = [ kubectl pkgs.busybox kubectlPod2 ];
|
||||
config.Entrypoint = "/bin/sh";
|
||||
};
|
||||
|
||||
test = ''
|
||||
$master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");
|
||||
$master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");
|
||||
|
||||
$one->execute("docker load < ${kubectlImage}");
|
||||
|
||||
$master->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}");
|
||||
$master->waitUntilSucceeds("kubectl apply -f ${roRole}");
|
||||
$master->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}");
|
||||
$master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}");
|
||||
|
||||
$master->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
|
||||
|
||||
$master->succeed("kubectl exec -ti kubectl -- kubectl get pods");
|
||||
$master->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");
|
||||
$master->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");
|
||||
'';
|
||||
|
||||
in makeTest {
|
||||
name = "kubernetes-rbac";
|
||||
|
||||
nodes = {
|
||||
master =
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
mkMerge [
|
||||
{
|
||||
virtualisation.memorySize = 768;
|
||||
virtualisation.diskSize = 4096;
|
||||
networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}];
|
||||
networking.primaryIPAddress = mkForce servers.master;
|
||||
}
|
||||
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
|
||||
(import ./kubernetes-master.nix { inherit pkgs config certs; })
|
||||
];
|
||||
|
||||
one =
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
mkMerge [
|
||||
{
|
||||
virtualisation.memorySize = 768;
|
||||
virtualisation.diskSize = 4096;
|
||||
networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}];
|
||||
networking.primaryIPAddress = mkForce servers.one;
|
||||
services.kubernetes.roles = ["node"];
|
||||
}
|
||||
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
|
||||
];
|
||||
base = {
|
||||
name = "rbac";
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
startAll;
|
||||
singlenode = base // {
|
||||
test = ''
|
||||
$machine1->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready");
|
||||
|
||||
${test}
|
||||
'';
|
||||
$machine1->execute("docker load < ${kubectlImage}");
|
||||
|
||||
$machine1->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}");
|
||||
$machine1->waitUntilSucceeds("kubectl apply -f ${roRole}");
|
||||
$machine1->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}");
|
||||
$machine1->waitUntilSucceeds("kubectl create -f ${kubectlPod}");
|
||||
|
||||
$machine1->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
|
||||
|
||||
$machine1->succeed("kubectl exec -ti kubectl -- kubectl get pods");
|
||||
$machine1->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");
|
||||
$machine1->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");
|
||||
'';
|
||||
};
|
||||
|
||||
multinode = base // {
|
||||
test = ''
|
||||
$machine1->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready");
|
||||
$machine1->waitUntilSucceeds("kubectl get node machine2.my.zyx | grep -w Ready");
|
||||
|
||||
$machine2->execute("docker load < ${kubectlImage}");
|
||||
|
||||
$machine1->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}");
|
||||
$machine1->waitUntilSucceeds("kubectl apply -f ${roRole}");
|
||||
$machine1->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}");
|
||||
$machine1->waitUntilSucceeds("kubectl create -f ${kubectlPod}");
|
||||
|
||||
$machine1->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
|
||||
|
||||
$machine1->succeed("kubectl exec -ti kubectl -- kubectl get pods");
|
||||
$machine1->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");
|
||||
$machine1->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");
|
||||
'';
|
||||
};
|
||||
|
||||
in {
|
||||
singlenode = mkKubernetesSingleNodeTest singlenode;
|
||||
multinode = mkKubernetesMultiNodeTest multinode;
|
||||
}
|
||||
|
||||
@ -1,75 +0,0 @@
|
||||
{ system ? builtins.currentSystem }:
|
||||
|
||||
with import ../../lib/testing.nix { inherit system; };
|
||||
with import ../../lib/qemu-flags.nix;
|
||||
with pkgs.lib;
|
||||
|
||||
let
|
||||
redisPod = pkgs.writeText "redis-master-pod.json" (builtins.toJSON {
|
||||
kind = "Pod";
|
||||
apiVersion = "v1";
|
||||
metadata.name = "redis";
|
||||
metadata.labels.name = "redis";
|
||||
spec.containers = [{
|
||||
name = "redis";
|
||||
image = "redis";
|
||||
args = ["--bind" "0.0.0.0"];
|
||||
imagePullPolicy = "Never";
|
||||
ports = [{
|
||||
name = "redis-server";
|
||||
containerPort = 6379;
|
||||
}];
|
||||
}];
|
||||
});
|
||||
|
||||
redisService = pkgs.writeText "redis-service.json" (builtins.toJSON {
|
||||
kind = "Service";
|
||||
apiVersion = "v1";
|
||||
metadata.name = "redis";
|
||||
spec = {
|
||||
ports = [{port = 6379; targetPort = 6379;}];
|
||||
selector = {name = "redis";};
|
||||
};
|
||||
});
|
||||
|
||||
redisImage = pkgs.dockerTools.buildImage {
|
||||
name = "redis";
|
||||
tag = "latest";
|
||||
contents = pkgs.redis;
|
||||
config.Entrypoint = "/bin/redis-server";
|
||||
};
|
||||
|
||||
testSimplePod = ''
|
||||
$kubernetes->execute("docker load < ${redisImage}");
|
||||
$kubernetes->waitUntilSucceeds("kubectl create -f ${redisPod}");
|
||||
$kubernetes->succeed("kubectl create -f ${redisService}");
|
||||
$kubernetes->waitUntilSucceeds("kubectl get pod redis | grep Running");
|
||||
$kubernetes->succeed("nc -z \$\(dig redis.default.svc.cluster.local +short\) 6379");
|
||||
'';
|
||||
in makeTest {
|
||||
name = "kubernetes-singlenode";
|
||||
|
||||
nodes = {
|
||||
kubernetes =
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
{
|
||||
virtualisation.memorySize = 768;
|
||||
virtualisation.diskSize = 2048;
|
||||
|
||||
programs.bash.enableCompletion = true;
|
||||
environment.systemPackages = with pkgs; [ netcat bind ];
|
||||
|
||||
services.kubernetes.roles = ["master" "node"];
|
||||
services.kubernetes.dns.port = 4453;
|
||||
|
||||
services.dnsmasq.enable = true;
|
||||
services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
startAll;
|
||||
|
||||
${testSimplePod}
|
||||
'';
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user