WIP: nixos/kubernetes: Fix tests on Hydra and OfBorg#37199
WIP: nixos/kubernetes: Fix tests on Hydra and OfBorg#37199srhb wants to merge 6 commits intoNixOS:masterfrom
Conversation
|
I think I've now incorporated the changes as suggested by the discussions in #36739 (plus the few commits that would permit the whole thing to succeed if certs.nix is fixed.) The problem at hand now is that every certificate/key file that kube components are now pointing at are actually files containing the paths to the cert/key instead of the actual cert/key data. I do not think I'll manage to resolve this. |
|
I'm also taking a stab at this. This is what I have so far: diff --git a/nixos/release.nix b/nixos/release.nix
index 6a3fcea1768..842bf264c08 100644
--- a/nixos/release.nix
+++ b/nixos/release.nix
@@ -296,7 +296,11 @@ in rec {
tests.kernel-copperhead = callTest tests/kernel-copperhead.nix {};
tests.kernel-latest = callTest tests/kernel-latest.nix {};
tests.kernel-lts = callTest tests/kernel-lts.nix {};
- tests.kubernetes = callSubTestsOnTheseSystems ["x86_64-linux"] tests/kubernetes/default.nix {};
+ #tests.kubernetes = callSubTestsOnTheseSystems ["x86_64-linux"] tests/kubernetes/default.nix {};
+ tests.kubernetes.dns = callSubTestsOnTheseSystems ["x86_64-linux"] tests/kubernetes/dns.nix {};
+ ## kubernetes.e2e should eventually replace kubernetes.rbac when it works
+ #tests.kubernetes.e2e = callSubTestsOnTheseSystems ["x86_64-linux"] tests/kubernetes/e2e.nix {};
+ tests.kubernetes.rbac = callSubTestsOnTheseSystems ["x86_64-linux"] tests/kubernetes/rbac.nix {};
tests.latestKernel.login = callTest tests/login.nix { latestKernel = true; };
tests.ldap = callTest tests/ldap.nix {};
#tests.lightdm = callTest tests/lightdm.nix {};
diff --git a/nixos/tests/kubernetes/certs.nix b/nixos/tests/kubernetes/certs.nix
index d3eff910c46..846b2ca6dbf 100644
--- a/nixos/tests/kubernetes/certs.nix
+++ b/nixos/tests/kubernetes/certs.nix
@@ -7,28 +7,59 @@
}:
let
runWithCFSSL = name: cmd:
- builtins.fromJSON (builtins.readFile (
- pkgs.runCommand "${name}-cfss.json" {
- buildInputs = [ pkgs.cfssl ];
- } "cfssl ${cmd} > $out"
- ));
+ let secrets = pkgs.runCommand "${name}-cfss.json" {
+ buildInputs = [ pkgs.cfssl pkgs.jq ];
+ outputs = [ "out" "cert" "key" "csr" ];
+ }
+ ''
+ (
+ echo "${cmd}"
+ cfssl ${cmd} > tmp
+ cat tmp | jq -r .key > $key
+ cat tmp | jq -r .cert > $cert
+ cat tmp | jq -r .csr > $csr
+
+ touch $out
+ ) 2>&1 | fold -w 80 -s
+ '';
+ in {
+ key = secrets.key;
+ cert = secrets.cert;
+ csr = secrets.csr;
+ };
writeCFSSL = content:
pkgs.runCommand content.name {
- buildInputs = [ pkgs.cfssl ];
+ buildInputs = [ pkgs.cfssl pkgs.jq ];
} ''
mkdir -p $out
cd $out
- cat ${writeFile content} | cfssljson -bare ${content.name}
+
+ json=${pkgs.lib.escapeShellArg (builtins.toJSON content)}
+
+ # for a given $field in the json, treat the associated value as a
+ # file path and substitute the contents thereof into the $json
+ # object.
+ expandFileField() {
+ local field=$1
+ local path="$(echo "$json" | jq -r ".$field")"
+ json="$(echo "$json" | jq --arg val "$(cat "$path")" ".$field = \$val")"
+ }
+
+ ${pkgs.lib.optionalString (content ? key) "expandFileField key"}
+ ${pkgs.lib.optionalString (content ? ca) "expandFileField ca"}
+ ${pkgs.lib.optionalString (content ? cert) "expandFileField cert"}
+
+ echo "$json" | cfssljson -bare ${content.name}
'';
noCSR = content: pkgs.lib.filterAttrs (n: v: n != "csr") content;
noKey = content: pkgs.lib.filterAttrs (n: v: n != "key") content;
- writeFile = content: pkgs.writeText "content" (
- if pkgs.lib.isAttrs content then builtins.toJSON content
- else toString content
- );
+ writeFile = content:
+ if pkgs.lib.isDerivation content
+ then content
+ else pkgs.writeText "content" (builtins.toJSON content);
createServingCertKey = { ca, cn, hosts? [], size ? 2048, name ? cn }:
noCSR (
@srhb The changes I made to The test are now running, and they're spewing a whole lot of text. Currently combing through it to see what's up. |
|
The tests seem to get stuck here: Note that last line. I'm not a K8S guru, so I'll have to give some thought to both
|
|
Ah, so this is a problem. Early in the logs: Here's the Nix responsible for that service: systemd.services.kubelet-bootstrap = {
description = "Boostrap Kubelet";
wantedBy = ["kubernetes.target"];
after = ["docker.service" "network.target"];
path = with pkgs; [ docker ];
script = ''
${concatMapStrings (img: ''
echo "Seeding docker image: ${img}"
docker load <${img}
'') cfg.kubelet.seedDockerImages}
rm /opt/cni/bin/* || true
${concatMapStrings (package: ''
echo "Linking cni package: ${package}"
ln -fs ${package.plugins}/* /opt/cni/bin
'') cfg.kubelet.cni.packages}
'';
serviceConfig = {
Slice = "kubernetes.slice";
Type = "oneshot";
};
};So that first log line was benign. Let's now look at that directory that was linked: That's no good -- that's supposed to contain all of the reference plugins, not the |
|
This seems to be the culprit: # Allways include cni plugins
services.kubernetes.kubelet.cni.packages = [pkgs.cni];should be # Allways include cni plugins
services.kubernetes.kubelet.cni.packages = [pkgs.cni.plugins];The EDIT: That was actually fine before -- I didn't catch the However, that's not sufficient. Let's look at that output:
So we can revert that commit, or (preferably, IMO) add a new package for the reference plugins (and we probably should remove the |
* Fix reference CNI plugins
* The plugins were split out of the upstream cni repo around version
0.6.0
* Fix RBAC and DNS tests
* Fix broken apiVersion fields
* Change plugin linking to look in ${package}/bin rather than
${package.plugins}
* Initial work towards a working e2e test
* Test still fails, but at least the expression evaluates now
Continues @srhb's work in NixOS#37199
Fixes NixOS#37199
* Fix reference CNI plugins
* The plugins were split out of the upstream cni repo around version
0.6.0
* Fix RBAC and DNS tests
* Fix broken apiVersion fields
* Change plugin linking to look in ${package}/bin rather than
${package.plugins}
* Initial work towards a working e2e test
* Test still fails, but at least the expression evaluates now
Continues @srhb's work in NixOS#37199
Fixes NixOS#37199
(cherry picked from commit 709b6f6)
Motivation for this change
Fix the test issues discussed in #36739
Things done
build-use-sandboxinnix.confon non-NixOS)nix-shell -p nox --run "nox-review wip"./result/bin/)