Skip to content

Commit 83b27f6

Browse files
EkleogEkleog
committed
tests: split into a separate all-tests.nix file
This will make the list much easier to re-use, eg. for `nixosTests` The drawback is that this approaches makes the ``` nix-build release.nix -A tests.opensmtpd.x86_64-linux ``` command about twice as slow (3s to 6s): it now has to evaluate `nixpkgs` once for each architecture, instead of just having the hardcoded list of tests that allowed to say “ok just evaluate for x86_64-linux”. On the other hand, complete evaluation of `release.nix` should be much faster because we no longer import `nixpkgs` for each test: testing with the following command went from 30s to 18s, and that's just for a few tests. ``` time nix-instantiate --eval --strict nixos/release.nix -A tests.nat ``` I initially wanted to test on the whole `release.nix`, but there are too many broken tests and it takes too long to eval them all, especially compared to the fact that the current implementation breaks some setup. Given developers can just `nix-build nixos/tests/my-test.nix`, it sounds like an overall win.
1 parent 6c68fbd commit 83b27f6

File tree

3 files changed

+232
-218
lines changed

3 files changed

+232
-218
lines changed

‎nixos/release.nix‎

Lines changed: 14 additions & 216 deletions
Original file line numberDiff line numberDiff line change
@@ -14,28 +14,19 @@ let
1414
versionSuffix =
1515
(if stableBranch then "." else "pre") + "${toString nixpkgs.revCount}.${nixpkgs.shortRev}";
1616

17-
importTest = fn: args: system: import fn ({
18-
inherit system;
19-
} // args);
20-
21-
# Note: only supportedSystems are considered.
22-
callTestOnMatchingSystems = systems: fn: args:
23-
forMatchingSystems
24-
(intersectLists supportedSystems systems)
25-
(system: hydraJob (importTest fn args system));
26-
callTest = callTestOnMatchingSystems supportedSystems;
27-
28-
callSubTests = callSubTestsOnMatchingSystems supportedSystems;
29-
callSubTestsOnMatchingSystems = systems: fn: args: let
30-
discover = attrs: let
31-
subTests = filterAttrs (const (hasAttr "test")) attrs;
32-
in mapAttrs (const (t: hydraJob t.test)) subTests;
33-
34-
discoverForSystem = system: mapAttrs (_: test: {
35-
${system} = test;
36-
}) (discover (importTest fn args system));
37-
38-
in foldAttrs mergeAttrs {} (map discoverForSystem (intersectLists systems supportedSystems));
17+
# Run the tests for each platform. You can run a test by doing
18+
# e.g. ‘nix-build -A tests.login.x86_64-linux’, or equivalently,
19+
# ‘nix-build tests/login.nix -A result’.
20+
allTestsForSystem = system:
21+
import ./tests/all-tests.nix {
22+
inherit system;
23+
pkgs = import nixpkgs { inherit system; };
24+
callTest = t: {
25+
${system} = hydraJob t.test;
26+
};
27+
};
28+
allTests =
29+
foldAttrs recursiveUpdate {} (map allTestsForSystem supportedSystems);
3930

4031
pkgs = import nixpkgs { system = "x86_64-linux"; };
4132

@@ -245,200 +236,7 @@ in rec {
245236
};
246237
*/
247238

248-
249-
# Run the tests for each platform. You can run a test by doing
250-
# e.g. ‘nix-build -A tests.login.x86_64-linux’, or equivalently,
251-
# ‘nix-build tests/login.nix -A result’.
252-
tests.atd = callTest tests/atd.nix {};
253-
tests.acme = callTest tests/acme.nix {};
254-
tests.avahi = callTest tests/avahi.nix {};
255-
tests.beegfs = callTest tests/beegfs.nix {};
256-
tests.upnp = callTest tests/upnp.nix {};
257-
tests.bittorrent = callTest tests/bittorrent.nix {};
258-
tests.bind = callTest tests/bind.nix {};
259-
#tests.blivet = callTest tests/blivet.nix {}; # broken since 2017-07024
260-
tests.boot = callSubTests tests/boot.nix {};
261-
tests.boot-stage1 = callTest tests/boot-stage1.nix {};
262-
tests.borgbackup = callTest tests/borgbackup.nix {};
263-
tests.buildbot = callSubTests tests/buildbot.nix {};
264-
tests.cadvisor = callTestOnMatchingSystems ["x86_64-linux"] tests/cadvisor.nix {};
265-
tests.ceph = callTestOnMatchingSystems ["x86_64-linux"] tests/ceph.nix {};
266-
tests.certmgr = callSubTests tests/certmgr.nix {};
267-
tests.cfssl = callTestOnMatchingSystems ["x86_64-linux"] tests/cfssl.nix {};
268-
tests.chromium = (callSubTestsOnMatchingSystems ["x86_64-linux"] tests/chromium.nix {}).stable or {};
269-
tests.cjdns = callTest tests/cjdns.nix {};
270-
tests.cloud-init = callTest tests/cloud-init.nix {};
271-
tests.codimd = callTest tests/codimd.nix {};
272-
tests.containers-ipv4 = callTest tests/containers-ipv4.nix {};
273-
tests.containers-ipv6 = callTest tests/containers-ipv6.nix {};
274-
tests.containers-bridge = callTest tests/containers-bridge.nix {};
275-
tests.containers-imperative = callTest tests/containers-imperative.nix {};
276-
tests.containers-extra_veth = callTest tests/containers-extra_veth.nix {};
277-
tests.containers-physical_interfaces = callTest tests/containers-physical_interfaces.nix {};
278-
tests.containers-restart_networking = callTest tests/containers-restart_networking.nix {};
279-
tests.containers-tmpfs = callTest tests/containers-tmpfs.nix {};
280-
tests.containers-hosts = callTest tests/containers-hosts.nix {};
281-
tests.containers-macvlans = callTest tests/containers-macvlans.nix {};
282-
tests.couchdb = callTest tests/couchdb.nix {};
283-
tests.deluge = callTest tests/deluge.nix {};
284-
tests.dhparams = callTest tests/dhparams.nix {};
285-
tests.docker = callTestOnMatchingSystems ["x86_64-linux"] tests/docker.nix {};
286-
tests.docker-tools = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-tools.nix {};
287-
tests.docker-tools-overlay = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-tools-overlay.nix {};
288-
tests.docker-edge = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-edge.nix {};
289-
tests.docker-preloader = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-preloader.nix {};
290-
tests.docker-registry = callTest tests/docker-registry.nix {};
291-
tests.dovecot = callTest tests/dovecot.nix {};
292-
tests.dnscrypt-proxy = callTestOnMatchingSystems ["x86_64-linux"] tests/dnscrypt-proxy.nix {};
293-
tests.ecryptfs = callTest tests/ecryptfs.nix {};
294-
tests.etcd = callTestOnMatchingSystems ["x86_64-linux"] tests/etcd.nix {};
295-
tests.ec2-nixops = (callSubTestsOnMatchingSystems ["x86_64-linux"] tests/ec2.nix {}).boot-ec2-nixops or {};
296-
# ec2-config doesn't work in a sandbox as the simulated ec2 instance needs network access
297-
#tests.ec2-config = (callSubTestsOnMatchingSystems ["x86_64-linux"] tests/ec2.nix {}).boot-ec2-config or {};
298-
tests.elk = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/elk.nix {};
299-
tests.env = callTest tests/env.nix {};
300-
tests.ferm = callTest tests/ferm.nix {};
301-
tests.firefox = callTest tests/firefox.nix {};
302-
tests.flatpak = callTest tests/flatpak.nix {};
303-
tests.firewall = callTest tests/firewall.nix {};
304-
tests.fsck = callTest tests/fsck.nix {};
305-
tests.fwupd = callTest tests/fwupd.nix {};
306-
tests.gdk-pixbuf = callTest tests/gdk-pixbuf.nix {};
307-
tests.gitea = callSubTests tests/gitea.nix {};
308-
tests.gitlab = callTest tests/gitlab.nix {};
309-
tests.gitolite = callTest tests/gitolite.nix {};
310-
tests.gjs = callTest tests/gjs.nix {};
311-
tests.gocd-agent = callTest tests/gocd-agent.nix {};
312-
tests.gocd-server = callTest tests/gocd-server.nix {};
313-
tests.gnome3 = callTest tests/gnome3.nix {};
314-
tests.gnome3-gdm = callTest tests/gnome3-gdm.nix {};
315-
tests.grafana = callTest tests/grafana.nix {};
316-
tests.graphite = callTest tests/graphite.nix {};
317-
tests.hadoop.hdfs = callTestOnMatchingSystems [ "x86_64-linux" ] tests/hadoop/hdfs.nix {};
318-
tests.hadoop.yarn = callTestOnMatchingSystems [ "x86_64-linux" ] tests/hadoop/yarn.nix {};
319-
tests.hardened = callTest tests/hardened.nix { };
320-
tests.haproxy = callTest tests/haproxy.nix {};
321-
tests.hibernate = callTest tests/hibernate.nix {};
322-
tests.hitch = callTest tests/hitch {};
323-
tests.home-assistant = callTest tests/home-assistant.nix { };
324-
tests.hound = callTest tests/hound.nix {};
325-
tests.hocker-fetchdocker = callTest tests/hocker-fetchdocker {};
326-
tests.hydra = callTest tests/hydra {};
327-
tests.i3wm = callTest tests/i3wm.nix {};
328-
tests.iftop = callTest tests/iftop.nix {};
329-
tests.initrd-network-ssh = callTest tests/initrd-network-ssh {};
330-
tests.installer = callSubTests tests/installer.nix {};
331-
tests.influxdb = callTest tests/influxdb.nix {};
332-
tests.ipv6 = callTest tests/ipv6.nix {};
333-
tests.jenkins = callTest tests/jenkins.nix {};
334-
tests.ostree = callTest tests/ostree.nix {};
335-
tests.osquery = callTest tests/osquery.nix {};
336-
tests.plasma5 = callTest tests/plasma5.nix {};
337-
tests.plotinus = callTest tests/plotinus.nix {};
338-
tests.keymap = callSubTests tests/keymap.nix {};
339-
tests.incron = callTest tests/incron.nix {};
340-
tests.initrdNetwork = callTest tests/initrd-network.nix {};
341-
tests.kafka = callSubTests tests/kafka.nix {};
342-
tests.kernel-latest = callTest tests/kernel-latest.nix {};
343-
tests.kernel-lts = callTest tests/kernel-lts.nix {};
344-
tests.kubernetes.dns = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/kubernetes/dns.nix {};
345-
## kubernetes.e2e should eventually replace kubernetes.rbac when it works
346-
#tests.kubernetes.e2e = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/kubernetes/e2e.nix {};
347-
tests.kubernetes.rbac = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/kubernetes/rbac.nix {};
348-
tests.latestKernel.login = callTest tests/login.nix { latestKernel = true; };
349-
tests.ldap = callTest tests/ldap.nix {};
350-
#tests.lightdm = callTest tests/lightdm.nix {};
351-
tests.login = callTest tests/login.nix {};
352-
#tests.logstash = callTest tests/logstash.nix {};
353-
tests.mathics = callTest tests/mathics.nix {};
354-
tests.matrix-synapse = callTest tests/matrix-synapse.nix {};
355-
tests.memcached = callTest tests/memcached.nix {};
356-
tests.mesos = callTest tests/mesos.nix {};
357-
tests.misc = callTest tests/misc.nix {};
358-
tests.mongodb = callTest tests/mongodb.nix {};
359-
tests.mpd = callTest tests/mpd.nix {};
360-
tests.mumble = callTest tests/mumble.nix {};
361-
tests.munin = callTest tests/munin.nix {};
362-
tests.mutableUsers = callTest tests/mutable-users.nix {};
363-
tests.mysql = callTest tests/mysql.nix {};
364-
tests.mysqlBackup = callTest tests/mysql-backup.nix {};
365-
tests.mysqlReplication = callTest tests/mysql-replication.nix {};
366-
tests.nat.firewall = callTest tests/nat.nix { withFirewall = true; };
367-
tests.nat.firewall-conntrack = callTest tests/nat.nix { withFirewall = true; withConntrackHelpers = true; };
368-
tests.nat.standalone = callTest tests/nat.nix { withFirewall = false; };
369-
tests.netdata = callTest tests/netdata.nix { };
370-
tests.networking.networkd = callSubTests tests/networking.nix { networkd = true; };
371-
tests.networking.scripted = callSubTests tests/networking.nix { networkd = false; };
372-
tests.nextcloud = callSubTests tests/nextcloud { };
373-
# TODO: put in networking.nix after the test becomes more complete
374-
tests.networkingProxy = callTest tests/networking-proxy.nix {};
375-
tests.nexus = callTest tests/nexus.nix { };
376-
tests.nfs3 = callTest tests/nfs.nix { version = 3; };
377-
tests.nfs4 = callTest tests/nfs.nix { version = 4; };
378-
tests.nginx = callTest tests/nginx.nix { };
379-
tests.nghttpx = callTest tests/nghttpx.nix { };
380-
tests.nix-ssh-serve = callTest tests/nix-ssh-serve.nix { };
381-
tests.novacomd = callTestOnMatchingSystems ["x86_64-linux"] tests/novacomd.nix { };
382-
tests.leaps = callTest tests/leaps.nix { };
383-
tests.nsd = callTest tests/nsd.nix {};
384-
tests.openssh = callTest tests/openssh.nix {};
385-
tests.openldap = callTest tests/openldap.nix {};
386-
tests.opensmtpd = callTest tests/opensmtpd.nix {};
387-
tests.owncloud = callTest tests/owncloud.nix {};
388-
tests.pam-oath-login = callTest tests/pam-oath-login.nix {};
389-
tests.peerflix = callTest tests/peerflix.nix {};
390-
tests.php-pcre = callTest tests/php-pcre.nix {};
391-
tests.postgresql = callSubTests tests/postgresql.nix {};
392-
tests.pgmanage = callTest tests/pgmanage.nix {};
393-
tests.postgis = callTest tests/postgis.nix {};
394-
tests.powerdns = callTest tests/powerdns.nix {};
395-
tests.pgjwt = callTest tests/pgjwt.nix {};
396-
tests.predictable-interface-names = callSubTests tests/predictable-interface-names.nix {};
397-
tests.printing = callTest tests/printing.nix {};
398-
tests.prometheus = callTest tests/prometheus.nix {};
399-
tests.prometheus-exporters = callTest tests/prometheus-exporters.nix {};
400-
tests.prosody = callTest tests/prosody.nix {};
401-
tests.proxy = callTest tests/proxy.nix {};
402-
tests.quagga = callTest tests/quagga.nix {};
403-
tests.quake3 = callTest tests/quake3.nix {};
404-
tests.rabbitmq = callTest tests/rabbitmq.nix {};
405-
tests.radicale = callTest tests/radicale.nix {};
406-
tests.redmine = callTest tests/redmine.nix {};
407-
tests.rspamd = callSubTests tests/rspamd.nix {};
408-
tests.rsyslogd = callSubTests tests/rsyslogd.nix {};
409-
tests.runInMachine = callTest tests/run-in-machine.nix {};
410-
tests.rxe = callTest tests/rxe.nix {};
411-
tests.samba = callTest tests/samba.nix {};
412-
tests.sddm = callSubTests tests/sddm.nix {};
413-
tests.simple = callTest tests/simple.nix {};
414-
tests.slim = callTest tests/slim.nix {};
415-
tests.slurm = callTest tests/slurm.nix {};
416-
tests.smokeping = callTest tests/smokeping.nix {};
417-
tests.snapper = callTest tests/snapper.nix {};
418-
tests.solr = callTest tests/solr.nix {};
419-
#tests.statsd = callTest tests/statsd.nix {}; # statsd is broken: #45946
420-
tests.strongswan-swanctl = callTest tests/strongswan-swanctl.nix {};
421-
tests.sudo = callTest tests/sudo.nix {};
422-
tests.systemd = callTest tests/systemd.nix {};
423-
tests.switchTest = callTest tests/switch-test.nix {};
424-
tests.taskserver = callTest tests/taskserver.nix {};
425-
tests.tomcat = callTest tests/tomcat.nix {};
426-
tests.tor = callTest tests/tor.nix {};
427-
tests.transmission = callTest tests/transmission.nix {};
428-
tests.udisks2 = callTest tests/udisks2.nix {};
429-
tests.vault = callTest tests/vault.nix {};
430-
tests.virtualbox = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/virtualbox.nix {};
431-
tests.wordpress = callTest tests/wordpress.nix {};
432-
tests.xautolock = callTest tests/xautolock.nix {};
433-
tests.xdg-desktop-portal = callTest tests/xdg-desktop-portal.nix {};
434-
tests.xfce = callTest tests/xfce.nix {};
435-
tests.xmonad = callTest tests/xmonad.nix {};
436-
tests.xrdp = callTest tests/xrdp.nix {};
437-
tests.xss-lock = callTest tests/xss-lock.nix {};
438-
tests.yabar = callTest tests/yabar.nix {};
439-
tests.zookeeper = callTest tests/zookeeper.nix {};
440-
tests.morty = callTest tests/morty.nix { };
441-
tests.bcachefs = callTest tests/bcachefs.nix { };
239+
tests = allTests;
442240

443241
/* Build a bunch of typical closures so that Hydra can keep track of
444242
the evolution of closure sizes. */

0 commit comments

Comments
 (0)