Skip to content

Commit fc614c3

Browse files
committed
nixos/documentation: split options doc build
most modules can be evaluated for their documentation in a very restricted environment that doesn't include all of nixpkgs. this evaluation can then be cached and reused for subsequent builds, merging only documentation that has changed into the cached set. since nixos ships with a large number of modules of which only a few are used in any given config this can save evaluation a huge percentage of nixos options available in any given config. in tests of this caching, despite having to copy most of nixos/, saves about 80% of the time needed to build the system manual, or about two second on the machine used for testing. build time for a full system config shrank from 9.4s to 7.4s, while turning documentation off entirely shortened the build to 7.1s.
1 parent 55daffc commit fc614c3

36 files changed

Lines changed: 384 additions & 22 deletions

lib/options.nix

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ rec {
177177
docOption = rec {
178178
loc = opt.loc;
179179
name = showOption opt.loc;
180-
description = opt.description or (lib.warn "Option `${name}' has no description." "This option has no description.");
180+
description = opt.description or null;
181181
declarations = filter (x: x != unknownModule) opt.declarations;
182182
internal = opt.internal or false;
183183
visible =

nixos/doc/manual/default.nix

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{ pkgs, options, config, version, revision, extraSources ? [] }:
1+
{ pkgs, options, config, version, revision, extraSources ? [], baseOptionsJSON ? null, prefix ? ../../.. }:
22

33
with pkgs;
44

@@ -11,11 +11,11 @@ let
1111
#
1212
# E.g. if some `options` came from modules in ${pkgs.customModules}/nix,
1313
# you'd need to include `extraSources = [ pkgs.customModules ]`
14-
prefixesToStrip = map (p: "${toString p}/") ([ ../../.. ] ++ extraSources);
14+
prefixesToStrip = map (p: "${toString p}/") ([ prefix ] ++ extraSources);
1515
stripAnyPrefixes = lib.flip (lib.foldr lib.removePrefix) prefixesToStrip;
1616

1717
optionsDoc = buildPackages.nixosOptionsDoc {
18-
inherit options revision;
18+
inherit options revision baseOptionsJSON;
1919
transformOptions = opt: opt // {
2020
# Clean up declaration sites to not refer to the NixOS source tree.
2121
declarations = map stripAnyPrefixes opt.declarations;
@@ -161,7 +161,7 @@ let
161161
in rec {
162162
inherit generatedSources;
163163

164-
inherit (optionsDoc) optionsJSON optionsDocBook;
164+
inherit (optionsDoc) optionsJSON optionsNix optionsDocBook;
165165

166166
# Generate the NixOS manual.
167167
manualHTML = runCommand "nixos-manual-html"

nixos/doc/manual/development/meta-attributes.section.md

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ extra information. Module meta attributes are defined in the `meta.nix`
55
special module.
66

77
`meta` is a top level attribute like `options` and `config`. Available
8-
meta-attributes are `maintainers` and `doc`.
8+
meta-attributes are `maintainers`, `doc`, and `buildDocsInSandbox`.
99

1010
Each of the meta-attributes must be defined at most once per module
1111
file.
@@ -24,6 +24,7 @@ file.
2424
meta = {
2525
maintainers = with lib.maintainers; [ ericsagnes ];
2626
doc = ./default.xml;
27+
buildDocsInSandbox = true;
2728
};
2829
}
2930
```
@@ -38,3 +39,28 @@ file.
3839
```ShellSession
3940
$ nix-build nixos/release.nix -A manual.x86_64-linux
4041
```
42+
43+
- `buildDocsInSandbox` indicates whether the option documentation for the
44+
module can be built in a derivation sandbox. This option is currently only
45+
honored for modules shipped by nixpkgs. User modules and modules taken from
46+
`NIXOS_EXTRA_MODULE_PATH` are always built outside of the sandbox, as has
47+
been the case in previous releases.
48+
49+
Building NixOS option documentation in a sandbox allows caching of the built
50+
documentation, which greatly decreases the amount of time needed to evaluate
51+
a system configuration that has NixOS documentation enabled. The sandbox also
52+
restricts which attributes may be referenced by documentation attributes
53+
(such as option descriptions) to the `options` and `lib` module arguments and
54+
the `pkgs.formats` attribute of the `pkgs` argument, `config` and the rest of
55+
`pkgs` are disallowed and will cause doc build failures when used. This
56+
restriction is necessary because we cannot reproduce the full nixpkgs
57+
instantiation with configuration and overlays from a system configuration
58+
inside the sandbox. The `options` argument only includes options of modules
59+
that are also built inside the sandbox, referencing an option of a module
60+
that isn't built in the sandbox is also forbidden.
61+
62+
The default is `true` and should usually not be changed; set it to `false`
63+
only if the module requires access to `pkgs` in its documentation (e.g.
64+
because it loads information from a linked package to build an option type)
65+
or if its documentation depends on other modules that also aren't sandboxed
66+
(e.g. by using types defined in the other module).

nixos/doc/manual/from_md/development/meta-attributes.section.xml

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88
<para>
99
<literal>meta</literal> is a top level attribute like
1010
<literal>options</literal> and <literal>config</literal>. Available
11-
meta-attributes are <literal>maintainers</literal> and
12-
<literal>doc</literal>.
11+
meta-attributes are <literal>maintainers</literal>,
12+
<literal>doc</literal>, and <literal>buildDocsInSandbox</literal>.
1313
</para>
1414
<para>
1515
Each of the meta-attributes must be defined at most once per module
@@ -29,6 +29,7 @@
2929
meta = {
3030
maintainers = with lib.maintainers; [ ericsagnes ];
3131
doc = ./default.xml;
32+
buildDocsInSandbox = true;
3233
};
3334
}
3435
</programlisting>
@@ -51,5 +52,44 @@
5152
$ nix-build nixos/release.nix -A manual.x86_64-linux
5253
</programlisting>
5354
</listitem>
55+
<listitem>
56+
<para>
57+
<literal>buildDocsInSandbox</literal> indicates whether the
58+
option documentation for the module can be built in a derivation
59+
sandbox. This option is currently only honored for modules
60+
shipped by nixpkgs. User modules and modules taken from
61+
<literal>NIXOS_EXTRA_MODULE_PATH</literal> are always built
62+
outside of the sandbox, as has been the case in previous
63+
releases.
64+
</para>
65+
<para>
66+
Building NixOS option documentation in a sandbox allows caching
67+
of the built documentation, which greatly decreases the amount
68+
of time needed to evaluate a system configuration that has NixOS
69+
documentation enabled. The sandbox also restricts which
70+
attributes may be referenced by documentation attributes (such
71+
as option descriptions) to the <literal>options</literal> and
72+
<literal>lib</literal> module arguments and the
73+
<literal>pkgs.formats</literal> attribute of the
74+
<literal>pkgs</literal> argument, <literal>config</literal> and
75+
the rest of <literal>pkgs</literal> are disallowed and will
76+
cause doc build failures when used. This restriction is
77+
necessary because we cannot reproduce the full nixpkgs
78+
instantiation with configuration and overlays from a system
79+
configuration inside the sandbox. The <literal>options</literal>
80+
argument only includes options of modules that are also built
81+
inside the sandbox, referencing an option of a module that isn’t
82+
built in the sandbox is also forbidden.
83+
</para>
84+
<para>
85+
The default is <literal>true</literal> and should usually not be
86+
changed; set it to <literal>false</literal> only if the module
87+
requires access to <literal>pkgs</literal> in its documentation
88+
(e.g. because it loads information from a linked package to
89+
build an option type) or if its documentation depends on other
90+
modules that also aren’t sandboxed (e.g. by using types defined
91+
in the other module).
92+
</para>
93+
</listitem>
5494
</itemizedlist>
5595
</section>
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
{ libPath
2+
, pkgsLibPath
3+
, nixosPath
4+
, modules
5+
, stateVersion
6+
, release
7+
}:
8+
9+
let
10+
lib = import libPath;
11+
modulesPath = "${nixosPath}/modules";
12+
# dummy pkgs set that contains no packages, only `pkgs.lib` from the full set.
13+
# not having `pkgs.lib` causes all users of `pkgs.formats` to fail.
14+
pkgs = import pkgsLibPath {
15+
inherit lib;
16+
pkgs = null;
17+
};
18+
utils = import "${nixosPath}/lib/utils.nix" {
19+
inherit config lib;
20+
pkgs = null;
21+
};
22+
# this is used both as a module and as specialArgs.
23+
# as a module it sets the _module special values, as specialArgs it makes `config`
24+
# unusable. this causes documentation attributes depending on `config` to fail.
25+
config = {
26+
_module.check = false;
27+
_module.args = {};
28+
system.stateVersion = stateVersion;
29+
};
30+
eval = lib.evalModules {
31+
modules = (map (m: "${modulesPath}/${m}") modules) ++ [
32+
config
33+
];
34+
specialArgs = {
35+
inherit config pkgs utils;
36+
};
37+
};
38+
docs = import "${nixosPath}/doc/manual" {
39+
pkgs = pkgs // {
40+
inherit lib;
41+
# duplicate of the declaration in all-packages.nix
42+
buildPackages.nixosOptionsDoc = attrs:
43+
(import "${nixosPath}/lib/make-options-doc")
44+
({ inherit pkgs lib; } // attrs);
45+
};
46+
config = config.config;
47+
options = eval.options;
48+
version = release;
49+
revision = "release-${release}";
50+
prefix = modulesPath;
51+
};
52+
in
53+
docs.optionsNix

nixos/lib/make-options-doc/default.nix

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@
2121
, options
2222
, transformOptions ? lib.id # function for additional tranformations of the options
2323
, revision ? "" # Specify revision for the options
24+
# a set of options the docs we are generating will be merged into, as if by recursiveUpdate.
25+
# used to split the options doc build into a static part (nixos/modules) and a dynamic part
26+
# (non-nixos modules imported via configuration.nix, other module sources).
27+
, baseOptionsJSON ? null
2428
}:
2529

2630
let
@@ -99,13 +103,23 @@ in rec {
99103
optionsJSON = pkgs.runCommand "options.json"
100104
{ meta.description = "List of NixOS options in JSON format";
101105
buildInputs = [ pkgs.brotli ];
106+
options = builtins.toFile "options.json"
107+
(builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix));
102108
}
103109
''
104110
# Export list of options in different format.
105111
dst=$out/share/doc/nixos
106112
mkdir -p $dst
107113
108-
cp ${builtins.toFile "options.json" (builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix))} $dst/options.json
114+
${
115+
if baseOptionsJSON == null
116+
then "cp $options $dst/options.json"
117+
else ''
118+
${pkgs.python3Minimal}/bin/python ${./mergeJSON.py} \
119+
${baseOptionsJSON} $options \
120+
> $dst/options.json
121+
''
122+
}
109123
110124
brotli -9 < $dst/options.json > $dst/options.json.br
111125
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
import collections
2+
import json
3+
import sys
4+
5+
class Key:
6+
def __init__(self, path):
7+
self.path = path
8+
def __hash__(self):
9+
result = 0
10+
for id in self.path:
11+
result ^= hash(id)
12+
return result
13+
def __eq__(self, other):
14+
return type(self) is type(other) and self.path == other.path
15+
16+
Option = collections.namedtuple('Option', ['name', 'value'])
17+
18+
# pivot a dict of options keyed by their display name to a dict keyed by their path
19+
def pivot(options):
20+
result = dict()
21+
for (name, opt) in options.items():
22+
result[Key(opt['loc'])] = Option(name, opt)
23+
return result
24+
25+
# pivot back to indexed-by-full-name
26+
# like the docbook build we'll just fail if multiple options with differing locs
27+
# render to the same option name.
28+
def unpivot(options):
29+
result = dict()
30+
for (key, opt) in options.items():
31+
if opt.name in result:
32+
raise RuntimeError(
33+
'multiple options with colliding ids found',
34+
opt.name,
35+
result[opt.name]['loc'],
36+
opt.value['loc'],
37+
)
38+
result[opt.name] = opt.value
39+
return result
40+
41+
options = pivot(json.load(open(sys.argv[1], 'r')))
42+
overrides = pivot(json.load(open(sys.argv[2], 'r')))
43+
44+
# fix up declaration paths in lazy options, since we don't eval them from a full nixpkgs dir
45+
for (k, v) in options.items():
46+
v.value['declarations'] = list(map(lambda s: f'nixos/modules/{s}', v.value['declarations']))
47+
48+
# merge both descriptions
49+
for (k, v) in overrides.items():
50+
cur = options.setdefault(k, v).value
51+
for (ok, ov) in v.value.items():
52+
if ok == 'declarations':
53+
decls = cur[ok]
54+
for d in ov:
55+
if d not in decls:
56+
decls += [d]
57+
elif ok == "type":
58+
# ignore types of placeholder options
59+
if ov != "_unspecified" or cur[ok] == "_unspecified":
60+
cur[ok] = ov
61+
elif ov is not None or cur.get(ok, None) is None:
62+
cur[ok] = ov
63+
64+
# check that every option has a description
65+
# TODO: nixos-rebuild with flakes may hide the warning, maybe turn on -L by default for those?
66+
for (k, v) in options.items():
67+
if v.value.get('description', None) is None:
68+
print(f"\x1b[1;31mwarning: option {v.name} has no description\x1b[0m", file=sys.stderr)
69+
v.value['description'] = "This option has no description."
70+
71+
json.dump(unpivot(options), fp=sys.stdout)

nixos/modules/config/qt5.nix

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,4 +101,7 @@ in
101101
environment.systemPackages = packages;
102102

103103
};
104+
105+
# uses relatedPackages
106+
meta.buildDocsInSandbox = false;
104107
}

nixos/modules/i18n/input-method/fcitx.nix

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,7 @@ in
4040
};
4141
services.xserver.displayManager.sessionCommands = "${fcitxPackage}/bin/fcitx";
4242
};
43+
44+
# uses attributes of the linked package
45+
meta.buildDocsInSandbox = false;
4346
}

nixos/modules/i18n/input-method/ibus.nix

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,4 +80,7 @@ in
8080
ibusPackage
8181
];
8282
};
83+
84+
# uses attributes of the linked package
85+
meta.buildDocsInSandbox = false;
8386
}

0 commit comments

Comments
 (0)