Skip to content

Commit 33d12e5

Browse files
committed
pythonPackages: ensure all derivations provide python modules
This adds a test to ensure no new uses of `buildPythonApplication` can be added to `python-packages.nix`. Python packages can be grouped into two groups: 1) applications and 2) packages providing importable modules. In `python-packages.nix` we only want to have 2). 1) should be in the top-level package set. To achieve this, all setup hooks need to be marked as being a setup hook. For the setup hooks in the Python packages set this is done by creating a new builder, `makePythonHook`. Because there were issues with splicing, the file importing all the hooks is converted to an extension. All non-packages were moved out of `python-packages.nix` into `python-packages-base.nix`. The `keep` argument to `makeScopeWithSplicing was cleaned up as well; there is no need to keep this one manually in sync reducing the risk of breaking cross-compilation.
1 parent 99bcead commit 33d12e5

6 files changed

Lines changed: 185 additions & 242 deletions

File tree

pkgs/development/interpreters/python/default.nix

Lines changed: 20 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,19 @@
3131
, pythonAttr ? null
3232
, self # is pythonOnHostForTarget
3333
}: let
34-
pythonPackages = callPackage
34+
pythonPackages = let
35+
ensurePythonModules = items: let
36+
providesSetupHook = lib.attrByPath [ "provides" "setupHook"] false;
37+
notValid = value: (lib.isDerivation value) && !((pythonPackages.hasPythonModule value) || (providesSetupHook value));
38+
func = name: value: if !(notValid value) then value else throw "${name} should use `buildPythonPackage` or `toPythonModule` if it is to be part of the Python packages set.";
39+
in lib.mapAttrs func items;
40+
in ensurePythonModules (callPackage
3541
# Function that when called
3642
# - imports python-packages.nix
3743
# - adds spliced package sets to the package set
3844
# - applies overrides from `packageOverrides` and `pythonPackagesOverlays`.
3945
({ pkgs, stdenv, python, overrides }: let
40-
pythonPackagesFun = import ../../../top-level/python-packages.nix {
46+
pythonPackagesFun = import ./python-packages-base.nix {
4147
inherit stdenv pkgs lib;
4248
python = self;
4349
};
@@ -48,59 +54,31 @@
4854
selfHostHost = pythonOnHostForHost.pkgs;
4955
selfTargetTarget = pythonOnTargetForTarget.pkgs or {}; # There is no Python TargetTarget.
5056
};
51-
keep = self: {
52-
# TODO maybe only define these here so nothing is needed to be kept in sync.
53-
inherit (self)
54-
isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy3k isPyPy pythonAtLeast pythonOlder
55-
python bootstrapped-pip buildPythonPackage buildPythonApplication
56-
fetchPypi
57-
hasPythonModule requiredPythonModules makePythonPath disabledIf
58-
toPythonModule toPythonApplication
59-
buildSetupcfg
60-
61-
condaInstallHook
62-
condaUnpackHook
63-
eggUnpackHook
64-
eggBuildHook
65-
eggInstallHook
66-
flitBuildHook
67-
pipBuildHook
68-
pipInstallHook
69-
pytestCheckHook
70-
pythonCatchConflictsHook
71-
pythonImportsCheckHook
72-
pythonNamespacesHook
73-
pythonRecompileBytecodeHook
74-
pythonRemoveBinBytecodeHook
75-
pythonRemoveTestsDirHook
76-
setuptoolsBuildHook
77-
setuptoolsCheckHook
78-
venvShellHook
79-
wheelUnpackHook
80-
81-
wrapPython
82-
83-
pythonPackages
84-
85-
recursivePthLoader
86-
;
87-
};
57+
hooks = import ./hooks/default.nix;
58+
keep = lib.extends hooks pythonPackagesFun;
8859
extra = _: {};
8960
optionalExtensions = cond: as: if cond then as else [];
61+
pythonExtension = import ../../../top-level/python-packages.nix;
9062
python2Extension = import ../../../top-level/python2-packages.nix;
91-
extensions = lib.composeManyExtensions ((optionalExtensions (!self.isPy3k) [python2Extension]) ++ pythonPackagesExtensions ++ [ overrides ]);
63+
extensions = lib.composeManyExtensions ([
64+
pythonExtension
65+
] ++ (optionalExtensions (!self.isPy3k) [
66+
python2Extension
67+
]) ++ pythonPackagesExtensions ++ [
68+
overrides
69+
]);
9270
aliases = self: super: lib.optionalAttrs config.allowAliases (import ../../../top-level/python-aliases.nix lib self super);
9371
in lib.makeScopeWithSplicing
9472
splicePackages
9573
newScope
9674
otherSplices
9775
keep
9876
extra
99-
(lib.extends (lib.composeExtensions aliases extensions) pythonPackagesFun))
77+
(lib.extends (lib.composeExtensions aliases extensions) keep))
10078
{
10179
overrides = packageOverrides;
10280
python = self;
103-
};
81+
});
10482
in rec {
10583
isPy27 = pythonVersion == "2.7";
10684
isPy35 = pythonVersion == "3.5";

pkgs/development/interpreters/python/hooks/default.nix

Lines changed: 56 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,123 +1,117 @@
1-
# Hooks for building Python packages.
2-
{ python
3-
, lib
4-
, makeSetupHook
5-
, disabledIf
6-
, isPy3k
7-
}:
1+
self: super: with self;
82

93
let
10-
callPackage = python.pythonForBuild.pkgs.callPackage;
11-
pythonInterpreter = python.pythonForBuild.interpreter;
12-
pythonSitePackages = python.sitePackages;
13-
pythonCheckInterpreter = python.interpreter;
4+
pythonInterpreter = super.python.pythonForBuild.interpreter;
5+
pythonSitePackages = super.python.sitePackages;
6+
pythonCheckInterpreter = super.python.interpreter;
147
setuppy = ../run_setup.py;
15-
in rec {
8+
in {
9+
makePythonHook = args: pkgs.makeSetupHook ({passthru.provides.setupHook = true; } // args);
1610

17-
condaInstallHook = callPackage ({ gnutar, lbzip2 }:
18-
makeSetupHook {
11+
condaInstallHook = callPackage ({ makePythonHook, gnutar, lbzip2 }:
12+
makePythonHook {
1913
name = "conda-install-hook";
2014
deps = [ gnutar lbzip2 ];
2115
substitutions = {
2216
inherit pythonSitePackages;
2317
};
2418
} ./conda-install-hook.sh) {};
2519

26-
condaUnpackHook = callPackage ({}:
27-
makeSetupHook {
20+
condaUnpackHook = callPackage ({ makePythonHook }:
21+
makePythonHook {
2822
name = "conda-unpack-hook";
2923
deps = [];
3024
} ./conda-unpack-hook.sh) {};
3125

32-
eggBuildHook = callPackage ({ }:
33-
makeSetupHook {
26+
eggBuildHook = callPackage ({ makePythonHook }:
27+
makePythonHook {
3428
name = "egg-build-hook.sh";
3529
deps = [ ];
3630
} ./egg-build-hook.sh) {};
3731

38-
eggInstallHook = callPackage ({ setuptools }:
39-
makeSetupHook {
32+
eggInstallHook = callPackage ({ makePythonHook, setuptools }:
33+
makePythonHook {
4034
name = "egg-install-hook.sh";
4135
deps = [ setuptools ];
4236
substitutions = {
4337
inherit pythonInterpreter pythonSitePackages;
4438
};
4539
} ./egg-install-hook.sh) {};
4640

47-
eggUnpackHook = callPackage ({ }:
48-
makeSetupHook {
41+
eggUnpackHook = callPackage ({ makePythonHook, }:
42+
makePythonHook {
4943
name = "egg-unpack-hook.sh";
5044
deps = [ ];
5145
} ./egg-unpack-hook.sh) {};
5246

53-
flitBuildHook = callPackage ({ flit }:
54-
makeSetupHook {
47+
flitBuildHook = callPackage ({ makePythonHook, flit }:
48+
makePythonHook {
5549
name = "flit-build-hook";
5650
deps = [ flit ];
5751
substitutions = {
5852
inherit pythonInterpreter;
5953
};
6054
} ./flit-build-hook.sh) {};
6155

62-
pipBuildHook = callPackage ({ pip, wheel }:
63-
makeSetupHook {
56+
pipBuildHook = callPackage ({ makePythonHook, pip, wheel }:
57+
makePythonHook {
6458
name = "pip-build-hook.sh";
6559
deps = [ pip wheel ];
6660
substitutions = {
6761
inherit pythonInterpreter pythonSitePackages;
6862
};
6963
} ./pip-build-hook.sh) {};
7064

71-
pipInstallHook = callPackage ({ pip }:
72-
makeSetupHook {
65+
pipInstallHook = callPackage ({ makePythonHook, pip }:
66+
makePythonHook {
7367
name = "pip-install-hook";
7468
deps = [ pip ];
7569
substitutions = {
7670
inherit pythonInterpreter pythonSitePackages;
7771
};
7872
} ./pip-install-hook.sh) {};
7973

80-
pytestCheckHook = callPackage ({ pytest }:
81-
makeSetupHook {
74+
pytestCheckHook = callPackage ({ makePythonHook, pytest }:
75+
makePythonHook {
8276
name = "pytest-check-hook";
8377
deps = [ pytest ];
8478
substitutions = {
8579
inherit pythonCheckInterpreter;
8680
};
8781
} ./pytest-check-hook.sh) {};
8882

89-
pythonCatchConflictsHook = callPackage ({ setuptools }:
90-
makeSetupHook {
83+
pythonCatchConflictsHook = callPackage ({ makePythonHook, setuptools }:
84+
makePythonHook {
9185
name = "python-catch-conflicts-hook";
9286
substitutions = {
9387
inherit pythonInterpreter pythonSitePackages setuptools;
9488
catchConflicts=../catch_conflicts/catch_conflicts.py;
9589
};
9690
} ./python-catch-conflicts-hook.sh) {};
9791

98-
pythonImportsCheckHook = callPackage ({}:
99-
makeSetupHook {
92+
pythonImportsCheckHook = callPackage ({ makePythonHook }:
93+
makePythonHook {
10094
name = "python-imports-check-hook.sh";
10195
substitutions = {
10296
inherit pythonCheckInterpreter;
10397
};
10498
} ./python-imports-check-hook.sh) {};
10599

106-
pythonNamespacesHook = callPackage ({ findutils }:
107-
makeSetupHook {
100+
pythonNamespacesHook = callPackage ({ makePythonHook, findutils }:
101+
makePythonHook {
108102
name = "python-namespaces-hook.sh";
109103
substitutions = {
110104
inherit pythonSitePackages findutils;
111105
};
112106
} ./python-namespaces-hook.sh) {};
113107

114-
pythonOutputDistHook = callPackage ({ }:
115-
makeSetupHook {
108+
pythonOutputDistHook = callPackage ({ makePythonHook }:
109+
makePythonHook {
116110
name = "python-output-dist-hook";
117111
} ./python-output-dist-hook.sh ) {};
118112

119-
pythonRecompileBytecodeHook = callPackage ({ }:
120-
makeSetupHook {
113+
pythonRecompileBytecodeHook = callPackage ({ makePythonHook }:
114+
makePythonHook {
121115
name = "python-recompile-bytecode-hook";
122116
substitutions = {
123117
inherit pythonInterpreter pythonSitePackages;
@@ -126,71 +120,75 @@ in rec {
126120
};
127121
} ./python-recompile-bytecode-hook.sh ) {};
128122

129-
pythonRelaxDepsHook = callPackage ({ wheel }:
130-
makeSetupHook {
123+
pythonRelaxDepsHook = callPackage ({ makePythonHook, wheel }:
124+
makePythonHook {
131125
name = "python-relax-deps-hook";
132126
deps = [ wheel ];
133127
substitutions = {
134128
inherit pythonInterpreter;
135129
};
136130
} ./python-relax-deps-hook.sh) {};
137131

138-
pythonRemoveBinBytecodeHook = callPackage ({ }:
139-
makeSetupHook {
132+
pythonRemoveBinBytecodeHook = callPackage ({ makePythonHook }:
133+
makePythonHook {
140134
name = "python-remove-bin-bytecode-hook";
141135
} ./python-remove-bin-bytecode-hook.sh) {};
142136

143-
pythonRemoveTestsDirHook = callPackage ({ }:
144-
makeSetupHook {
137+
pythonRemoveTestsDirHook = callPackage ({ makePythonHook }:
138+
makePythonHook {
145139
name = "python-remove-tests-dir-hook";
146140
substitutions = {
147141
inherit pythonSitePackages;
148142
};
149143
} ./python-remove-tests-dir-hook.sh) {};
150144

151-
setuptoolsBuildHook = callPackage ({ setuptools, wheel }:
152-
makeSetupHook {
145+
setuptoolsBuildHook = callPackage ({ makePythonHook, setuptools, wheel }:
146+
makePythonHook {
153147
name = "setuptools-setup-hook";
154148
deps = [ setuptools wheel ];
155149
substitutions = {
156150
inherit pythonInterpreter pythonSitePackages setuppy;
157151
};
158152
} ./setuptools-build-hook.sh) {};
159153

160-
setuptoolsCheckHook = callPackage ({ setuptools }:
161-
makeSetupHook {
154+
setuptoolsCheckHook = callPackage ({ makePythonHook, setuptools }:
155+
makePythonHook {
162156
name = "setuptools-check-hook";
163157
deps = [ setuptools ];
164158
substitutions = {
165159
inherit pythonCheckInterpreter setuppy;
166160
};
167161
} ./setuptools-check-hook.sh) {};
168162

169-
unittestCheckHook = callPackage ({ }:
170-
makeSetupHook {
163+
unittestCheckHook = callPackage ({ makePythonHook }:
164+
makePythonHook {
171165
name = "unittest-check-hook";
172166
substitutions = {
173167
inherit pythonCheckInterpreter;
174168
};
175169
} ./unittest-check-hook.sh) {};
176170

177-
venvShellHook = disabledIf (!isPy3k) (callPackage ({ ensureNewerSourcesForZipFilesHook }:
178-
makeSetupHook {
171+
venvShellHook = disabledIf (!isPy3k) (callPackage ({ makePythonHook, ensureNewerSourcesForZipFilesHook }:
172+
makePythonHook {
179173
name = "venv-shell-hook";
180174
deps = [ ensureNewerSourcesForZipFilesHook ];
181175
substitutions = {
182176
inherit pythonInterpreter;
183177
};
184178
} ./venv-shell-hook.sh) {});
185179

186-
wheelUnpackHook = callPackage ({ wheel }:
187-
makeSetupHook {
180+
wheelUnpackHook = callPackage ({ makePythonHook, wheel }:
181+
makePythonHook {
188182
name = "wheel-unpack-hook.sh";
189183
deps = [ wheel ];
190184
} ./wheel-unpack-hook.sh) {};
191185

192-
sphinxHook = callPackage ({ sphinx, installShellFiles }:
193-
makeSetupHook {
186+
wrapPython = callPackage ../wrap-python.nix {
187+
inherit (pkgs.buildPackages) makeWrapper;
188+
};
189+
190+
sphinxHook = callPackage ({ makePythonHook, sphinx, installShellFiles }:
191+
makePythonHook {
194192
name = "python${python.pythonVersion}-sphinx-hook";
195193
deps = [ sphinx installShellFiles ];
196194
} ./sphinx-hook.sh) {};

0 commit comments

Comments
 (0)