pkgutil.py
1
"""Utilities to support packages."""
2
3
from collections import namedtuple
4
from functools import singledispatch as simplegeneric
5
import importlib
6
import importlib.util
7
import importlib.machinery
8
import os
10
import sys
11
from types import ModuleType
12
import warnings
13
14
__all__ = [
15
'get_importer', 'iter_importers', 'get_loader', 'find_loader',
16
'walk_packages', 'iter_modules', 'get_data',
17
'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
18
'ModuleInfo',
22
ModuleInfo = namedtuple('ModuleInfo', 'module_finder name ispkg')
23
ModuleInfo.__doc__ = 'A namedtuple with minimal info about a module.'
24
25
26
def _get_spec(finder, name):
27
"""Return the finder-specific module spec."""
28
# Works with legacy finders.
29
try:
30
find_spec = finder.find_spec
31
except AttributeError:
32
loader = finder.find_module(name)
33
if loader is None:
34
return None
35
return importlib.util.spec_from_loader(name, loader)
36
else:
37
return find_spec(name)
38
39
40
def read_code(stream):
41
# This helper is needed in order for the PEP 302 emulation to
42
# correctly handle compiled files
43
import marshal
44
45
magic = stream.read(4)
46
if magic != importlib.util.MAGIC_NUMBER:
49
stream.read(8) # Skip timestamp and size
50
return marshal.load(stream)
51
52
53
def walk_packages(path=None, prefix='', onerror=None):
54
"""Yields ModuleInfo for all modules recursively
55
on path, or, if path is None, all accessible modules.
56
57
'path' should be either None or a list of paths to look for
58
modules in.
59
60
'prefix' is a string to output on the front of every module name
61
on output.
62
63
Note that this function must import all *packages* (NOT all
64
modules!) on the given path, in order to access the __path__
65
attribute to find submodules.
66
67
'onerror' is a function which gets called with one argument (the
68
name of the package which was being imported) if any exception
69
occurs while trying to import a package. If no onerror function is
70
supplied, ImportErrors are caught and ignored, while all other
71
exceptions are propagated, terminating the search.
72
73
Examples:
74
75
# list all modules python can access
76
walk_packages()
77
78
# list all submodules of ctypes
79
walk_packages(ctypes.__path__, ctypes.__name__+'.')
80
"""
82
def seen(p, m={}):
83
if p in m:
84
return True
87
for info in iter_modules(path, prefix):
88
yield info
90
if info.ispkg:
92
__import__(info.name)
93
except ImportError:
94
if onerror is not None:
95
onerror(info.name)
96
except Exception:
97
if onerror is not None:
98
onerror(info.name)
99
else:
100
raise
102
path = getattr(sys.modules[info.name], '__path__', None) or []
103
104
# don't traverse path items we've seen before
105
path = [p for p in path if not seen(p)]
106
107
yield from walk_packages(path, info.name+'.', onerror)
108
109
110
def iter_modules(path=None, prefix=''):
111
"""Yields ModuleInfo for all submodules on path,
112
or, if path is None, all top-level modules on sys.path.
113
114
'path' should be either None or a list of paths to look for
115
modules in.
116
117
'prefix' is a string to output on the front of every module name
118
on output.
119
"""
120
if path is None:
121
importers = iter_importers()
122
else:
123
importers = map(get_importer, path)
124
125
yielded = {}
126
for i in importers:
127
for name, ispkg in iter_importer_modules(i, prefix):
128
if name not in yielded:
129
yielded[name] = 1
130
yield ModuleInfo(i, name, ispkg)
133
@simplegeneric
134
def iter_importer_modules(importer, prefix=''):
135
if not hasattr(importer, 'iter_modules'):
136
return []
137
return importer.iter_modules(prefix)
138
139
140
# Implement a file walker for the normal importlib path hook
141
def _iter_file_finder_modules(importer, prefix=''):
142
if importer.path is None or not os.path.isdir(importer.path):
143
return
144
145
yielded = {}
146
import inspect
147
try:
148
filenames = os.listdir(importer.path)
149
except OSError:
150
# ignore unreadable directories like import does
151
filenames = []
152
filenames.sort() # handle packages before same-named modules
153
154
for fn in filenames:
155
modname = inspect.getmodulename(fn)
156
if modname=='__init__' or modname in yielded:
157
continue
158
159
path = os.path.join(importer.path, fn)
160
ispkg = False
161
162
if not modname and os.path.isdir(path) and '.' not in fn:
163
modname = fn
164
try:
165
dircontents = os.listdir(path)
166
except OSError:
167
# ignore unreadable directories like import does
168
dircontents = []
169
for fn in dircontents:
170
subname = inspect.getmodulename(fn)
171
if subname=='__init__':
172
ispkg = True
173
break
174
else:
175
continue # not a package
176
177
if modname and '.' not in modname:
178
yielded[modname] = 1
179
yield prefix + modname, ispkg
180
181
iter_importer_modules.register(
182
importlib.machinery.FileFinder, _iter_file_finder_modules)
184
185
def _import_imp():
186
global imp
187
with warnings.catch_warnings():
188
warnings.simplefilter('ignore', DeprecationWarning)
189
imp = importlib.import_module('imp')
190
191
class ImpImporter:
192
"""PEP 302 Finder that wraps Python's "classic" import algorithm
194
ImpImporter(dirname) produces a PEP 302 finder that searches that
195
directory. ImpImporter(None) produces a PEP 302 finder that searches
196
the current sys.path, plus any modules that are frozen or built-in.
197
198
Note that ImpImporter does not currently support being used by placement
199
on sys.meta_path.
200
"""
201
202
def __init__(self, path=None):
203
global imp
204
warnings.warn("This emulation is deprecated, use 'importlib' instead",
205
DeprecationWarning)
206
_import_imp()
207
self.path = path
208
209
def find_module(self, fullname, path=None):
210
# Note: we ignore 'path' argument since it is only used via meta_path
211
subname = fullname.split(".")[-1]
212
if subname != fullname and self.path is None:
213
return None
214
if self.path is None:
215
path = None
216
else:
217
path = [os.path.realpath(self.path)]
218
try:
219
file, filename, etc = imp.find_module(subname, path)
220
except ImportError:
221
return None
222
return ImpLoader(fullname, file, filename, etc)
223
224
def iter_modules(self, prefix=''):
225
if self.path is None or not os.path.isdir(self.path):
226
return
227
228
yielded = {}
229
import inspect
230
try:
231
filenames = os.listdir(self.path)
232
except OSError:
233
# ignore unreadable directories like import does
234
filenames = []
235
filenames.sort() # handle packages before same-named modules
236
237
for fn in filenames:
238
modname = inspect.getmodulename(fn)
239
if modname=='__init__' or modname in yielded:
240
continue
241
242
path = os.path.join(self.path, fn)
243
ispkg = False
244
245
if not modname and os.path.isdir(path) and '.' not in fn:
246
modname = fn
247
try:
248
dircontents = os.listdir(path)
249
except OSError:
250
# ignore unreadable directories like import does
251
dircontents = []
252
for fn in dircontents:
253
subname = inspect.getmodulename(fn)
254
if subname=='__init__':
255
ispkg = True
256
break
257
else:
258
continue # not a package
259
260
if modname and '.' not in modname:
261
yielded[modname] = 1
262
yield prefix + modname, ispkg
263
264
265
class ImpLoader:
266
"""PEP 302 Loader that wraps Python's "classic" import algorithm
267
"""
268
code = source = None
269
270
def __init__(self, fullname, file, filename, etc):
271
warnings.warn("This emulation is deprecated, use 'importlib' instead",
272
DeprecationWarning)
273
_import_imp()
274
self.file = file
275
self.filename = filename
276
self.fullname = fullname
277
self.etc = etc
278
279
def load_module(self, fullname):
280
self._reopen()
281
try:
282
mod = imp.load_module(fullname, self.file, self.filename, self.etc)
283
finally:
284
if self.file:
285
self.file.close()
286
# Note: we don't set __loader__ because we want the module to look
287
# normal; i.e. this is just a wrapper for standard import machinery
288
return mod
289
290
def get_data(self, pathname):
291
with open(pathname, "rb") as file:
292
return file.read()
293
294
def _reopen(self):
295
if self.file and self.file.closed:
296
mod_type = self.etc[2]
297
if mod_type==imp.PY_SOURCE:
298
self.file = open(self.filename, 'r')
299
elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
300
self.file = open(self.filename, 'rb')
301
302
def _fix_name(self, fullname):
303
if fullname is None:
304
fullname = self.fullname
305
elif fullname != self.fullname:
306
raise ImportError("Loader for module %s cannot handle "
307
"module %s" % (self.fullname, fullname))
308
return fullname
309
310
def is_package(self, fullname):
311
fullname = self._fix_name(fullname)
312
return self.etc[2]==imp.PKG_DIRECTORY
313
314
def get_code(self, fullname=None):
315
fullname = self._fix_name(fullname)
316
if self.code is None:
317
mod_type = self.etc[2]
318
if mod_type==imp.PY_SOURCE:
319
source = self.get_source(fullname)
320
self.code = compile(source, self.filename, 'exec')
321
elif mod_type==imp.PY_COMPILED:
322
self._reopen()
323
try:
324
self.code = read_code(self.file)
325
finally:
326
self.file.close()
327
elif mod_type==imp.PKG_DIRECTORY:
328
self.code = self._get_delegate().get_code()
329
return self.code
330
331
def get_source(self, fullname=None):
332
fullname = self._fix_name(fullname)
333
if self.source is None:
334
mod_type = self.etc[2]
335
if mod_type==imp.PY_SOURCE:
336
self._reopen()
337
try:
338
self.source = self.file.read()
339
finally:
340
self.file.close()
341
elif mod_type==imp.PY_COMPILED:
342
if os.path.exists(self.filename[:-1]):
343
with open(self.filename[:-1], 'r') as f:
344
self.source = f.read()
345
elif mod_type==imp.PKG_DIRECTORY:
346
self.source = self._get_delegate().get_source()
347
return self.source
348
349
def _get_delegate(self):
350
finder = ImpImporter(self.filename)
351
spec = _get_spec(finder, '__init__')
352
return spec.loader
353
354
def get_filename(self, fullname=None):
355
fullname = self._fix_name(fullname)
356
mod_type = self.etc[2]
357
if mod_type==imp.PKG_DIRECTORY:
358
return self._get_delegate().get_filename()
359
elif mod_type in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
360
return self.filename
361
return None
362
363
364
try:
365
import zipimport
366
from zipimport import zipimporter
367
368
def iter_zipimport_modules(importer, prefix=''):
369
dirlist = sorted(zipimport._zip_directory_cache[importer.archive])
370
_prefix = importer.prefix
371
plen = len(_prefix)
372
yielded = {}
373
import inspect
374
for fn in dirlist:
375
if not fn.startswith(_prefix):
376
continue
377
378
fn = fn[plen:].split(os.sep)
379
380
if len(fn)==2 and fn[1].startswith('__init__.py'):
381
if fn[0] not in yielded:
382
yielded[fn[0]] = 1
383
yield prefix + fn[0], True
384
385
if len(fn)!=1:
386
continue
387
388
modname = inspect.getmodulename(fn[0])
389
if modname=='__init__':
390
continue
391
392
if modname and '.' not in modname and modname not in yielded:
393
yielded[modname] = 1
394
yield prefix + modname, False
395
396
iter_importer_modules.register(zipimporter, iter_zipimport_modules)
397
398
except ImportError:
399
pass
400
401
402
def get_importer(path_item):
403
"""Retrieve a finder for the given path item
405
The returned finder is cached in sys.path_importer_cache
406
if it was newly created by a path hook.
407
408
The cache (or part of it) can be cleared manually if a
409
rescan of sys.path_hooks is necessary.
410
"""
411
try:
412
importer = sys.path_importer_cache[path_item]
413
except KeyError:
414
for path_hook in sys.path_hooks:
415
try:
416
importer = path_hook(path_item)
417
sys.path_importer_cache.setdefault(path_item, importer)
418
break
419
except ImportError:
420
pass
421
else:
422
importer = None
423
return importer
424
425
426
def iter_importers(fullname=""):
427
"""Yield finders for the given module name
429
If fullname contains a '.', the finders will be for the package
430
containing fullname, otherwise they will be all registered top level
431
finders (i.e. those on both sys.meta_path and sys.path_hooks).
433
If the named module is in a package, that package is imported as a side
434
effect of invoking this function.
436
If no module name is specified, all top level finders are produced.
437
"""
438
if fullname.startswith('.'):
439
msg = "Relative module name {!r} not supported".format(fullname)
440
raise ImportError(msg)
441
if '.' in fullname:
442
# Get the containing package's __path__
443
pkg_name = fullname.rpartition(".")[0]
444
pkg = importlib.import_module(pkg_name)
445
path = getattr(pkg, '__path__', None)
446
if path is None:
447
return
449
yield from sys.meta_path
450
path = sys.path
451
for item in path:
452
yield get_importer(item)
453
455
def get_loader(module_or_name):
456
"""Get a "loader" object for module_or_name
458
Returns None if the module cannot be found or imported.
459
If the named module is not already imported, its containing package
460
(if any) is imported, in order to establish the package __path__.
461
"""
462
if module_or_name in sys.modules:
463
module_or_name = sys.modules[module_or_name]
464
if module_or_name is None:
465
return None
466
if isinstance(module_or_name, ModuleType):
467
module = module_or_name
468
loader = getattr(module, '__loader__', None)
469
if loader is not None:
470
return loader
471
if getattr(module, '__spec__', None) is None:
472
return None
473
fullname = module.__name__
474
else:
475
fullname = module_or_name
476
return find_loader(fullname)
477
479
def find_loader(fullname):
480
"""Find a "loader" object for fullname
482
This is a backwards compatibility wrapper around
483
importlib.util.find_spec that converts most failures to ImportError
484
and only returns the loader rather than the full spec
486
if fullname.startswith('.'):
487
msg = "Relative module name {!r} not supported".format(fullname)
488
raise ImportError(msg)
489
try:
490
spec = importlib.util.find_spec(fullname)
491
except (ImportError, AttributeError, TypeError, ValueError) as ex:
492
# This hack fixes an impedance mismatch between pkgutil and
493
# importlib, where the latter raises other errors for cases where
494
# pkgutil previously raised ImportError
495
msg = "Error while finding loader for {!r} ({}: {})"
496
raise ImportError(msg.format(fullname, type(ex), ex)) from ex
497
return spec.loader if spec is not None else None
499
500
def extend_path(path, name):
501
"""Extend a package's path.
502
503
Intended use is to place the following code in a package's __init__.py:
504
505
from pkgutil import extend_path
506
__path__ = extend_path(__path__, __name__)
507
508
This will add to the package's __path__ all subdirectories of
509
directories on sys.path named after the package. This is useful
510
if one wants to distribute different parts of a single logical
511
package as multiple directories.
512
513
It also looks for *.pkg files beginning where * matches the name
514
argument. This feature is similar to *.pth files (see site.py),
515
except that it doesn't special-case lines starting with 'import'.
516
A *.pkg file is trusted at face value: apart from checking for
517
duplicates, all entries found in a *.pkg file are added to the
518
path, regardless of whether they are exist the filesystem. (This
519
is a feature.)
520
521
If the input path is not a list (as is the case for frozen
522
packages) it is returned unchanged. The input path is not
523
modified; an extended copy is returned. Items are only appended
524
to the copy at the end.
525
526
It is assumed that sys.path is a sequence. Items of sys.path that
527
are not (unicode or 8-bit) strings referring to existing
528
directories are ignored. Unicode items of sys.path that cause
529
errors when used as filenames may cause this function to raise an
530
exception (in line with os.path.isdir() behavior).
531
"""
532
533
if not isinstance(path, list):
534
# This could happen e.g. when this is called from inside a
535
# frozen package. Return the path unchanged in that case.
536
return path
537
538
sname_pkg = name + ".pkg"
539
540
path = path[:] # Start with a copy of the existing path
541
542
parent_package, _, final_name = name.rpartition('.')
543
if parent_package:
544
try:
545
search_path = sys.modules[parent_package].__path__
546
except (KeyError, AttributeError):
547
# We can't do anything: find_loader() returns None when
548
# passed a dotted name.
549
return path
550
else:
551
search_path = sys.path
552
553
for dir in search_path:
554
if not isinstance(dir, str):
556
557
finder = get_importer(dir)
558
if finder is not None:
559
portions = []
560
if hasattr(finder, 'find_spec'):
561
spec = finder.find_spec(final_name)
562
if spec is not None:
563
portions = spec.submodule_search_locations or []
564
# Is this finder PEP 420 compliant?
565
elif hasattr(finder, 'find_loader'):
566
_, portions = finder.find_loader(final_name)
567
568
for portion in portions:
569
# XXX This may still add duplicate entries to path on
570
# case-insensitive filesystems
571
if portion not in path:
572
path.append(portion)
573
574
# XXX Is this the right thing for subpackages like zope.app?
575
# It looks for a file named "zope.app.pkg"
576
pkgfile = os.path.join(dir, sname_pkg)
577
if os.path.isfile(pkgfile):
578
try:
579
f = open(pkgfile)
580
except OSError as msg:
581
sys.stderr.write("Can't open %s: %s\n" %
582
(pkgfile, msg))
583
else:
584
with f:
585
for line in f:
586
line = line.rstrip('\n')
587
if not line or line.startswith('#'):
588
continue
589
path.append(line) # Don't check for existence!
590
591
return path
594
def get_data(package, resource):
595
"""Get a resource from a package.
596
597
This is a wrapper round the PEP 302 loader get_data API. The package
598
argument should be the name of a package, in standard module format
599
(foo.bar). The resource argument should be in the form of a relative
600
filename, using '/' as the path separator. The parent directory name '..'
601
is not allowed, and nor is a rooted name (starting with a '/').
602
603
The function returns a binary string, which is the contents of the
604
specified resource.
605
606
For packages located in the filesystem, which have already been imported,
607
this is the rough equivalent of
608
609
d = os.path.dirname(sys.modules[package].__file__)
610
data = open(os.path.join(d, resource), 'rb').read()
611
612
If the package cannot be located or loaded, or it uses a PEP 302 loader
613
which does not support get_data(), then None is returned.
614
"""
615
616
spec = importlib.util.find_spec(package)
617
if spec is None:
618
return None
619
loader = spec.loader
620
if loader is None or not hasattr(loader, 'get_data'):
621
return None
622
# XXX needs test
623
mod = (sys.modules.get(package) or
624
importlib._bootstrap._load(spec))
625
if mod is None or not hasattr(mod, '__file__'):
626
return None
627
628
# Modify the resource name to be compatible with the loader.get_data
629
# signature - an os.path format "filename" starting with the dirname of
630
# the package's __file__
631
parts = resource.split('/')
632
parts.insert(0, os.path.dirname(mod.__file__))
633
resource_name = os.path.join(*parts)
634
return loader.get_data(resource_name)