/lib/pkg_resources.py

https://bitbucket.org/cistrome/cistrome-harvard/ · Python · 2625 lines · 2241 code · 206 blank · 178 comment · 167 complexity · 42537d2e5dd1311cff8b1e28342369c5 MD5 · raw file

Large files are truncated click here to view the full file

  1. """Package resource API
  2. --------------------
  3. A resource is a logical file contained within a package, or a logical
  4. subdirectory thereof. The package resource API expects resource names
  5. to have their path parts separated with ``/``, *not* whatever the local
  6. path separator is. Do not use os.path operations to manipulate resource
  7. names being passed into the API.
  8. The package resource API is designed to work with normal filesystem packages,
  9. .egg files, and unpacked .egg files. It can also work in a limited way with
  10. .zip files and with custom PEP 302 loaders that support the ``get_data()``
  11. method.
  12. """
  13. import sys, os, zipimport, time, re, imp
  14. try:
  15. frozenset
  16. except NameError:
  17. from sets import ImmutableSet as frozenset
  18. # capture these to bypass sandboxing
  19. from os import utime, rename, unlink, mkdir
  20. from os import open as os_open
  21. from os.path import isdir, split
  22. def _bypass_ensure_directory(name, mode=0777):
  23. # Sandbox-bypassing version of ensure_directory()
  24. dirname, filename = split(name)
  25. if dirname and filename and not isdir(dirname):
  26. _bypass_ensure_directory(dirname)
  27. mkdir(dirname, mode)
  28. _state_vars = {}
  29. def _declare_state(vartype, **kw):
  30. g = globals()
  31. for name, val in kw.iteritems():
  32. g[name] = val
  33. _state_vars[name] = vartype
  34. def __getstate__():
  35. state = {}
  36. g = globals()
  37. for k, v in _state_vars.iteritems():
  38. state[k] = g['_sget_'+v](g[k])
  39. return state
  40. def __setstate__(state):
  41. g = globals()
  42. for k, v in state.iteritems():
  43. g['_sset_'+_state_vars[k]](k, g[k], v)
  44. return state
  45. def _sget_dict(val):
  46. return val.copy()
  47. def _sset_dict(key, ob, state):
  48. ob.clear()
  49. ob.update(state)
  50. def _sget_object(val):
  51. return val.__getstate__()
  52. def _sset_object(key, ob, state):
  53. ob.__setstate__(state)
  54. _sget_none = _sset_none = lambda *args: None
  55. def get_supported_platform():
  56. """Return this platform's maximum compatible version.
  57. distutils.util.get_platform() normally reports the minimum version
  58. of Mac OS X that would be required to *use* extensions produced by
  59. distutils. But what we want when checking compatibility is to know the
  60. version of Mac OS X that we are *running*. To allow usage of packages that
  61. explicitly require a newer version of Mac OS X, we must also know the
  62. current version of the OS.
  63. If this condition occurs for any other platform with a version in its
  64. platform strings, this function should be extended accordingly.
  65. """
  66. plat = get_build_platform(); m = macosVersionString.match(plat)
  67. if m is not None and sys.platform == "darwin":
  68. try:
  69. plat = 'macosx-%s-%s' % ('.'.join(_macosx_vers()[:2]), m.group(3))
  70. except ValueError:
  71. pass # not Mac OS X
  72. return plat
  73. __all__ = [
  74. # Basic resource access and distribution/entry point discovery
  75. 'require', 'run_script', 'get_provider', 'get_distribution',
  76. 'load_entry_point', 'get_entry_map', 'get_entry_info', 'iter_entry_points',
  77. 'resource_string', 'resource_stream', 'resource_filename',
  78. 'resource_listdir', 'resource_exists', 'resource_isdir',
  79. # Environmental control
  80. 'declare_namespace', 'working_set', 'add_activation_listener',
  81. 'find_distributions', 'set_extraction_path', 'cleanup_resources',
  82. 'get_default_cache',
  83. # Primary implementation classes
  84. 'Environment', 'WorkingSet', 'ResourceManager',
  85. 'Distribution', 'Requirement', 'EntryPoint',
  86. # Exceptions
  87. 'ResolutionError','VersionConflict','DistributionNotFound','UnknownExtra',
  88. 'ExtractionError',
  89. # Parsing functions and string utilities
  90. 'parse_requirements', 'parse_version', 'safe_name', 'safe_version',
  91. 'get_platform', 'compatible_platforms', 'yield_lines', 'split_sections',
  92. 'safe_extra', 'to_filename',
  93. # filesystem utilities
  94. 'ensure_directory', 'normalize_path',
  95. # Distribution "precedence" constants
  96. 'EGG_DIST', 'BINARY_DIST', 'SOURCE_DIST', 'CHECKOUT_DIST', 'DEVELOP_DIST',
  97. # "Provider" interfaces, implementations, and registration/lookup APIs
  98. 'IMetadataProvider', 'IResourceProvider', 'FileMetadata',
  99. 'PathMetadata', 'EggMetadata', 'EmptyProvider', 'empty_provider',
  100. 'NullProvider', 'EggProvider', 'DefaultProvider', 'ZipProvider',
  101. 'register_finder', 'register_namespace_handler', 'register_loader_type',
  102. 'fixup_namespace_packages', 'get_importer',
  103. # Deprecated/backward compatibility only
  104. 'run_main', 'AvailableDistributions',
  105. ]
  106. class ResolutionError(Exception):
  107. """Abstract base for dependency resolution errors"""
  108. def __repr__(self): return self.__class__.__name__+repr(self.args)
  109. class VersionConflict(ResolutionError):
  110. """An already-installed version conflicts with the requested version"""
  111. class DistributionNotFound(ResolutionError):
  112. """A requested distribution was not found"""
  113. class UnknownExtra(ResolutionError):
  114. """Distribution doesn't have an "extra feature" of the given name"""
  115. _provider_factories = {}
  116. PY_MAJOR = sys.version[:3]
  117. EGG_DIST = 3
  118. BINARY_DIST = 2
  119. SOURCE_DIST = 1
  120. CHECKOUT_DIST = 0
  121. DEVELOP_DIST = -1
  122. def register_loader_type(loader_type, provider_factory):
  123. """Register `provider_factory` to make providers for `loader_type`
  124. `loader_type` is the type or class of a PEP 302 ``module.__loader__``,
  125. and `provider_factory` is a function that, passed a *module* object,
  126. returns an ``IResourceProvider`` for that module.
  127. """
  128. _provider_factories[loader_type] = provider_factory
  129. def get_provider(moduleOrReq):
  130. """Return an IResourceProvider for the named module or requirement"""
  131. if isinstance(moduleOrReq,Requirement):
  132. return working_set.find(moduleOrReq) or require(str(moduleOrReq))[0]
  133. try:
  134. module = sys.modules[moduleOrReq]
  135. except KeyError:
  136. __import__(moduleOrReq)
  137. module = sys.modules[moduleOrReq]
  138. loader = getattr(module, '__loader__', None)
  139. return _find_adapter(_provider_factories, loader)(module)
  140. def _macosx_vers(_cache=[]):
  141. if not _cache:
  142. from platform import mac_ver
  143. _cache.append(mac_ver()[0].split('.'))
  144. return _cache[0]
  145. def _macosx_arch(machine):
  146. return {'PowerPC':'ppc', 'Power_Macintosh':'ppc'}.get(machine,machine)
  147. def get_build_platform():
  148. """Return this platform's string for platform-specific distributions
  149. XXX Currently this is the same as ``distutils.util.get_platform()``, but it
  150. needs some hacks for Linux and Mac OS X.
  151. """
  152. from distutils.util import get_platform
  153. plat = get_platform()
  154. if sys.platform == "darwin" and not plat.startswith('macosx-'):
  155. try:
  156. version = _macosx_vers()
  157. machine = os.uname()[4].replace(" ", "_")
  158. return "macosx-%d.%d-%s" % (int(version[0]), int(version[1]),
  159. _macosx_arch(machine))
  160. except ValueError:
  161. # if someone is running a non-Mac darwin system, this will fall
  162. # through to the default implementation
  163. pass
  164. return plat
  165. macosVersionString = re.compile(r"macosx-(\d+)\.(\d+)-(.*)")
  166. darwinVersionString = re.compile(r"darwin-(\d+)\.(\d+)\.(\d+)-(.*)")
  167. get_platform = get_build_platform # XXX backward compat
  168. def compatible_platforms(provided,required):
  169. """Can code for the `provided` platform run on the `required` platform?
  170. Returns true if either platform is ``None``, or the platforms are equal.
  171. XXX Needs compatibility checks for Linux and other unixy OSes.
  172. """
  173. if provided is None or required is None or provided==required:
  174. return True # easy case
  175. # Mac OS X special cases
  176. reqMac = macosVersionString.match(required)
  177. if reqMac:
  178. provMac = macosVersionString.match(provided)
  179. # is this a Mac package?
  180. if not provMac:
  181. # this is backwards compatibility for packages built before
  182. # setuptools 0.6. All packages built after this point will
  183. # use the new macosx designation.
  184. provDarwin = darwinVersionString.match(provided)
  185. if provDarwin:
  186. dversion = int(provDarwin.group(1))
  187. macosversion = "%s.%s" % (reqMac.group(1), reqMac.group(2))
  188. if dversion == 7 and macosversion >= "10.3" or \
  189. dversion == 8 and macosversion >= "10.4":
  190. #import warnings
  191. #warnings.warn("Mac eggs should be rebuilt to "
  192. # "use the macosx designation instead of darwin.",
  193. # category=DeprecationWarning)
  194. return True
  195. return False # egg isn't macosx or legacy darwin
  196. # are they the same major version and machine type?
  197. if provMac.group(1) != reqMac.group(1) or \
  198. provMac.group(3) != reqMac.group(3):
  199. return False
  200. # is the required OS major update >= the provided one?
  201. if int(provMac.group(2)) > int(reqMac.group(2)):
  202. return False
  203. return True
  204. # XXX Linux and other platforms' special cases should go here
  205. return False
  206. def run_script(dist_spec, script_name):
  207. """Locate distribution `dist_spec` and run its `script_name` script"""
  208. ns = sys._getframe(1).f_globals
  209. name = ns['__name__']
  210. ns.clear()
  211. ns['__name__'] = name
  212. require(dist_spec)[0].run_script(script_name, ns)
  213. run_main = run_script # backward compatibility
  214. def get_distribution(dist):
  215. """Return a current distribution object for a Requirement or string"""
  216. if isinstance(dist,basestring): dist = Requirement.parse(dist)
  217. if isinstance(dist,Requirement): dist = get_provider(dist)
  218. if not isinstance(dist,Distribution):
  219. raise TypeError("Expected string, Requirement, or Distribution", dist)
  220. return dist
  221. def load_entry_point(dist, group, name):
  222. """Return `name` entry point of `group` for `dist` or raise ImportError"""
  223. return get_distribution(dist).load_entry_point(group, name)
  224. def get_entry_map(dist, group=None):
  225. """Return the entry point map for `group`, or the full entry map"""
  226. return get_distribution(dist).get_entry_map(group)
  227. def get_entry_info(dist, group, name):
  228. """Return the EntryPoint object for `group`+`name`, or ``None``"""
  229. return get_distribution(dist).get_entry_info(group, name)
  230. class IMetadataProvider:
  231. def has_metadata(name):
  232. """Does the package's distribution contain the named metadata?"""
  233. def get_metadata(name):
  234. """The named metadata resource as a string"""
  235. def get_metadata_lines(name):
  236. """Yield named metadata resource as list of non-blank non-comment lines
  237. Leading and trailing whitespace is stripped from each line, and lines
  238. with ``#`` as the first non-blank character are omitted."""
  239. def metadata_isdir(name):
  240. """Is the named metadata a directory? (like ``os.path.isdir()``)"""
  241. def metadata_listdir(name):
  242. """List of metadata names in the directory (like ``os.listdir()``)"""
  243. def run_script(script_name, namespace):
  244. """Execute the named script in the supplied namespace dictionary"""
  245. class IResourceProvider(IMetadataProvider):
  246. """An object that provides access to package resources"""
  247. def get_resource_filename(manager, resource_name):
  248. """Return a true filesystem path for `resource_name`
  249. `manager` must be an ``IResourceManager``"""
  250. def get_resource_stream(manager, resource_name):
  251. """Return a readable file-like object for `resource_name`
  252. `manager` must be an ``IResourceManager``"""
  253. def get_resource_string(manager, resource_name):
  254. """Return a string containing the contents of `resource_name`
  255. `manager` must be an ``IResourceManager``"""
  256. def has_resource(resource_name):
  257. """Does the package contain the named resource?"""
  258. def resource_isdir(resource_name):
  259. """Is the named resource a directory? (like ``os.path.isdir()``)"""
  260. def resource_listdir(resource_name):
  261. """List of resource names in the directory (like ``os.listdir()``)"""
  262. class WorkingSet(object):
  263. """A collection of active distributions on sys.path (or a similar list)"""
  264. def __init__(self, entries=None):
  265. """Create working set from list of path entries (default=sys.path)"""
  266. self.entries = []
  267. self.entry_keys = {}
  268. self.by_key = {}
  269. self.callbacks = []
  270. if entries is None:
  271. entries = sys.path
  272. for entry in entries:
  273. self.add_entry(entry)
  274. def add_entry(self, entry):
  275. """Add a path item to ``.entries``, finding any distributions on it
  276. ``find_distributions(entry, True)`` is used to find distributions
  277. corresponding to the path entry, and they are added. `entry` is
  278. always appended to ``.entries``, even if it is already present.
  279. (This is because ``sys.path`` can contain the same value more than
  280. once, and the ``.entries`` of the ``sys.path`` WorkingSet should always
  281. equal ``sys.path``.)
  282. """
  283. self.entry_keys.setdefault(entry, [])
  284. self.entries.append(entry)
  285. for dist in find_distributions(entry, True):
  286. self.add(dist, entry, False)
  287. def __contains__(self,dist):
  288. """True if `dist` is the active distribution for its project"""
  289. return self.by_key.get(dist.key) == dist
  290. def find(self, req):
  291. """Find a distribution matching requirement `req`
  292. If there is an active distribution for the requested project, this
  293. returns it as long as it meets the version requirement specified by
  294. `req`. But, if there is an active distribution for the project and it
  295. does *not* meet the `req` requirement, ``VersionConflict`` is raised.
  296. If there is no active distribution for the requested project, ``None``
  297. is returned.
  298. """
  299. dist = self.by_key.get(req.key)
  300. if dist is not None and dist not in req:
  301. raise VersionConflict(dist,req) # XXX add more info
  302. else:
  303. return dist
  304. def iter_entry_points(self, group, name=None):
  305. """Yield entry point objects from `group` matching `name`
  306. If `name` is None, yields all entry points in `group` from all
  307. distributions in the working set, otherwise only ones matching
  308. both `group` and `name` are yielded (in distribution order).
  309. """
  310. for dist in self:
  311. entries = dist.get_entry_map(group)
  312. if name is None:
  313. for ep in entries.values():
  314. yield ep
  315. elif name in entries:
  316. yield entries[name]
  317. def run_script(self, requires, script_name):
  318. """Locate distribution for `requires` and run `script_name` script"""
  319. ns = sys._getframe(1).f_globals
  320. name = ns['__name__']
  321. ns.clear()
  322. ns['__name__'] = name
  323. self.require(requires)[0].run_script(script_name, ns)
  324. def __iter__(self):
  325. """Yield distributions for non-duplicate projects in the working set
  326. The yield order is the order in which the items' path entries were
  327. added to the working set.
  328. """
  329. seen = {}
  330. for item in self.entries:
  331. for key in self.entry_keys[item]:
  332. if key not in seen:
  333. seen[key]=1
  334. yield self.by_key[key]
  335. def add(self, dist, entry=None, insert=True):
  336. """Add `dist` to working set, associated with `entry`
  337. If `entry` is unspecified, it defaults to the ``.location`` of `dist`.
  338. On exit from this routine, `entry` is added to the end of the working
  339. set's ``.entries`` (if it wasn't already present).
  340. `dist` is only added to the working set if it's for a project that
  341. doesn't already have a distribution in the set. If it's added, any
  342. callbacks registered with the ``subscribe()`` method will be called.
  343. """
  344. if insert:
  345. dist.insert_on(self.entries, entry)
  346. if entry is None:
  347. entry = dist.location
  348. keys = self.entry_keys.setdefault(entry,[])
  349. keys2 = self.entry_keys.setdefault(dist.location,[])
  350. if dist.key in self.by_key:
  351. return # ignore hidden distros
  352. self.by_key[dist.key] = dist
  353. if dist.key not in keys:
  354. keys.append(dist.key)
  355. if dist.key not in keys2:
  356. keys2.append(dist.key)
  357. self._added_new(dist)
  358. def resolve(self, requirements, env=None, installer=None):
  359. """List all distributions needed to (recursively) meet `requirements`
  360. `requirements` must be a sequence of ``Requirement`` objects. `env`,
  361. if supplied, should be an ``Environment`` instance. If
  362. not supplied, it defaults to all distributions available within any
  363. entry or distribution in the working set. `installer`, if supplied,
  364. will be invoked with each requirement that cannot be met by an
  365. already-installed distribution; it should return a ``Distribution`` or
  366. ``None``.
  367. """
  368. requirements = list(requirements)[::-1] # set up the stack
  369. processed = {} # set of processed requirements
  370. best = {} # key -> dist
  371. to_activate = []
  372. while requirements:
  373. req = requirements.pop(0) # process dependencies breadth-first
  374. if req in processed:
  375. # Ignore cyclic or redundant dependencies
  376. continue
  377. dist = best.get(req.key)
  378. if dist is None:
  379. # Find the best distribution and add it to the map
  380. dist = self.by_key.get(req.key)
  381. if dist is None:
  382. if env is None:
  383. env = Environment(self.entries)
  384. dist = best[req.key] = env.best_match(req, self, installer)
  385. if dist is None:
  386. raise DistributionNotFound(req) # XXX put more info here
  387. to_activate.append(dist)
  388. if dist not in req:
  389. # Oops, the "best" so far conflicts with a dependency
  390. raise VersionConflict(dist,req) # XXX put more info here
  391. requirements.extend(dist.requires(req.extras)[::-1])
  392. processed[req] = True
  393. return to_activate # return list of distros to activate
  394. def find_plugins(self,
  395. plugin_env, full_env=None, installer=None, fallback=True
  396. ):
  397. """Find all activatable distributions in `plugin_env`
  398. Example usage::
  399. distributions, errors = working_set.find_plugins(
  400. Environment(plugin_dirlist)
  401. )
  402. map(working_set.add, distributions) # add plugins+libs to sys.path
  403. print "Couldn't load", errors # display errors
  404. The `plugin_env` should be an ``Environment`` instance that contains
  405. only distributions that are in the project's "plugin directory" or
  406. directories. The `full_env`, if supplied, should be an ``Environment``
  407. contains all currently-available distributions. If `full_env` is not
  408. supplied, one is created automatically from the ``WorkingSet`` this
  409. method is called on, which will typically mean that every directory on
  410. ``sys.path`` will be scanned for distributions.
  411. `installer` is a standard installer callback as used by the
  412. ``resolve()`` method. The `fallback` flag indicates whether we should
  413. attempt to resolve older versions of a plugin if the newest version
  414. cannot be resolved.
  415. This method returns a 2-tuple: (`distributions`, `error_info`), where
  416. `distributions` is a list of the distributions found in `plugin_env`
  417. that were loadable, along with any other distributions that are needed
  418. to resolve their dependencies. `error_info` is a dictionary mapping
  419. unloadable plugin distributions to an exception instance describing the
  420. error that occurred. Usually this will be a ``DistributionNotFound`` or
  421. ``VersionConflict`` instance.
  422. """
  423. plugin_projects = list(plugin_env)
  424. plugin_projects.sort() # scan project names in alphabetic order
  425. error_info = {}
  426. distributions = {}
  427. if full_env is None:
  428. env = Environment(self.entries)
  429. env += plugin_env
  430. else:
  431. env = full_env + plugin_env
  432. shadow_set = self.__class__([])
  433. map(shadow_set.add, self) # put all our entries in shadow_set
  434. for project_name in plugin_projects:
  435. for dist in plugin_env[project_name]:
  436. req = [dist.as_requirement()]
  437. try:
  438. resolvees = shadow_set.resolve(req, env, installer)
  439. except ResolutionError,v:
  440. error_info[dist] = v # save error info
  441. if fallback:
  442. continue # try the next older version of project
  443. else:
  444. break # give up on this project, keep going
  445. else:
  446. map(shadow_set.add, resolvees)
  447. distributions.update(dict.fromkeys(resolvees))
  448. # success, no need to try any more versions of this project
  449. break
  450. distributions = list(distributions)
  451. distributions.sort()
  452. return distributions, error_info
  453. def require(self, *requirements):
  454. """Ensure that distributions matching `requirements` are activated
  455. `requirements` must be a string or a (possibly-nested) sequence
  456. thereof, specifying the distributions and versions required. The
  457. return value is a sequence of the distributions that needed to be
  458. activated to fulfill the requirements; all relevant distributions are
  459. included, even if they were already activated in this working set.
  460. """
  461. needed = self.resolve(parse_requirements(requirements))
  462. for dist in needed:
  463. self.add(dist)
  464. return needed
  465. def subscribe(self, callback):
  466. """Invoke `callback` for all distributions (including existing ones)"""
  467. if callback in self.callbacks:
  468. return
  469. self.callbacks.append(callback)
  470. for dist in self:
  471. callback(dist)
  472. def _added_new(self, dist):
  473. for callback in self.callbacks:
  474. callback(dist)
  475. def __getstate__(self):
  476. return (
  477. self.entries[:], self.entry_keys.copy(), self.by_key.copy(),
  478. self.callbacks[:]
  479. )
  480. def __setstate__(self, (entries, keys, by_key, callbacks)):
  481. self.entries = entries[:]
  482. self.entry_keys = keys.copy()
  483. self.by_key = by_key.copy()
  484. self.callbacks = callbacks[:]
  485. class Environment(object):
  486. """Searchable snapshot of distributions on a search path"""
  487. def __init__(self, search_path=None, platform=get_supported_platform(), python=PY_MAJOR):
  488. """Snapshot distributions available on a search path
  489. Any distributions found on `search_path` are added to the environment.
  490. `search_path` should be a sequence of ``sys.path`` items. If not
  491. supplied, ``sys.path`` is used.
  492. `platform` is an optional string specifying the name of the platform
  493. that platform-specific distributions must be compatible with. If
  494. unspecified, it defaults to the current platform. `python` is an
  495. optional string naming the desired version of Python (e.g. ``'2.4'``);
  496. it defaults to the current version.
  497. You may explicitly set `platform` (and/or `python`) to ``None`` if you
  498. wish to map *all* distributions, not just those compatible with the
  499. running platform or Python version.
  500. """
  501. self._distmap = {}
  502. self._cache = {}
  503. self.platform = platform
  504. self.python = python
  505. self.scan(search_path)
  506. def can_add(self, dist):
  507. """Is distribution `dist` acceptable for this environment?
  508. The distribution must match the platform and python version
  509. requirements specified when this environment was created, or False
  510. is returned.
  511. """
  512. return (self.python is None or dist.py_version is None
  513. or dist.py_version==self.python) \
  514. and compatible_platforms(dist.platform,self.platform)
  515. def remove(self, dist):
  516. """Remove `dist` from the environment"""
  517. self._distmap[dist.key].remove(dist)
  518. def scan(self, search_path=None):
  519. """Scan `search_path` for distributions usable in this environment
  520. Any distributions found are added to the environment.
  521. `search_path` should be a sequence of ``sys.path`` items. If not
  522. supplied, ``sys.path`` is used. Only distributions conforming to
  523. the platform/python version defined at initialization are added.
  524. """
  525. if search_path is None:
  526. search_path = sys.path
  527. for item in search_path:
  528. for dist in find_distributions(item):
  529. self.add(dist)
  530. def __getitem__(self,project_name):
  531. """Return a newest-to-oldest list of distributions for `project_name`
  532. """
  533. try:
  534. return self._cache[project_name]
  535. except KeyError:
  536. project_name = project_name.lower()
  537. if project_name not in self._distmap:
  538. return []
  539. if project_name not in self._cache:
  540. dists = self._cache[project_name] = self._distmap[project_name]
  541. _sort_dists(dists)
  542. return self._cache[project_name]
  543. def add(self,dist):
  544. """Add `dist` if we ``can_add()`` it and it isn't already added"""
  545. if self.can_add(dist) and dist.has_version():
  546. dists = self._distmap.setdefault(dist.key,[])
  547. if dist not in dists:
  548. dists.append(dist)
  549. if dist.key in self._cache:
  550. _sort_dists(self._cache[dist.key])
  551. def best_match(self, req, working_set, installer=None):
  552. """Find distribution best matching `req` and usable on `working_set`
  553. This calls the ``find(req)`` method of the `working_set` to see if a
  554. suitable distribution is already active. (This may raise
  555. ``VersionConflict`` if an unsuitable version of the project is already
  556. active in the specified `working_set`.) If a suitable distribution
  557. isn't active, this method returns the newest distribution in the
  558. environment that meets the ``Requirement`` in `req`. If no suitable
  559. distribution is found, and `installer` is supplied, then the result of
  560. calling the environment's ``obtain(req, installer)`` method will be
  561. returned.
  562. """
  563. dist = working_set.find(req)
  564. if dist is not None:
  565. return dist
  566. for dist in self[req.key]:
  567. if dist in req:
  568. return dist
  569. return self.obtain(req, installer) # try and download/install
  570. def obtain(self, requirement, installer=None):
  571. """Obtain a distribution matching `requirement` (e.g. via download)
  572. Obtain a distro that matches requirement (e.g. via download). In the
  573. base ``Environment`` class, this routine just returns
  574. ``installer(requirement)``, unless `installer` is None, in which case
  575. None is returned instead. This method is a hook that allows subclasses
  576. to attempt other ways of obtaining a distribution before falling back
  577. to the `installer` argument."""
  578. if installer is not None:
  579. return installer(requirement)
  580. def __iter__(self):
  581. """Yield the unique project names of the available distributions"""
  582. for key in self._distmap.keys():
  583. if self[key]: yield key
  584. def __iadd__(self, other):
  585. """In-place addition of a distribution or environment"""
  586. if isinstance(other,Distribution):
  587. self.add(other)
  588. elif isinstance(other,Environment):
  589. for project in other:
  590. for dist in other[project]:
  591. self.add(dist)
  592. else:
  593. raise TypeError("Can't add %r to environment" % (other,))
  594. return self
  595. def __add__(self, other):
  596. """Add an environment or distribution to an environment"""
  597. new = self.__class__([], platform=None, python=None)
  598. for env in self, other:
  599. new += env
  600. return new
  601. AvailableDistributions = Environment # XXX backward compatibility
  602. class ExtractionError(RuntimeError):
  603. """An error occurred extracting a resource
  604. The following attributes are available from instances of this exception:
  605. manager
  606. The resource manager that raised this exception
  607. cache_path
  608. The base directory for resource extraction
  609. original_error
  610. The exception instance that caused extraction to fail
  611. """
  612. class ResourceManager:
  613. """Manage resource extraction and packages"""
  614. extraction_path = None
  615. def __init__(self):
  616. self.cached_files = {}
  617. def resource_exists(self, package_or_requirement, resource_name):
  618. """Does the named resource exist?"""
  619. return get_provider(package_or_requirement).has_resource(resource_name)
  620. def resource_isdir(self, package_or_requirement, resource_name):
  621. """Is the named resource an existing directory?"""
  622. return get_provider(package_or_requirement).resource_isdir(
  623. resource_name
  624. )
  625. def resource_filename(self, package_or_requirement, resource_name):
  626. """Return a true filesystem path for specified resource"""
  627. return get_provider(package_or_requirement).get_resource_filename(
  628. self, resource_name
  629. )
  630. def resource_stream(self, package_or_requirement, resource_name):
  631. """Return a readable file-like object for specified resource"""
  632. return get_provider(package_or_requirement).get_resource_stream(
  633. self, resource_name
  634. )
  635. def resource_string(self, package_or_requirement, resource_name):
  636. """Return specified resource as a string"""
  637. return get_provider(package_or_requirement).get_resource_string(
  638. self, resource_name
  639. )
  640. def resource_listdir(self, package_or_requirement, resource_name):
  641. """List the contents of the named resource directory"""
  642. return get_provider(package_or_requirement).resource_listdir(
  643. resource_name
  644. )
  645. def extraction_error(self):
  646. """Give an error message for problems extracting file(s)"""
  647. old_exc = sys.exc_info()[1]
  648. cache_path = self.extraction_path or get_default_cache()
  649. err = ExtractionError("""Can't extract file(s) to egg cache
  650. The following error occurred while trying to extract file(s) to the Python egg
  651. cache:
  652. %s
  653. The Python egg cache directory is currently set to:
  654. %s
  655. Perhaps your account does not have write access to this directory? You can
  656. change the cache directory by setting the PYTHON_EGG_CACHE environment
  657. variable to point to an accessible directory.
  658. """ % (old_exc, cache_path)
  659. )
  660. err.manager = self
  661. err.cache_path = cache_path
  662. err.original_error = old_exc
  663. raise err
  664. def get_cache_path(self, archive_name, names=()):
  665. """Return absolute location in cache for `archive_name` and `names`
  666. The parent directory of the resulting path will be created if it does
  667. not already exist. `archive_name` should be the base filename of the
  668. enclosing egg (which may not be the name of the enclosing zipfile!),
  669. including its ".egg" extension. `names`, if provided, should be a
  670. sequence of path name parts "under" the egg's extraction location.
  671. This method should only be called by resource providers that need to
  672. obtain an extraction location, and only for names they intend to
  673. extract, as it tracks the generated names for possible cleanup later.
  674. """
  675. extract_path = self.extraction_path or get_default_cache()
  676. target_path = os.path.join(extract_path, archive_name+'-tmp', *names)
  677. try:
  678. _bypass_ensure_directory(target_path)
  679. except:
  680. self.extraction_error()
  681. self.cached_files[target_path] = 1
  682. return target_path
  683. def postprocess(self, tempname, filename):
  684. """Perform any platform-specific postprocessing of `tempname`
  685. This is where Mac header rewrites should be done; other platforms don't
  686. have anything special they should do.
  687. Resource providers should call this method ONLY after successfully
  688. extracting a compressed resource. They must NOT call it on resources
  689. that are already in the filesystem.
  690. `tempname` is the current (temporary) name of the file, and `filename`
  691. is the name it will be renamed to by the caller after this routine
  692. returns.
  693. """
  694. if os.name == 'posix':
  695. # Make the resource executable
  696. mode = ((os.stat(tempname).st_mode) | 0555) & 07777
  697. os.chmod(tempname, mode)
  698. def set_extraction_path(self, path):
  699. """Set the base path where resources will be extracted to, if needed.
  700. If you do not call this routine before any extractions take place, the
  701. path defaults to the return value of ``get_default_cache()``. (Which
  702. is based on the ``PYTHON_EGG_CACHE`` environment variable, with various
  703. platform-specific fallbacks. See that routine's documentation for more
  704. details.)
  705. Resources are extracted to subdirectories of this path based upon
  706. information given by the ``IResourceProvider``. You may set this to a
  707. temporary directory, but then you must call ``cleanup_resources()`` to
  708. delete the extracted files when done. There is no guarantee that
  709. ``cleanup_resources()`` will be able to remove all extracted files.
  710. (Note: you may not change the extraction path for a given resource
  711. manager once resources have been extracted, unless you first call
  712. ``cleanup_resources()``.)
  713. """
  714. if self.cached_files:
  715. raise ValueError(
  716. "Can't change extraction path, files already extracted"
  717. )
  718. self.extraction_path = path
  719. def cleanup_resources(self, force=False):
  720. """
  721. Delete all extracted resource files and directories, returning a list
  722. of the file and directory names that could not be successfully removed.
  723. This function does not have any concurrency protection, so it should
  724. generally only be called when the extraction path is a temporary
  725. directory exclusive to a single process. This method is not
  726. automatically called; you must call it explicitly or register it as an
  727. ``atexit`` function if you wish to ensure cleanup of a temporary
  728. directory used for extractions.
  729. """
  730. # XXX
  731. def get_default_cache():
  732. """Determine the default cache location
  733. This returns the ``PYTHON_EGG_CACHE`` environment variable, if set.
  734. Otherwise, on Windows, it returns a "Python-Eggs" subdirectory of the
  735. "Application Data" directory. On all other systems, it's "~/.python-eggs".
  736. """
  737. try:
  738. return os.environ['PYTHON_EGG_CACHE']
  739. except KeyError:
  740. pass
  741. if os.name!='nt':
  742. return os.path.expanduser('~/.python-eggs')
  743. app_data = 'Application Data' # XXX this may be locale-specific!
  744. app_homes = [
  745. (('APPDATA',), None), # best option, should be locale-safe
  746. (('USERPROFILE',), app_data),
  747. (('HOMEDRIVE','HOMEPATH'), app_data),
  748. (('HOMEPATH',), app_data),
  749. (('HOME',), None),
  750. (('WINDIR',), app_data), # 95/98/ME
  751. ]
  752. for keys, subdir in app_homes:
  753. dirname = ''
  754. for key in keys:
  755. if key in os.environ:
  756. dirname = os.path.join(dirname, os.environ[key])
  757. else:
  758. break
  759. else:
  760. if subdir:
  761. dirname = os.path.join(dirname,subdir)
  762. return os.path.join(dirname, 'Python-Eggs')
  763. else:
  764. raise RuntimeError(
  765. "Please set the PYTHON_EGG_CACHE enviroment variable"
  766. )
  767. def safe_name(name):
  768. """Convert an arbitrary string to a standard distribution name
  769. Any runs of non-alphanumeric/. characters are replaced with a single '-'.
  770. """
  771. return re.sub('[^A-Za-z0-9.]+', '-', name)
  772. def safe_version(version):
  773. """Convert an arbitrary string to a standard version string
  774. Spaces become dots, and all other non-alphanumeric characters become
  775. dashes, with runs of multiple dashes condensed to a single dash.
  776. """
  777. version = version.replace(' ','.')
  778. return re.sub('[^A-Za-z0-9.]+', '-', version)
  779. def safe_extra(extra):
  780. """Convert an arbitrary string to a standard 'extra' name
  781. Any runs of non-alphanumeric characters are replaced with a single '_',
  782. and the result is always lowercased.
  783. """
  784. return re.sub('[^A-Za-z0-9.]+', '_', extra).lower()
  785. def to_filename(name):
  786. """Convert a project or version name to its filename-escaped form
  787. Any '-' characters are currently replaced with '_'.
  788. """
  789. return name.replace('-','_')
  790. class NullProvider:
  791. """Try to implement resources and metadata for arbitrary PEP 302 loaders"""
  792. egg_name = None
  793. egg_info = None
  794. loader = None
  795. def __init__(self, module):
  796. self.loader = getattr(module, '__loader__', None)
  797. self.module_path = os.path.dirname(getattr(module, '__file__', ''))
  798. def get_resource_filename(self, manager, resource_name):
  799. return self._fn(self.module_path, resource_name)
  800. def get_resource_stream(self, manager, resource_name):
  801. return StringIO(self.get_resource_string(manager, resource_name))
  802. def get_resource_string(self, manager, resource_name):
  803. return self._get(self._fn(self.module_path, resource_name))
  804. def has_resource(self, resource_name):
  805. return self._has(self._fn(self.module_path, resource_name))
  806. def has_metadata(self, name):
  807. return self.egg_info and self._has(self._fn(self.egg_info,name))
  808. def get_metadata(self, name):
  809. if not self.egg_info:
  810. return ""
  811. return self._get(self._fn(self.egg_info,name))
  812. def get_metadata_lines(self, name):
  813. return yield_lines(self.get_metadata(name))
  814. def resource_isdir(self,resource_name):
  815. return self._isdir(self._fn(self.module_path, resource_name))
  816. def metadata_isdir(self,name):
  817. return self.egg_info and self._isdir(self._fn(self.egg_info,name))
  818. def resource_listdir(self,resource_name):
  819. return self._listdir(self._fn(self.module_path,resource_name))
  820. def metadata_listdir(self,name):
  821. if self.egg_info:
  822. return self._listdir(self._fn(self.egg_info,name))
  823. return []
  824. def run_script(self,script_name,namespace):
  825. script = 'scripts/'+script_name
  826. if not self.has_metadata(script):
  827. raise ResolutionError("No script named %r" % script_name)
  828. script_text = self.get_metadata(script).replace('\r\n','\n')
  829. script_text = script_text.replace('\r','\n')
  830. script_filename = self._fn(self.egg_info,script)
  831. namespace['__file__'] = script_filename
  832. if os.path.exists(script_filename):
  833. execfile(script_filename, namespace, namespace)
  834. else:
  835. from linecache import cache
  836. cache[script_filename] = (
  837. len(script_text), 0, script_text.split('\n'), script_filename
  838. )
  839. script_code = compile(script_text,script_filename,'exec')
  840. exec script_code in namespace, namespace
  841. def _has(self, path):
  842. raise NotImplementedError(
  843. "Can't perform this operation for unregistered loader type"
  844. )
  845. def _isdir(self, path):
  846. raise NotImplementedError(
  847. "Can't perform this operation for unregistered loader type"
  848. )
  849. def _listdir(self, path):
  850. raise NotImplementedError(
  851. "Can't perform this operation for unregistered loader type"
  852. )
  853. def _fn(self, base, resource_name):
  854. if resource_name:
  855. return os.path.join(base, *resource_name.split('/'))
  856. return base
  857. def _get(self, path):
  858. if hasattr(self.loader, 'get_data'):
  859. return self.loader.get_data(path)
  860. raise NotImplementedError(
  861. "Can't perform this operation for loaders without 'get_data()'"
  862. )
  863. register_loader_type(object, NullProvider)
  864. class EggProvider(NullProvider):
  865. """Provider based on a virtual filesystem"""
  866. def __init__(self,module):
  867. NullProvider.__init__(self,module)
  868. self._setup_prefix()
  869. def _setup_prefix(self):
  870. # we assume here that our metadata may be nested inside a "basket"
  871. # of multiple eggs; that's why we use module_path instead of .archive
  872. path = self.module_path
  873. old = None
  874. while path!=old:
  875. if path.lower().endswith('.egg'):
  876. self.egg_name = os.path.basename(path)
  877. self.egg_info = os.path.join(path, 'EGG-INFO')
  878. self.egg_root = path
  879. break
  880. old = path
  881. path, base = os.path.split(path)
  882. class DefaultProvider(EggProvider):
  883. """Provides access to package resources in the filesystem"""
  884. def _has(self, path):
  885. return os.path.exists(path)
  886. def _isdir(self,path):
  887. return os.path.isdir(path)
  888. def _listdir(self,path):
  889. return os.listdir(path)
  890. def get_resource_stream(self, manager, resource_name):
  891. return open(self._fn(self.module_path, resource_name), 'rb')
  892. def _get(self, path):
  893. stream = open(path, 'rb')
  894. try:
  895. return stream.read()
  896. finally:
  897. stream.close()
  898. register_loader_type(type(None), DefaultProvider)
  899. class EmptyProvider(NullProvider):
  900. """Provider that returns nothing for all requests"""
  901. _isdir = _has = lambda self,path: False
  902. _get = lambda self,path: ''
  903. _listdir = lambda self,path: []
  904. module_path = None
  905. def __init__(self):
  906. pass
  907. empty_provider = EmptyProvider()
  908. class ZipProvider(EggProvider):
  909. """Resource support for zips and eggs"""
  910. eagers = None
  911. def __init__(self, module):
  912. EggProvider.__init__(self,module)
  913. self.zipinfo = zipimport._zip_directory_cache[self.loader.archive]
  914. self.zip_pre = self.loader.archive+os.sep
  915. def _zipinfo_name(self, fspath):
  916. # Convert a virtual filename (full path to file) into a zipfile subpath
  917. # usable with the zipimport directory cache for our target archive
  918. if fspath.startswith(self.zip_pre):
  919. return fspath[len(self.zip_pre):]
  920. raise AssertionError(
  921. "%s is not a subpath of %s" % (fspath,self.zip_pre)
  922. )
  923. def _parts(self,zip_path):
  924. # Convert a zipfile subpath into an egg-relative path part list
  925. fspath = self.zip_pre+zip_path # pseudo-fs path
  926. if fspath.startswith(self.egg_root+os.sep):
  927. return fspath[len(self.egg_root)+1:].split(os.sep)
  928. raise AssertionError(
  929. "%s is not a subpath of %s" % (fspath,self.egg_root)
  930. )
  931. def get_resource_filename(self, manager, resource_name):
  932. if not self.egg_name:
  933. raise NotImplementedError(
  934. "resource_filename() only supported for .egg, not .zip"
  935. )
  936. # no need to lock for extraction, since we use temp names
  937. zip_path = self._resource_to_zip(resource_name)
  938. eagers = self._get_eager_resources()
  939. if '/'.join(self._parts(zip_path)) in eagers:
  940. for name in eagers:
  941. self._extract_resource(manager, self._eager_to_zip(name))
  942. return self._extract_resource(manager, zip_path)
  943. def _extract_resource(self, manager, zip_path):
  944. if zip_path in self._index():
  945. for name in self._index()[zip_path]:
  946. last = self._extract_resource(
  947. manager, os.path.join(zip_path, name)
  948. )
  949. return os.path.dirname(last) # return the extracted directory name
  950. zip_stat = self.zipinfo[zip_path]
  951. t,d,size = zip_stat[5], zip_stat[6], zip_stat[3]
  952. date_time = (
  953. (d>>9)+1980, (d>>5)&0xF, d&0x1F, # ymd
  954. (t&0xFFFF)>>11, (t>>5)&0x3F, (t&0x1F) * 2, 0, 0, -1 # hms, etc.
  955. )
  956. timestamp = time.mktime(date_time)
  957. try:
  958. real_path = manager.get_cache_path(
  959. self.egg_name, self._parts(zip_path)
  960. )
  961. if os.path.isfile(real_path):
  962. stat = os.stat(real_path)
  963. if stat.st_size==size and stat.st_mtime==timestamp:
  964. # size and stamp match, don't bother extracting
  965. return real_path
  966. outf, tmpnam = _mkstemp(".$extract", dir=os.path.dirname(real_path))
  967. os.write(outf, self.loader.get_data(zip_path))
  968. os.close(outf)
  969. utime(tmpnam, (timestamp,timestamp))
  970. manager.postprocess(tmpnam, real_path)
  971. try:
  972. rename(tmpnam, real_path)
  973. except os.error:
  974. if os.path.isfile(real_path):
  975. stat = os.stat(real_path)
  976. if stat.st_size==size and stat.st_mtime==timestamp:
  977. # size and stamp match, somebody did it just ahead of
  978. # us, so we're done
  979. return real_path
  980. elif os.name=='nt': # Windows, del old file and retry
  981. unlink(real_path)
  982. rename(tmpnam, real_path)
  983. return real_path
  984. raise
  985. except os.error:
  986. manager.extraction_error() # report a user-friendly error
  987. return real_path
  988. def _get_eager_resources(self):
  989. if self.eagers is None:
  990. eagers = []
  991. for name in ('native_libs.txt', 'eager_resources.txt'):
  992. if self.has_metadata(name):
  993. eagers.extend(self.get_metadata_lines(name))
  994. self.eagers = eagers
  995. return self.eagers
  996. def _index(self):
  997. try:
  998. return self._dirindex
  999. except AttributeError:
  1000. ind = {}
  1001. for path in self.zipinfo:
  1002. parts = path.split(os.sep)
  1003. while parts:
  1004. parent = os.sep.join(parts[:-1])
  1005. if parent in ind:
  1006. ind[parent].append(parts[-1])
  1007. break
  1008. else:
  1009. ind[parent] = [parts.pop()]
  1010. self._dirindex = ind
  1011. return ind
  1012. def _has(self, fspath):
  1013. zip_path = self._zipinfo_name(fspath)
  1014. return zip_path in self.zipinfo or zip_path in self._index()
  1015. def _isdir(self,fspath):
  1016. return self._zipinfo_name(fspath) in self._index()
  1017. def _listdir(self,fspath):
  1018. return list(self._index().get(self._zipinfo_name(fspath), ()))
  1019. def _eager_to_zip(self,resource_name):
  1020. return self._zipinfo_name(self._fn(self.egg_root,resource_name))
  1021. def _resource_to_zip(self,resource_name):
  1022. return self._zipinfo_name(self._fn(self.module_path,resource_name))
  1023. register_loader_type(zipimport.zipimporter, ZipProvider)
  1024. class FileMetadata(EmptyProvider):
  1025. """Metadata handler for standalone PKG-INFO files
  1026. Usage::
  1027. metadata = FileMetadata("/path/to/PKG-INFO")
  1028. This provider rejects all data and metadata requests except for PKG-INFO,
  1029. which is treated as existing, and will be the contents of the file at
  1030. the provided location.
  1031. """
  1032. def __init__(self,path):
  1033. self.path = path
  1034. def has_metadata(self,name):
  1035. return name=='PKG-INFO'
  1036. def get_metadata(self,name):
  1037. if name=='PKG-INFO':
  1038. return open(self.path,'rU').read()
  1039. raise KeyError("No metadata except PKG-INFO is available")
  1040. def get_metadata_lines(self,name):
  1041. return yield_lines(self.get_metadata(name))
  1042. class PathMetadata(DefaultProvider):
  1043. """Metadata provider for egg directories
  1044. Usage::
  1045. # Development eggs:
  1046. egg_info = "/path/to/PackageName.egg-info"
  1047. base_dir = os.path.dirname(egg_info)
  1048. metadata = PathMetadata(base_dir, egg_info)
  1049. dist_name = os.path.splitext(os.path.basename(egg_info))[0]
  1050. dist = Distribution(basedir,project_name=dist_name,metadata=metadata)
  1051. # Unpacked egg directories:
  1052. egg_path = "/path/to/PackageName-ver-pyver-etc.egg"
  1053. metadata = PathMetadata(egg_path, os.path.join(egg_path,'EGG-INFO'))
  1054. dist = Distribution.from_filename(egg_path, metadata=metadata)
  1055. """
  1056. def __init__(self, path, egg_info):
  1057. self.module_path = path
  1058. self.egg_info = egg_info
  1059. class EggMetadata(ZipProvider):
  1060. """Metadata provider for .egg files"""
  1061. def __init__(self, importer):
  1062. """Create a metadata provider from a zipimporter"""
  1063. self.zipinfo = zipimport._zip_directory_cache[importer.archive]
  1064. self.zip_pre = importer.archive+os.sep
  1065. self.loader = importer
  1066. if importer.prefix:
  1067. self.modul