/manage/easy_install.py

#############################################################################
#
# Copyright (c) 2005 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Python easy_install API

This module provides a high-level Python API for installing packages.
It doesn't install scripts.  It uses setuptools and requires it to be
installed.
"""

import distutils.errors
import fnmatch
import glob
import logging
import os
import pkg_resources
import py_compile
import re
import setuptools.archive_util
import setuptools.command.setopt
import setuptools.package_index
import shutil
import subprocess
import sys
import tempfile
import warnings
#import zc.buildout
import zipimport

_oprp = getattr(os.path, 'realpath', lambda path: path)
def realpath(path):
    return os.path.normcase(os.path.abspath(_oprp(path)))

default_index_url = os.environ.get(
    'buildout-testing-index-url',
    'http://pypi.python.org/simple',
    )

logger = logging.getLogger('easy_install')

url_match = re.compile('[a-z0-9+.-]+://').match

is_win32 = sys.platform == 'win32'
is_jython = sys.platform.startswith('java')
is_distribute = (
    pkg_resources.Requirement.parse('setuptools').key=='distribute')

BROKEN_DASH_S_WARNING = (
    'Buildout has been asked to exclude or limit site-packages so that '
    'builds can be repeatable when using a system Python.  However, '
    'the chosen Python executable has a broken implementation of -S (see '
    'https://bugs.launchpad.net/virtualenv/+bug/572545 for an example '
    "problem) and this breaks buildout's ability to isolate site-packages.  "
    "If the executable already has a clean site-packages (e.g., "
    "using virtualenv's ``--no-site-packages`` option) you may be getting "
    'equivalent repeatability.  To silence this warning, use the -s argument '
    'to the buildout script.  Alternatively, use a Python executable with a '
    'working -S (such as a standard Python binary).')

if is_jython:
    import java.lang.System
    jython_os_name = (java.lang.System.getProperties()['os.name']).lower()

setuptools_loc = pkg_resources.working_set.find(
    pkg_resources.Requirement.parse('setuptools')
    ).location

# Include buildout and setuptools eggs in paths.  We prevent dupes just to
# keep from duplicating any log messages about them.
#buildout_loc = pkg_resources.working_set.find(
#    pkg_resources.Requirement.parse('zc.buildout')).location
buildout_and_setuptools_path = [setuptools_loc]
if os.path.normpath(setuptools_loc) != os.path.normpath(buildout_loc):
    buildout_and_setuptools_path.append(buildout_loc)

def _has_broken_dash_S(executable):
    """Detect https://bugs.launchpad.net/virtualenv/+bug/572545 ."""
    # The first attempt here was to simply have the executable attempt to import
    # ConfigParser and return the return code. That worked except for tests on
    # Windows, where the return code was wrong for the fake Python executable
    # generated by the virtualenv.txt test, apparently because setuptools' .exe
    # file does not pass the -script.py's returncode back properly, at least in
    # some circumstances. Therefore...print statements.
    stdout, stderr = subprocess.Popen(
        [executable, '-Sc',
         'try:\n'
         '    import ConfigParser\n'
         'except ImportError:\n'
         '    print 1\n'
         'else:\n'
         '    print 0\n'],
        stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
    return bool(int(stdout.strip()))

def _get_system_paths(executable):
    """Return lists of standard lib and site paths for executable.
    """
    # We want to get a list of the site packages, which is not easy.
    # The canonical way to do this is to use
    # distutils.sysconfig.get_python_lib(), but that only returns a
    # single path, which does not reflect reality for many system
    # Pythons, which have multiple additions.  Instead, we start Python
    # with -S, which does not import site.py and set up the extra paths
    # like site-packages or (Ubuntu/Debian) dist-packages and
    # python-support. We then compare that sys.path with the normal one
    # (minus user packages if this is Python 2.6, because we don't
    # support those (yet?).  The set of the normal one minus the set of
    # the ones in ``python -S`` is the set of packages that are
    # effectively site-packages.
    #
    # The given executable might not be the current executable, so it is
    # appropriate to do another subprocess to figure out what the
    # additional site-package paths are. Moreover, even if this
    # executable *is* the current executable, this code might be run in
    # the context of code that has manipulated the sys.path--for
    # instance, to add local zc.buildout or setuptools eggs.
    def get_sys_path(*args, **kwargs):
        cmd = [executable]
        cmd.extend(args)
        cmd.extend([
            "-c", "import sys, os;"
            "print repr([os.path.normpath(p) for p in sys.path if p])"])
        # Windows needs some (as yet to be determined) part of the real env.
        env = os.environ.copy()
        # We need to make sure that PYTHONPATH, which will often be set
        # to include a custom buildout-generated site.py, is not set, or
        # else we will not get an accurate sys.path for the executable.
        env.pop('PYTHONPATH', None)
        env.update(kwargs)
        _proc = subprocess.Popen(
            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
        stdout, stderr = _proc.communicate();
        if _proc.returncode:
            raise RuntimeError(
                'error trying to get system packages:\n%s' % (stderr,))
        res = eval(stdout.strip())
        try:
            res.remove('.')
        except ValueError:
            pass
        return res
    stdlib = get_sys_path('-S') # stdlib only
    no_user_paths = get_sys_path(PYTHONNOUSERSITE='x')
    site_paths = [p for p in no_user_paths if p not in stdlib]
    return (stdlib, site_paths)

def _get_version_info(executable):
    cmd = [executable, '-Sc',
           'import sys; print(repr(tuple(x for x in sys.version_info)))']
    _proc = subprocess.Popen(
        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout, stderr = _proc.communicate();
    if _proc.returncode:
        raise RuntimeError(
            'error trying to get system packages:\n%s' % (stderr,))
    return eval(stdout.strip())


_versions = {sys.executable: '%d.%d' % sys.version_info[:2]}
def _get_version(executable):
    try:
        return _versions[executable]
    except KeyError:
        cmd = _safe_arg(executable) + ' -V'
        p = subprocess.Popen(cmd,
                             shell=True,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT,
                             close_fds=not is_win32)
        i, o = (p.stdin, p.stdout)
        i.close()
        version = o.read().strip()
        o.close()
        pystring, version = version.split()
        assert pystring == 'Python'
        version = re.match('(\d[.]\d)([.].*\d)?$', version).group(1)
        _versions[executable] = version
        return version

FILE_SCHEME = re.compile('file://', re.I).match


class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
    """Will allow urls that are local to the system.

    No matter what is allow_hosts.
    """
    def url_ok(self, url, fatal=False):
        if FILE_SCHEME(url):
            return True
        return setuptools.package_index.PackageIndex.url_ok(self, url, False)


_indexes = {}
def _get_index(executable, index_url, find_links, allow_hosts=('*',),
               path=None):
    # If path is None, the index will use sys.path.  If you provide an empty
    # path ([]), it will complain uselessly about missing index pages for
    # packages found in the paths that you expect to use.  Therefore, this path
    # is always the same as the _env path in the Installer.
    key = executable, index_url, tuple(find_links)
    index = _indexes.get(key)
    if index is not None:
        return index

    if index_url is None:
        index_url = default_index_url
    index = AllowHostsPackageIndex(
        index_url, hosts=allow_hosts, search_path=path,
        python=_get_version(executable)
        )

    if find_links:
        index.add_find_links(find_links)

    _indexes[key] = index
    return index

clear_index_cache = _indexes.clear

if is_win32:
    # work around spawn lamosity on windows
    # XXX need safe quoting (see the subprocess.list2cmdline) and test
    def _safe_arg(arg):
        return '"%s"' % arg
else:
    _safe_arg = str

# The following string is used to run easy_install in
# Installer._call_easy_install.  It is usually started with python -S
# (that is, don't import site at start).  That flag, and all of the code
# in this snippet above the last two lines, exist to work around a
# relatively rare problem.  If
#
# - your buildout configuration is trying to install a package that is within
#   a namespace package, and
#
# - you use a Python that has a different version of this package
#   installed in in its site-packages using
#   --single-version-externally-managed (that is, using the mechanism
#   sometimes used by system packagers:
#   http://peak.telecommunity.com/DevCenter/setuptools#install-command ), and
#
# - the new package tries to do sys.path tricks in the setup.py to get a
#   __version__,
#
# then the older package will be loaded first, making the setup version
# the wrong number. While very arguably packages simply shouldn't do
# the sys.path tricks, some do, and we don't want buildout to fall over
# when they do.
#
# The namespace packages installed in site-packages with
# --single-version-externally-managed use a mechanism that cause them to
# be processed when site.py is imported  (see
# http://mail.python.org/pipermail/distutils-sig/2009-May/011730.html
# for another description of the problem).  Simply starting Python with
# -S addresses the problem in Python 2.4 and 2.5, but Python 2.6's
# distutils imports a value from the site module, so we unfortunately
# have to do more drastic surgery in the _easy_install_preface code below.
#
# Here's an example of the .pth files created by setuptools when using that
# flag:
#
# import sys,new,os;
# p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('<NAMESPACE>',));
# ie = os.path.exists(os.path.join(p,'__init__.py'));
# m = not ie and sys.modules.setdefault('<NAMESPACE>',new.module('<NAMESPACE>'));
# mp = (m or []) and m.__dict__.setdefault('__path__',[]);
# (p not in mp) and mp.append(p)
#
# The code, below, then, runs under -S, indicating that site.py should
# not be loaded initially.  It gets the initial sys.path under these
# circumstances, and then imports site (because Python 2.6's distutils
# will want it, as mentioned above). It then reinstates the old sys.path
# value. Then it removes namespace packages (created by the setuptools
# code above) from sys.modules.  It identifies namespace packages by
# iterating over every loaded module.  It first looks if there is a
# __path__, so it is a package; and then it sees if that __path__ does
# not have an __init__.py.  (Note that PEP 382,
# http://www.python.org/dev/peps/pep-0382, makes it possible to have a
# namespace package that has an __init__.py, but also should make it
# unnecessary for site.py to preprocess these packages, so it should be
# fine, as far as can be guessed as of this writing.)  Finally, it
# imports easy_install and runs it.
_easy_install_preface = '''\
import sys,os;\
p = sys.path[:];\
import site;\
sys.path[:] = p;\
[sys.modules.pop(k) for k, v in sys.modules.items()\
 if hasattr(v, '__path__') and len(v.__path__)==1 and\
 not os.path.exists(os.path.join(v.__path__[0],'__init__.py'))];'''
_easy_install_cmd = (
    'from setuptools.command.easy_install import main;main()')


class Installer:

    _versions = {}
    _download_cache = None
    _install_from_cache = False
    _prefer_final = True
    _use_dependency_links = True
    _allow_picked_versions = True
    _always_unzip = False
    _include_site_packages = True
    _allowed_eggs_from_site_packages = ('*',)

    def __init__(self,
                 dest=None,
                 links=(),
                 index=None,
                 executable=sys.executable,
                 always_unzip=None,
                 path=None,
                 newest=True,
                 versions=None,
                 use_dependency_links=None,
                 allow_hosts=('*',),
                 include_site_packages=None,
                 allowed_eggs_from_site_packages=None,
                 prefer_final=None,
                 ):
        self._dest = dest
        self._allow_hosts = allow_hosts

        if self._install_from_cache:
            if not self._download_cache:
                raise ValueError("install_from_cache set to true with no"
                                 " download cache")
            links = ()
            index = 'file://' + self._download_cache

        if use_dependency_links is not None:
            self._use_dependency_links = use_dependency_links
        if prefer_final is not None:
            self._prefer_final = prefer_final
        self._links = links = list(_fix_file_links(links))
        if self._download_cache and (self._download_cache not in links):
            links.insert(0, self._download_cache)

        self._index_url = index
        self._executable = executable
        self._has_broken_dash_S = _has_broken_dash_S(self._executable)
        if always_unzip is not None:
            self._always_unzip = always_unzip
        path = (path and path[:] or [])
        if include_site_packages is not None:
            self._include_site_packages = include_site_packages
        if allowed_eggs_from_site_packages is not None:
            self._allowed_eggs_from_site_packages = tuple(
                allowed_eggs_from_site_packages)
        if self._has_broken_dash_S:
            if (not self._include_site_packages or
                self._allowed_eggs_from_site_packages != ('*',)):
                # We can't do this if the executable has a broken -S.
                warnings.warn(BROKEN_DASH_S_WARNING)
                self._include_site_packages = True
                self._allowed_eggs_from_site_packages = ('*',)
            self._easy_install_cmd = _easy_install_cmd
        else:
            self._easy_install_cmd = _easy_install_preface + _easy_install_cmd
        self._easy_install_cmd = _safe_arg(self._easy_install_cmd)
        stdlib, self._site_packages = _get_system_paths(executable)
        version_info = _get_version_info(executable)
        if version_info == sys.version_info:
            # Maybe we can add the buildout and setuptools path.  If we
            # are including site_packages, we only have to include the extra
            # bits here, so we don't duplicate.  On the other hand, if we
            # are not including site_packages, we only want to include the
            # parts that are not in site_packages, so the code is the same.
            path.extend(
                set(buildout_and_setuptools_path).difference(
                    self._site_packages))
        if self._include_site_packages:
            path.extend(self._site_packages)
        if dest is not None and dest not in path:
            path.insert(0, dest)
        self._path = path
        if self._dest is None:
            newest = False
        self._newest = newest
        self._env = pkg_resources.Environment(path,
                                              python=_get_version(executable))
        self._index = _get_index(executable, index, links, self._allow_hosts,
                                 self._path)

        if versions is not None:
            self._versions = versions

    _allowed_eggs_from_site_packages_regex = None
    def allow_site_package_egg(self, name):
        if (not self._include_site_packages or
            not self._allowed_eggs_from_site_packages):
            # If the answer is a blanket "no," perform a shortcut.
            return False
        if self._allowed_eggs_from_site_packages_regex is None:
            pattern = '(%s)' % (
                '|'.join(
                    fnmatch.translate(name)
                    for name in self._allowed_eggs_from_site_packages),
                )
            self._allowed_eggs_from_site_packages_regex = re.compile(pattern)
        return bool(self._allowed_eggs_from_site_packages_regex.match(name))

    def _satisfied(self, req, source=None):
        # We get all distributions that match the given requirement.  If we are
        # not supposed to include site-packages for the given egg, we also
        # filter those out. Even if include_site_packages is False and so we
        # have excluded site packages from the _env's paths (see
        # Installer.__init__), we need to do the filtering here because an
        # .egg-link, such as one for setuptools or zc.buildout installed by
        # zc.buildout.buildout.Buildout.bootstrap, can indirectly include a
        # path in our _site_packages.
        dists = [dist for dist in self._env[req.project_name] if (
                    dist in req and (
                        dist.location not in self._site_packages or
                        self.allow_site_package_egg(dist.project_name))
                    )
                ]
        if not dists:
            logger.debug('We have no distributions for %s that satisfies %r.',
                         req.project_name, str(req))

            return None, self._obtain(req, source)

        # Note that dists are sorted from best to worst, as promised by
        # env.__getitem__

        for dist in dists:
            if (dist.precedence == pkg_resources.DEVELOP_DIST and
                dist.location not in self._site_packages):
                # System eggs are sometimes installed as develop eggs.
                # Those are not the kind of develop eggs we are looking for
                # here: we want ones that the buildout itself has locally as
                # develop eggs.
                logger.debug('We have a develop egg: %s', dist)
                return dist, None

        # Special common case, we have a specification for a single version:
        specs = req.specs
        if len(specs) == 1 and specs[0][0] == '==':
            logger.debug('We have the distribution that satisfies %r.',
                         str(req))
            return dists[0], None

        if self._prefer_final:
            fdists = [dist for dist in dists
                      if _final_version(dist.parsed_version)
                      ]
            if fdists:
                # There are final dists, so only use those
                dists = fdists

        if not self._newest:
            # We don't need the newest, so we'll use the newest one we
            # find, which is the first returned by
            # Environment.__getitem__.
            return dists[0], None

        best_we_have = dists[0] # Because dists are sorted from best to worst

        # We have some installed distros.  There might, theoretically, be
        # newer ones.  Let's find out which ones are available and see if
        # any are newer.  We only do this if we're willing to install
        # something, which is only true if dest is not None:

        if self._dest is not None:
            best_available = self._obtain(req, source)
        else:
            best_available = None

        if best_available is None:
            # That's a bit odd.  There aren't any distros available.
            # We should use the best one we have that meets the requirement.
            logger.debug(
                'There are no distros available that meet %r.\n'
                'Using our best, %s.',
                str(req), best_available)
            return best_we_have, None

        if self._prefer_final:
            if _final_version(best_available.parsed_version):
                if _final_version(best_we_have.parsed_version):
                    if (best_we_have.parsed_version
                        <
                        best_available.parsed_version
                        ):
                        return None, best_available
                else:
                    return None, best_available
            else:
                if (not _final_version(best_we_have.parsed_version)
                    and
                    (best_we_have.parsed_version
                     <
                     best_available.parsed_version
                     )
                    ):
                    return None, best_available
        else:
            if (best_we_have.parsed_version
                <
                best_available.parsed_version
                ):
                return None, best_available

        logger.debug(
            'We have the best distribution that satisfies %r.',
            str(req))
        return best_we_have, None

    def _load_dist(self, dist):
        dists = pkg_resources.Environment(
            dist.location,
            python=_get_version(self._executable),
            )[dist.project_name]
        assert len(dists) == 1
        return dists[0]

    def _call_easy_install(self, spec, ws, dest, dist):

        tmp = tempfile.mkdtemp(dir=dest)
        try:
            path = setuptools_loc

            args = ('-c', self._easy_install_cmd, '-mUNxd', _safe_arg(tmp))
            if not self._has_broken_dash_S:
                args = ('-S',) + args
            if self._always_unzip:
                args += ('-Z', )
            level = logger.getEffectiveLevel()
            if level > 0:
                args += ('-q', )
            elif level < 0:
                args += ('-v', )

            args += (_safe_arg(spec), )

            if level <= logging.DEBUG:
                logger.debug('Running easy_install:\n%s "%s"\npath=%s\n',
                             self._executable, '" "'.join(args), path)

            if is_jython:
                extra_env = dict(os.environ, PYTHONPATH=path)
            else:
                args += (dict(os.environ, PYTHONPATH=path), )

            sys.stdout.flush() # We want any pending output first

            if is_jython:
                exit_code = subprocess.Popen(
                [_safe_arg(self._executable)] + list(args),
                env=extra_env).wait()
            else:
                exit_code = os.spawnle(
                    os.P_WAIT, self._executable, _safe_arg (self._executable),
                    *args)

            dists = []
            env = pkg_resources.Environment(
                [tmp],
                python=_get_version(self._executable),
                )
            for project in env:
                dists.extend(env[project])

            if exit_code:
                logger.error(
                    "An error occurred when trying to install %s. "
                    "Look above this message for any errors that "
                    "were output by easy_install.",
                    dist)

            if not dists:
                raise UserError("Couldn't install: %s" % dist)

            if len(dists) > 1:
                logger.warn("Installing %s\n"
                            "caused multiple distributions to be installed:\n"
                            "%s\n",
                            dist, '\n'.join(map(str, dists)))
            else:
                d = dists[0]
                if d.project_name != dist.project_name:
                    logger.warn("Installing %s\n"
                                "Caused installation of a distribution:\n"
                                "%s\n"
                                "with a different project name.",
                                dist, d)
                if d.version != dist.version:
                    logger.warn("Installing %s\n"
                                "Caused installation of a distribution:\n"
                                "%s\n"
                                "with a different version.",
                                dist, d)

            result = []
            for d in dists:
                newloc = os.path.join(dest, os.path.basename(d.location))
                if os.path.exists(newloc):
                    if os.path.isdir(newloc):
                        shutil.rmtree(newloc)
                    else:
                        os.remove(newloc)
                os.rename(d.location, newloc)

                [d] = pkg_resources.Environment(
                    [newloc],
                    python=_get_version(self._executable),
                    )[d.project_name]

                result.append(d)

            return result

        finally:
            shutil.rmtree(tmp)

    def _obtain(self, requirement, source=None):
        # initialize out index for this project:
        index = self._index

        if index.obtain(requirement) is None:
            # Nothing is available.
            return None

        # Filter the available dists for the requirement and source flag.  If
        # we are not supposed to include site-packages for the given egg, we
        # also filter those out. Even if include_site_packages is False and so
        # we have excluded site packages from the _env's paths (see
        # Installer.__init__), we need to do the filtering here because an
        # .egg-link, such as one for setuptools or zc.buildout installed by
        # zc.buildout.buildout.Buildout.bootstrap, can indirectly include a
        # path in our _site_packages.
        dists = [dist for dist in index[requirement.project_name] if (
                    dist in requirement and (
                        dist.location not in self._site_packages or
                        self.allow_site_package_egg(dist.project_name))
                    and (
                        (not source) or
                        (dist.precedence == pkg_resources.SOURCE_DIST))
                    )
                 ]

        # If we prefer final dists, filter for final and use the
        # result if it is non empty.
        if self._prefer_final:
            fdists = [dist for dist in dists
                      if _final_version(dist.parsed_version)
                      ]
            if fdists:
                # There are final dists, so only use those
                dists = fdists

        # Now find the best one:
        best = []
        bestv = ()
        for dist in dists:
            distv = dist.parsed_version
            if distv > bestv:
                best = [dist]
                bestv = distv
            elif distv == bestv:
                best.append(dist)

        if not best:
            return None

        if len(best) == 1:
            return best[0]

        if self._download_cache:
            for dist in best:
                if (realpath(os.path.dirname(dist.location))
                    ==
                    self._download_cache
                    ):
                    return dist

        best.sort()
        return best[-1]

    def _fetch(self, dist, tmp, download_cache):
        if (download_cache
            and (realpath(os.path.dirname(dist.location)) == download_cache)
            ):
            return dist

        new_location = self._index.download(dist.location, tmp)
        if (download_cache
            and (realpath(new_location) == realpath(dist.location))
            and os.path.isfile(new_location)
            ):
            # setuptools avoids making extra copies, but we want to copy
            # to the download cache
            shutil.copy2(new_location, tmp)
            new_location = os.path.join(tmp, os.path.basename(new_location))

        return dist.clone(location=new_location)

    def _get_dist(self, requirement, ws, always_unzip):

        __doing__ = 'Getting distribution for %r.', str(requirement)

        # Maybe an existing dist is already the best dist that satisfies the
        # requirement
        dist, avail = self._satisfied(requirement)

        if dist is None:
            if self._dest is not None:
                logger.info(*__doing__)

            # Retrieve the dist:
            if avail is None:
                raise MissingDistribution(requirement, ws)

            # We may overwrite distributions, so clear importer
            # cache.
            sys.path_importer_cache.clear()

            tmp = self._download_cache
            if tmp is None:
                tmp = tempfile.mkdtemp('get_dist')

            try:
                dist = self._fetch(avail, tmp, self._download_cache)

                if dist is None:
                    raise UserError(
                        "Couldn't download distribution %s." % avail)

                if dist.precedence == pkg_resources.EGG_DIST:
                    # It's already an egg, just fetch it into the dest

                    newloc = os.path.join(
                        self._dest, os.path.basename(dist.location))

                    if os.path.isdir(dist.location):
                        # we got a directory. It must have been
                        # obtained locally.  Just copy it.
                        shutil.copytree(dist.location, newloc)
                    else:

                        if self._always_unzip:
                            should_unzip = True
                        else:
                            metadata = pkg_resources.EggMetadata(
                                zipimport.zipimporter(dist.location)
                                )
                            should_unzip = (
                                metadata.has_metadata('not-zip-safe')
                                or
                                not metadata.has_metadata('zip-safe')
                                )

                        if should_unzip:
                            setuptools.archive_util.unpack_archive(
                                dist.location, newloc)
                        else:
                            shutil.copyfile(dist.location, newloc)

                    redo_pyc(newloc)

                    # Getting the dist from the environment causes the
                    # distribution meta data to be read.  Cloning isn't
                    # good enough.
                    dists = pkg_resources.Environment(
                        [newloc],
                        python=_get_version(self._executable),
                        )[dist.project_name]
                else:
                    # It's some other kind of dist.  We'll let easy_install
                    # deal with it:
                    dists = self._call_easy_install(
                        dist.location, ws, self._dest, dist)
                    for dist in dists:
                        redo_pyc(dist.location)

            finally:
                if tmp != self._download_cache:
                    shutil.rmtree(tmp)

            self._env.scan([self._dest])
            dist = self._env.best_match(requirement, ws)
            logger.info("Got %s.", dist)

        else:
            dists = [dist]

        for dist in dists:
            if (dist.has_metadata('dependency_links.txt')
                and not self._install_from_cache
                and self._use_dependency_links
                ):
                for link in dist.get_metadata_lines('dependency_links.txt'):
                    link = link.strip()
                    if link not in self._links:
                        logger.debug('Adding find link %r from %s', link, dist)
                        self._links.append(link)
                        self._index = _get_index(self._executable,
                                                 self._index_url, self._links,
                                                 self._allow_hosts, self._path)

        for dist in dists:
            # Check whether we picked a version and, if we did, report it:
            if not (
                dist.precedence == pkg_resources.DEVELOP_DIST
                or
                (len(requirement.specs) == 1
                 and
                 requirement.specs[0][0] == '==')
                ):
                logger.debug('Picked: %s = %s',
                             dist.project_name, dist.version)
                if not self._allow_picked_versions:
                    raise UserError(
                        'Picked: %s = %s' % (dist.project_name, dist.version)
                        )

        return dists

    def _maybe_add_setuptools(self, ws, dist):
        if dist.has_metadata('namespace_packages.txt'):
            for r in dist.requires():
                if r.project_name in ('setuptools', 'distribute'):
                    break
            else:
                # We have a namespace package but no requirement for setuptools
                if dist.precedence == pkg_resources.DEVELOP_DIST:
                    logger.warn(
                        "Develop distribution: %s\n"
                        "uses namespace packages but the distribution "
                        "does not require setuptools.",
                        dist)
                requirement = self._constrain(
                    pkg_resources.Requirement.parse('setuptools')
                    )
                if ws.find(requirement) is None:
                    for dist in self._get_dist(requirement, ws, False):
                        ws.add(dist)


    def _constrain(self, requirement):
        if is_distribute and requirement.key == 'setuptools':
            requirement = pkg_resources.Requirement.parse('distribute')
        version = self._versions.get(requirement.project_name)
        if version:
            if version not in requirement:
                logger.error("The version, %s, is not consistent with the "
                             "requirement, %r.", version, str(requirement))
                raise IncompatibleVersionError("Bad version", version)

            requirement = pkg_resources.Requirement.parse(
                "%s[%s] ==%s" % (requirement.project_name,
                               ','.join(requirement.extras),
                               version))

        return requirement

    def install(self, specs, working_set=None):

        logger.debug('Installing %s.', repr(specs)[1:-1])

        path = self._path
        destination = self._dest
        if destination is not None and destination not in path:
            path.insert(0, destination)

        requirements = [self._constrain(pkg_resources.Requirement.parse(spec))
                        for spec in specs]



        if working_set is None:
            ws = pkg_resources.WorkingSet([])
        else:
            ws = working_set

        for requirement in requirements:
            for dist in self._get_dist(requirement, ws, self._always_unzip):
                ws.add(dist)
                self._maybe_add_setuptools(ws, dist)

        # OK, we have the requested distributions and they're in the working
        # set, but they may have unmet requirements.  We'll resolve these
        # requirements. This is code modified from
        # pkg_resources.WorkingSet.resolve.  We can't reuse that code directly
        # because we have to constrain our requirements (see
        # versions_section_ignored_for_dependency_in_favor_of_site_packages in
        # zc.buildout.tests).
        requirements.reverse() # Set up the stack.
        processed = {}  # This is a set of processed requirements.
        best = {}  # This is a mapping of key -> dist.
        # Note that we don't use the existing environment, because we want
        # to look for new eggs unless what we have is the best that
        # matches the requirement.
        env = pkg_resources.Environment(ws.entries)
        while requirements:
            # Process dependencies breadth-first.
            req = self._constrain(requirements.pop(0))
            if req in processed:
                # Ignore cyclic or redundant dependencies.
                continue
            dist = best.get(req.key)
            if dist is None:
                # Find the best distribution and add it to the map.
                dist = ws.by_key.get(req.key)
                if dist is None:
                    try:
                        dist = best[req.key] = env.best_match(req, ws)
                    except pkg_resources.VersionConflict, err:
                        raise VersionConflict(err, ws)
                    if dist is None or (
                        dist.location in self._site_packages and not
                        self.allow_site_package_egg(dist.project_name)):
                        # If we didn't find a distribution in the
                        # environment, or what we found is from site
                        # packages and not allowed to be there, try
                        # again.
                        if destination:
                            logger.debug('Getting required %r', str(req))
                        else:
                            logger.debug('Adding required %r', str(req))
                        _log_requirement(ws, req)
                        for dist in self._get_dist(req,
                                                   ws, self._always_unzip):
                            ws.add(dist)
                            self._maybe_add_setuptools(ws, dist)
            if dist not in req:
                # Oops, the "best" so far conflicts with a dependency.
                raise VersionConflict(
                    pkg_resources.VersionConflict(dist, req), ws)
            requirements.extend(dist.requires(req.extras)[::-1])
            processed[req] = True
            if dist.location in self._site_packages:
                logger.debug('Egg from site-packages: %s', dist)
        return ws

    def build(self, spec, build_ext):

        requirement = self._constrain(pkg_resources.Requirement.parse(spec))

        dist, avail = self._satisfied(requirement, 1)
        if dist is not None:
            return [dist.location]

        # Retrieve the dist:
        if avail is None:
            raise UserError(
                "Couldn't find a source distribution for %r."
                % str(requirement))

        logger.debug('Building %r', spec)

        tmp = self._download_cache
        if tmp is None:
            tmp = tempfile.mkdtemp('get_dist')

        try:
            dist = self._fetch(avail, tmp, self._download_cache)

            build_tmp = tempfile.mkdtemp('build')
            try:
                setuptools.archive_util.unpack_archive(dist.location,
                                                       build_tmp)
                if os.path.exists(os.path.join(build_tmp, 'setup.py')):
                    base = build_tmp
                else:
                    setups = glob.glob(
                        os.path.join(build_tmp, '*', 'setup.py'))
                    if not setups:
                        raise distutils.errors.DistutilsError(
                            "Couldn't find a setup script in %s"
                            % os.path.basename(dist.location)
                            )
                    if len(setups) > 1:
                        raise distutils.errors.DistutilsError(
                            "Multiple setup scripts in %s"
                            % os.path.basename(dist.location)
                            )
                    base = os.path.dirname(setups[0])

                setup_cfg = os.path.join(base, 'setup.cfg')
                if not os.path.exists(setup_cfg):
                    f = open(setup_cfg, 'w')
                    f.close()
                setuptools.command.setopt.edit_config(
                    setup_cfg, dict(build_ext=build_ext))

                dists = self._call_easy_install(
                    base, pkg_resources.WorkingSet(),
                    self._dest, dist)

                for dist in dists:
                    redo_pyc(dist.location)

                return [dist.location for dist in dists]
            finally:
                shutil.rmtree(build_tmp)

        finally:
            if tmp != self._download_cache:
                shutil.rmtree(tmp)

def default_versions(versions=None):
    old = Installer._versions
    if versions is not None:
        Installer._versions = versions
    return old

def download_cache(path=-1):
    old = Installer._download_cache
    if path != -1:
        if path:
            path = realpath(path)
        Installer._download_cache = path
    return old

def install_from_cache(setting=None):
    old = Installer._install_from_cache
    if setting is not None:
        Installer._install_from_cache = bool(setting)
    return old

def prefer_final(setting=None):
    old = Installer._prefer_final
    if setting is not None:
        Installer._prefer_final = bool(setting)
    return old

def include_site_packages(setting=None):
    old = Installer._include_site_packages
    if setting is not None:
        Installer._include_site_packages = bool(setting)
    return old

def allowed_eggs_from_site_packages(setting=None):
    old = Installer._allowed_eggs_from_site_packages
    if setting is not None:
        Installer._allowed_eggs_from_site_packages = tuple(setting)
    return old

def use_dependency_links(setting=None):
    old = Installer._use_dependency_links
    if setting is not None:
        Installer._use_dependency_links = bool(setting)
    return old

def allow_picked_versions(setting=None):
    old = Installer._allow_picked_versions
    if setting is not None:
        Installer._allow_picked_versions = bool(setting)
    return old

def always_unzip(setting=None):
    old = Installer._always_unzip
    if setting is not None:
        Installer._always_unzip = bool(setting)
    return old

def install(specs, dest,
            links=(), index=None,
            executable=sys.executable, always_unzip=None,
            path=None, working_set=None, newest=True, versions=None,
            use_dependency_links=None, allow_hosts=('*',),
            include_site_packages=None, allowed_eggs_from_site_packages=None,
            prefer_final=None):
    installer = Installer(
        dest, links, index, executable, always_unzip, path, newest,
        versions, use_dependency_links, allow_hosts=allow_hosts,
        include_site_packages=include_site_packages,
        allowed_eggs_from_site_packages=allowed_eggs_from_site_packages,
        prefer_final=prefer_final)
    return installer.install(specs, working_set)


def build(spec, dest, build_ext,
          links=(), index=None,
          executable=sys.executable,
          path=None, newest=True, versions=None, allow_hosts=('*',),
          include_site_packages=None, allowed_eggs_from_site_packages=None):
    installer = Installer(
        dest, links, index, executable, True, path, newest, versions,
        allow_hosts=allow_hosts,
        include_site_packages=include_site_packages,
        allowed_eggs_from_site_packages=allowed_eggs_from_site_packages)
    return installer.build(spec, build_ext)



def _rm(*paths):
    for path in paths:
        if os.path.isdir(path):
            shutil.rmtree(path)
        elif os.path.exists(path):
            os.remove(path)

def _copyeggs(src, dest, suffix, undo):
    result = []
    undo.append(lambda : _rm(*result))
    for name in os.listdir(src):
        if name.endswith(suffix):
            new = os.path.join(dest, name)
            _rm(new)
            os.rename(os.path.join(src, name), new)
            result.append(new)

    assert len(result) == 1, str(result)
    undo.pop()

    return result[0]

def develop(setup, dest,
            build_ext=None,
            executable=sys.executable):

    if os.path.isdir(setup):
        directory = setup
        setup = os.path.join(directory, 'setup.py')
    else:
        directory = os.path.dirname(setup)

    undo = []
    try:
        if build_ext:
            setup_cfg = os.path.join(directory, 'setup.cfg')
            if os.path.exists(setup_cfg):
                os.rename(setup_cfg, setup_cfg+'-develop-aside')
                def restore_old_setup():
                    if os.path.exists(setup_cfg):
                        os.remove(setup_cfg)
                    os.rename(setup_cfg+'-develop-aside', setup_cfg)
                undo.append(restore_old_setup)
            else:
                open(setup_cfg, 'w')
                undo.append(lambda: os.remove(setup_cfg))
            setuptools.command.setopt.edit_config(
                setup_cfg, dict(build_ext=build_ext))

        fd, tsetup = tempfile.mkstemp()
        undo.append(lambda: os.remove(tsetup))
        undo.append(lambda: os.close(fd))

        os.write(fd, runsetup_template % dict(
            setuptools=setuptools_loc,
            setupdir=directory,
            setup=setup,
            __file__ = setup,
            ))

        tmp3 = tempfile.mkdtemp('build', dir=dest)
        undo.append(lambda : shutil.rmtree(tmp3))

        args = [
            _safe_arg(tsetup),
            '-q', 'develop', '-mxN',
            '-d', _safe_arg(tmp3),
            ]

        log_level = logger.getEffectiveLevel()
        if log_level <= 0:
            if log_level == 0:
                del args[1]
            else:
                args[1] == '-v'
        if log_level < logging.DEBUG:
            logger.debug("in: %r\n%s", directory, ' '.join(args))

        if is_jython:
            assert subprocess.Popen([_safe_arg(executable)] + args).wait() == 0
        else:
            assert os.spawnl(os.P_WAIT, executable, _safe_arg(executable),
                             *args) == 0

        return _copyeggs(tmp3, dest, '.egg-link', undo)

    finally:
        undo.reverse()
        [f() for f in undo]

def working_set(specs, executable, path, include_site_packages=None,
                allowed_eggs_from_site_packages=None, prefer_final=None):
    return install(
        specs, None, executable=executable, path=path,
        include_site_packages=include_site_packages,
        allowed_eggs_from_site_packages=allowed_eggs_from_site_packages,
        prefer_final=prefer_final)

############################################################################
# Script generation functions

def scripts(
    reqs, working_set, executable, dest,
    scripts=None,
    extra_paths=(),
    arguments='',
    interpreter=None,
    initialization='',
    relative_paths=False,
    ):
    """Generate scripts and/or an interpreter.

    See sitepackage_safe_scripts for a version that can be used with a Python
    that has code installed in site-packages. It has more options and a
    different approach.
    """
    path = _get_path(working_set, extra_paths)
    if initialization:
        initialization = '\n'+initialization+'\n'
    generated = _generate_scripts(
        reqs, working_set, dest, path, scripts, relative_paths,
        initialization, executable, arguments)
    if interpreter:
        sname = os.path.join(dest, interpreter)
        spath, rpsetup = _relative_path_and_setup(sname, path, relative_paths)
        generated.extend(
            _pyscript(spath, sname, executable, rpsetup))
    return generated

# We need to give an alternate name to the ``scripts`` function so that it
# can be referenced within sitepackage_safe_scripts, which uses ``scripts``
# as an argument name.
_original_scripts_function = scripts

def sitepackage_safe_scripts(
    dest, working_set, executable, site_py_dest,
    reqs=(),
    scripts=None,
    interpreter=None,
    extra_paths=(),
    initialization='',
    include_site_packages=False,
    exec_sitecustomize=False,
    relative_paths=False,
    script_arguments='',
    script_initialization='',
    ):
    """Generate scripts and/or an interpreter from a system Python.

    This accomplishes the same job as the ``scripts`` function, above,
    but it does so in an alternative way that allows safely including
    Python site packages, if desired, and  choosing to execute the Python's
    sitecustomize.
    """
    if _has_broken_dash_S(executable):
        if not include_site_packages:
            warnings.warn(BROKEN_DASH_S_WARNING)
        return _original_scripts_function(
            reqs, working_set, executable, dest, scripts, extra_paths,
            script_arguments, interpreter, initialization, relative_paths)
    generated = []
    generated.append(_generate_sitecustomize(
        site_py_dest, executable, initialization, exec_sitecustomize))
    generated.append(_generate_site(
        site_py_dest, working_set, executable, extra_paths,
        include_site_packages, relative_paths))
    script_initialization = _script_initialization_template % dict(
        site_py_dest=site_py_dest,
        script_initialization=script_initialization)
    if not script_initialization.endswith('\n'):
        script_initialization += '\n'
    generated.extend(_generate_scripts(
        reqs, working_set, dest, [site_py_dest], scripts, relative_paths,
        script_initialization, executable, script_arguments, block_site=True))
    if interpreter:
        generated.extend(_generate_interpreter(
            interpreter, dest, executable, site_py_dest, relative_paths))
    return generated

_script_initialization_template = '''
import os
path = sys.path[0]
if os.environ.get('PYTHONPATH'):
    path = os.pathsep.join([path, os.environ['PYTHONPATH