/pip.py
Python | 4841 lines | 4605 code | 129 blank | 107 comment | 414 complexity | 2b5001e3688a34df698c0efd6a2a249f MD5 | raw file
- #!/usr/bin/env python
- import sys
- import os
- import errno
- import stat
- import optparse
- import pkg_resources
- import urllib2
- import urllib
- import mimetypes
- import zipfile
- import tarfile
- import tempfile
- import subprocess
- import posixpath
- import re
- import shutil
- import fnmatch
- import operator
- import copy
- try:
- from hashlib import md5
- except ImportError:
- import md5 as md5_module
- md5 = md5_module.new
- import urlparse
- from email.FeedParser import FeedParser
- import traceback
- from cStringIO import StringIO
- import socket
- from Queue import Queue
- from Queue import Empty as QueueEmpty
- import threading
- import httplib
- import time
- import logging
- import ConfigParser
- from distutils.util import strtobool
- from distutils import sysconfig
- class InstallationError(Exception):
- """General exception during installation"""
- class UninstallationError(Exception):
- """General exception during uninstallation"""
- class DistributionNotFound(InstallationError):
- """Raised when a distribution cannot be found to satisfy a requirement"""
- class BadCommand(Exception):
- """Raised when virtualenv or a command is not found"""
- try:
- any
- except NameError:
- def any(seq):
- for item in seq:
- if item:
- return True
- return False
- if getattr(sys, 'real_prefix', None):
- ## FIXME: is build/ a good name?
- build_prefix = os.path.join(sys.prefix, 'build')
- src_prefix = os.path.join(sys.prefix, 'src')
- else:
- ## FIXME: this isn't a very good default
- build_prefix = os.path.join(os.getcwd(), 'build')
- src_prefix = os.path.join(os.getcwd(), 'src')
- # FIXME doesn't account for venv linked to global site-packages
- site_packages = sysconfig.get_python_lib()
- user_dir = os.path.expanduser('~')
- if sys.platform == 'win32':
- bin_py = os.path.join(sys.prefix, 'Scripts')
- # buildout uses 'bin' on Windows too?
- if not os.path.exists(bin_py):
- bin_py = os.path.join(sys.prefix, 'bin')
- config_dir = os.environ.get('APPDATA', user_dir) # Use %APPDATA% for roaming
- default_config_file = os.path.join(config_dir, 'pip', 'pip.ini')
- else:
- bin_py = os.path.join(sys.prefix, 'bin')
- default_config_file = os.path.join(user_dir, '.pip', 'pip.conf')
- # Forcing to use /usr/local/bin for standard Mac OS X framework installs
- if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/':
- bin_py = '/usr/local/bin'
- class UpdatingDefaultsHelpFormatter(optparse.IndentedHelpFormatter):
- """Custom help formatter for use in ConfigOptionParser that updates
- the defaults before expanding them, allowing them to show up correctly
- in the help listing"""
- def expand_default(self, option):
- if self.parser is not None:
- self.parser.update_defaults(self.parser.defaults)
- return optparse.IndentedHelpFormatter.expand_default(self, option)
- class ConfigOptionParser(optparse.OptionParser):
- """Custom option parser which updates its defaults by by checking the
- configuration files and environmental variables"""
- def __init__(self, *args, **kwargs):
- self.config = ConfigParser.RawConfigParser()
- self.name = kwargs.pop('name')
- self.files = self.get_config_files()
- self.config.read(self.files)
- assert self.name
- optparse.OptionParser.__init__(self, *args, **kwargs)
- def get_config_files(self):
- config_file = os.environ.get('PIP_CONFIG_FILE', False)
- if config_file and os.path.exists(config_file):
- return [config_file]
- return [default_config_file]
- def update_defaults(self, defaults):
- """Updates the given defaults with values from the config files and
- the environ. Does a little special handling for certain types of
- options (lists)."""
- # Then go and look for the other sources of configuration:
- config = {}
- # 1. config files
- for section in ('global', self.name):
- config.update(dict(self.get_config_section(section)))
- # 2. environmental variables
- config.update(dict(self.get_environ_vars()))
- # Then set the options with those values
- for key, val in config.iteritems():
- key = key.replace('_', '-')
- if not key.startswith('--'):
- key = '--%s' % key # only prefer long opts
- option = self.get_option(key)
- if option is not None:
- # ignore empty values
- if not val:
- continue
- # handle multiline configs
- if option.action == 'append':
- val = val.split()
- else:
- option.nargs = 1
- if option.action in ('store_true', 'store_false', 'count'):
- val = strtobool(val)
- try:
- val = option.convert_value(key, val)
- except optparse.OptionValueError, e:
- print ("An error occured during configuration: %s" % e)
- sys.exit(3)
- defaults[option.dest] = val
- return defaults
- def get_config_section(self, name):
- """Get a section of a configuration"""
- if self.config.has_section(name):
- return self.config.items(name)
- return []
- def get_environ_vars(self, prefix='PIP_'):
- """Returns a generator with all environmental vars with prefix PIP_"""
- for key, val in os.environ.iteritems():
- if key.startswith(prefix):
- yield (key.replace(prefix, '').lower(), val)
- def get_default_values(self):
- """Overridding to make updating the defaults after instantiation of
- the option parser possible, update_defaults() does the dirty work."""
- if not self.process_default_values:
- # Old, pre-Optik 1.5 behaviour.
- return optparse.Values(self.defaults)
- defaults = self.update_defaults(self.defaults.copy()) # ours
- for option in self._get_all_options():
- default = defaults.get(option.dest)
- if isinstance(default, basestring):
- opt_str = option.get_opt_string()
- defaults[option.dest] = option.check_value(opt_str, default)
- return optparse.Values(defaults)
- try:
- pip_dist = pkg_resources.get_distribution('pip')
- version = '%s from %s (python %s)' % (
- pip_dist, pip_dist.location, sys.version[:3])
- except pkg_resources.DistributionNotFound:
- # when running pip.py without installing
- version=None
- def rmtree_errorhandler(func, path, exc_info):
- """On Windows, the files in .svn are read-only, so when rmtree() tries to
- remove them, an exception is thrown. We catch that here, remove the
- read-only attribute, and hopefully continue without problems."""
- exctype, value = exc_info[:2]
- # lookin for a windows error
- if exctype is not WindowsError or 'Access is denied' not in str(value):
- raise
- # file type should currently be read only
- if ((os.stat(path).st_mode & stat.S_IREAD) != stat.S_IREAD):
- raise
- # convert to read/write
- os.chmod(path, stat.S_IWRITE)
- # use the original function to repeat the operation
- func(path)
- class VcsSupport(object):
- _registry = {}
- schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp']
- def __init__(self):
- # Register more schemes with urlparse for various version control systems
- urlparse.uses_netloc.extend(self.schemes)
- urlparse.uses_fragment.extend(self.schemes)
- super(VcsSupport, self).__init__()
- def __iter__(self):
- return self._registry.__iter__()
- @property
- def backends(self):
- return self._registry.values()
- @property
- def dirnames(self):
- return [backend.dirname for backend in self.backends]
- @property
- def all_schemes(self):
- schemes = []
- for backend in self.backends:
- schemes.extend(backend.schemes)
- return schemes
- def register(self, cls):
- if not hasattr(cls, 'name'):
- logger.warn('Cannot register VCS %s' % cls.__name__)
- return
- if cls.name not in self._registry:
- self._registry[cls.name] = cls
- def unregister(self, cls=None, name=None):
- if name in self._registry:
- del self._registry[name]
- elif cls in self._registry.values():
- del self._registry[cls.name]
- else:
- logger.warn('Cannot unregister because no class or name given')
- def get_backend_name(self, location):
- """
- Return the name of the version control backend if found at given
- location, e.g. vcs.get_backend_name('/path/to/vcs/checkout')
- """
- for vc_type in self._registry.values():
- path = os.path.join(location, vc_type.dirname)
- if os.path.exists(path):
- return vc_type.name
- return None
- def get_backend(self, name):
- name = name.lower()
- if name in self._registry:
- return self._registry[name]
- def get_backend_from_location(self, location):
- vc_type = self.get_backend_name(location)
- if vc_type:
- return self.get_backend(vc_type)
- return None
- vcs = VcsSupport()
- parser = ConfigOptionParser(
- usage='%prog COMMAND [OPTIONS]',
- version=version,
- add_help_option=False,
- formatter=UpdatingDefaultsHelpFormatter(),
- name='global')
- parser.add_option(
- '-h', '--help',
- dest='help',
- action='store_true',
- help='Show help')
- parser.add_option(
- '-E', '--environment',
- dest='venv',
- metavar='DIR',
- help='virtualenv environment to run pip in (either give the '
- 'interpreter or the environment base directory)')
- parser.add_option(
- '-s', '--enable-site-packages',
- dest='site_packages',
- action='store_true',
- help='Include site-packages in virtualenv if one is to be '
- 'created. Ignored if --environment is not used or '
- 'the virtualenv already exists.')
- parser.add_option(
- # Defines a default root directory for virtualenvs, relative
- # virtualenvs names/paths are considered relative to it.
- '--virtualenv-base',
- dest='venv_base',
- type='str',
- default='',
- help=optparse.SUPPRESS_HELP)
- parser.add_option(
- # Run only if inside a virtualenv, bail if not.
- '--require-virtualenv', '--require-venv',
- dest='require_venv',
- action='store_true',
- default=False,
- help=optparse.SUPPRESS_HELP)
- parser.add_option(
- # Use automatically an activated virtualenv instead of installing
- # globally. -E will be ignored if used.
- '--respect-virtualenv', '--respect-venv',
- dest='respect_venv',
- action='store_true',
- default=False,
- help=optparse.SUPPRESS_HELP)
- parser.add_option(
- '-v', '--verbose',
- dest='verbose',
- action='count',
- default=0,
- help='Give more output')
- parser.add_option(
- '-q', '--quiet',
- dest='quiet',
- action='count',
- default=0,
- help='Give less output')
- parser.add_option(
- '--log',
- dest='log',
- metavar='FILENAME',
- help='Log file where a complete (maximum verbosity) record will be kept')
- parser.add_option(
- # Writes the log levels explicitely to the log'
- '--log-explicit-levels',
- dest='log_explicit_levels',
- action='store_true',
- default=False,
- help=optparse.SUPPRESS_HELP)
- parser.add_option(
- # The default log file
- '--local-log', '--log-file',
- dest='log_file',
- metavar='FILENAME',
- default='./pip-log.txt',
- help=optparse.SUPPRESS_HELP)
- parser.add_option(
- '--proxy',
- dest='proxy',
- type='str',
- default='',
- help="Specify a proxy in the form user:passwd@proxy.server:port. "
- "Note that the user:password@ is optional and required only if you "
- "are behind an authenticated proxy. If you provide "
- "user@proxy.server:port then you will be prompted for a password.")
- parser.add_option(
- '--timeout', '--default-timeout',
- metavar='SECONDS',
- dest='timeout',
- type='float',
- default=15,
- help='Set the socket timeout (default %default seconds)')
- parser.add_option(
- # The default version control system for editables, e.g. 'svn'
- '--default-vcs',
- dest='default_vcs',
- type='str',
- default='',
- help=optparse.SUPPRESS_HELP)
- parser.add_option(
- # A regex to be used to skip requirements
- '--skip-requirements-regex',
- dest='skip_requirements_regex',
- type='str',
- default='',
- help=optparse.SUPPRESS_HELP)
- parser.disable_interspersed_args()
- _commands = {}
- class Command(object):
- name = None
- usage = None
- hidden = False
- def __init__(self):
- assert self.name
- self.parser = ConfigOptionParser(
- usage=self.usage,
- prog='%s %s' % (sys.argv[0], self.name),
- version=parser.version,
- formatter=UpdatingDefaultsHelpFormatter(),
- name=self.name)
- for option in parser.option_list:
- if not option.dest or option.dest == 'help':
- # -h, --version, etc
- continue
- self.parser.add_option(option)
- _commands[self.name] = self
- def merge_options(self, initial_options, options):
- # Make sure we have all global options carried over
- for attr in ['log', 'venv', 'proxy', 'venv_base', 'require_venv',
- 'respect_venv', 'log_explicit_levels', 'log_file',
- 'timeout', 'default_vcs', 'skip_requirements_regex']:
- setattr(options, attr, getattr(initial_options, attr) or getattr(options, attr))
- options.quiet += initial_options.quiet
- options.verbose += initial_options.verbose
- def main(self, complete_args, args, initial_options):
- global logger
- options, args = self.parser.parse_args(args)
- self.merge_options(initial_options, options)
- if options.require_venv and not options.venv:
- # If a venv is required check if it can really be found
- if not os.environ.get('VIRTUAL_ENV'):
- print 'Could not find an activated virtualenv (required).'
- sys.exit(3)
- # Automatically install in currently activated venv if required
- options.respect_venv = True
- if args and args[-1] == '___VENV_RESTART___':
- ## FIXME: We don't do anything this this value yet:
- venv_location = args[-2]
- args = args[:-2]
- options.venv = None
- else:
- # If given the option to respect the activated environment
- # check if no venv is given as a command line parameter
- if options.respect_venv and os.environ.get('VIRTUAL_ENV'):
- if options.venv and os.path.exists(options.venv):
- # Make sure command line venv and environmental are the same
- if (os.path.realpath(os.path.expanduser(options.venv)) !=
- os.path.realpath(os.environ.get('VIRTUAL_ENV'))):
- print ("Given virtualenv (%s) doesn't match "
- "currently activated virtualenv (%s)."
- % (options.venv, os.environ.get('VIRTUAL_ENV')))
- sys.exit(3)
- else:
- options.venv = os.environ.get('VIRTUAL_ENV')
- print 'Using already activated environment %s' % options.venv
- level = 1 # Notify
- level += options.verbose
- level -= options.quiet
- level = Logger.level_for_integer(4-level)
- complete_log = []
- logger = Logger([(level, sys.stdout),
- (Logger.DEBUG, complete_log.append)])
- if options.log_explicit_levels:
- logger.explicit_levels = True
- if options.venv:
- if options.verbose > 0:
- # The logger isn't setup yet
- print 'Running in environment %s' % options.venv
- site_packages=False
- if options.site_packages:
- site_packages=True
- restart_in_venv(options.venv, options.venv_base, site_packages,
- complete_args)
- # restart_in_venv should actually never return, but for clarity...
- return
- ## FIXME: not sure if this sure come before or after venv restart
- if options.log:
- log_fp = open_logfile_append(options.log)
- logger.consumers.append((logger.DEBUG, log_fp))
- else:
- log_fp = None
- socket.setdefaulttimeout(options.timeout or None)
- setup_proxy_handler(options.proxy)
- exit = 0
- try:
- self.run(options, args)
- except (InstallationError, UninstallationError), e:
- logger.fatal(str(e))
- logger.info('Exception information:\n%s' % format_exc())
- exit = 1
- except:
- logger.fatal('Exception:\n%s' % format_exc())
- exit = 2
- if log_fp is not None:
- log_fp.close()
- if exit:
- log_fn = options.log_file
- text = '\n'.join(complete_log)
- logger.fatal('Storing complete log in %s' % log_fn)
- log_fp = open_logfile_append(log_fn)
- log_fp.write(text)
- log_fp.close()
- return exit
- class HelpCommand(Command):
- name = 'help'
- usage = '%prog'
- summary = 'Show available commands'
- def run(self, options, args):
- if args:
- ## FIXME: handle errors better here
- command = args[0]
- if command not in _commands:
- raise InstallationError('No command with the name: %s' % command)
- command = _commands[command]
- command.parser.print_help()
- return
- parser.print_help()
- print
- print 'Commands available:'
- commands = list(set(_commands.values()))
- commands.sort(key=lambda x: x.name)
- for command in commands:
- if command.hidden:
- continue
- print ' %s: %s' % (command.name, command.summary)
- HelpCommand()
- class InstallCommand(Command):
- name = 'install'
- usage = '%prog [OPTIONS] PACKAGE_NAMES...'
- summary = 'Install packages'
- bundle = False
- def __init__(self):
- super(InstallCommand, self).__init__()
- self.parser.add_option(
- '-e', '--editable',
- dest='editables',
- action='append',
- default=[],
- metavar='VCS+REPOS_URL[@REV]#egg=PACKAGE',
- help='Install a package directly from a checkout. Source will be checked '
- 'out into src/PACKAGE (lower-case) and installed in-place (using '
- 'setup.py develop). You can run this on an existing directory/checkout (like '
- 'pip install -e src/mycheckout). This option may be provided multiple times. '
- 'Possible values for VCS are: svn, git, hg and bzr.')
- self.parser.add_option(
- '-r', '--requirement',
- dest='requirements',
- action='append',
- default=[],
- metavar='FILENAME',
- help='Install all the packages listed in the given requirements file. '
- 'This option can be used multiple times.')
- self.parser.add_option(
- '-f', '--find-links',
- dest='find_links',
- action='append',
- default=[],
- metavar='URL',
- help='URL to look for packages at')
- self.parser.add_option(
- '-i', '--index-url', '--pypi-url',
- dest='index_url',
- metavar='URL',
- default='http://pypi.python.org/simple',
- help='Base URL of Python Package Index (default %default)')
- self.parser.add_option(
- '--extra-index-url',
- dest='extra_index_urls',
- metavar='URL',
- action='append',
- default=[],
- help='Extra URLs of package indexes to use in addition to --index-url')
- self.parser.add_option(
- '--no-index',
- dest='no_index',
- action='store_true',
- default=False,
- help='Ignore package index (only looking at --find-links URLs instead)')
- self.parser.add_option(
- '-b', '--build', '--build-dir', '--build-directory',
- dest='build_dir',
- metavar='DIR',
- default=None,
- help='Unpack packages into DIR (default %s) and build from there' % build_prefix)
- self.parser.add_option(
- '-d', '--download', '--download-dir', '--download-directory',
- dest='download_dir',
- metavar='DIR',
- default=None,
- help='Download packages into DIR instead of installing them')
- self.parser.add_option(
- '--download-cache',
- dest='download_cache',
- metavar='DIR',
- default=None,
- help='Cache downloaded packages in DIR')
- self.parser.add_option(
- '--src', '--source', '--source-dir', '--source-directory',
- dest='src_dir',
- metavar='DIR',
- default=None,
- help='Check out --editable packages into DIR (default %s)' % src_prefix)
- self.parser.add_option(
- '-U', '--upgrade',
- dest='upgrade',
- action='store_true',
- help='Upgrade all packages to the newest available version')
- self.parser.add_option(
- '-I', '--ignore-installed',
- dest='ignore_installed',
- action='store_true',
- help='Ignore the installed packages (reinstalling instead)')
- self.parser.add_option(
- '--no-deps', '--no-dependencies',
- dest='ignore_dependencies',
- action='store_true',
- default=False,
- help='Ignore package dependencies')
- self.parser.add_option(
- '--no-install',
- dest='no_install',
- action='store_true',
- help="Download and unpack all packages, but don't actually install them")
- self.parser.add_option(
- '--install-option',
- dest='install_options',
- action='append',
- help="Extra arguments to be supplied to the setup.py install "
- "command (use like --install-option=\"--install-scripts=/usr/local/bin\"). "
- "Use multiple --install-option options to pass multiple options to setup.py install. "
- "If you are using an option with a directory path, be sure to use absolute path.")
- def run(self, options, args):
- if not options.build_dir:
- options.build_dir = build_prefix
- if not options.src_dir:
- options.src_dir = src_prefix
- if options.download_dir:
- options.no_install = True
- options.ignore_installed = True
- else:
- options.build_dir = os.path.abspath(options.build_dir)
- options.src_dir = os.path.abspath(options.src_dir)
- install_options = options.install_options or []
- index_urls = [options.index_url] + options.extra_index_urls
- if options.no_index:
- logger.notify('Ignoring indexes: %s' % ','.join(index_urls))
- index_urls = []
- finder = PackageFinder(
- find_links=options.find_links,
- index_urls=index_urls)
- requirement_set = RequirementSet(
- build_dir=options.build_dir,
- src_dir=options.src_dir,
- download_dir=options.download_dir,
- download_cache=options.download_cache,
- upgrade=options.upgrade,
- ignore_installed=options.ignore_installed,
- ignore_dependencies=options.ignore_dependencies)
- for name in args:
- requirement_set.add_requirement(
- InstallRequirement.from_line(name, None))
- for name in options.editables:
- requirement_set.add_requirement(
- InstallRequirement.from_editable(name, default_vcs=options.default_vcs))
- for filename in options.requirements:
- for req in parse_requirements(filename, finder=finder, options=options):
- requirement_set.add_requirement(req)
- requirement_set.install_files(finder, force_root_egg_info=self.bundle)
- if not options.no_install and not self.bundle:
- requirement_set.install(install_options)
- installed = ' '.join([req.name for req in
- requirement_set.successfully_installed])
- if installed:
- logger.notify('Successfully installed %s' % installed)
- elif not self.bundle:
- downloaded = ' '.join([req.name for req in
- requirement_set.successfully_downloaded])
- if downloaded:
- logger.notify('Successfully downloaded %s' % downloaded)
- return requirement_set
- InstallCommand()
- class UninstallCommand(Command):
- name = 'uninstall'
- usage = '%prog [OPTIONS] PACKAGE_NAMES ...'
- summary = 'Uninstall packages'
- def __init__(self):
- super(UninstallCommand, self).__init__()
- self.parser.add_option(
- '-r', '--requirement',
- dest='requirements',
- action='append',
- default=[],
- metavar='FILENAME',
- help='Uninstall all the packages listed in the given requirements file. '
- 'This option can be used multiple times.')
- self.parser.add_option(
- '-y', '--yes',
- dest='yes',
- action='store_true',
- help="Don't ask for confirmation of uninstall deletions.")
- def run(self, options, args):
- requirement_set = RequirementSet(
- build_dir=None,
- src_dir=None,
- download_dir=None)
- for name in args:
- requirement_set.add_requirement(
- InstallRequirement.from_line(name))
- for filename in options.requirements:
- for req in parse_requirements(filename, options=options):
- requirement_set.add_requirement(req)
- requirement_set.uninstall(auto_confirm=options.yes)
- UninstallCommand()
- class BundleCommand(InstallCommand):
- name = 'bundle'
- usage = '%prog [OPTIONS] BUNDLE_NAME.pybundle PACKAGE_NAMES...'
- summary = 'Create pybundles (archives containing multiple packages)'
- bundle = True
- def __init__(self):
- super(BundleCommand, self).__init__()
- def run(self, options, args):
- if not args:
- raise InstallationError('You must give a bundle filename')
- if not options.build_dir:
- options.build_dir = backup_dir(build_prefix, '-bundle')
- if not options.src_dir:
- options.src_dir = backup_dir(src_prefix, '-bundle')
- # We have to get everything when creating a bundle:
- options.ignore_installed = True
- logger.notify('Putting temporary build files in %s and source/develop files in %s'
- % (display_path(options.build_dir), display_path(options.src_dir)))
- bundle_filename = args[0]
- args = args[1:]
- requirement_set = super(BundleCommand, self).run(options, args)
- # FIXME: here it has to do something
- requirement_set.create_bundle(bundle_filename)
- logger.notify('Created bundle in %s' % bundle_filename)
- return requirement_set
- BundleCommand()
- class FreezeCommand(Command):
- name = 'freeze'
- usage = '%prog [OPTIONS]'
- summary = 'Output all currently installed packages (exact versions) to stdout'
- def __init__(self):
- super(FreezeCommand, self).__init__()
- self.parser.add_option(
- '-r', '--requirement',
- dest='requirement',
- action='store',
- default=None,
- metavar='FILENAME',
- help='Use the given requirements file as a hint about how to generate the new frozen requirements')
- self.parser.add_option(
- '-f', '--find-links',
- dest='find_links',
- action='append',
- default=[],
- metavar='URL',
- help='URL for finding packages, which will be added to the frozen requirements file')
- def run(self, options, args):
- requirement = options.requirement
- find_links = options.find_links or []
- ## FIXME: Obviously this should be settable:
- find_tags = False
- skip_match = None
- skip_regex = options.skip_requirements_regex
- if skip_regex:
- skip_match = re.compile(skip_regex)
- logger.move_stdout_to_stderr()
- dependency_links = []
- f = sys.stdout
- for dist in pkg_resources.working_set:
- if dist.has_metadata('dependency_links.txt'):
- dependency_links.extend(dist.get_metadata_lines('dependency_links.txt'))
- for link in find_links:
- if '#egg=' in link:
- dependency_links.append(link)
- for link in find_links:
- f.write('-f %s\n' % link)
- installations = {}
- for dist in pkg_resources.working_set:
- if dist.key in ('setuptools', 'pip', 'python'):
- ## FIXME: also skip virtualenv?
- continue
- req = FrozenRequirement.from_dist(dist, dependency_links, find_tags=find_tags)
- installations[req.name] = req
- if requirement:
- req_f = open(requirement)
- for line in req_f:
- if not line.strip() or line.strip().startswith('#'):
- f.write(line)
- continue
- if skip_match and skip_match.search(line):
- f.write(line)
- continue
- elif line.startswith('-e') or line.startswith('--editable'):
- if line.startswith('-e'):
- line = line[2:].strip()
- else:
- line = line[len('--editable'):].strip().lstrip('=')
- line_req = InstallRequirement.from_editable(line, default_vcs=options.default_vcs)
- elif (line.startswith('-r') or line.startswith('--requirement')
- or line.startswith('-Z') or line.startswith('--always-unzip')):
- logger.debug('Skipping line %r' % line.strip())
- continue
- else:
- line_req = InstallRequirement.from_line(line)
- if not line_req.name:
- logger.notify("Skipping line because it's not clear what it would install: %s"
- % line.strip())
- logger.notify(" (add #egg=PackageName to the URL to avoid this warning)")
- continue
- if line_req.name not in installations:
- logger.warn("Requirement file contains %s, but that package is not installed"
- % line.strip())
- continue
- f.write(str(installations[line_req.name]))
- del installations[line_req.name]
- f.write('## The following requirements were added by pip --freeze:\n')
- for installation in sorted(installations.values(), key=lambda x: x.name):
- f.write(str(installation))
- FreezeCommand()
- class ZipCommand(Command):
- name = 'zip'
- usage = '%prog [OPTIONS] PACKAGE_NAMES...'
- summary = 'Zip individual packages'
- def __init__(self):
- super(ZipCommand, self).__init__()
- if self.name == 'zip':
- self.parser.add_option(
- '--unzip',
- action='store_true',
- dest='unzip',
- help='Unzip (rather than zip) a package')
- else:
- self.parser.add_option(
- '--zip',
- action='store_false',
- dest='unzip',
- default=True,
- help='Zip (rather than unzip) a package')
- self.parser.add_option(
- '--no-pyc',
- action='store_true',
- dest='no_pyc',
- help='Do not include .pyc files in zip files (useful on Google App Engine)')
- self.parser.add_option(
- '-l', '--list',
- action='store_true',
- dest='list',
- help='List the packages available, and their zip status')
- self.parser.add_option(
- '--sort-files',
- action='store_true',
- dest='sort_files',
- help='With --list, sort packages according to how many files they contain')
- self.parser.add_option(
- '--path',
- action='append',
- dest='paths',
- help='Restrict operations to the given paths (may include wildcards)')
- self.parser.add_option(
- '-n', '--simulate',
- action='store_true',
- help='Do not actually perform the zip/unzip operation')
- def paths(self):
- """All the entries of sys.path, possibly restricted by --path"""
- if not self.select_paths:
- return sys.path
- result = []
- match_any = set()
- for path in sys.path:
- path = os.path.normcase(os.path.abspath(path))
- for match in self.select_paths:
- match = os.path.normcase(os.path.abspath(match))
- if '*' in match:
- if re.search(fnmatch.translate(match+'*'), path):
- result.append(path)
- match_any.add(match)
- break
- else:
- if path.startswith(match):
- result.append(path)
- match_any.add(match)
- break
- else:
- logger.debug("Skipping path %s because it doesn't match %s"
- % (path, ', '.join(self.select_paths)))
- for match in self.select_paths:
- if match not in match_any and '*' not in match:
- result.append(match)
- logger.debug("Adding path %s because it doesn't match anything already on sys.path"
- % match)
- return result
- def run(self, options, args):
- self.select_paths = options.paths
- self.simulate = options.simulate
- if options.list:
- return self.list(options, args)
- if not args:
- raise InstallationError(
- 'You must give at least one package to zip or unzip')
- packages = []
- for arg in args:
- module_name, filename = self.find_package(arg)
- if options.unzip and os.path.isdir(filename):
- raise InstallationError(
- 'The module %s (in %s) is not a zip file; cannot be unzipped'
- % (module_name, filename))
- elif not options.unzip and not os.path.isdir(filename):
- raise InstallationError(
- 'The module %s (in %s) is not a directory; cannot be zipped'
- % (module_name, filename))
- packages.append((module_name, filename))
- last_status = None
- for module_name, filename in packages:
- if options.unzip:
- last_status = self.unzip_package(module_name, filename)
- else:
- last_status = self.zip_package(module_name, filename, options.no_pyc)
- return last_status
- def unzip_package(self, module_name, filename):
- zip_filename = os.path.dirname(filename)
- if not os.path.isfile(zip_filename) and zipfile.is_zipfile(zip_filename):
- raise InstallationError(
- 'Module %s (in %s) isn\'t located in a zip file in %s'
- % (module_name, filename, zip_filename))
- package_path = os.path.dirname(zip_filename)
- if not package_path in self.paths():
- logger.warn(
- 'Unpacking %s into %s, but %s is not on sys.path'
- % (display_path(zip_filename), display_path(package_path),
- display_path(package_path)))
- logger.notify('Unzipping %s (in %s)' % (module_name, display_path(zip_filename)))
- if self.simulate:
- logger.notify('Skipping remaining operations because of --simulate')
- return
- logger.indent += 2
- try:
- ## FIXME: this should be undoable:
- zip = zipfile.ZipFile(zip_filename)
- to_save = []
- for name in zip.namelist():
- if name.startswith('%s/' % module_name):
- content = zip.read(name)
- dest = os.path.join(package_path, name)
- if not os.path.exists(os.path.dirname(dest)):
- os.makedirs(os.path.dirname(dest))
- if not content and dest.endswith('/'):
- if not os.path.exists(dest):
- os.makedirs(dest)
- else:
- f = open(dest, 'wb')
- f.write(content)
- f.close()
- else:
- to_save.append((name, zip.read(name)))
- zip.close()
- if not to_save:
- logger.info('Removing now-empty zip file %s' % display_path(zip_filename))
- os.unlink(zip_filename)
- self.remove_filename_from_pth(zip_filename)
- else:
- logger.info('Removing entries in %s/ from zip file %s' % (module_name, display_path(zip_filename)))
- zip = zipfile.ZipFile(zip_filename, 'w')
- for name, content in to_save:
- zip.writestr(name, content)
- zip.close()
- finally:
- logger.indent -= 2
- def zip_package(self, module_name, filename, no_pyc):
- orig_filename = filename
- logger.notify('Zip %s (in %s)' % (module_name, display_path(filename)))
- logger.indent += 2
- if filename.endswith('.egg'):
- dest_filename = filename
- else:
- dest_filename = filename + '.zip'
- try:
- ## FIXME: I think this needs to be undoable:
- if filename == dest_filename:
- filename = backup_dir(orig_filename)
- logger.notify('Moving %s aside to %s' % (orig_filename, filename))
- if not self.simulate:
- shutil.move(orig_filename, filename)
- try:
- logger.info('Creating zip file in %s' % display_path(dest_filename))
- if not self.simulate:
- zip = zipfile.ZipFile(dest_filename, 'w')
- zip.writestr(module_name + '/', '')
- for dirpath, dirnames, filenames in os.walk(filename):
- if no_pyc:
- filenames = [f for f in filenames
- if not f.lower().endswith('.pyc')]
- for fns, is_dir in [(dirnames, True), (filenames, False)]:
- for fn in fns:
- full = os.path.join(dirpath, fn)
- dest = os.path.join(module_name, dirpath[len(filename):].lstrip(os.path.sep), fn)
- if is_dir:
- zip.writestr(dest+'/', '')
- else:
- zip.write(full, dest)
- zip.close()
- logger.info('Removing old directory %s' % display_path(filename))
- if not self.simulate:
- shutil.rmtree(filename)
- except:
- ## FIXME: need to do an undo here
- raise
- ## FIXME: should also be undone:
- self.add_filename_to_pth(dest_filename)
- finally:
- logger.indent -= 2
- def remove_filename_from_pth(self, filename):
- for pth in self.pth_files():
- f = open(pth, 'r')
- lines = f.readlines()
- f.close()
- new_lines = [
- l for l in lines if l.strip() != filename]
- if lines != new_lines:
- logger.info('Removing reference to %s from .pth file %s'
- % (display_path(filename), display_path(pth)))
- if not filter(None, new_lines):
- logger.info('%s file would be empty: deleting' % display_path(pth))
- if not self.simulate:
- os.unlink(pth)
- else:
- if not self.simulate:
- f = open(pth, 'w')
- f.writelines(new_lines)
- f.close()
- return
- logger.warn('Cannot find a reference to %s in any .pth file' % display_path(filename))
- def add_filename_to_pth(self, filename):
- path = os.path.dirname(filename)
- dest = os.path.join(path, filename + '.pth')
- if path not in self.paths():
- logger.warn('Adding .pth file %s, but it is not on sys.path' % display_path(dest))
- if not self.simulate:
- if os.path.exists(dest):
- f = open(dest)
- lines = f.readlines()
- f.close()
- if lines and not lines[-1].endswith('\n'):
- lines[-1] += '\n'
- lines.append(filename+'\n')
- else:
- lines = [filename + '\n']
- f = open(dest, 'w')
- f.writelines(lines)
- f.close()
- def pth_files(self):
- for path in self.paths():
- if not os.path.exists(path) or not os.path.isdir(path):
- continue
- for filename in os.listdir(path):
- if filename.endswith('.pth'):
- yield os.path.join(path, filename)
- def find_package(self, package):
- for path in self.paths():
- full = os.path.join(path, package)
- if os.path.exists(full):
- return package, full
- if not os.path.isdir(path) and zipfile.is_zipfile(path):
- zip = zipfile.ZipFile(path, 'r')
- try:
- zip.read('%s/__init__.py' % package)
- except KeyError:
- pass
- else:
- zip.close()
- return package, full
- zip.close()
- ## FIXME: need special error for package.py case:
- raise InstallationError(
- 'No package with the name %s found' % package)
- def list(self, options, args):
- if args:
- raise InstallationError(
- 'You cannot give an argument with --list')
- for path in sorted(self.paths()):
- if not os.path.exists(path):
- continue
- basename = os.path.basename(path.rstrip(os.path.sep))
- if os.path.isfile(path) and zipfile.is_zipfile(path):
- if os.path.dirname(path) not in self.paths():
- logger.notify('Zipped egg: %s' % display_path(path))
- continue
- if (basename != 'site-packages'
- and not path.replace('\\', '/').endswith('lib/python')):
- continue
- logger.notify('In %s:' % display_path(path))
- logger.indent += 2
- zipped = []
- unzipped = []
- try:
- for filename in sorted(os.listdir(path)):
- ext = os.path.splitext(filename)[1].lower()
- if ext in ('.pth', '.egg-info', '.egg-link'):
- continue
- if ext == '.py':
- logger.info('Not displaying %s: not a package' % display_path(filename))
- continue
- full = os.path.join(path, filename)
- if os.path.isdir(full):
- unzipped.append((filename, self.count_package(full)))
- elif zipfile.is_zipfile(full):
- zipped.append(filename)
- else:
- logger.info('Unknown file: %s' % display_path(filename))
- if zipped:
- logger.notify('Zipped packages:')
- logger.indent += 2
- try:
- for filename in zipped:
- logger.notify(filename)
- finally:
- logger.indent -= 2
- else:
- logger.notify('No zipped packages.')
- if unzipped:
- if options.sort_files:
- unzipped.sort(key=lambda x: -x[1])
- logger.notify('Unzipped packages:')
- logger.indent += 2
- try:
- for filename, count in unzipped:
- logger.notify('%s (%i files)' % (filename, count))
- finally:
- logger.indent -= 2
- else:
- logger.notify('No unzipped packages.')
- finally:
- logger.indent -= 2
- def count_package(self, path):
- total = 0
- for dirpath, dirnames, filenames in os.walk(path):
- filenames = [f for f in filenames
- if not f.lower().endswith('.pyc')]
- total += len(filenames)
- return total
- ZipCommand()
- class UnzipCommand(ZipCommand):
- name = 'unzip'
- summary = 'Unzip individual packages'
- UnzipCommand()
- BASE_COMPLETION = """
- # pip %(shell)s completion start%(script)s# pip %(shell)s completion end
- """
- COMPLETION_SCRIPTS = {
- 'bash': """
- _pip_completion()
- {
- COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]}" \\
- COMP_CWORD=$COMP_CWORD \\
- PIP_AUTO_COMPLETE=1 $1 ) )
- }
- complete -o default -F _pip_completion pip
- """, 'zsh': """
- function _pip_completion {
- local words cword
- read -Ac words
- read -cn cword
- reply=( $( COMP_WORDS="$words[*]" \\
- COMP_CWORD=$(( cword-1 )) \\
- PIP_AUTO_COMPLETE=1 $words[1] ) )
- }
- compctl -K _pip_completion pip
- """
- }
- class CompletionCommand(Command):
- name = 'completion'
- summary = 'A helper command to be used for command completion'
- hidden = True
- def __init__(self):
- super(CompletionCommand, self).__init__()
- self.parser.add_option(
- '--bash', '-b',
- action='store_const',
- const='bash',
- dest='shell',
- help='Emit completion code for bash')
- self.parser.add_option(
- '--zsh', '-z',
- action='store_const',
- const='zsh',
- dest='shell',
- help='Emit completion code for zsh')
- def run(self, options, args):
- """Prints the completion code of the given shell"""
- if options.shell in ('bash', 'zsh'):
- script = COMPLETION_SCRIPTS.get(options.shell, '')
- print BASE_COMPLETION % {'script': script, 'shell': options.shell}
- else:
- print 'ERROR: You must pass --bash or --zsh'
- CompletionCommand()
- def autocomplete():
- """Command and option completion for the main option parser (and options)
- and its subcommands (and options).
- Enable by sourcing one of the completion shell scripts (bash or zsh).
- """
- # Don't complete if user hasn't sourced bash_completion file.
- if not os.environ.has_key('PIP_AUTO_COMPLETE'):
- return
- cwords = os.environ['COMP_WORDS'].split()[1:]
- cword = int(os.environ['COMP_CWORD'])
- try:
- current = cwords[cword-1]
- except IndexError:
- current = ''
- subcommands = [cmd for cmd, cls in _commands.items() if not cls.hidden]
- options = []
- # subcommand
- if cword == 1:
- # show options of main parser only when necessary
- if current.startswith('-') or current.startswith('--'):
- subcommands += [opt.get_opt_string()
- for opt in parser.option_list
- if opt.help != optparse.SUPPRESS_HELP]
- print ' '.join(filter(lambda x: x.startswith(current), subcommands))
- # subcommand options
- # special case: the 'help' subcommand has no options
- elif cwords[0] in subcommands and cwords[0] != 'help':
- subcommand = _commands.get(cwords[0])
- options += [(opt.get_opt_string(), opt.nargs)
- for opt in subcommand.parser.option_list
- if opt.help != optparse.SUPPRESS_HELP]
- # filter out previously specified options from available options
- prev_opts = [x.split('=')[0] for x in cwords[1:cword-1]]
- options = filter(lambda (x, v): x not in prev_opts, options)
- # filter options by current input
- options = [(k, v) for k, v in options if k.startswith(current)]
- for option in options:
- opt_label = option[0]
- # append '=' to options which require args
- if option[1]:
- opt_label += '='
- print opt_label
- sys.exit(1)
- def main(initial_args=None):
- if initial_args is None:
- initial_args = sys.argv[1:]
- autocomplete()
- options, args = parser.parse_args(initial_args)
- if options.help and not args:
- args = ['help']
- if not args:
- parser.error('You must give a command (use "pip help" see a list of commands)')
- command = args[0].lower()
- ## FIXME: search for a command match?
- if command not in _commands:
- parser.error('No command by the name %(script)s %(arg)s\n (maybe you meant "%(script)s install %(arg)s")'
- % dict(script=os.path.basename(sys.argv[0]), arg=command))
- command = _commands[command]
- return command.main(initial_args, args[1:], options)
- def get_proxy(proxystr=''):
- """Get the proxy given the option passed on the command line. If an
- empty string is passed it looks at the HTTP_PROXY environment
- variable."""
- if not proxystr:
- proxystr = os.environ.get('HTTP_PROXY', '')
- if proxystr:
- if '@' in proxystr:
- user_password, server_port = proxystr.split('@', 1)
- if ':' in user_password:
- user, password = user_password.split(':', 1)
- else:
- user = user_password
- import getpass
- prompt = 'Password for %s@%s: ' % (user, server_port)
- password = urllib.quote(getpass.getpass(prompt))
- return '%s:%s@%s' % (user, password, server_port)
- else:
- return proxystr
- else:
- return None
- def setup_proxy_handler(proxystr=''):
- """Set the proxy handler given the option passed on the command
- line. If an empty string is passed it looks at the HTTP_PROXY
- environment variable. """
- proxy = get_proxy(proxystr)
- if proxy:
- proxy_support = urllib2.ProxyHandler({"http": proxy, "ftp": proxy})
- opener = urllib2.build_opener(proxy_support, urllib2.CacheFTPHandler)
- urllib2.install_opener(opener)
- def format_exc(exc_info=None):
- if exc_info is None:
- exc_info = sys.exc_info()
- out = StringIO()
- traceback.print_exception(*exc_info, **dict(file=out))
- return out.getvalue()
- def restart_in_venv(venv, base, site_packages, args):
- """
- Restart this script using the interpreter in the given virtual environment
- """
- if base and not os.path.isabs(venv) and not venv.startswith('~'):
- base = os.path.expanduser(base)
- # ensure we have an abs basepath at this point:
- # a relative one makes no sense (or does it?)
- if os.path.isabs(base):
- venv = os.path.join(base, venv)
- if venv.startswith('~'):
- venv = os.path.expanduser(venv)
- if not os.path.exists(venv):
- try:
- import virtualenv
- except ImportError:
- print 'The virtual environment does not exist: %s' % venv
- print 'and virtualenv is not installed, so a new environment cannot be created'
- sys.exit(3)
- print 'Creating new virtualenv environment in %s' % venv
- virtualenv.logger = logger
- logger.indent += 2
- virtualenv.create_environment(venv, site_packages=site_packages)
- if sys.platform == 'win32':
- python = os.path.join(venv, 'Scripts', 'python.exe')
- # check for bin directory which is used in buildouts
- if not os.path.exists(python):
- python = os.path.join(venv, 'bin', 'python.exe')
- else:
- python = os.path.join(venv, 'bin', 'python')
- if not os.path.exists(python):
- python = venv
- if not os.path.exists(python):
- raise BadCommand('Cannot find virtual environment interpreter at %s' % python)
- base = os.path.dirname(os.path.dirname(python))
- file = __file__
- if file.endswith('.pyc'):
- file = file[:-1]
- proc = subprocess.Popen(
- [python, file] + args + [base, '___VENV_RESTART___'])
- proc.wait()
- sys.exit(proc.returncode)
- class PackageFinder(object):
- """This finds packages.
- This is meant to match easy_install's technique for looking for
- packages, by reading pages and looking for appropriate links
- """
- failure_limit = 3
- def __init__(self, find_links, index_urls):
- self.find_links = find_links
- self.index_urls = index_urls
- self.dependency_links = []
- self.cache = PageCache()
- # These are boring links that have already been logged somehow:
- self.logged_links = set()
- def add_dependency_links(self, links):
- ## FIXME: this shouldn't be global list this, it should only
- ## apply to requirements of the package that specifies the
- ## dependency_links value
- ## FIXME: also, we should track comes_from (i.e., use Link)
- self.dependency_links.extend(links)
- def find_requirement(self, req, upgrade):
- url_name = req.url_name
- # Only check main index if index URL is given:
- main_index_url = None
- if self.index_urls:
- # Check that we have the url_name correctly spelled:
- main_index_url = Link(posixpath.join(self.index_urls[0], url_name))
- # This will also cache the page, so it's okay that we get it again later:
- page = self._get_page(main_index_url, req)
- if page is None:
- url_name = self._find_url_name(Link(self.index_urls[0]), url_name, req) or req.url_name
- def mkurl_pypi_url(url):
- loc = posixpath.join(url, url_name)
- # For maximum compatibility with easy_install, ensure the path
- # ends in a trailing slash. Although this isn't in the spec
- # (and PyPI can handle it without the slash) some other index
- # implementations might break if they relied on easy_install's behavior.
- if not loc.endswith('/'):
- loc = loc + '/'
- return loc
- if url_name is not None:
- locations = [
- mkurl_pypi_url(url)
- for url in self.index_urls] + self.find_links
- else:
- locations = list(self.find_links)
- locations.extend(self.dependency_links)
- for version in req.absolute_versions:
- if url_name is not None and main_index_url is not None:
- locations = [
- posixpath.join(main_index_url.url, version)] + locations
- file_locations = []
- url_locations = []
- for url in locations:
- if url.startswith('file:'):
- fn = url_to_filename(url)
- if os.path.isdir(fn):
- path = os.path.realpath(fn)
- for item in os.listdir(path):
- file_locations.append(
- filename_to_url2(os.path.join(path, item)))
- elif os.path.isfile(fn):
- file_locations.append(filename_to_url2(fn))
- else:
- url_locations.append(url)
- locations = [Link(url) for url in url_locations]
- logger.debug('URLs to search for versions for %s:' % req)
- for location in locations:
- logger.debug('* %s' % location)
- found_versions = []
- found_versions.extend(
- self._package_versions(
- [Link(url, '-f') for url in self.find_links], req.name.lower()))
- page_versions = []
- for page in self._get_pages(locations, req):
- logger.debug('Analyzing links from page %s' % page.url)
- logger.indent += 2
- try:
- page_versions.extend(self._package_versions(page.links, req.name.lower()))
- finally:
- logger.indent -= 2
- dependency_versions = list(self._package_versions(
- [Link(url) for url in self.dependency_links], req.name.lower()))
- if dependency_versions:
- logger.info('dependency_links found: %s' % ', '.join([link.url for parsed, link, version in dependency_versions]))
- file_versions = list(self._package_versions(
- [Link(url) for url in file_locations], req.name.lower()))
- if not found_versions and not page_versions and not dependency_versions and not file_versions:
- logger.fatal('Could not find any downloads that satisfy the requirement %s' % req)
- raise DistributionNotFound('No distributions at all found for %s' % req)
- if req.satisfied_by is not None:
- found_versions.append((req.satisfied_by.parsed_version, Inf, req.satisfied_by.version))
- if file_versions:
- file_versions.sort(reverse=True)
- logger.info('Local files found: %s' % ', '.join([url_to_filename(link.url) for parsed, link, version in file_versions]))
- found_versions = file_versions + found_versions
- all_versions = found_versions + page_versions + dependency_versions
- applicable_versions = []
- for (parsed_version, link, version) in all_versions:
- if version not in req.req:
- logger.info("Ignoring link %s, version %s doesn't match %s"
- % (link, version, ','.join([''.join(s) for s in req.req.specs])))
- continue
- applicable_versions.append((link, version))
- applicable_versions = sorted(applicable_versions, key=operator.itemgetter(1),
- cmp=lambda x, y : cmp(pkg_resources.parse_version(y), pkg_resources.parse_version(x))
- )
- existing_applicable = bool([link for link, version in applicable_versions if link is Inf])
- if not upgrade and existing_applicable:
- if applicable_versions[0][1] is Inf:
- logger.info('Existing installed version (%s) is most up-to-date and satisfies requirement'
- % req.satisfied_by.version)
- else:
- logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)'
- % (req.satisfied_by.version, applicable_versions[0][1]))
- return None
- if not applicable_versions:
- logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)'
- % (req, ', '.join([version for parsed_version, link, version in found_versions])))
- raise DistributionNotFound('No distributions matching the version for %s' % req)
- if applicable_versions[0][0] is Inf:
- # We have an existing version, and its the best version
- logger.info('Installed version (%s) is most up-to-date (past versions: %s)'
- % (req.satisfied_by.version, ', '.join([version for link, version in applicable_versions[1:]]) or 'none'))
- return None
- if len(applicable_versions) > 1:
- logger.info('Using version %s (newest of versions: %s)' %
- (applicable_versions[0][1], ', '.join([version for link, version in applicable_versions])))
- return applicable_versions[0][0]
- def _find_url_name(self, index_url, url_name, req):
- """Finds the true URL name of a package, when the given name isn't quite correct.
- This is usually used to implement case-insensitivity."""
- if not index_url.url.endswith('/'):
- # Vaguely part of the PyPI API... weird but true.
- ## FIXME: bad to modify this?
- index_url.url += '/'
- page = self._get_page(index_url, req)
- if page is None:
- logger.fatal('Cannot fetch index base URL %s' % index_url)
- return
- norm_name = normalize_name(req.url_name)
- for link in page.links:
- base = posixpath.basename(link.path.rstrip('/'))
- if norm_name == normalize_name(base):
- logger.notify('Real name of requirement %s is %s' % (url_name, base))
- return base
- return None
- def _get_pages(self, locations, req):
- """Yields (page, page_url) from the given locations, skipping
- locations that have errors, and adding download/homepage links"""
- pending_queue = Queue()
- for location in locations:
- pending_queue.put(location)
- done = []
- seen = set()
- threads = []
- for i in range(min(10, len(locations))):
- t = threading.Thread(target=self._get_queued_page, args=(req, pending_queue, done, seen))
- t.setDaemon(True)
- threads.append(t)
- t.start()
- for t in threads:
- t.join()
- return done
- _log_lock = threading.Lock()
- def _get_queued_page(self, req, pending_queue, done, seen):
- while 1:
- try:
- location = pending_queue.get(False)
- except QueueEmpty:
- return
- if location in seen:
- continue
- seen.add(location)
- page = self._get_page(location, req)
- if page is None:
- continue
- done.append(page)
- for link in page.rel_links():
- pending_queue.put(link)
- _egg_fragment_re = re.compile(r'#egg=([^&]*)')
- _egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.-]+)', re.I)
- _py_version_re = re.compile(r'-py([123]\.[0-9])$')
- def _sort_links(self, links):
- "Brings links in order, non-egg links first, egg links second"
- eggs, no_eggs = [], []
- for link in links:
- if link.egg_fragment:
- eggs.append(link)
- else:
- no_eggs.append(link)
- return no_eggs + eggs
- def _package_versions(self, links, search_name):
- seen_links = {}
- for link in self._sort_links(links):
- if link.url in seen_links:
- continue
- seen_links[link.url] = None
- if link.egg_fragment:
- egg_info = link.egg_fragment
- else:
- path = link.path
- egg_info, ext = link.splitext()
- if not ext:
- if link not in self.logged_links:
- logger.debug('Skipping link %s; not a file' % link)
- self.logged_links.add(link)
- continue
- if egg_info.endswith('.tar'):
- # Special double-extension case:
- egg_info = egg_info[:-4]
- ext = '.tar' + ext
- if ext not in ('.tar.gz', '.tar.bz2', '.tar', '.tgz', '.zip'):
- if link not in self.logged_links:
- logger.debug('Skipping link %s; unknown archive format: %s' % (link, ext))
- self.logged_links.add(link)
- continue
- version = self._egg_info_matches(egg_info, search_name, link)
- if version is None:
- logger.debug('Skipping link %s; wrong project name (not %s)' % (link, search_name))
- continue
- match = self._py_version_re.search(version)
- if match:
- version = version[:match.start()]
- py_version = match.group(1)
- if py_version != sys.version[:3]:
- logger.debug('Skipping %s because Python version is incorrect' % link)
- continue
- logger.debug('Found link %s, version: %s' % (link, version))
- yield (pkg_resources.parse_version(version),
- link,
- version)
- def _egg_info_matches(self, egg_info, search_name, link):
- match = self._egg_info_re.search(egg_info)
- if not match:
- logger.debug('Could not parse version from link: %s' % link)
- return None
- name = match.group(0).lower()
- # To match the "safe" name that pkg_resources creates:
- name = name.replace('_', '-')
- if name.startswith(search_name.lower()):
- return match.group(0)[len(search_name):].lstrip('-')
- else:
- return None
- def _get_page(self, link, req):
- return HTMLPage.get_page(link, req, cache=self.cache)
- class InstallRequirement(object):
- def __init__(self, req, comes_from, source_dir=None, editable=False,
- url=None, update=True):
- if isinstance(req, basestring):
- req = pkg_resources.Requirement.parse(req)
- self.req = req
- self.comes_from = comes_from
- self.source_dir = source_dir
- self.editable = editable
- self.url = url
- self._egg_info_path = None
- # This holds the pkg_resources.Distribution object if this requirement
- # is already available:
- self.satisfied_by = None
- # This hold the pkg_resources.Distribution object if this requirement
- # conflicts with another installed distribution:
- self.conflicts_with = None
- self._temp_build_dir = None
- self._is_bundle = None
- # True if the editable should be updated:
- self.update = update
- # Set to True after successful installation
- self.install_succeeded = None
- # UninstallPathSet of uninstalled distribution (for possible rollback)
- self.uninstalled = None
- @classmethod
- def from_editable(cls, editable_req, comes_from=None, default_vcs=None):
- name, url = parse_editable(editable_req, default_vcs)
- if url.startswith('file:'):
- source_dir = url_to_filename(url)
- else:
- source_dir = None
- return cls(name, comes_from, source_dir=source_dir, editable=True, url=url)
- @classmethod
- def from_line(cls, name, comes_from=None):
- """Creates an InstallRequirement from a name, which might be a
- requirement, filename, or URL.
- """
- url = None
- name = name.strip()
- req = name
- if is_url(name):
- url = name
- ## FIXME: I think getting the requirement here is a bad idea:
- #req = get_requirement_from_url(url)
- req = None
- elif is_filename(name):
- if not os.path.exists(name):
- logger.warn('Requirement %r looks like a filename, but the file does not exist'
- % name)
- url = filename_to_url(name)
- #req = get_requirement_from_url(url)
- req = None
- return cls(req, comes_from, url=url)
- def __str__(self):
- if self.req:
- s = str(self.req)
- if self.url:
- s += ' from %s' % self.url
- else:
- s = self.url
- if self.satisfied_by is not None:
- s += ' in %s' % display_path(self.satisfied_by.location)
- if self.comes_from:
- if isinstance(self.comes_from, basestring):
- comes_from = self.comes_from
- else:
- comes_from = self.comes_from.from_path()
- if comes_from:
- s += ' (from %s)' % comes_from
- return s
- def from_path(self):
- if self.req is None:
- return None
- s = str(self.req)
- if self.comes_from:
- if isinstance(self.comes_from, basestring):
- comes_from = self.comes_from
- else:
- comes_from = self.comes_from.from_path()
- if comes_from:
- s += '->' + comes_from
- return s
- def build_location(self, build_dir, unpack=True):
- if self._temp_build_dir is not None:
- return self._temp_build_dir
- if self.req is None:
- self._temp_build_dir = tempfile.mkdtemp('-build', 'pip-')
- self._ideal_build_dir = build_dir
- return self._temp_build_dir
- if self.editable:
- name = self.name.lower()
- else:
- name = self.name
- # FIXME: Is there a better place to create the build_dir? (hg and bzr need this)
- if not os.path.exists(build_dir):
- os.makedirs(build_dir)
- return os.path.join(build_dir, name)
- def correct_build_location(self):
- """If the build location was a temporary directory, this will move it
- to a new more permanent location"""
- if self.source_dir is not None:
- return
- assert self.req is not None
- assert self._temp_build_dir
- old_location = self._temp_build_dir
- new_build_dir = self._ideal_build_dir
- del self._ideal_build_dir
- if self.editable:
- name = self.name.lower()
- else:
- name = self.name
- new_location = os.path.join(new_build_dir, name)
- if not os.path.exists(new_build_dir):
- logger.debug('Creating directory %s' % new_build_dir)
- os.makedirs(new_build_dir)
- if os.path.exists(new_location):
- raise InstallationError(
- 'A package already exists in %s; please remove it to continue'
- % display_path(new_location))
- logger.debug('Moving package %s from %s to new location %s'
- % (self, display_path(old_location), display_path(new_location)))
- shutil.move(old_location, new_location)
- self._temp_build_dir = new_location
- self.source_dir = new_location
- self._egg_info_path = None
- @property
- def name(self):
- if self.req is None:
- return None
- return self.req.project_name
- @property
- def url_name(self):
- if self.req is None:
- return None
- return urllib.quote(self.req.unsafe_name)
- @property
- def setup_py(self):
- return os.path.join(self.source_dir, 'setup.py')
- def run_egg_info(self, force_root_egg_info=False):
- assert self.source_dir
- if self.name:
- logger.notify('Running setup.py egg_info for package %s' % self.name)
- else:
- logger.notify('Running setup.py egg_info for package from %s' % self.url)
- logger.indent += 2
- try:
- script = self._run_setup_py
- script = script.replace('__SETUP_PY__', repr(self.setup_py))
- script = script.replace('__PKG_NAME__', repr(self.name))
- # We can't put the .egg-info files at the root, because then the source code will be mistaken
- # for an installed egg, causing problems
- if self.editable or force_root_egg_info:
- egg_base_option = []
- else:
- egg_info_dir = os.path.join(self.source_dir, 'pip-egg-info')
- if not os.path.exists(egg_info_dir):
- os.makedirs(egg_info_dir)
- egg_base_option = ['--egg-base', 'pip-egg-info']
- call_subprocess(
- [sys.executable, '-c', script, 'egg_info'] + egg_base_option,
- cwd=self.source_dir, filter_stdout=self._filter_install, show_stdout=False,
- command_level=Logger.VERBOSE_DEBUG,
- command_desc='python setup.py egg_info')
- finally:
- logger.indent -= 2
- if not self.req:
- self.req = pkg_resources.Requirement.parse(self.pkg_info()['Name'])
- self.correct_build_location()
- ## FIXME: This is a lame hack, entirely for PasteScript which has
- ## a self-provided entry point that causes this awkwardness
- _run_setup_py = """
- __file__ = __SETUP_PY__
- from setuptools.command import egg_info
- def replacement_run(self):
- self.mkpath(self.egg_info)
- installer = self.distribution.fetch_build_egg
- for ep in egg_info.iter_entry_points('egg_info.writers'):
- # require=False is the change we're making:
- writer = ep.load(require=False)
- if writer:
- writer(self, ep.name, egg_info.os.path.join(self.egg_info,ep.name))
- self.find_sources()
- egg_info.egg_info.run = replacement_run
- execfile(__file__)
- """
- def egg_info_data(self, filename):
- if self.satisfied_by is not None:
- if not self.satisfied_by.has_metadata(filename):
- return None
- return self.satisfied_by.get_metadata(filename)
- assert self.source_dir
- filename = self.egg_info_path(filename)
- if not os.path.exists(filename):
- return None
- fp = open(filename, 'r')
- data = fp.read()
- fp.close()
- return data
- def egg_info_path(self, filename):
- if self._egg_info_path is None:
- if self.editable:
- base = self.source_dir
- else:
- base = os.path.join(self.source_dir, 'pip-egg-info')
- filenames = os.listdir(base)
- if self.editable:
- filenames = []
- for root, dirs, files in os.walk(base):
- for dir in vcs.dirnames:
- if dir in dirs:
- dirs.remove(dir)
- filenames.extend([os.path.join(root, dir)
- for dir in dirs])
- filenames = [f for f in filenames if f.endswith('.egg-info')]
- assert filenames, "No files/directories in %s (from %s)" % (base, filename)
- assert len(filenames) == 1, "Unexpected files/directories in %s: %s" % (base, ' '.join(filenames))
- self._egg_info_path = os.path.join(base, filenames[0])
- return os.path.join(self._egg_info_path, filename)
- def egg_info_lines(self, filename):
- data = self.egg_info_data(filename)
- if not data:
- return []
- result = []
- for line in data.splitlines():
- line = line.strip()
- if not line or line.startswith('#'):
- continue
- result.append(line)
- return result
- def pkg_info(self):
- p = FeedParser()
- data = self.egg_info_data('PKG-INFO')
- if not data:
- logger.warn('No PKG-INFO file found in %s' % display_path(self.egg_info_path('PKG-INFO')))
- p.feed(data or '')
- return p.close()
- @property
- def dependency_links(self):
- return self.egg_info_lines('dependency_links.txt')
- _requirements_section_re = re.compile(r'\[(.*?)\]')
- def requirements(self, extras=()):
- in_extra = None
- for line in self.egg_info_lines('requires.txt'):
- match = self._requirements_section_re.match(line)
- if match:
- in_extra = match.group(1)
- continue
- if in_extra and in_extra not in extras:
- # Skip requirement for an extra we aren't requiring
- continue
- yield line
- @property
- def absolute_versions(self):
- for qualifier, version in self.req.specs:
- if qualifier == '==':
- yield version
- @property
- def installed_version(self):
- return self.pkg_info()['version']
- def assert_source_matches_version(self):
- assert self.source_dir
- if self.comes_from is None:
- # We don't check the versions of things explicitly installed.
- # This makes, e.g., "pip Package==dev" possible
- return
- version = self.installed_version
- if version not in self.req:
- logger.fatal(
- 'Source in %s has the version %s, which does not match the requirement %s'
- % (display_path(self.source_dir), version, self))
- raise InstallationError(
- 'Source in %s has version %s that conflicts with %s'
- % (display_path(self.source_dir), version, self))
- else:
- logger.debug('Source in %s has version %s, which satisfies requirement %s'
- % (display_path(self.source_dir), version, self))
- def update_editable(self, obtain=True):
- if not self.url:
- logger.info("Cannot update repository at %s; repository location is unknown" % self.source_dir)
- return
- assert self.editable
- assert self.source_dir
- if self.url.startswith('file:'):
- # Static paths don't get updated
- return
- assert '+' in self.url, "bad url: %r" % self.url
- if not self.update:
- return
- vc_type, url = self.url.split('+', 1)
- backend = vcs.get_backend(vc_type)
- if backend:
- vcs_backend = backend(self.url)
- if obtain:
- vcs_backend.obtain(self.source_dir)
- else:
- vcs_backend.export(self.source_dir)
- else:
- assert 0, (
- 'Unexpected version control type (in %s): %s'
- % (self.url, vc_type))
- def uninstall(self, auto_confirm=False):
- """
- Uninstall the distribution currently satisfying this requirement.
- Prompts before removing or modifying files unless
- ``auto_confirm`` is True.
- Refuses to delete or modify files outside of ``sys.prefix`` -
- thus uninstallation within a virtual environment can only
- modify that virtual environment, even if the virtualenv is
- linked to global site-packages.
- """
- if not self.check_if_exists():
- raise UninstallationError("Cannot uninstall requirement %s, not installed" % (self.name,))
- dist = self.satisfied_by or self.conflicts_with
- paths_to_remove = UninstallPathSet(dist, sys.prefix)
- pip_egg_info_path = os.path.join(dist.location,
- dist.egg_name()) + '.egg-info'
- easy_install_egg = dist.egg_name() + '.egg'
- # This won't find a globally-installed develop egg if
- # we're in a virtualenv.
- # (There doesn't seem to be any metadata in the
- # Distribution object for a develop egg that points back
- # to its .egg-link and easy-install.pth files). That's
- # OK, because we restrict ourselves to making changes
- # within sys.prefix anyway.
- develop_egg_link = os.path.join(site_packages,
- dist.project_name) + '.egg-link'
- if os.path.exists(pip_egg_info_path):
- # package installed by pip
- paths_to_remove.add(pip_egg_info_path)
- if dist.has_metadata('installed-files.txt'):
- for installed_file in dist.get_metadata('installed-files.txt').splitlines():
- path = os.path.normpath(os.path.join(pip_egg_info_path, installed_file))
- if os.path.exists(path):
- paths_to_remove.add(path)
- if dist.has_metadata('top_level.txt'):
- for top_level_pkg in [p for p
- in dist.get_metadata('top_level.txt').splitlines()
- if p]:
- path = os.path.join(dist.location, top_level_pkg)
- if os.path.exists(path):
- paths_to_remove.add(path)
- elif os.path.exists(path + '.py'):
- paths_to_remove.add(path + '.py')
- if os.path.exists(path + '.pyc'):
- paths_to_remove.add(path + '.pyc')
- elif dist.location.endswith(easy_install_egg):
- # package installed by easy_install
- paths_to_remove.add(dist.location)
- easy_install_pth = os.path.join(os.path.dirname(dist.location),
- 'easy-install.pth')
- paths_to_remove.add_pth(easy_install_pth, './' + easy_install_egg)
- elif os.path.isfile(develop_egg_link):
- # develop egg
- fh = open(develop_egg_link, 'r')
- link_pointer = os.path.normcase(fh.readline().strip())
- fh.close()
- assert (link_pointer == dist.location), 'Egg-link %s does not match installed location of %s (at %s)' % (link_pointer, self.name, dist.location)
- paths_to_remove.add(develop_egg_link)
- easy_install_pth = os.path.join(os.path.dirname(develop_egg_link),
- 'easy-install.pth')
- paths_to_remove.add_pth(easy_install_pth, dist.location)
- # fix location (so we can uninstall links to sources outside venv)
- paths_to_remove.location = develop_egg_link
- # find distutils scripts= scripts
- if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'):
- for script in dist.metadata_listdir('scripts'):
- paths_to_remove.add(os.path.join(bin_py, script))
- if sys.platform == 'win32':
- paths_to_remove.add(os.path.join(bin_py, script) + '.bat')
- # find console_scripts
- if dist.has_metadata('entry_points.txt'):
- config = ConfigParser.SafeConfigParser()
- config.readfp(FakeFile(dist.get_metadata_lines('entry_points.txt')))
- for name, value in config.items('console_scripts'):
- paths_to_remove.add(os.path.join(bin_py, name))
- if sys.platform == 'win32':
- paths_to_remove.add(os.path.join(bin_py, name) + '.exe')
- paths_to_remove.add(os.path.join(bin_py, name) + '-script.py')
- paths_to_remove.remove(auto_confirm)
- self.uninstalled = paths_to_remove
- def rollback_uninstall(self):
- if self.uninstalled:
- self.uninstalled.rollback()
- else:
- logger.error("Can't rollback %s, nothing uninstalled."
- % (self.project_name,))
- def archive(self, build_dir):
- assert self.source_dir
- create_archive = True
- archive_name = '%s-%s.zip' % (self.name, self.installed_version)
- archive_path = os.path.join(build_dir, archive_name)
- if os.path.exists(archive_path):
- response = ask('The file %s exists. (i)gnore, (w)ipe, (b)ackup '
- % display_path(archive_path), ('i', 'w', 'b'))
- if response == 'i':
- create_archive = False
- elif response == 'w':
- logger.warn('Deleting %s' % display_path(archive_path))
- os.remove(archive_path)
- elif response == 'b':
- dest_file = backup_dir(archive_path)
- logger.warn('Backing up %s to %s'
- % (display_path(archive_path), display_path(dest_file)))
- shutil.move(archive_path, dest_file)
- if create_archive:
- zip = zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED)
- dir = os.path.normcase(os.path.abspath(self.source_dir))
- for dirpath, dirnames, filenames in os.walk(dir):
- if 'pip-egg-info' in dirnames:
- dirnames.remove('pip-egg-info')
- for dirname in dirnames:
- dirname = os.path.join(dirpath, dirname)
- name = self._clean_zip_name(dirname, dir)
- zipdir = zipfile.ZipInfo(self.name + '/' + name + '/')
- zipdir.external_attr = 0755 << 16L
- zip.writestr(zipdir, '')
- for filename in filenames:
- if filename == 'pip-delete-this-directory.txt':
- continue
- filename = os.path.join(dirpath, filename)
- name = self._clean_zip_name(filename, dir)
- zip.write(filename, self.name + '/' + name)
- zip.close()
- logger.indent -= 2
- logger.notify('Saved %s' % display_path(archive_path))
- def _clean_zip_name(self, name, prefix):
- assert name.startswith(prefix+'/'), (
- "name %r doesn't start with prefix %r" % (name, prefix))
- name = name[len(prefix)+1:]
- name = name.replace(os.path.sep, '/')
- return name
- def install(self, install_options):
- if self.editable:
- self.install_editable()
- return
- temp_location = tempfile.mkdtemp('-record', 'pip-')
- record_filename = os.path.join(temp_location, 'install-record.txt')
- ## FIXME: I'm not sure if this is a reasonable location; probably not
- ## but we can't put it in the default location, as that is a virtualenv symlink that isn't writable
- header_dir = os.path.join(os.path.dirname(os.path.dirname(self.source_dir)), 'lib', 'include')
- logger.notify('Running setup.py install for %s' % self.name)
- logger.indent += 2
- try:
- call_subprocess(
- [sys.executable, '-c',
- "import setuptools; __file__=%r; execfile(%r)" % (self.setup_py, self.setup_py),
- 'install', '--single-version-externally-managed', '--record', record_filename,
- '--install-headers', header_dir] + install_options,
- cwd=self.source_dir, filter_stdout=self._filter_install, show_stdout=False)
- finally:
- logger.indent -= 2
- self.install_succeeded = True
- f = open(record_filename)
- for line in f:
- line = line.strip()
- if line.endswith('.egg-info'):
- egg_info_dir = line
- break
- else:
- logger.warn('Could not find .egg-info directory in install record for %s' % self)
- ## FIXME: put the record somewhere
- return
- f.close()
- new_lines = []
- f = open(record_filename)
- for line in f:
- filename = line.strip()
- if os.path.isdir(filename):
- filename += os.path.sep
- new_lines.append(make_path_relative(filename, egg_info_dir))
- f.close()
- f = open(os.path.join(egg_info_dir, 'installed-files.txt'), 'w')
- f.write('\n'.join(new_lines)+'\n')
- f.close()
- def remove_temporary_source(self):
- """Remove the source files from this requirement, if they are marked
- for deletion"""
- if self.is_bundle or os.path.exists(self.delete_marker_filename):
- logger.info('Removing source in %s' % self.source_dir)
- if self.source_dir:
- shutil.rmtree(self.source_dir, ignore_errors=True, onerror=rmtree_errorhandler)
- self.source_dir = None
- if self._temp_build_dir and os.path.exists(self._temp_build_dir):
- shutil.rmtree(self._temp_build_dir, ignore_errors=True, onerror=rmtree_errorhandler)
- self._temp_build_dir = None
- def install_editable(self):
- logger.notify('Running setup.py develop for %s' % self.name)
- logger.indent += 2
- try:
- ## FIXME: should we do --install-headers here too?
- call_subprocess(
- [sys.executable, '-c',
- "import setuptools; __file__=%r; execfile(%r)" % (self.setup_py, self.setup_py),
- 'develop', '--no-deps'], cwd=self.source_dir, filter_stdout=self._filter_install,
- show_stdout=False)
- finally:
- logger.indent -= 2
- self.install_succeeded = True
- def _filter_install(self, line):
- level = Logger.NOTIFY
- for regex in [r'^running .*', r'^writing .*', '^creating .*', '^[Cc]opying .*',
- r'^reading .*', r"^removing .*\.egg-info' \(and everything under it\)$",
- r'^byte-compiling ',
- # Not sure what this warning is, but it seems harmless:
- r"^warning: manifest_maker: standard file '-c' not found$"]:
- if re.search(regex, line.strip()):
- level = Logger.INFO
- break
- return (level, line)
- def check_if_exists(self):
- """Find an installed distribution that satisfies or conflicts
- with this requirement, and set self.satisfied_by or
- self.conflicts_with appropriately."""
- if self.req is None:
- return False
- try:
- self.satisfied_by = pkg_resources.get_distribution(self.req)
- except pkg_resources.DistributionNotFound:
- return False
- except pkg_resources.VersionConflict:
- self.conflicts_with = pkg_resources.get_distribution(self.req.project_name)
- return True
- @property
- def is_bundle(self):
- if self._is_bundle is not None:
- return self._is_bundle
- base = self._temp_build_dir
- if not base:
- ## FIXME: this doesn't seem right:
- return False
- self._is_bundle = (os.path.exists(os.path.join(base, 'pip-manifest.txt'))
- or os.path.exists(os.path.join(base, 'pyinstall-manifest.txt')))
- return self._is_bundle
- def bundle_requirements(self):
- for dest_dir in self._bundle_editable_dirs:
- package = os.path.basename(dest_dir)
- ## FIXME: svnism:
- for vcs_backend in vcs.backends:
- url = rev = None
- vcs_bundle_file = os.path.join(
- dest_dir, vcs_backend.bundle_file)
- if os.path.exists(vcs_bundle_file):
- vc_type = vcs_backend.name
- fp = open(vcs_bundle_file)
- content = fp.read()
- fp.close()
- url, rev = vcs_backend().parse_vcs_bundle_file(content)
- break
- if url:
- url = '%s+%s@%s' % (vc_type, url, rev)
- else:
- url = None
- yield InstallRequirement(
- package, self, editable=True, url=url,
- update=False, source_dir=dest_dir)
- for dest_dir in self._bundle_build_dirs:
- package = os.path.basename(dest_dir)
- yield InstallRequirement(
- package, self,
- source_dir=dest_dir)
- def move_bundle_files(self, dest_build_dir, dest_src_dir):
- base = self._temp_build_dir
- assert base
- src_dir = os.path.join(base, 'src')
- build_dir = os.path.join(base, 'build')
- bundle_build_dirs = []
- bundle_editable_dirs = []
- for source_dir, dest_dir, dir_collection in [
- (src_dir, dest_src_dir, bundle_editable_dirs),
- (build_dir, dest_build_dir, bundle_build_dirs)]:
- if os.path.exists(source_dir):
- for dirname in os.listdir(source_dir):
- dest = os.path.join(dest_dir, dirname)
- dir_collection.append(dest)
- if os.path.exists(dest):
- logger.warn('The directory %s (containing package %s) already exists; cannot move source from bundle %s'
- % (dest, dirname, self))
- continue
- if not os.path.exists(dest_dir):
- logger.info('Creating directory %s' % dest_dir)
- os.makedirs(dest_dir)
- shutil.move(os.path.join(source_dir, dirname), dest)
- if not os.listdir(source_dir):
- os.rmdir(source_dir)
- self._temp_build_dir = None
- self._bundle_build_dirs = bundle_build_dirs
- self._bundle_editable_dirs = bundle_editable_dirs
- @property
- def delete_marker_filename(self):
- assert self.source_dir
- return os.path.join(self.source_dir, 'pip-delete-this-directory.txt')
- DELETE_MARKER_MESSAGE = '''\
- This file is placed here by pip to indicate the source was put
- here by pip.
- Once this package is successfully installed this source code will be
- deleted (unless you remove this file).
- '''
- class RequirementSet(object):
- def __init__(self, build_dir, src_dir, download_dir, download_cache=None,
- upgrade=False, ignore_installed=False,
- ignore_dependencies=False):
- self.build_dir = build_dir
- self.src_dir = src_dir
- self.download_dir = download_dir
- self.download_cache = download_cache
- self.upgrade = upgrade
- self.ignore_installed = ignore_installed
- self.requirements = {}
- # Mapping of alias: real_name
- self.requirement_aliases = {}
- self.unnamed_requirements = []
- self.ignore_dependencies = ignore_dependencies
- self.successfully_downloaded = []
- self.successfully_installed = []
- def __str__(self):
- reqs = [req for req in self.requirements.values()
- if not req.comes_from]
- reqs.sort(key=lambda req: req.name.lower())
- return ' '.join([str(req.req) for req in reqs])
- def add_requirement(self, install_req):
- name = install_req.name
- if not name:
- self.unnamed_requirements.append(install_req)
- else:
- if self.has_requirement(name):
- raise InstallationError(
- 'Double requirement given: %s (aready in %s, name=%r)'
- % (install_req, self.get_requirement(name), name))
- self.requirements[name] = install_req
- ## FIXME: what about other normalizations? E.g., _ vs. -?
- if name.lower() != name:
- self.requirement_aliases[name.lower()] = name
- def has_requirement(self, project_name):
- for name in project_name, project_name.lower():
- if name in self.requirements or name in self.requirement_aliases:
- return True
- return False
- @property
- def is_download(self):
- if self.download_dir:
- self.download_dir = os.path.expanduser(self.download_dir)
- if os.path.exists(self.download_dir):
- return True
- else:
- logger.fatal('Could not find download directory')
- raise InstallationError(
- "Could not find or access download directory '%s'"
- % display_path(self.download_dir))
- return False
- def get_requirement(self, project_name):
- for name in project_name, project_name.lower():
- if name in self.requirements:
- return self.requirements[name]
- if name in self.requirement_aliases:
- return self.requirements[self.requirement_aliases[name]]
- raise KeyError("No project with the name %r" % project_name)
- def uninstall(self, auto_confirm=False):
- for req in self.requirements.values():
- req.uninstall(auto_confirm=auto_confirm)
- def install_files(self, finder, force_root_egg_info=False):
- unnamed = list(self.unnamed_requirements)
- reqs = self.requirements.values()
- while reqs or unnamed:
- if unnamed:
- req_to_install = unnamed.pop(0)
- else:
- req_to_install = reqs.pop(0)
- install = True
- if not self.ignore_installed and not req_to_install.editable:
- req_to_install.check_if_exists()
- if req_to_install.satisfied_by:
- if self.upgrade:
- req_to_install.conflicts_with = req_to_install.satisfied_by
- req_to_install.satisfied_by = None
- else:
- install = False
- if req_to_install.satisfied_by:
- logger.notify('Requirement already satisfied '
- '(use --upgrade to upgrade): %s'
- % req_to_install)
- if req_to_install.editable:
- logger.notify('Obtaining %s' % req_to_install)
- elif install:
- if req_to_install.url and req_to_install.url.lower().startswith('file:'):
- logger.notify('Unpacking %s' % display_path(url_to_filename(req_to_install.url)))
- else:
- logger.notify('Downloading/unpacking %s' % req_to_install)
- logger.indent += 2
- is_bundle = False
- try:
- if req_to_install.editable:
- if req_to_install.source_dir is None:
- location = req_to_install.build_location(self.src_dir)
- req_to_install.source_dir = location
- else:
- location = req_to_install.source_dir
- if not os.path.exists(self.build_dir):
- os.makedirs(self.build_dir)
- req_to_install.update_editable(not self.is_download)
- if self.is_download:
- req_to_install.run_egg_info()
- req_to_install.archive(self.download_dir)
- else:
- req_to_install.run_egg_info()
- elif install:
- location = req_to_install.build_location(self.build_dir, not self.is_download)
- ## FIXME: is the existance of the checkout good enough to use it? I don't think so.
- unpack = True
- if not os.path.exists(os.path.join(location, 'setup.py')):
- ## FIXME: this won't upgrade when there's an existing package unpacked in `location`
- if req_to_install.url is None:
- url = finder.find_requirement(req_to_install, upgrade=self.upgrade)
- else:
- ## FIXME: should req_to_install.url already be a link?
- url = Link(req_to_install.url)
- assert url
- if url:
- try:
- self.unpack_url(url, location, self.is_download)
- except urllib2.HTTPError, e:
- logger.fatal('Could not install requirement %s because of error %s'
- % (req_to_install, e))
- raise InstallationError(
- 'Could not install requirement %s because of HTTP error %s for URL %s'
- % (req_to_install, e, url))
- else:
- unpack = False
- if unpack:
- is_bundle = req_to_install.is_bundle
- url = None
- if is_bundle:
- req_to_install.move_bundle_files(self.build_dir, self.src_dir)
- for subreq in req_to_install.bundle_requirements():
- reqs.append(subreq)
- self.add_requirement(subreq)
- elif self.is_download:
- req_to_install.source_dir = location
- if url and url.scheme in vcs.all_schemes:
- req_to_install.run_egg_info()
- req_to_install.archive(self.download_dir)
- else:
- req_to_install.source_dir = location
- req_to_install.run_egg_info()
- if force_root_egg_info:
- # We need to run this to make sure that the .egg-info/
- # directory is created for packing in the bundle
- req_to_install.run_egg_info(force_root_egg_info=True)
- req_to_install.assert_source_matches_version()
- f = open(req_to_install.delete_marker_filename, 'w')
- f.write(DELETE_MARKER_MESSAGE)
- f.close()
- if not is_bundle and not self.is_download:
- ## FIXME: shouldn't be globally added:
- finder.add_dependency_links(req_to_install.dependency_links)
- ## FIXME: add extras in here:
- if not self.ignore_dependencies:
- for req in req_to_install.requirements():
- try:
- name = pkg_resources.Requirement.parse(req).project_name
- except ValueError, e:
- ## FIXME: proper warning
- logger.error('Invalid requirement: %r (%s) in requirement %s' % (req, e, req_to_install))
- continue
- if self.has_requirement(name):
- ## FIXME: check for conflict
- continue
- subreq = InstallRequirement(req, req_to_install)
- reqs.append(subreq)
- self.add_requirement(subreq)
- if req_to_install.name not in self.requirements:
- self.requirements[req_to_install.name] = req_to_install
- else:
- req_to_install.remove_temporary_source()
- if install:
- self.successfully_downloaded.append(req_to_install)
- finally:
- logger.indent -= 2
- def unpack_url(self, link, location, only_download=False):
- if only_download:
- location = self.download_dir
- for backend in vcs.backends:
- if link.scheme in backend.schemes:
- vcs_backend = backend(link.url)
- if only_download:
- vcs_backend.export(location)
- else:
- vcs_backend.unpack(location)
- return
- dir = tempfile.mkdtemp()
- if link.url.lower().startswith('file:'):
- source = url_to_filename(link.url)
- content_type = mimetypes.guess_type(source)[0]
- self.unpack_file(source, location, content_type, link)
- return
- md5_hash = link.md5_hash
- target_url = link.url.split('#', 1)[0]
- target_file = None
- if self.download_cache:
- if not os.path.isdir(self.download_cache):
- logger.indent -= 2
- logger.notify('Creating supposed download cache at %s' % self.download_cache)
- logger.indent += 2
- os.makedirs(self.download_cache)
- target_file = os.path.join(self.download_cache,
- urllib.quote(target_url, ''))
- if (target_file and os.path.exists(target_file)
- and os.path.exists(target_file+'.content-type')):
- fp = open(target_file+'.content-type')
- content_type = fp.read().strip()
- fp.close()
- if md5_hash:
- download_hash = md5()
- fp = open(target_file, 'rb')
- while 1:
- chunk = fp.read(4096)
- if not chunk:
- break
- download_hash.update(chunk)
- fp.close()
- temp_location = target_file
- logger.notify('Using download cache from %s' % target_file)
- else:
- try:
- resp = urllib2.urlopen(target_url)
- except urllib2.HTTPError, e:
- logger.fatal("HTTP error %s while getting %s" % (e.code, link))
- raise
- except IOError, e:
- # Typically an FTP error
- logger.fatal("Error %s while getting %s" % (e, link))
- raise
- content_type = resp.info()['content-type']
- filename = link.filename
- ext = splitext(filename)[1]
- if not ext:
- ext = mimetypes.guess_extension(content_type)
- if ext:
- filename += ext
- if not ext and link.url != resp.geturl():
- ext = os.path.splitext(resp.geturl())[1]
- if ext:
- filename += ext
- temp_location = os.path.join(dir, filename)
- fp = open(temp_location, 'wb')
- if md5_hash:
- download_hash = md5()
- try:
- total_length = int(resp.info()['content-length'])
- except (ValueError, KeyError):
- total_length = 0
- downloaded = 0
- show_progress = total_length > 40*1000 or not total_length
- show_url = link.show_url
- try:
- if show_progress:
- ## FIXME: the URL can get really long in this message:
- if total_length:
- logger.start_progress('Downloading %s (%s): ' % (show_url, format_size(total_length)))
- else:
- logger.start_progress('Downloading %s (unknown size): ' % show_url)
- else:
- logger.notify('Downloading %s' % show_url)
- logger.debug('Downloading from URL %s' % link)
- while 1:
- chunk = resp.read(4096)
- if not chunk:
- break
- downloaded += len(chunk)
- if show_progress:
- if not total_length:
- logger.show_progress('%s' % format_size(downloaded))
- else:
- logger.show_progress('%3i%% %s' % (100*downloaded/total_length, format_size(downloaded)))
- if md5_hash:
- download_hash.update(chunk)
- fp.write(chunk)
- fp.close()
- finally:
- if show_progress:
- logger.end_progress('%s downloaded' % format_size(downloaded))
- if md5_hash:
- download_hash = download_hash.hexdigest()
- if download_hash != md5_hash:
- logger.fatal("MD5 hash of the package %s (%s) doesn't match the expected hash %s!"
- % (link, download_hash, md5_hash))
- raise InstallationError('Bad MD5 hash for package %s' % link)
- if only_download:
- self.copy_file(temp_location, location, content_type, link)
- else:
- self.unpack_file(temp_location, location, content_type, link)
- if target_file and target_file != temp_location:
- logger.notify('Storing download in cache at %s' % display_path(target_file))
- shutil.copyfile(temp_location, target_file)
- fp = open(target_file+'.content-type', 'w')
- fp.write(content_type)
- fp.close()
- os.unlink(temp_location)
- if target_file is None:
- os.unlink(temp_location)
- def copy_file(self, filename, location, content_type, link):
- copy = True
- download_location = os.path.join(location, link.filename)
- if os.path.exists(download_location):
- response = ask('The file %s exists. (i)gnore, (w)ipe, (b)ackup '
- % display_path(download_location), ('i', 'w', 'b'))
- if response == 'i':
- copy = False
- elif response == 'w':
- logger.warn('Deleting %s' % display_path(download_location))
- os.remove(download_location)
- elif response == 'b':
- dest_file = backup_dir(download_location)
- logger.warn('Backing up %s to %s'
- % (display_path(download_location), display_path(dest_file)))
- shutil.move(download_location, dest_file)
- if copy:
- shutil.copy(filename, download_location)
- logger.indent -= 2
- logger.notify('Saved %s' % display_path(download_location))
- def unpack_file(self, filename, location, content_type, link):
- if (content_type == 'application/zip'
- or filename.endswith('.zip')
- or filename.endswith('.pybundle')
- or zipfile.is_zipfile(filename)):
- self.unzip_file(filename, location, flatten=not filename.endswith('.pybundle'))
- elif (content_type == 'application/x-gzip'
- or tarfile.is_tarfile(filename)
- or splitext(filename)[1].lower() in ('.tar', '.tar.gz', '.tar.bz2', '.tgz', '.tbz')):
- self.untar_file(filename, location)
- elif (content_type and content_type.startswith('text/html')
- and is_svn_page(file_contents(filename))):
- # We don't really care about this
- Subversion('svn+' + link.url).unpack(location)
- else:
- ## FIXME: handle?
- ## FIXME: magic signatures?
- logger.fatal('Cannot unpack file %s (downloaded from %s, content-type: %s); cannot detect archive format'
- % (filename, location, content_type))
- raise InstallationError('Cannot determine archive format of %s' % location)
- def unzip_file(self, filename, location, flatten=True):
- """Unzip the file (zip file located at filename) to the destination
- location"""
- if not os.path.exists(location):
- os.makedirs(location)
- zipfp = open(filename, 'rb')
- try:
- zip = zipfile.ZipFile(zipfp)
- leading = has_leading_dir(zip.namelist()) and flatten
- for name in zip.namelist():
- data = zip.read(name)
- fn = name
- if leading:
- fn = split_leading_dir(name)[1]
- fn = os.path.join(location, fn)
- dir = os.path.dirname(fn)
- if not os.path.exists(dir):
- os.makedirs(dir)
- if fn.endswith('/') or fn.endswith('\\'):
- # A directory
- if not os.path.exists(fn):
- os.makedirs(fn)
- else:
- fp = open(fn, 'wb')
- try:
- fp.write(data)
- finally:
- fp.close()
- finally:
- zipfp.close()
- def untar_file(self, filename, location):
- """Untar the file (tar file located at filename) to the destination location"""
- if not os.path.exists(location):
- os.makedirs(location)
- if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'):
- mode = 'r:gz'
- elif filename.lower().endswith('.bz2') or filename.lower().endswith('.tbz'):
- mode = 'r:bz2'
- elif filename.lower().endswith('.tar'):
- mode = 'r'
- else:
- logger.warn('Cannot determine compression type for file %s' % filename)
- mode = 'r:*'
- tar = tarfile.open(filename, mode)
- try:
- leading = has_leading_dir([member.name for member in tar.getmembers()])
- for member in tar.getmembers():
- fn = member.name
- if leading:
- fn = split_leading_dir(fn)[1]
- path = os.path.join(location, fn)
- if member.isdir():
- if not os.path.exists(path):
- os.makedirs(path)
- else:
- try:
- fp = tar.extractfile(member)
- except (KeyError, AttributeError), e:
- # Some corrupt tar files seem to produce this
- # (specifically bad symlinks)
- logger.warn(
- 'In the tar file %s the member %s is invalid: %s'
- % (filename, member.name, e))
- continue
- if not os.path.exists(os.path.dirname(path)):
- os.makedirs(os.path.dirname(path))
- destfp = open(path, 'wb')
- try:
- shutil.copyfileobj(fp, destfp)
- finally:
- destfp.close()
- fp.close()
- finally:
- tar.close()
- def install(self, install_options):
- """Install everything in this set (after having downloaded and unpacked the packages)"""
- to_install = sorted([r for r in self.requirements.values()
- if self.upgrade or not r.satisfied_by],
- key=lambda p: p.name.lower())
- if to_install:
- logger.notify('Installing collected packages: %s' % (', '.join([req.name for req in to_install])))
- logger.indent += 2
- try:
- for requirement in to_install:
- if requirement.conflicts_with:
- logger.notify('Found existing installation: %s'
- % requirement.conflicts_with)
- logger.indent += 2
- try:
- requirement.uninstall(auto_confirm=True)
- finally:
- logger.indent -= 2
- try:
- requirement.install(install_options)
- except:
- # if install did not succeed, rollback previous uninstall
- if requirement.conflicts_with and not requirement.install_succeeded:
- requirement.rollback_uninstall()
- raise
- requirement.remove_temporary_source()
- finally:
- logger.indent -= 2
- self.successfully_installed = to_install
- def create_bundle(self, bundle_filename):
- ## FIXME: can't decide which is better; zip is easier to read
- ## random files from, but tar.bz2 is smaller and not as lame a
- ## format.
- ## FIXME: this file should really include a manifest of the
- ## packages, maybe some other metadata files. It would make
- ## it easier to detect as well.
- zip = zipfile.ZipFile(bundle_filename, 'w', zipfile.ZIP_DEFLATED)
- vcs_dirs = []
- for dir, basename in (self.build_dir, 'build'), (self.src_dir, 'src'):
- dir = os.path.normcase(os.path.abspath(dir))
- for dirpath, dirnames, filenames in os.walk(dir):
- for backend in vcs.backends:
- vcs_backend = backend()
- vcs_url = vcs_rev = None
- if vcs_backend.dirname in dirnames:
- for vcs_dir in vcs_dirs:
- if dirpath.startswith(vcs_dir):
- # vcs bundle file already in parent directory
- break
- else:
- vcs_url, vcs_rev = vcs_backend.get_info(
- os.path.join(dir, dirpath))
- vcs_dirs.append(dirpath)
- vcs_bundle_file = vcs_backend.bundle_file
- vcs_guide = vcs_backend.guide % {'url': vcs_url,
- 'rev': vcs_rev}
- dirnames.remove(vcs_backend.dirname)
- break
- if 'pip-egg-info' in dirnames:
- dirnames.remove('pip-egg-info')
- for dirname in dirnames:
- dirname = os.path.join(dirpath, dirname)
- name = self._clean_zip_name(dirname, dir)
- zip.writestr(basename + '/' + name + '/', '')
- for filename in filenames:
- if filename == 'pip-delete-this-directory.txt':
- continue
- filename = os.path.join(dirpath, filename)
- name = self._clean_zip_name(filename, dir)
- zip.write(filename, basename + '/' + name)
- if vcs_url:
- name = os.path.join(dirpath, vcs_bundle_file)
- name = self._clean_zip_name(name, dir)
- zip.writestr(basename + '/' + name, vcs_guide)
- zip.writestr('pip-manifest.txt', self.bundle_requirements())
- zip.close()
- # Unlike installation, this will always delete the build directories
- logger.info('Removing temporary build dir %s and source dir %s'
- % (self.build_dir, self.src_dir))
- for dir in self.build_dir, self.src_dir:
- if os.path.exists(dir):
- shutil.rmtree(dir)
- BUNDLE_HEADER = '''\
- # This is a pip bundle file, that contains many source packages
- # that can be installed as a group. You can install this like:
- # pip this_file.zip
- # The rest of the file contains a list of all the packages included:
- '''
- def bundle_requirements(self):
- parts = [self.BUNDLE_HEADER]
- for req in sorted(
- [req for req in self.requirements.values()
- if not req.comes_from],
- key=lambda x: x.name):
- parts.append('%s==%s\n' % (req.name, req.installed_version))
- parts.append('# These packages were installed to satisfy the above requirements:\n')
- for req in sorted(
- [req for req in self.requirements.values()
- if req.comes_from],
- key=lambda x: x.name):
- parts.append('%s==%s\n' % (req.name, req.installed_version))
- ## FIXME: should we do something with self.unnamed_requirements?
- return ''.join(parts)
- def _clean_zip_name(self, name, prefix):
- assert name.startswith(prefix+'/'), (
- "name %r doesn't start with prefix %r" % (name, prefix))
- name = name[len(prefix)+1:]
- name = name.replace(os.path.sep, '/')
- return name
- class HTMLPage(object):
- """Represents one page, along with its URL"""
- ## FIXME: these regexes are horrible hacks:
- _homepage_re = re.compile(r'<th>\s*home\s*page', re.I)
- _download_re = re.compile(r'<th>\s*download\s+url', re.I)
- ## These aren't so aweful:
- _rel_re = re.compile("""<[^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*>""", re.I)
- _href_re = re.compile('href=(?:"([^"]*)"|\'([^\']*)\'|([^>\\s\\n]*))', re.I|re.S)
- _base_re = re.compile(r"""<base\s+href\s*=\s*['"]?([^'">]+)""", re.I)
- def __init__(self, content, url, headers=None):
- self.content = content
- self.url = url
- self.headers = headers
- def __str__(self):
- return self.url
- @classmethod
- def get_page(cls, link, req, cache=None, skip_archives=True):
- url = link.url
- url = url.split('#', 1)[0]
- if cache.too_many_failures(url):
- return None
- if url.lower().startswith('svn'):
- logger.debug('Cannot look at svn URL %s' % link)
- return None
- if cache is not None:
- inst = cache.get_page(url)
- if inst is not None:
- return inst
- try:
- if skip_archives:
- if cache is not None:
- if cache.is_archive(url):
- return None
- filename = link.filename
- for bad_ext in ['.tar', '.tar.gz', '.tar.bz2', '.tgz', '.zip']:
- if filename.endswith(bad_ext):
- content_type = cls._get_content_type(url)
- if content_type.lower().startswith('text/html'):
- break
- else:
- logger.debug('Skipping page %s because of Content-Type: %s' % (link, content_type))
- if cache is not None:
- cache.set_is_archive(url)
- return None
- logger.debug('Getting page %s' % url)
- resp = urllib2.urlopen(url)
- real_url = resp.geturl()
- headers = resp.info()
- inst = cls(resp.read(), real_url, headers)
- except (urllib2.HTTPError, urllib2.URLError, socket.timeout, socket.error), e:
- desc = str(e)
- if isinstance(e, socket.timeout):
- log_meth = logger.info
- level =1
- desc = 'timed out'
- elif isinstance(e, urllib2.URLError):
- log_meth = logger.info
- if hasattr(e, 'reason') and isinstance(e.reason, socket.timeout):
- desc = 'timed out'
- level = 1
- else:
- level = 2
- elif isinstance(e, urllib2.HTTPError) and e.code == 404:
- ## FIXME: notify?
- log_meth = logger.info
- level = 2
- else:
- log_meth = logger.info
- level = 1
- log_meth('Could not fetch URL %s: %s' % (link, desc))
- log_meth('Will skip URL %s when looking for download links for %s' % (link.url, req))
- if cache is not None:
- cache.add_page_failure(url, level)
- return None
- if cache is not None:
- cache.add_page([url, real_url], inst)
- return inst
- @staticmethod
- def _get_content_type(url):
- """Get the Content-Type of the given url, using a HEAD request"""
- scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
- if scheme == 'http':
- ConnClass = httplib.HTTPConnection
- elif scheme == 'https':
- ConnClass = httplib.HTTPSConnection
- else:
- ## FIXME: some warning or something?
- ## assertion error?
- return ''
- if query:
- path += '?' + query
- conn = ConnClass(netloc)
- try:
- conn.request('HEAD', path, headers={'Host': netloc})
- resp = conn.getresponse()
- if resp.status != 200:
- ## FIXME: doesn't handle redirects
- return ''
- return resp.getheader('Content-Type') or ''
- finally:
- conn.close()
- @property
- def base_url(self):
- if not hasattr(self, "_base_url"):
- match = self._base_re.search(self.content)
- if match:
- self._base_url = match.group(1)
- else:
- self._base_url = self.url
- return self._base_url
- @property
- def links(self):
- """Yields all links in the page"""
- for match in self._href_re.finditer(self.content):
- url = match.group(1) or match.group(2) or match.group(3)
- url = self.clean_link(urlparse.urljoin(self.base_url, url))
- yield Link(url, self)
- def rel_links(self):
- for url in self.explicit_rel_links():
- yield url
- for url in self.scraped_rel_links():
- yield url
- def explicit_rel_links(self, rels=('homepage', 'download')):
- """Yields all links with the given relations"""
- for match in self._rel_re.finditer(self.content):
- found_rels = match.group(1).lower().split()
- for rel in rels:
- if rel in found_rels:
- break
- else:
- continue
- match = self._href_re.search(match.group(0))
- if not match:
- continue
- url = match.group(1) or match.group(2) or match.group(3)
- url = self.clean_link(urlparse.urljoin(self.base_url, url))
- yield Link(url, self)
- def scraped_rel_links(self):
- for regex in (self._homepage_re, self._download_re):
- match = regex.search(self.content)
- if not match:
- continue
- href_match = self._href_re.search(self.content, pos=match.end())
- if not href_match:
- continue
- url = match.group(1) or match.group(2) or match.group(3)
- if not url:
- continue
- url = self.clean_link(urlparse.urljoin(self.base_url, url))
- yield Link(url, self)
- _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I)
- def clean_link(self, url):
- """Makes sure a link is fully encoded. That is, if a ' ' shows up in
- the link, it will be rewritten to %20 (while not over-quoting
- % or other characters)."""
- return self._clean_re.sub(
- lambda match: '%%%2x' % ord(match.group(0)), url)
- class PageCache(object):
- """Cache of HTML pages"""
- failure_limit = 3
- def __init__(self):
- self._failures = {}
- self._pages = {}
- self._archives = {}
- def too_many_failures(self, url):
- return self._failures.get(url, 0) >= self.failure_limit
- def get_page(self, url):
- return self._pages.get(url)
- def is_archive(self, url):
- return self._archives.get(url, False)
- def set_is_archive(self, url, value=True):
- self._archives[url] = value
- def add_page_failure(self, url, level):
- self._failures[url] = self._failures.get(url, 0)+level
- def add_page(self, urls, page):
- for url in urls:
- self._pages[url] = page
- class Link(object):
- def __init__(self, url, comes_from=None):
- self.url = url
- self.comes_from = comes_from
- def __str__(self):
- if self.comes_from:
- return '%s (from %s)' % (self.url, self.comes_from)
- else:
- return self.url
- def __repr__(self):
- return '<Link %s>' % self
- @property
- def filename(self):
- url = self.url
- url = url.split('#', 1)[0]
- url = url.split('?', 1)[0]
- url = url.rstrip('/')
- name = posixpath.basename(url)
- assert name, (
- 'URL %r produced no filename' % url)
- return name
- @property
- def scheme(self):
- return urlparse.urlsplit(self.url)[0]
- @property
- def path(self):
- return urlparse.urlsplit(self.url)[2]
- def splitext(self):
- return splitext(posixpath.basename(self.path.rstrip('/')))
- _egg_fragment_re = re.compile(r'#egg=([^&]*)')
- @property
- def egg_fragment(self):
- match = self._egg_fragment_re.search(self.url)
- if not match:
- return None
- return match.group(1)
- _md5_re = re.compile(r'md5=([a-f0-9]+)')
- @property
- def md5_hash(self):
- match = self._md5_re.search(self.url)
- if match:
- return match.group(1)
- return None
- @property
- def show_url(self):
- return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0])
- ############################################################
- ## Writing freeze files
- class FrozenRequirement(object):
- def __init__(self, name, req, editable, comments=()):
- self.name = name
- self.req = req
- self.editable = editable
- self.comments = comments
- _rev_re = re.compile(r'-r(\d+)$')
- _date_re = re.compile(r'-(20\d\d\d\d\d\d)$')
- @classmethod
- def from_dist(cls, dist, dependency_links, find_tags=False):
- location = os.path.normcase(os.path.abspath(dist.location))
- comments = []
- if vcs.get_backend_name(location):
- editable = True
- req = get_src_requirement(dist, location, find_tags)
- if req is None:
- logger.warn('Could not determine repository location of %s' % location)
- comments.append('## !! Could not determine repository location')
- req = dist.as_requirement()
- editable = False
- else:
- editable = False
- req = dist.as_requirement()
- specs = req.specs
- assert len(specs) == 1 and specs[0][0] == '=='
- version = specs[0][1]
- ver_match = cls._rev_re.search(version)
- date_match = cls._date_re.search(version)
- if ver_match or date_match:
- svn_backend = vcs.get_backend('svn')
- if svn_backend:
- svn_location = svn_backend(
- ).get_location(dist, dependency_links)
- if not svn_location:
- logger.warn(
- 'Warning: cannot find svn location for %s' % req)
- comments.append('## FIXME: could not find svn URL in dependency_links for this package:')
- else:
- comments.append('# Installing as editable to satisfy requirement %s:' % req)
- if ver_match:
- rev = ver_match.group(1)
- else:
- rev = '{%s}' % date_match.group(1)
- editable = True
- req = 'svn+%s@%s#egg=%s' % (svn_location, rev, cls.egg_name(dist))
- return cls(dist.project_name, req, editable, comments)
- @staticmethod
- def egg_name(dist):
- name = dist.egg_name()
- match = re.search(r'-py\d\.\d$', name)
- if match:
- name = name[:match.start()]
- return name
- def __str__(self):
- req = self.req
- if self.editable:
- req = '-e %s' % req
- return '\n'.join(list(self.comments)+[str(req)])+'\n'
- class VersionControl(object):
- name = ''
- dirname = ''
- def __init__(self, url=None, *args, **kwargs):
- self.url = url
- self._cmd = None
- super(VersionControl, self).__init__(*args, **kwargs)
- def _filter(self, line):
- return (Logger.INFO, line)
- @property
- def cmd(self):
- if self._cmd is not None:
- return self._cmd
- command = find_command(self.name)
- if command is None:
- raise BadCommand('Cannot find command %s' % self.name)
- logger.info('Found command %s at %s' % (self.name, command))
- self._cmd = command
- return command
- def get_url_rev(self):
- """
- Returns the correct repository URL and revision by parsing the given
- repository URL
- """
- url = self.url.split('+', 1)[1]
- scheme, netloc, path, query, frag = urlparse.urlsplit(url)
- rev = None
- if '@' in path:
- path, rev = path.rsplit('@', 1)
- url = urlparse.urlunsplit((scheme, netloc, path, query, ''))
- return url, rev
- def get_info(self, location):
- """
- Returns (url, revision), where both are strings
- """
- assert not location.rstrip('/').endswith(self.dirname), 'Bad directory: %s' % location
- return self.get_url(location), self.get_revision(location)
- def normalize_url(self, url):
- """
- Normalize a URL for comparison by unquoting it and removing any trailing slash.
- """
- return urllib.unquote(url).rstrip('/')
- def compare_urls(self, url1, url2):
- """
- Compare two repo URLs for identity, ignoring incidental differences.
- """
- return (self.normalize_url(url1) == self.normalize_url(url2))
- def parse_vcs_bundle_file(self, content):
- """
- Takes the contents of the bundled text file that explains how to revert
- the stripped off version control data of the given package and returns
- the URL and revision of it.
- """
- raise NotImplementedError
- def obtain(self, dest):
- """
- Called when installing or updating an editable package, takes the
- source path of the checkout.
- """
- raise NotImplementedError
- def switch(self, dest, url, rev_options):
- """
- Switch the repo at ``dest`` to point to ``URL``.
- """
- raise NotImplemented
- def update(self, dest, rev_options):
- """
- Update an already-existing repo to the given ``rev_options``.
- """
- raise NotImplementedError
- def check_destination(self, dest, url, rev_options, rev_display):
- """
- Prepare a location to receive a checkout/clone.
- Return True if the location is ready for (and requires) a
- checkout/clone, False otherwise.
- """
- checkout = True
- prompt = False
- if os.path.exists(dest):
- checkout = False
- if os.path.exists(os.path.join(dest, self.dirname)):
- existing_url = self.get_url(dest)
- if self.compare_urls(existing_url, url):
- logger.info('%s in %s exists, and has correct URL (%s)'
- % (self.repo_name.title(), display_path(dest), url))
- logger.notify('Updating %s %s%s'
- % (display_path(dest), self.repo_name, rev_display))
- self.update(dest, rev_options)
- else:
- logger.warn('%s %s in %s exists with URL %s'
- % (self.name, self.repo_name, display_path(dest), existing_url))
- prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', ('s', 'i', 'w', 'b'))
- else:
- logger.warn('Directory %s already exists, and is not a %s %s.'
- % (dest, self.name, self.repo_name))
- prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b'))
- if prompt:
- logger.warn('The plan is to install the %s repository %s'
- % (self.name, url))
- response = ask('What to do? %s' % prompt[0], prompt[1])
- if response == 's':
- logger.notify('Switching %s %s to %s%s'
- % (self.repo_name, display_path(dest), url, rev_display))
- self.switch(dest, url, rev_options)
- elif response == 'i':
- # do nothing
- pass
- elif response == 'w':
- logger.warn('Deleting %s' % display_path(dest))
- shutil.rmtree(dest)
- checkout = True
- elif response == 'b':
- dest_dir = backup_dir(dest)
- logger.warn('Backing up %s to %s'
- % (display_path(dest), dest_dir))
- shutil.move(dest, dest_dir)
- checkout = True
- return checkout
- def unpack(self, location):
- raise NotImplementedError
- def get_src_requirement(self, dist, location, find_tags=False):
- raise NotImplementedError
- _svn_xml_url_re = re.compile('url="([^"]+)"')
- _svn_rev_re = re.compile('committed-rev="(\d+)"')
- _svn_url_re = re.compile(r'URL: (.+)')
- _svn_revision_re = re.compile(r'Revision: (.+)')
- class Subversion(VersionControl):
- name = 'svn'
- dirname = '.svn'
- repo_name = 'checkout'
- schemes = ('svn', 'svn+ssh', 'svn+http', 'svn+https')
- bundle_file = 'svn-checkout.txt'
- guide = ('# This was an svn checkout; to make it a checkout again run:\n'
- 'svn checkout --force -r %(rev)s %(url)s .\n')
- def get_info(self, location):
- """Returns (url, revision), where both are strings"""
- assert not location.rstrip('/').endswith(self.dirname), 'Bad directory: %s' % location
- output = call_subprocess(
- ['svn', 'info', location], show_stdout=False, extra_environ={'LANG': 'C'})
- match = _svn_url_re.search(output)
- if not match:
- logger.warn('Cannot determine URL of svn checkout %s' % display_path(location))
- logger.info('Output that cannot be parsed: \n%s' % output)
- return None, None
- url = match.group(1).strip()
- match = _svn_revision_re.search(output)
- if not match:
- logger.warn('Cannot determine revision of svn checkout %s' % display_path(location))
- logger.info('Output that cannot be parsed: \n%s' % output)
- return url, None
- return url, match.group(1)
- def get_url(self, location):
- return self.get_info(location)[0]
- def get_revision(self, location):
- return self.get_info(location)[1]
- def parse_vcs_bundle_file(self, content):
- for line in content.splitlines():
- if not line.strip() or line.strip().startswith('#'):
- continue
- match = re.search(r'^-r\s*([^ ])?', line)
- if not match:
- return None, None
- rev = match.group(1)
- rest = line[match.end():].strip().split(None, 1)[0]
- return rest, rev
- return None, None
- def unpack(self, location):
- """Check out the svn repository at the url to the destination location"""
- url, rev = self.get_url_rev()
- logger.notify('Checking out svn repository %s to %s' % (url, location))
- logger.indent += 2
- try:
- if os.path.exists(location):
- # Subversion doesn't like to check out over an existing directory
- # --force fixes this, but was only added in svn 1.5
- shutil.rmtree(location, onerror=rmtree_errorhandler)
- call_subprocess(
- ['svn', 'checkout', url, location],
- filter_stdout=self._filter, show_stdout=False)
- finally:
- logger.indent -= 2
- def export(self, location):
- """Export the svn repository at the url to the destination location"""
- url, rev = self.get_url_rev()
- logger.notify('Checking out svn repository %s to %s' % (url, location))
- logger.indent += 2
- try:
- if os.path.exists(location):
- # Subversion doesn't like to check out over an existing directory
- # --force fixes this, but was only added in svn 1.5
- shutil.rmtree(location, onerror=rmtree_errorhandler)
- call_subprocess(
- ['svn', 'export', url, location],
- filter_stdout=self._filter, show_stdout=False)
- finally:
- logger.indent -= 2
- def switch(self, dest, url, rev_options):
- call_subprocess(
- ['svn', 'switch'] + rev_options + [url, dest])
- def update(self, dest, rev_options):
- call_subprocess(
- ['svn', 'update'] + rev_options + [dest])
- def obtain(self, dest):
- url, rev = self.get_url_rev()
- if rev:
- rev_options = ['-r', rev]
- rev_display = ' (to revision %s)' % rev
- else:
- rev_options = []
- rev_display = ''
- if self.check_destination(dest, url, rev_options, rev_display):
- logger.notify('Checking out %s%s to %s'
- % (url, rev_display, display_path(dest)))
- call_subprocess(
- ['svn', 'checkout', '-q'] + rev_options + [url, dest])
- def get_location(self, dist, dependency_links):
- egg_fragment_re = re.compile(r'#egg=(.*)$')
- for url in dependency_links:
- egg_fragment = Link(url).egg_fragment
- if not egg_fragment:
- continue
- if '-' in egg_fragment:
- ## FIXME: will this work when a package has - in the name?
- key = '-'.join(egg_fragment.split('-')[:-1]).lower()
- else:
- key = egg_fragment
- if key == dist.key:
- return url.split('#', 1)[0]
- return None
- def get_revision(self, location):
- """
- Return the maximum revision for all files under a given location
- """
- # Note: taken from setuptools.command.egg_info
- revision = 0
- for base, dirs, files in os.walk(location):
- if self.dirname not in dirs:
- dirs[:] = []
- continue # no sense walking uncontrolled subdirs
- dirs.remove(self.dirname)
- entries_fn = os.path.join(base, self.dirname, 'entries')
- if not os.path.exists(entries_fn):
- ## FIXME: should we warn?
- continue
- f = open(entries_fn)
- data = f.read()
- f.close()
- if data.startswith('8') or data.startswith('9') or data.startswith('10'):
- data = map(str.splitlines,data.split('\n\x0c\n'))
- del data[0][0] # get rid of the '8'
- dirurl = data[0][3]
- revs = [int(d[9]) for d in data if len(d)>9 and d[9]]+[0]
- if revs:
- localrev = max(revs)
- else:
- localrev = 0
- elif data.startswith('<?xml'):
- dirurl = _svn_xml_url_re.search(data).group(1) # get repository URL
- revs = [int(m.group(1)) for m in _svn_rev_re.finditer(data)]+[0]
- if revs:
- localrev = max(revs)
- else:
- localrev = 0
- else:
- logger.warn("Unrecognized .svn/entries format; skipping %s", base)
- dirs[:] = []
- continue
- if base == location:
- base_url = dirurl+'/' # save the root url
- elif not dirurl.startswith(base_url):
- dirs[:] = []
- continue # not part of the same svn tree, skip it
- revision = max(revision, localrev)
- return revision
- def get_url(self, location):
- # In cases where the source is in a subdirectory, not alongside setup.py
- # we have to look up in the location until we find a real setup.py
- orig_location = location
- while not os.path.exists(os.path.join(location, 'setup.py')):
- last_location = location
- location = os.path.dirname(location)
- if location == last_location:
- # We've traversed up to the root of the filesystem without finding setup.py
- logger.warn("Could not find setup.py for directory %s (tried all parent directories)"
- % orig_location)
- return None
- f = open(os.path.join(location, self.dirname, 'entries'))
- data = f.read()
- f.close()
- if data.startswith('8') or data.startswith('9') or data.startswith('10'):
- data = map(str.splitlines,data.split('\n\x0c\n'))
- del data[0][0] # get rid of the '8'
- return data[0][3]
- elif data.startswith('<?xml'):
- match = _svn_xml_url_re.search(data)
- if not match:
- raise ValueError('Badly formatted data: %r' % data)
- return match.group(1) # get repository URL
- else:
- logger.warn("Unrecognized .svn/entries format in %s" % location)
- # Or raise exception?
- return None
- def get_tag_revs(self, svn_tag_url):
- stdout = call_subprocess(
- ['svn', 'ls', '-v', svn_tag_url], show_stdout=False)
- results = []
- for line in stdout.splitlines():
- parts = line.split()
- rev = int(parts[0])
- tag = parts[-1].strip('/')
- results.append((tag, rev))
- return results
- def find_tag_match(self, rev, tag_revs):
- best_match_rev = None
- best_tag = None
- for tag, tag_rev in tag_revs:
- if (tag_rev > rev and
- (best_match_rev is None or best_match_rev > tag_rev)):
- # FIXME: Is best_match > tag_rev really possible?
- # or is it a sign something is wacky?
- best_match_rev = tag_rev
- best_tag = tag
- return best_tag
- def get_src_requirement(self, dist, location, find_tags=False):
- repo = self.get_url(location)
- if repo is None:
- return None
- parts = repo.split('/')
- ## FIXME: why not project name?
- egg_project_name = dist.egg_name().split('-', 1)[0]
- rev = self.get_revision(location)
- if parts[-2] in ('tags', 'tag'):
- # It's a tag, perfect!
- full_egg_name = '%s-%s' % (egg_project_name, parts[-1])
- elif parts[-2] in ('branches', 'branch'):
- # It's a branch :(
- full_egg_name = '%s-%s-r%s' % (dist.egg_name(), parts[-1], rev)
- elif parts[-1] == 'trunk':
- # Trunk :-/
- full_egg_name = '%s-dev_r%s' % (dist.egg_name(), rev)
- if find_tags:
- tag_url = '/'.join(parts[:-1]) + '/tags'
- tag_revs = self.get_tag_revs(tag_url)
- match = self.find_tag_match(rev, tag_revs)
- if match:
- logger.notify('trunk checkout %s seems to be equivalent to tag %s' % match)
- repo = '%s/%s' % (tag_url, match)
- full_egg_name = '%s-%s' % (egg_project_name, match)
- else:
- # Don't know what it is
- logger.warn('svn URL does not fit normal structure (tags/branches/trunk): %s' % repo)
- full_egg_name = '%s-dev_r%s' % (egg_project_name, rev)
- return 'svn+%s@%s#egg=%s' % (repo, rev, full_egg_name)
- vcs.register(Subversion)
- class Git(VersionControl):
- name = 'git'
- dirname = '.git'
- repo_name = 'clone'
- schemes = ('git', 'git+http', 'git+ssh', 'git+git')
- bundle_file = 'git-clone.txt'
- guide = ('# This was a Git repo; to make it a repo again run:\n'
- 'git init\ngit remote add origin %(url)s -f\ngit checkout %(rev)s\n')
- def parse_vcs_bundle_file(self, content):
- url = rev = None
- for line in content.splitlines():
- if not line.strip() or line.strip().startswith('#'):
- continue
- url_match = re.search(r'git\s*remote\s*add\s*origin(.*)\s*-f', line)
- if url_match:
- url = url_match.group(1).strip()
- rev_match = re.search(r'^git\s*checkout\s*-q\s*(.*)\s*', line)
- if rev_match:
- rev = rev_match.group(1).strip()
- if url and rev:
- return url, rev
- return None, None
- def unpack(self, location):
- """Clone the Git repository at the url to the destination location"""
- url, rev = self.get_url_rev()
- logger.notify('Cloning Git repository %s to %s' % (url, location))
- logger.indent += 2
- try:
- if os.path.exists(location):
- os.rmdir(location)
- call_subprocess(
- [self.cmd, 'clone', url, location],
- filter_stdout=self._filter, show_stdout=False)
- finally:
- logger.indent -= 2
- def export(self, location):
- """Export the Git repository at the url to the destination location"""
- temp_dir = tempfile.mkdtemp('-export', 'pip-')
- self.unpack(temp_dir)
- try:
- if not location.endswith('/'):
- location = location + '/'
- call_subprocess(
- [self.cmd, 'checkout-index', '-a', '-f', '--prefix', location],
- filter_stdout=self._filter, show_stdout=False, cwd=temp_dir)
- finally:
- shutil.rmtree(temp_dir)
- def check_rev_options(self, rev, dest, rev_options):
- """Check the revision options before checkout to compensate that tags
- and branches may need origin/ as a prefix"""
- if rev is None:
- # bail and use preset
- return rev_options
- revisions = self.get_tag_revs(dest)
- revisions.update(self.get_branch_revs(dest))
- if rev in revisions:
- # if rev is a sha
- return [rev]
- inverse_revisions = dict((v,k) for k, v in revisions.iteritems())
- if rev not in inverse_revisions: # is rev a name or tag?
- origin_rev = 'origin/%s' % rev
- if origin_rev in inverse_revisions:
- rev = inverse_revisions[origin_rev]
- else:
- logger.warn("Could not find a tag or branch '%s', assuming commit." % rev)
- return [rev]
- def switch(self, dest, url, rev_options):
- call_subprocess(
- [self.cmd, 'config', 'remote.origin.url', url], cwd=dest)
- call_subprocess(
- [self.cmd, 'checkout', '-q'] + rev_options, cwd=dest)
- def update(self, dest, rev_options):
- call_subprocess([self.cmd, 'fetch', '-q'], cwd=dest)
- call_subprocess(
- [self.cmd, 'checkout', '-q', '-f'] + rev_options, cwd=dest)
- def obtain(self, dest):
- url, rev = self.get_url_rev()
- if rev:
- rev_options = [rev]
- rev_display = ' (to %s)' % rev
- else:
- rev_options = ['origin/master']
- rev_display = ''
- if self.check_destination(dest, url, rev_options, rev_display):
- logger.notify('Cloning %s%s to %s' % (url, rev_display, display_path(dest)))
- call_subprocess(
- [self.cmd, 'clone', '-q', url, dest])
- rev_options = self.check_rev_options(rev, dest, rev_options)
- call_subprocess(
- [self.cmd, 'checkout', '-q'] + rev_options, cwd=dest)
- def get_url(self, location):
- url = call_subprocess(
- [self.cmd, 'config', 'remote.origin.url'],
- show_stdout=False, cwd=location)
- return url.strip()
- def get_revision(self, location):
- current_rev = call_subprocess(
- [self.cmd, 'rev-parse', 'HEAD'], show_stdout=False, cwd=location)
- return current_rev.strip()
- def get_tag_revs(self, location):
- tags = call_subprocess(
- [self.cmd, 'tag'], show_stdout=False, cwd=location)
- tag_revs = []
- for line in tags.splitlines():
- tag = line.strip()
- rev = call_subprocess(
- [self.cmd, 'rev-parse', tag], show_stdout=False, cwd=location)
- tag_revs.append((rev.strip(), tag))
- tag_revs = dict(tag_revs)
- return tag_revs
- def get_branch_revs(self, location):
- branches = call_subprocess(
- [self.cmd, 'branch', '-r'], show_stdout=False, cwd=location)
- branch_revs = []
- for line in branches.splitlines():
- line = line.split('->')[0].strip()
- branch = "".join([b for b in line.split() if b != '*'])
- rev = call_subprocess(
- [self.cmd, 'rev-parse', branch], show_stdout=False, cwd=location)
- branch_revs.append((rev.strip(), branch))
- branch_revs = dict(branch_revs)
- return branch_revs
- def get_src_requirement(self, dist, location, find_tags):
- repo = self.get_url(location)
- if not repo.lower().startswith('git:'):
- repo = 'git+' + repo
- egg_project_name = dist.egg_name().split('-', 1)[0]
- if not repo:
- return None
- current_rev = self.get_revision(location)
- tag_revs = self.get_tag_revs(location)
- branch_revs = self.get_branch_revs(location)
- if current_rev in tag_revs:
- # It's a tag
- full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev])
- elif (current_rev in branch_revs and
- branch_revs[current_rev] != 'origin/master'):
- # It's the head of a branch
- full_egg_name = '%s-%s' % (dist.egg_name(),
- branch_revs[current_rev].replace('origin/', ''))
- else:
- full_egg_name = '%s-dev' % dist.egg_name()
- return '%s@%s#egg=%s' % (repo, current_rev, full_egg_name)
- def get_url_rev(self):
- """
- Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'.
- That's required because although they use SSH they sometimes doesn't
- work with a ssh:// scheme (e.g. Github). But we need a scheme for
- parsing. Hence we remove it again afterwards and return it as a stub.
- """
- if not '://' in self.url:
- self.url = self.url.replace('git+', 'git+ssh://')
- url, rev = super(Git, self).get_url_rev()
- url = url.replace('ssh://', '')
- return url, rev
- return super(Git, self).get_url_rev()
- vcs.register(Git)
- class Mercurial(VersionControl):
- name = 'hg'
- dirname = '.hg'
- repo_name = 'clone'
- schemes = ('hg', 'hg+http', 'hg+ssh')
- bundle_file = 'hg-clone.txt'
- guide = ('# This was a Mercurial repo; to make it a repo again run:\n'
- 'hg init\nhg pull %(url)s\nhg update -r %(rev)s\n')
- def parse_vcs_bundle_file(self, content):
- url = rev = None
- for line in content.splitlines():
- if not line.strip() or line.strip().startswith('#'):
- continue
- url_match = re.search(r'hg\s*pull\s*(.*)\s*', line)
- if url_match:
- url = url_match.group(1).strip()
- rev_match = re.search(r'^hg\s*update\s*-r\s*(.*)\s*', line)
- if rev_match:
- rev = rev_match.group(1).strip()
- if url and rev:
- return url, rev
- return None, None
- def unpack(self, location):
- """Clone the Hg repository at the url to the destination location"""
- url, rev = self.get_url_rev()
- logger.notify('Cloning Mercurial repository %s to %s' % (url, location))
- logger.indent += 2
- try:
- if os.path.exists(location):
- os.rmdir(location)
- call_subprocess(
- ['hg', 'clone', url, location],
- filter_stdout=self._filter, show_stdout=False)
- finally:
- logger.indent -= 2
- def export(self, location):
- """Export the Hg repository at the url to the destination location"""
- temp_dir = tempfile.mkdtemp('-export', 'pip-')
- self.unpack(temp_dir)
- try:
- call_subprocess(
- ['hg', 'archive', location],
- filter_stdout=self._filter, show_stdout=False, cwd=temp_dir)
- finally:
- shutil.rmtree(temp_dir)
- def switch(self, dest, url, rev_options):
- repo_config = os.path.join(dest, self.dirname, 'hgrc')
- config = ConfigParser.SafeConfigParser()
- try:
- config.read(repo_config)
- config.set('paths', 'default', url)
- config_file = open(repo_config, 'w')
- config.write(config_file)
- config_file.close()
- except (OSError, ConfigParser.NoSectionError), e:
- logger.warn(
- 'Could not switch Mercurial repository to %s: %s'
- % (url, e))
- else:
- call_subprocess(['hg', 'update', '-q'] + rev_options, cwd=dest)
- def update(self, dest, rev_options):
- call_subprocess(['hg', 'pull', '-q'], cwd=dest)
- call_subprocess(
- ['hg', 'update', '-q'] + rev_options, cwd=dest)
- def obtain(self, dest):
- url, rev = self.get_url_rev()
- if rev:
- rev_options = [rev]
- rev_display = ' (to revision %s)' % rev
- else:
- rev_options = []
- rev_display = ''
- if self.check_destination(dest, url, rev_options, rev_display):
- logger.notify('Cloning hg %s%s to %s'
- % (url, rev_display, display_path(dest)))
- call_subprocess(['hg', 'clone', '-q', url, dest])
- call_subprocess(['hg', 'update', '-q'] + rev_options, cwd=dest)
- def get_url(self, location):
- url = call_subprocess(
- ['hg', 'showconfig', 'paths.default'],
- show_stdout=False, cwd=location).strip()
- if url.startswith('/') or url.startswith('\\'):
- url = filename_to_url(url)
- return url.strip()
- def get_tag_revs(self, location):
- tags = call_subprocess(
- ['hg', 'tags'], show_stdout=False, cwd=location)
- tag_revs = []
- for line in tags.splitlines():
- tags_match = re.search(r'([\w\d\.-]+)\s*([\d]+):.*$', line)
- if tags_match:
- tag = tags_match.group(1)
- rev = tags_match.group(2)
- tag_revs.append((rev.strip(), tag.strip()))
- return dict(tag_revs)
- def get_branch_revs(self, location):
- branches = call_subprocess(
- ['hg', 'branches'], show_stdout=False, cwd=location)
- branch_revs = []
- for line in branches.splitlines():
- branches_match = re.search(r'([\w\d\.-]+)\s*([\d]+):.*$', line)
- if branches_match:
- branch = branches_match.group(1)
- rev = branches_match.group(2)
- branch_revs.append((rev.strip(), branch.strip()))
- return dict(branch_revs)
- def get_revision(self, location):
- current_revision = call_subprocess(
- ['hg', 'parents', '--template={rev}'],
- show_stdout=False, cwd=location).strip()
- return current_revision
- def get_revision_hash(self, location):
- current_rev_hash = call_subprocess(
- ['hg', 'parents', '--template={node}'],
- show_stdout=False, cwd=location).strip()
- return current_rev_hash
- def get_src_requirement(self, dist, location, find_tags):
- repo = self.get_url(location)
- if not repo.lower().startswith('hg:'):
- repo = 'hg+' + repo
- egg_project_name = dist.egg_name().split('-', 1)[0]
- if not repo:
- return None
- current_rev = self.get_revision(location)
- current_rev_hash = self.get_revision_hash(location)
- tag_revs = self.get_tag_revs(location)
- branch_revs = self.get_branch_revs(location)
- if current_rev in tag_revs:
- # It's a tag
- full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev])
- elif current_rev in branch_revs:
- # It's the tip of a branch
- full_egg_name = '%s-%s' % (dist.egg_name(), branch_revs[current_rev])
- else:
- full_egg_name = '%s-dev' % dist.egg_name()
- return '%s@%s#egg=%s' % (repo, current_rev_hash, full_egg_name)
- vcs.register(Mercurial)
- class Bazaar(VersionControl):
- name = 'bzr'
- dirname = '.bzr'
- repo_name = 'branch'
- bundle_file = 'bzr-branch.txt'
- schemes = ('bzr', 'bzr+http', 'bzr+https', 'bzr+ssh', 'bzr+sftp')
- guide = ('# This was a Bazaar branch; to make it a branch again run:\n'
- 'bzr branch -r %(rev)s %(url)s .\n')
- def parse_vcs_bundle_file(self, content):
- url = rev = None
- for line in content.splitlines():
- if not line.strip() or line.strip().startswith('#'):
- continue
- match = re.search(r'^bzr\s*branch\s*-r\s*(\d*)', line)
- if match:
- rev = match.group(1).strip()
- url = line[match.end():].strip().split(None, 1)[0]
- if url and rev:
- return url, rev
- return None, None
- def unpack(self, location):
- """Get the bzr branch at the url to the destination location"""
- url, rev = self.get_url_rev()
- logger.notify('Checking out bzr repository %s to %s' % (url, location))
- logger.indent += 2
- try:
- if os.path.exists(location):
- os.rmdir(location)
- call_subprocess(
- [self.cmd, 'branch', url, location],
- filter_stdout=self._filter, show_stdout=False)
- finally:
- logger.indent -= 2
- def export(self, location):
- """Export the Bazaar repository at the url to the destination location"""
- temp_dir = tempfile.mkdtemp('-export', 'pip-')
- self.unpack(temp_dir)
- if os.path.exists(location):
- # Remove the location to make sure Bazaar can export it correctly
- shutil.rmtree(location, onerror=rmtree_errorhandler)
- try:
- call_subprocess([self.cmd, 'export', location], cwd=temp_dir,
- filter_stdout=self._filter, show_stdout=False)
- finally:
- shutil.rmtree(temp_dir)
- def switch(self, dest, url, rev_options):
- call_subprocess([self.cmd, 'switch', url], cwd=dest)
- def update(self, dest, rev_options):
- call_subprocess(
- [self.cmd, 'pull', '-q'] + rev_options, cwd=dest)
- def obtain(self, dest):
- url, rev = self.get_url_rev()
- if rev:
- rev_options = ['-r', rev]
- rev_display = ' (to revision %s)' % rev
- else:
- rev_options = []
- rev_display = ''
- if self.check_destination(dest, url, rev_options, rev_display):
- logger.notify('Checking out %s%s to %s'
- % (url, rev_display, display_path(dest)))
- call_subprocess(
- [self.cmd, 'branch', '-q'] + rev_options + [url, dest])
- def get_url_rev(self):
- # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it
- url, rev = super(Bazaar, self).get_url_rev()
- if url.startswith('ssh://'):
- url = 'bzr+' + url
- return url, rev
- def get_url(self, location):
- urls = call_subprocess(
- [self.cmd, 'info'], show_stdout=False, cwd=location)
- for line in urls.splitlines():
- line = line.strip()
- for x in ('checkout of branch: ',
- 'parent branch: '):
- if line.startswith(x):
- return line.split(x)[1]
- return None
- def get_revision(self, location):
- revision = call_subprocess(
- [self.cmd, 'revno'], show_stdout=False, cwd=location)
- return revision.splitlines()[-1]
- def get_tag_revs(self, location):
- tags = call_subprocess(
- [self.cmd, 'tags'], show_stdout=False, cwd=location)
- tag_revs = []
- for line in tags.splitlines():
- tags_match = re.search(r'([.\w-]+)\s*(.*)$', line)
- if tags_match:
- tag = tags_match.group(1)
- rev = tags_match.group(2)
- tag_revs.append((rev.strip(), tag.strip()))
- return dict(tag_revs)
- def get_src_requirement(self, dist, location, find_tags):
- repo = self.get_url(location)
- if not repo.lower().startswith('bzr:'):
- repo = 'bzr+' + repo
- egg_project_name = dist.egg_name().split('-', 1)[0]
- if not repo:
- return None
- current_rev = self.get_revision(location)
- tag_revs = self.get_tag_revs(location)
- if current_rev in tag_revs:
- # It's a tag
- tag = tag_revs.get(current_rev, current_rev)
- full_egg_name = '%s-%s' % (egg_project_name, tag_revs[current_rev])
- else:
- full_egg_name = '%s-dev_r%s' % (dist.egg_name(), current_rev)
- return '%s@%s#egg=%s' % (repo, current_rev, full_egg_name)
- vcs.register(Bazaar)
- def get_src_requirement(dist, location, find_tags):
- version_control = vcs.get_backend_from_location(location)
- if version_control:
- return version_control().get_src_requirement(dist, location, find_tags)
- logger.warn('cannot determine version of editable source in %s (is not SVN checkout, Git clone, Mercurial clone or Bazaar branch)' % location)
- return dist.as_requirement()
- ############################################################
- ## Requirement files
- _scheme_re = re.compile(r'^(http|https|file):', re.I)
- _url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I)
- def get_file_content(url, comes_from=None):
- """Gets the content of a file; it may be a filename, file: URL, or
- http: URL. Returns (location, content)"""
- match = _scheme_re.search(url)
- if match:
- scheme = match.group(1).lower()
- if (scheme == 'file' and comes_from
- and comes_from.startswith('http')):
- raise InstallationError(
- 'Requirements file %s references URL %s, which is local'
- % (comes_from, url))
- if scheme == 'file':
- path = url.split(':', 1)[1]
- path = path.replace('\\', '/')
- match = _url_slash_drive_re.match(path)
- if match:
- path = match.group(1) + ':' + path.split('|', 1)[1]
- path = urllib.unquote(path)
- if path.startswith('/'):
- path = '/' + path.lstrip('/')
- url = path
- else:
- ## FIXME: catch some errors
- resp = urllib2.urlopen(url)
- return resp.geturl(), resp.read()
- f = open(url)
- content = f.read()
- f.close()
- return url, content
- def parse_requirements(filename, finder=None, comes_from=None, options=None):
- skip_match = None
- skip_regex = options.skip_requirements_regex
- if skip_regex:
- skip_match = re.compile(skip_regex)
- filename, content = get_file_content(filename, comes_from=comes_from)
- for line_number, line in enumerate(content.splitlines()):
- line_number += 1
- line = line.strip()
- if not line or line.startswith('#'):
- continue
- if skip_match and skip_match.search(line):
- continue
- if line.startswith('-r') or line.startswith('--requirement'):
- if line.startswith('-r'):
- req_url = line[2:].strip()
- else:
- req_url = line[len('--requirement'):].strip().strip('=')
- if _scheme_re.search(filename):
- # Relative to a URL
- req_url = urlparse.urljoin(filename, url)
- elif not _scheme_re.search(req_url):
- req_url = os.path.join(os.path.dirname(filename), req_url)
- for item in parse_requirements(req_url, finder, comes_from=filename, options=options):
- yield item
- elif line.startswith('-Z') or line.startswith('--always-unzip'):
- # No longer used, but previously these were used in
- # requirement files, so we'll ignore.
- pass
- elif finder and line.startswith('-f') or line.startswith('--find-links'):
- if line.startswith('-f'):
- line = line[2:].strip()
- else:
- line = line[len('--find-links'):].strip().lstrip('=')
- ## FIXME: it would be nice to keep track of the source of
- ## the find_links:
- finder.find_links.append(line)
- elif line.startswith('-i') or line.startswith('--index-url'):
- if line.startswith('-i'):
- line = line[2:].strip()
- else:
- line = line[len('--index-url'):].strip().lstrip('=')
- finder.index_urls = [line]
- elif line.startswith('--extra-index-url'):
- line = line[len('--extra-index-url'):].strip().lstrip('=')
- finder.index_urls.append(line)
- else:
- comes_from = '-r %s (line %s)' % (filename, line_number)
- if line.startswith('-e') or line.startswith('--editable'):
- if line.startswith('-e'):
- line = line[2:].strip()
- else:
- line = line[len('--editable'):].strip()
- req = InstallRequirement.from_editable(
- line, comes_from=comes_from, default_vcs=options.default_vcs)
- else:
- req = InstallRequirement.from_line(line, comes_from)
- yield req
- ############################################################
- ## Logging
- class Logger(object):
- """
- Logging object for use in command-line script. Allows ranges of
- levels, to avoid some redundancy of displayed information.
- """
- VERBOSE_DEBUG = logging.DEBUG-1
- DEBUG = logging.DEBUG
- INFO = logging.INFO
- NOTIFY = (logging.INFO+logging.WARN)/2
- WARN = WARNING = logging.WARN
- ERROR = logging.ERROR
- FATAL = logging.FATAL
- LEVELS = [VERBOSE_DEBUG, DEBUG, INFO, NOTIFY, WARN, ERROR, FATAL]
- def __init__(self, consumers):
- self.consumers = consumers
- self.indent = 0
- self.explicit_levels = False
- self.in_progress = None
- self.in_progress_hanging = False
- def debug(self, msg, *args, **kw):
- self.log(self.DEBUG, msg, *args, **kw)
- def info(self, msg, *args, **kw):
- self.log(self.INFO, msg, *args, **kw)
- def notify(self, msg, *args, **kw):
- self.log(self.NOTIFY, msg, *args, **kw)
- def warn(self, msg, *args, **kw):
- self.log(self.WARN, msg, *args, **kw)
- def error(self, msg, *args, **kw):
- self.log(self.WARN, msg, *args, **kw)
- def fatal(self, msg, *args, **kw):
- self.log(self.FATAL, msg, *args, **kw)
- def log(self, level, msg, *args, **kw):
- if args:
- if kw:
- raise TypeError(
- "You may give positional or keyword arguments, not both")
- args = args or kw
- rendered = None
- for consumer_level, consumer in self.consumers:
- if self.level_matches(level, consumer_level):
- if (self.in_progress_hanging
- and consumer in (sys.stdout, sys.stderr)):
- self.in_progress_hanging = False
- sys.stdout.write('\n')
- sys.stdout.flush()
- if rendered is None:
- if args:
- rendered = msg % args
- else:
- rendered = msg
- rendered = ' '*self.indent + rendered
- if self.explicit_levels:
- ## FIXME: should this be a name, not a level number?
- rendered = '%02i %s' % (level, rendered)
- if hasattr(consumer, 'write'):
- consumer.write(rendered+'\n')
- else:
- consumer(rendered)
- def start_progress(self, msg):
- assert not self.in_progress, (
- "Tried to start_progress(%r) while in_progress %r"
- % (msg, self.in_progress))
- if self.level_matches(self.NOTIFY, self._stdout_level()):
- sys.stdout.write(' '*self.indent + msg)
- sys.stdout.flush()
- self.in_progress_hanging = True
- else:
- self.in_progress_hanging = False
- self.in_progress = msg
- self.last_message = None
- def end_progress(self, msg='done.'):
- assert self.in_progress, (
- "Tried to end_progress without start_progress")
- if self.stdout_level_matches(self.NOTIFY):
- if not self.in_progress_hanging:
- # Some message has been printed out since start_progress
- sys.stdout.write('...' + self.in_progress + msg + '\n')
- sys.stdout.flush()
- else:
- # These erase any messages shown with show_progress (besides .'s)
- logger.show_progress('')
- logger.show_progress('')
- sys.stdout.write(msg + '\n')
- sys.stdout.flush()
- self.in_progress = None
- self.in_progress_hanging = False
- def show_progress(self, message=None):
- """If we are in a progress scope, and no log messages have been
- shown, write out another '.'"""
- if self.in_progress_hanging:
- if message is None:
- sys.stdout.write('.')
- sys.stdout.flush()
- else:
- if self.last_message:
- padding = ' ' * max(0, len(self.last_message)-len(message))
- else:
- padding = ''
- sys.stdout.write('\r%s%s%s%s' % (' '*self.indent, self.in_progress, message, padding))
- sys.stdout.flush()
- self.last_message = message
- def stdout_level_matches(self, level):
- """Returns true if a message at this level will go to stdout"""
- return self.level_matches(level, self._stdout_level())
- def _stdout_level(self):
- """Returns the level that stdout runs at"""
- for level, consumer in self.consumers:
- if consumer is sys.stdout:
- return level
- return self.FATAL
- def level_matches(self, level, consumer_level):
- """
- >>> l = Logger()
- >>> l.level_matches(3, 4)
- False
- >>> l.level_matches(3, 2)
- True
- >>> l.level_matches(slice(None, 3), 3)
- False
- >>> l.level_matches(slice(None, 3), 2)
- True
- >>> l.level_matches(slice(1, 3), 1)
- True
- >>> l.level_matches(slice(2, 3), 1)
- False
- """
- if isinstance(level, slice):
- start, stop = level.start, level.stop
- if start is not None and start > consumer_level:
- return False
- if stop is not None or stop <= consumer_level:
- return False
- return True
- else:
- return level >= consumer_level
- @classmethod
- def level_for_integer(cls, level):
- levels = cls.LEVELS
- if level < 0:
- return levels[0]
- if level >= len(levels):
- return levels[-1]
- return levels[level]
- def move_stdout_to_stderr(self):
- to_remove = []
- to_add = []
- for consumer_level, consumer in self.consumers:
- if consumer == sys.stdout:
- to_remove.append((consumer_level, consumer))
- to_add.append((consumer_level, sys.stderr))
- for item in to_remove:
- self.consumers.remove(item)
- self.consumers.extend(to_add)
- def call_subprocess(cmd, show_stdout=True,
- filter_stdout=None, cwd=None,
- raise_on_returncode=True,
- command_level=Logger.DEBUG, command_desc=None,
- extra_environ=None):
- if command_desc is None:
- cmd_parts = []
- for part in cmd:
- if ' ' in part or '\n' in part or '"' in part or "'" in part:
- part = '"%s"' % part.replace('"', '\\"')
- cmd_parts.append(part)
- command_desc = ' '.join(cmd_parts)
- if show_stdout:
- stdout = None
- else:
- stdout = subprocess.PIPE
- logger.log(command_level, "Running command %s" % command_desc)
- env = os.environ.copy()
- if extra_environ:
- env.update(extra_environ)
- try:
- proc = subprocess.Popen(
- cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout,
- cwd=cwd, env=env)
- except Exception, e:
- logger.fatal(
- "Error %s while executing command %s" % (e, command_desc))
- raise
- all_output = []
- if stdout is not None:
- stdout = proc.stdout
- while 1:
- line = stdout.readline()
- if not line:
- break
- line = line.rstrip()
- all_output.append(line + '\n')
- if filter_stdout:
- level = filter_stdout(line)
- if isinstance(level, tuple):
- level, line = level
- logger.log(level, line)
- if not logger.stdout_level_matches(level):
- logger.show_progress()
- else:
- logger.info(line)
- else:
- returned_stdout, returned_stderr = proc.communicate()
- all_output = [returned_stdout or '']
- proc.wait()
- if proc.returncode:
- if raise_on_returncode:
- if all_output:
- logger.notify('Complete output from command %s:' % command_desc)
- logger.notify('\n'.join(all_output) + '\n----------------------------------------')
- raise InstallationError(
- "Command %s failed with error code %s"
- % (command_desc, proc.returncode))
- else:
- logger.warn(
- "Command %s had error code %s"
- % (command_desc, proc.returncode))
- if stdout is not None:
- return ''.join(all_output)
- ############################################################
- ## Utility functions
- def is_svn_page(html):
- """Returns true if the page appears to be the index page of an svn repository"""
- return (re.search(r'<title>[^<]*Revision \d+:', html)
- and re.search(r'Powered by (?:<a[^>]*?>)?Subversion', html, re.I))
- def file_contents(filename):
- fp = open(filename, 'rb')
- try:
- return fp.read()
- finally:
- fp.close()
- def split_leading_dir(path):
- path = str(path)
- path = path.lstrip('/').lstrip('\\')
- if '/' in path and (('\\' in path and path.find('/') < path.find('\\'))
- or '\\' not in path):
- return path.split('/', 1)
- elif '\\' in path:
- return path.split('\\', 1)
- else:
- return path, ''
- def has_leading_dir(paths):
- """Returns true if all the paths have the same leading path name
- (i.e., everything is in one subdirectory in an archive)"""
- common_prefix = None
- for path in paths:
- prefix, rest = split_leading_dir(path)
- if not prefix:
- return False
- elif common_prefix is None:
- common_prefix = prefix
- elif prefix != common_prefix:
- return False
- return True
- def format_size(bytes):
- if bytes > 1000*1000:
- return '%.1fMb' % (bytes/1000.0/1000)
- elif bytes > 10*1000:
- return '%iKb' % (bytes/1000)
- elif bytes > 1000:
- return '%.1fKb' % (bytes/1000.0)
- else:
- return '%ibytes' % bytes
- _normalize_re = re.compile(r'[^a-z]', re.I)
- def normalize_name(name):
- return _normalize_re.sub('-', name.lower())
- def make_path_relative(path, rel_to):
- """
- Make a filename relative, where the filename path, and it is
- relative to rel_to
- >>> make_relative_path('/usr/share/something/a-file.pth',
- ... '/usr/share/another-place/src/Directory')
- '../../../something/a-file.pth'
- >>> make_relative_path('/usr/share/something/a-file.pth',
- ... '/home/user/src/Directory')
- '../../../usr/share/something/a-file.pth'
- >>> make_relative_path('/usr/share/a-file.pth', '/usr/share/')
- 'a-file.pth'
- """
- path_filename = os.path.basename(path)
- path = os.path.dirname(path)
- path = os.path.normpath(os.path.abspath(path))
- rel_to = os.path.normpath(os.path.abspath(rel_to))
- path_parts = path.strip(os.path.sep).split(os.path.sep)
- rel_to_parts = rel_to.strip(os.path.sep).split(os.path.sep)
- while path_parts and rel_to_parts and path_parts[0] == rel_to_parts[0]:
- path_parts.pop(0)
- rel_to_parts.pop(0)
- full_parts = ['..']*len(rel_to_parts) + path_parts + [path_filename]
- if full_parts == ['']:
- return '.' + os.path.sep
- return os.path.sep.join(full_parts)
- def display_path(path):
- """Gives the display value for a given path, making it relative to cwd
- if possible."""
- path = os.path.normcase(os.path.abspath(path))
- if path.startswith(os.getcwd() + os.path.sep):
- path = '.' + path[len(os.getcwd()):]
- return path
- def parse_editable(editable_req, default_vcs=None):
- """Parses svn+http://blahblah@rev#egg=Foobar into a requirement
- (Foobar) and a URL"""
- url = editable_req
- if os.path.isdir(url) and os.path.exists(os.path.join(url, 'setup.py')):
- # Treating it as code that has already been checked out
- url = filename_to_url(url)
- if url.lower().startswith('file:'):
- return None, url
- for version_control in vcs:
- if url.lower().startswith('%s:' % version_control):
- url = '%s+%s' % (version_control, url)
- if '+' not in url:
- if default_vcs:
- url = default_vcs + '+' + url
- else:
- raise InstallationError(
- '--editable=%s should be formatted with svn+URL, git+URL, hg+URL or bzr+URL' % editable_req)
- vc_type = url.split('+', 1)[0].lower()
- if not vcs.get_backend(vc_type):
- raise InstallationError(
- 'For --editable=%s only svn (svn+URL), Git (git+URL), Mercurial (hg+URL) and Bazaar (bzr+URL) is currently supported' % editable_req)
- match = re.search(r'(?:#|#.*?&)egg=([^&]*)', editable_req)
- if (not match or not match.group(1)) and vcs.get_backend(vc_type):
- parts = [p for p in editable_req.split('#', 1)[0].split('/') if p]
- if parts[-2] in ('tags', 'branches', 'tag', 'branch'):
- req = parts[-3]
- elif parts[-1] == 'trunk':
- req = parts[-2]
- else:
- raise InstallationError(
- '--editable=%s is not the right format; it must have #egg=Package'
- % editable_req)
- else:
- req = match.group(1)
- ## FIXME: use package_to_requirement?
- match = re.search(r'^(.*?)(?:-dev|-\d.*)', req)
- if match:
- # Strip off -dev, -0.2, etc.
- req = match.group(1)
- return req, url
- def backup_dir(dir, ext='.bak'):
- """Figure out the name of a directory to back up the given dir to
- (adding .bak, .bak2, etc)"""
- n = 1
- extension = ext
- while os.path.exists(dir + extension):
- n += 1
- extension = ext + str(n)
- return dir + extension
- def ask(message, options):
- """Ask the message interactively, with the given possible responses"""
- while 1:
- if os.environ.get('PIP_NO_INPUT'):
- raise Exception('No input was expected ($PIP_NO_INPUT set); question: %s' % message)
- response = raw_input(message)
- response = response.strip().lower()
- if response not in options:
- print 'Your response (%r) was not one of the expected responses: %s' % (
- response, ', '.join(options))
- else:
- return response
- def open_logfile_append(filename):
- """Open the named log file in append mode.
- If the file already exists, a separator will also be printed to
- the file to separate past activity from current activity.
- """
- exists = os.path.exists(filename)
- log_fp = open(filename, 'a')
- if exists:
- print >> log_fp, '-'*60
- print >> log_fp, '%s run on %s' % (sys.argv[0], time.strftime('%c'))
- return log_fp
- def is_url(name):
- """Returns true if the name looks like a URL"""
- if ':' not in name:
- return False
- scheme = name.split(':', 1)[0].lower()
- return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes
- def is_filename(name):
- if (splitext(name)[1].lower() in ('.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar', '.pybundle')
- and os.path.exists(name)):
- return True
- if os.path.sep not in name and '/' not in name:
- # Doesn't have any path components, probably a requirement like 'Foo'
- return False
- return True
- _drive_re = re.compile('^([a-z]):', re.I)
- _url_drive_re = re.compile('^([a-z])[:|]', re.I)
- def filename_to_url(filename):
- """
- Convert a path to a file: URL. The path will be made absolute.
- """
- filename = os.path.normcase(os.path.abspath(filename))
- if _drive_re.match(filename):
- filename = filename[0] + '|' + filename[2:]
- url = urllib.quote(filename)
- url = url.replace(os.path.sep, '/')
- url = url.lstrip('/')
- return 'file:///' + url
- def filename_to_url2(filename):
- """
- Convert a path to a file: URL. The path will be made absolute and have
- quoted path parts.
- """
- filename = os.path.normcase(os.path.abspath(filename))
- drive, filename = os.path.splitdrive(filename)
- filepath = filename.split(os.path.sep)
- url = '/'.join([urllib.quote(part) for part in filepath])
- if not drive:
- url = url.lstrip('/')
- return 'file:///' + drive + url
- def url_to_filename(url):
- """
- Convert a file: URL to a path.
- """
- assert url.startswith('file:'), (
- "You can only turn file: urls into filenames (not %r)" % url)
- filename = url[len('file:'):].lstrip('/')
- filename = urllib.unquote(filename)
- if _url_drive_re.match(filename):
- filename = filename[0] + ':' + filename[2:]
- else:
- filename = '/' + filename
- return filename
- def get_requirement_from_url(url):
- """Get a requirement from the URL, if possible. This looks for #egg
- in the URL"""
- link = Link(url)
- egg_info = link.egg_fragment
- if not egg_info:
- egg_info = splitext(link.filename)[0]
- return package_to_requirement(egg_info)
- def package_to_requirement(package_name):
- """Translate a name like Foo-1.2 to Foo==1.3"""
- match = re.search(r'^(.*?)(-dev|-\d.*)', package_name)
- if match:
- name = match.group(1)
- version = match.group(2)
- else:
- name = package_name
- version = ''
- if version:
- return '%s==%s' % (name, version)
- else:
- return name
- def is_framework_layout(path):
- """Return True if the current platform is the default Python of Mac OS X
- which installs scripts in /usr/local/bin"""
- return (sys.platform[:6] == 'darwin' and
- (path[:9] == '/Library/' or path[:16] == '/System/Library/'))
- def strip_prefix(path, prefix):
- """ If ``path`` begins with ``prefix``, return ``path`` with
- ``prefix`` stripped off. Otherwise return None."""
- prefixes = [prefix]
- # Yep, we are special casing the framework layout of MacPython here
- if is_framework_layout(sys.prefix):
- for location in ('/Library', '/usr/local'):
- if path.startswith(location):
- prefixes.append(location)
- for prefix in prefixes:
- if path.startswith(prefix):
- return prefix, path.replace(prefix + os.path.sep, '')
- return None, None
- class UninstallPathSet(object):
- """A set of file paths to be removed in the uninstallation of a
- requirement."""
- def __init__(self, dist, restrict_to_prefix):
- self.paths = set()
- self._refuse = set()
- self.pth = {}
- self.prefix = os.path.normcase(os.path.realpath(restrict_to_prefix))
- self.dist = dist
- self.location = dist.location
- self.save_dir = None
- self._moved_paths = []
- def _can_uninstall(self):
- prefix, stripped = strip_prefix(self.location, self.prefix)
- if not stripped:
- logger.notify("Not uninstalling %s at %s, outside environment %s"
- % (self.dist.project_name, self.dist.location,
- self.prefix))
- return False
- return True
- def add(self, path):
- path = os.path.abspath(path)
- if not os.path.exists(path):
- return
- prefix, stripped = strip_prefix(os.path.normcase(path), self.prefix)
- if stripped:
- self.paths.add((prefix, stripped))
- else:
- self._refuse.add((prefix, path))
- def add_pth(self, pth_file, entry):
- prefix, stripped = strip_prefix(os.path.normcase(pth_file), self.prefix)
- if stripped:
- entry = os.path.normcase(entry)
- if stripped not in self.pth:
- self.pth[stripped] = UninstallPthEntries(os.path.join(prefix, stripped))
- self.pth[stripped].add(os.path.normcase(entry))
- else:
- self._refuse.add((prefix, pth_file))
- def compact(self, paths):
- """Compact a path set to contain the minimal number of paths
- necessary to contain all paths in the set. If /a/path/ and
- /a/path/to/a/file.txt are both in the set, leave only the
- shorter path."""
- short_paths = set()
- def sort_set(x, y):
- prefix_x, path_x = x
- prefix_y, path_y = y
- return cmp(len(path_x), len(path_y))
- for prefix, path in sorted(paths, sort_set):
- if not any([(path.startswith(shortpath) and
- path[len(shortpath.rstrip(os.path.sep))] == os.path.sep)
- for shortprefix, shortpath in short_paths]):
- short_paths.add((prefix, path))
- return short_paths
- def remove(self, auto_confirm=False):
- """Remove paths in ``self.paths`` with confirmation (unless
- ``auto_confirm`` is True)."""
- if not self._can_uninstall():
- return
- logger.notify('Uninstalling %s:' % self.dist.project_name)
- logger.indent += 2
- paths = sorted(self.compact(self.paths))
- try:
- if auto_confirm:
- response = 'y'
- else:
- for prefix, path in paths:
- logger.notify(os.path.join(prefix, path))
- response = ask('Proceed (y/n)? ', ('y', 'n'))
- if self._refuse:
- logger.notify('Not removing or modifying (outside of prefix):')
- for prefix, path in self.compact(self._refuse):
- logger.notify(os.path.join(prefix, path))
- if response == 'y':
- self.save_dir = tempfile.mkdtemp('-uninstall', 'pip-')
- for prefix, path in paths:
- full_path = os.path.join(prefix, path)
- new_path = os.path.join(self.save_dir, path)
- new_dir = os.path.dirname(new_path)
- logger.info('Removing file or directory %s' % full_path)
- self._moved_paths.append((prefix, path))
- os.renames(full_path, new_path)
- for pth in self.pth.values():
- pth.remove()
- logger.notify('Successfully uninstalled %s' % self.dist.project_name)
- finally:
- logger.indent -= 2
- def rollback(self):
- """Rollback the changes previously made by remove()."""
- if self.save_dir is None:
- logger.error("Can't roll back %s; was not uninstalled" % self.dist.project_name)
- return False
- logger.notify('Rolling back uninstall of %s' % self.dist.project_name)
- for prefix, path in self._moved_paths:
- tmp_path = os.path.join(self.save_dir, path)
- real_path = os.path.join(prefix, path)
- logger.info('Replacing %s' % real_path)
- os.renames(tmp_path, real_path)
- for pth in self.pth:
- pth.rollback()
- def commit(self):
- """Remove temporary save dir: rollback will no longer be possible."""
- if self.save_dir is not None:
- shutil.rmtree(self.save_dir)
- self.save_dir = None
- self._moved_paths = []
- class UninstallPthEntries(object):
- def __init__(self, pth_file):
- if not os.path.isfile(pth_file):
- raise UninstallationError("Cannot remove entries from nonexistent file %s" % pth_file)
- self.file = pth_file
- self.entries = set()
- self._saved_lines = None
- def add(self, entry):
- self.entries.add(entry)
- def remove(self):
- logger.info('Removing pth entries from %s:' % self.file)
- fh = open(self.file, 'r')
- lines = fh.readlines()
- self._saved_lines = lines
- fh.close()
- try:
- for entry in self.entries:
- logger.info('Removing entry: %s' % entry)
- try:
- lines.remove(entry + '\n')
- except ValueError:
- pass
- finally:
- pass
- fh = open(self.file, 'w')
- fh.writelines(lines)
- fh.close()
- def rollback(self):
- if self._saved_lines is None:
- logger.error('Cannot roll back changes to %s, none were made' % self.file)
- return False
- logger.info('Rolling %s back to previous state' % self.file)
- fh = open(self.file, 'w')
- fh.writelines(self._saved_lines)
- fh.close()
- return True
- class FakeFile(object):
- """Wrap a list of lines in an object with readline() to make
- ConfigParser happy."""
- def __init__(self, lines):
- self._gen = (l for l in lines)
- def readline(self):
- try:
- return self._gen.next()
- except StopIteration:
- return ''
-
- def splitext(path):
- """Like os.path.splitext, but take off .tar too"""
- base, ext = posixpath.splitext(path)
- if base.lower().endswith('.tar'):
- ext = base[-4:] + ext
- base = base[:-4]
- return base, ext
- def find_command(cmd, paths=None, pathext=None):
- """Searches the PATH for the given command and returns its path"""
- if paths is None:
- paths = os.environ.get('PATH', []).split(os.pathsep)
- if isinstance(paths, basestring):
- paths = [paths]
- # check if there are funny path extensions for executables, e.g. Windows
- if pathext is None:
- pathext = os.environ.get('PATHEXT', '.COM;.EXE;.BAT;.CMD')
- pathext = [ext for ext in pathext.lower().split(os.pathsep)]
- # don't use extensions if the command ends with one of them
- if os.path.splitext(cmd)[1].lower() in pathext:
- pathext = ['']
- # check if we find the command on PATH
- for path in paths:
- # try without extension first
- cmd_path = os.path.join(path, cmd)
- for ext in pathext:
- # then including the extension
- cmd_path_ext = cmd_path + ext
- if os.path.exists(cmd_path_ext):
- return cmd_path_ext
- if os.path.exists(cmd_path):
- return cmd_path
- return None
- class _Inf(object):
- """I am bigger than everything!"""
- def __cmp__(self, a):
- if self is a:
- return 0
- return 1
- def __repr__(self):
- return 'Inf'
- Inf = _Inf()
- del _Inf
- if __name__ == '__main__':
- exit = main()
- if exit:
- sys.exit(exit)