/pip.py
Python | 4841 lines | 4605 code | 129 blank | 107 comment | 414 complexity | 2b5001e3688a34df698c0efd6a2a249f MD5 | raw file
Large files files are truncated, but you can click here to view the full file
- #!/usr/bin/env python
- import sys
- import os
- import errno
- import stat
- import optparse
- import pkg_resources
- import urllib2
- import urllib
- import mimetypes
- import zipfile
- import tarfile
- import tempfile
- import subprocess
- import posixpath
- import re
- import shutil
- import fnmatch
- import operator
- import copy
- try:
- from hashlib import md5
- except ImportError:
- import md5 as md5_module
- md5 = md5_module.new
- import urlparse
- from email.FeedParser import FeedParser
- import traceback
- from cStringIO import StringIO
- import socket
- from Queue import Queue
- from Queue import Empty as QueueEmpty
- import threading
- import httplib
- import time
- import logging
- import ConfigParser
- from distutils.util import strtobool
- from distutils import sysconfig
- class InstallationError(Exception):
- """General exception during installation"""
- class UninstallationError(Exception):
- """General exception during uninstallation"""
- class DistributionNotFound(InstallationError):
- """Raised when a distribution cannot be found to satisfy a requirement"""
- class BadCommand(Exception):
- """Raised when virtualenv or a command is not found"""
- try:
- any
- except NameError:
- def any(seq):
- for item in seq:
- if item:
- return True
- return False
- if getattr(sys, 'real_prefix', None):
- ## FIXME: is build/ a good name?
- build_prefix = os.path.join(sys.prefix, 'build')
- src_prefix = os.path.join(sys.prefix, 'src')
- else:
- ## FIXME: this isn't a very good default
- build_prefix = os.path.join(os.getcwd(), 'build')
- src_prefix = os.path.join(os.getcwd(), 'src')
- # FIXME doesn't account for venv linked to global site-packages
- site_packages = sysconfig.get_python_lib()
- user_dir = os.path.expanduser('~')
- if sys.platform == 'win32':
- bin_py = os.path.join(sys.prefix, 'Scripts')
- # buildout uses 'bin' on Windows too?
- if not os.path.exists(bin_py):
- bin_py = os.path.join(sys.prefix, 'bin')
- config_dir = os.environ.get('APPDATA', user_dir) # Use %APPDATA% for roaming
- default_config_file = os.path.join(config_dir, 'pip', 'pip.ini')
- else:
- bin_py = os.path.join(sys.prefix, 'bin')
- default_config_file = os.path.join(user_dir, '.pip', 'pip.conf')
- # Forcing to use /usr/local/bin for standard Mac OS X framework installs
- if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/':
- bin_py = '/usr/local/bin'
- class UpdatingDefaultsHelpFormatter(optparse.IndentedHelpFormatter):
- """Custom help formatter for use in ConfigOptionParser that updates
- the defaults before expanding them, allowing them to show up correctly
- in the help listing"""
- def expand_default(self, option):
- if self.parser is not None:
- self.parser.update_defaults(self.parser.defaults)
- return optparse.IndentedHelpFormatter.expand_default(self, option)
- class ConfigOptionParser(optparse.OptionParser):
- """Custom option parser which updates its defaults by by checking the
- configuration files and environmental variables"""
- def __init__(self, *args, **kwargs):
- self.config = ConfigParser.RawConfigParser()
- self.name = kwargs.pop('name')
- self.files = self.get_config_files()
- self.config.read(self.files)
- assert self.name
- optparse.OptionParser.__init__(self, *args, **kwargs)
- def get_config_files(self):
- config_file = os.environ.get('PIP_CONFIG_FILE', False)
- if config_file and os.path.exists(config_file):
- return [config_file]
- return [default_config_file]
- def update_defaults(self, defaults):
- """Updates the given defaults with values from the config files and
- the environ. Does a little special handling for certain types of
- options (lists)."""
- # Then go and look for the other sources of configuration:
- config = {}
- # 1. config files
- for section in ('global', self.name):
- config.update(dict(self.get_config_section(section)))
- # 2. environmental variables
- config.update(dict(self.get_environ_vars()))
- # Then set the options with those values
- for key, val in config.iteritems():
- key = key.replace('_', '-')
- if not key.startswith('--'):
- key = '--%s' % key # only prefer long opts
- option = self.get_option(key)
- if option is not None:
- # ignore empty values
- if not val:
- continue
- # handle multiline configs
- if option.action == 'append':
- val = val.split()
- else:
- option.nargs = 1
- if option.action in ('store_true', 'store_false', 'count'):
- val = strtobool(val)
- try:
- val = option.convert_value(key, val)
- except optparse.OptionValueError, e:
- print ("An error occured during configuration: %s" % e)
- sys.exit(3)
- defaults[option.dest] = val
- return defaults
- def get_config_section(self, name):
- """Get a section of a configuration"""
- if self.config.has_section(name):
- return self.config.items(name)
- return []
- def get_environ_vars(self, prefix='PIP_'):
- """Returns a generator with all environmental vars with prefix PIP_"""
- for key, val in os.environ.iteritems():
- if key.startswith(prefix):
- yield (key.replace(prefix, '').lower(), val)
- def get_default_values(self):
- """Overridding to make updating the defaults after instantiation of
- the option parser possible, update_defaults() does the dirty work."""
- if not self.process_default_values:
- # Old, pre-Optik 1.5 behaviour.
- return optparse.Values(self.defaults)
- defaults = self.update_defaults(self.defaults.copy()) # ours
- for option in self._get_all_options():
- default = defaults.get(option.dest)
- if isinstance(default, basestring):
- opt_str = option.get_opt_string()
- defaults[option.dest] = option.check_value(opt_str, default)
- return optparse.Values(defaults)
- try:
- pip_dist = pkg_resources.get_distribution('pip')
- version = '%s from %s (python %s)' % (
- pip_dist, pip_dist.location, sys.version[:3])
- except pkg_resources.DistributionNotFound:
- # when running pip.py without installing
- version=None
- def rmtree_errorhandler(func, path, exc_info):
- """On Windows, the files in .svn are read-only, so when rmtree() tries to
- remove them, an exception is thrown. We catch that here, remove the
- read-only attribute, and hopefully continue without problems."""
- exctype, value = exc_info[:2]
- # lookin for a windows error
- if exctype is not WindowsError or 'Access is denied' not in str(value):
- raise
- # file type should currently be read only
- if ((os.stat(path).st_mode & stat.S_IREAD) != stat.S_IREAD):
- raise
- # convert to read/write
- os.chmod(path, stat.S_IWRITE)
- # use the original function to repeat the operation
- func(path)
- class VcsSupport(object):
- _registry = {}
- schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp']
- def __init__(self):
- # Register more schemes with urlparse for various version control systems
- urlparse.uses_netloc.extend(self.schemes)
- urlparse.uses_fragment.extend(self.schemes)
- super(VcsSupport, self).__init__()
- def __iter__(self):
- return self._registry.__iter__()
- @property
- def backends(self):
- return self._registry.values()
- @property
- def dirnames(self):
- return [backend.dirname for backend in self.backends]
- @property
- def all_schemes(self):
- schemes = []
- for backend in self.backends:
- schemes.extend(backend.schemes)
- return schemes
- def register(self, cls):
- if not hasattr(cls, 'name'):
- logger.warn('Cannot register VCS %s' % cls.__name__)
- return
- if cls.name not in self._registry:
- self._registry[cls.name] = cls
- def unregister(self, cls=None, name=None):
- if name in self._registry:
- del self._registry[name]
- elif cls in self._registry.values():
- del self._registry[cls.name]
- else:
- logger.warn('Cannot unregister because no class or name given')
- def get_backend_name(self, location):
- """
- Return the name of the version control backend if found at given
- location, e.g. vcs.get_backend_name('/path/to/vcs/checkout')
- """
- for vc_type in self._registry.values():
- path = os.path.join(location, vc_type.dirname)
- if os.path.exists(path):
- return vc_type.name
- return None
- def get_backend(self, name):
- name = name.lower()
- if name in self._registry:
- return self._registry[name]
- def get_backend_from_location(self, location):
- vc_type = self.get_backend_name(location)
- if vc_type:
- return self.get_backend(vc_type)
- return None
- vcs = VcsSupport()
- parser = ConfigOptionParser(
- usage='%prog COMMAND [OPTIONS]',
- version=version,
- add_help_option=False,
- formatter=UpdatingDefaultsHelpFormatter(),
- name='global')
- parser.add_option(
- '-h', '--help',
- dest='help',
- action='store_true',
- help='Show help')
- parser.add_option(
- '-E', '--environment',
- dest='venv',
- metavar='DIR',
- help='virtualenv environment to run pip in (either give the '
- 'interpreter or the environment base directory)')
- parser.add_option(
- '-s', '--enable-site-packages',
- dest='site_packages',
- action='store_true',
- help='Include site-packages in virtualenv if one is to be '
- 'created. Ignored if --environment is not used or '
- 'the virtualenv already exists.')
- parser.add_option(
- # Defines a default root directory for virtualenvs, relative
- # virtualenvs names/paths are considered relative to it.
- '--virtualenv-base',
- dest='venv_base',
- type='str',
- default='',
- help=optparse.SUPPRESS_HELP)
- parser.add_option(
- # Run only if inside a virtualenv, bail if not.
- '--require-virtualenv', '--require-venv',
- dest='require_venv',
- action='store_true',
- default=False,
- help=optparse.SUPPRESS_HELP)
- parser.add_option(
- # Use automatically an activated virtualenv instead of installing
- # globally. -E will be ignored if used.
- '--respect-virtualenv', '--respect-venv',
- dest='respect_venv',
- action='store_true',
- default=False,
- help=optparse.SUPPRESS_HELP)
- parser.add_option(
- '-v', '--verbose',
- dest='verbose',
- action='count',
- default=0,
- help='Give more output')
- parser.add_option(
- '-q', '--quiet',
- dest='quiet',
- action='count',
- default=0,
- help='Give less output')
- parser.add_option(
- '--log',
- dest='log',
- metavar='FILENAME',
- help='Log file where a complete (maximum verbosity) record will be kept')
- parser.add_option(
- # Writes the log levels explicitely to the log'
- '--log-explicit-levels',
- dest='log_explicit_levels',
- action='store_true',
- default=False,
- help=optparse.SUPPRESS_HELP)
- parser.add_option(
- # The default log file
- '--local-log', '--log-file',
- dest='log_file',
- metavar='FILENAME',
- default='./pip-log.txt',
- help=optparse.SUPPRESS_HELP)
- parser.add_option(
- '--proxy',
- dest='proxy',
- type='str',
- default='',
- help="Specify a proxy in the form user:passwd@proxy.server:port. "
- "Note that the user:password@ is optional and required only if you "
- "are behind an authenticated proxy. If you provide "
- "user@proxy.server:port then you will be prompted for a password.")
- parser.add_option(
- '--timeout', '--default-timeout',
- metavar='SECONDS',
- dest='timeout',
- type='float',
- default=15,
- help='Set the socket timeout (default %default seconds)')
- parser.add_option(
- # The default version control system for editables, e.g. 'svn'
- '--default-vcs',
- dest='default_vcs',
- type='str',
- default='',
- help=optparse.SUPPRESS_HELP)
- parser.add_option(
- # A regex to be used to skip requirements
- '--skip-requirements-regex',
- dest='skip_requirements_regex',
- type='str',
- default='',
- help=optparse.SUPPRESS_HELP)
- parser.disable_interspersed_args()
- _commands = {}
- class Command(object):
- name = None
- usage = None
- hidden = False
- def __init__(self):
- assert self.name
- self.parser = ConfigOptionParser(
- usage=self.usage,
- prog='%s %s' % (sys.argv[0], self.name),
- version=parser.version,
- formatter=UpdatingDefaultsHelpFormatter(),
- name=self.name)
- for option in parser.option_list:
- if not option.dest or option.dest == 'help':
- # -h, --version, etc
- continue
- self.parser.add_option(option)
- _commands[self.name] = self
- def merge_options(self, initial_options, options):
- # Make sure we have all global options carried over
- for attr in ['log', 'venv', 'proxy', 'venv_base', 'require_venv',
- 'respect_venv', 'log_explicit_levels', 'log_file',
- 'timeout', 'default_vcs', 'skip_requirements_regex']:
- setattr(options, attr, getattr(initial_options, attr) or getattr(options, attr))
- options.quiet += initial_options.quiet
- options.verbose += initial_options.verbose
- def main(self, complete_args, args, initial_options):
- global logger
- options, args = self.parser.parse_args(args)
- self.merge_options(initial_options, options)
- if options.require_venv and not options.venv:
- # If a venv is required check if it can really be found
- if not os.environ.get('VIRTUAL_ENV'):
- print 'Could not find an activated virtualenv (required).'
- sys.exit(3)
- # Automatically install in currently activated venv if required
- options.respect_venv = True
- if args and args[-1] == '___VENV_RESTART___':
- ## FIXME: We don't do anything this this value yet:
- venv_location = args[-2]
- args = args[:-2]
- options.venv = None
- else:
- # If given the option to respect the activated environment
- # check if no venv is given as a command line parameter
- if options.respect_venv and os.environ.get('VIRTUAL_ENV'):
- if options.venv and os.path.exists(options.venv):
- # Make sure command line venv and environmental are the same
- if (os.path.realpath(os.path.expanduser(options.venv)) !=
- os.path.realpath(os.environ.get('VIRTUAL_ENV'))):
- print ("Given virtualenv (%s) doesn't match "
- "currently activated virtualenv (%s)."
- % (options.venv, os.environ.get('VIRTUAL_ENV')))
- sys.exit(3)
- else:
- options.venv = os.environ.get('VIRTUAL_ENV')
- print 'Using already activated environment %s' % options.venv
- level = 1 # Notify
- level += options.verbose
- level -= options.quiet
- level = Logger.level_for_integer(4-level)
- complete_log = []
- logger = Logger([(level, sys.stdout),
- (Logger.DEBUG, complete_log.append)])
- if options.log_explicit_levels:
- logger.explicit_levels = True
- if options.venv:
- if options.verbose > 0:
- # The logger isn't setup yet
- print 'Running in environment %s' % options.venv
- site_packages=False
- if options.site_packages:
- site_packages=True
- restart_in_venv(options.venv, options.venv_base, site_packages,
- complete_args)
- # restart_in_venv should actually never return, but for clarity...
- return
- ## FIXME: not sure if this sure come before or after venv restart
- if options.log:
- log_fp = open_logfile_append(options.log)
- logger.consumers.append((logger.DEBUG, log_fp))
- else:
- log_fp = None
- socket.setdefaulttimeout(options.timeout or None)
- setup_proxy_handler(options.proxy)
- exit = 0
- try:
- self.run(options, args)
- except (InstallationError, UninstallationError), e:
- logger.fatal(str(e))
- logger.info('Exception information:\n%s' % format_exc())
- exit = 1
- except:
- logger.fatal('Exception:\n%s' % format_exc())
- exit = 2
- if log_fp is not None:
- log_fp.close()
- if exit:
- log_fn = options.log_file
- text = '\n'.join(complete_log)
- logger.fatal('Storing complete log in %s' % log_fn)
- log_fp = open_logfile_append(log_fn)
- log_fp.write(text)
- log_fp.close()
- return exit
- class HelpCommand(Command):
- name = 'help'
- usage = '%prog'
- summary = 'Show available commands'
- def run(self, options, args):
- if args:
- ## FIXME: handle errors better here
- command = args[0]
- if command not in _commands:
- raise InstallationError('No command with the name: %s' % command)
- command = _commands[command]
- command.parser.print_help()
- return
- parser.print_help()
- print
- print 'Commands available:'
- commands = list(set(_commands.values()))
- commands.sort(key=lambda x: x.name)
- for command in commands:
- if command.hidden:
- continue
- print ' %s: %s' % (command.name, command.summary)
- HelpCommand()
- class InstallCommand(Command):
- name = 'install'
- usage = '%prog [OPTIONS] PACKAGE_NAMES...'
- summary = 'Install packages'
- bundle = False
- def __init__(self):
- super(InstallCommand, self).__init__()
- self.parser.add_option(
- '-e', '--editable',
- dest='editables',
- action='append',
- default=[],
- metavar='VCS+REPOS_URL[@REV]#egg=PACKAGE',
- help='Install a package directly from a checkout. Source will be checked '
- 'out into src/PACKAGE (lower-case) and installed in-place (using '
- 'setup.py develop). You can run this on an existing directory/checkout (like '
- 'pip install -e src/mycheckout). This option may be provided multiple times. '
- 'Possible values for VCS are: svn, git, hg and bzr.')
- self.parser.add_option(
- '-r', '--requirement',
- dest='requirements',
- action='append',
- default=[],
- metavar='FILENAME',
- help='Install all the packages listed in the given requirements file. '
- 'This option can be used multiple times.')
- self.parser.add_option(
- '-f', '--find-links',
- dest='find_links',
- action='append',
- default=[],
- metavar='URL',
- help='URL to look for packages at')
- self.parser.add_option(
- '-i', '--index-url', '--pypi-url',
- dest='index_url',
- metavar='URL',
- default='http://pypi.python.org/simple',
- help='Base URL of Python Package Index (default %default)')
- self.parser.add_option(
- '--extra-index-url',
- dest='extra_index_urls',
- metavar='URL',
- action='append',
- default=[],
- help='Extra URLs of package indexes to use in addition to --index-url')
- self.parser.add_option(
- '--no-index',
- dest='no_index',
- action='store_true',
- default=False,
- help='Ignore package index (only looking at --find-links URLs instead)')
- self.parser.add_option(
- '-b', '--build', '--build-dir', '--build-directory',
- dest='build_dir',
- metavar='DIR',
- default=None,
- help='Unpack packages into DIR (default %s) and build from there' % build_prefix)
- self.parser.add_option(
- '-d', '--download', '--download-dir', '--download-directory',
- dest='download_dir',
- metavar='DIR',
- default=None,
- help='Download packages into DIR instead of installing them')
- self.parser.add_option(
- '--download-cache',
- dest='download_cache',
- metavar='DIR',
- default=None,
- help='Cache downloaded packages in DIR')
- self.parser.add_option(
- '--src', '--source', '--source-dir', '--source-directory',
- dest='src_dir',
- metavar='DIR',
- default=None,
- help='Check out --editable packages into DIR (default %s)' % src_prefix)
- self.parser.add_option(
- '-U', '--upgrade',
- dest='upgrade',
- action='store_true',
- help='Upgrade all packages to the newest available version')
- self.parser.add_option(
- '-I', '--ignore-installed',
- dest='ignore_installed',
- action='store_true',
- help='Ignore the installed packages (reinstalling instead)')
- self.parser.add_option(
- '--no-deps', '--no-dependencies',
- dest='ignore_dependencies',
- action='store_true',
- default=False,
- help='Ignore package dependencies')
- self.parser.add_option(
- '--no-install',
- dest='no_install',
- action='store_true',
- help="Download and unpack all packages, but don't actually install them")
- self.parser.add_option(
- '--install-option',
- dest='install_options',
- action='append',
- help="Extra arguments to be supplied to the setup.py install "
- "command (use like --install-option=\"--install-scripts=/usr/local/bin\"). "
- "Use multiple --install-option options to pass multiple options to setup.py install. "
- "If you are using an option with a directory path, be sure to use absolute path.")
- def run(self, options, args):
- if not options.build_dir:
- options.build_dir = build_prefix
- if not options.src_dir:
- options.src_dir = src_prefix
- if options.download_dir:
- options.no_install = True
- options.ignore_installed = True
- else:
- options.build_dir = os.path.abspath(options.build_dir)
- options.src_dir = os.path.abspath(options.src_dir)
- install_options = options.install_options or []
- index_urls = [options.index_url] + options.extra_index_urls
- if options.no_index:
- logger.notify('Ignoring indexes: %s' % ','.join(index_urls))
- index_urls = []
- finder = PackageFinder(
- find_links=options.find_links,
- index_urls=index_urls)
- requirement_set = RequirementSet(
- build_dir=options.build_dir,
- src_dir=options.src_dir,
- download_dir=options.download_dir,
- download_cache=options.download_cache,
- upgrade=options.upgrade,
- ignore_installed=options.ignore_installed,
- ignore_dependencies=options.ignore_dependencies)
- for name in args:
- requirement_set.add_requirement(
- InstallRequirement.from_line(name, None))
- for name in options.editables:
- requirement_set.add_requirement(
- InstallRequirement.from_editable(name, default_vcs=options.default_vcs))
- for filename in options.requirements:
- for req in parse_requirements(filename, finder=finder, options=options):
- requirement_set.add_requirement(req)
- requirement_set.install_files(finder, force_root_egg_info=self.bundle)
- if not options.no_install and not self.bundle:
- requirement_set.install(install_options)
- installed = ' '.join([req.name for req in
- requirement_set.successfully_installed])
- if installed:
- logger.notify('Successfully installed %s' % installed)
- elif not self.bundle:
- downloaded = ' '.join([req.name for req in
- requirement_set.successfully_downloaded])
- if downloaded:
- logger.notify('Successfully downloaded %s' % downloaded)
- return requirement_set
- InstallCommand()
- class UninstallCommand(Command):
- name = 'uninstall'
- usage = '%prog [OPTIONS] PACKAGE_NAMES ...'
- summary = 'Uninstall packages'
- def __init__(self):
- super(UninstallCommand, self).__init__()
- self.parser.add_option(
- '-r', '--requirement',
- dest='requirements',
- action='append',
- default=[],
- metavar='FILENAME',
- help='Uninstall all the packages listed in the given requirements file. '
- 'This option can be used multiple times.')
- self.parser.add_option(
- '-y', '--yes',
- dest='yes',
- action='store_true',
- help="Don't ask for confirmation of uninstall deletions.")
- def run(self, options, args):
- requirement_set = RequirementSet(
- build_dir=None,
- src_dir=None,
- download_dir=None)
- for name in args:
- requirement_set.add_requirement(
- InstallRequirement.from_line(name))
- for filename in options.requirements:
- for req in parse_requirements(filename, options=options):
- requirement_set.add_requirement(req)
- requirement_set.uninstall(auto_confirm=options.yes)
- UninstallCommand()
- class BundleCommand(InstallCommand):
- name = 'bundle'
- usage = '%prog [OPTIONS] BUNDLE_NAME.pybundle PACKAGE_NAMES...'
- summary = 'Create pybundles (archives containing multiple packages)'
- bundle = True
- def __init__(self):
- super(BundleCommand, self).__init__()
- def run(self, options, args):
- if not args:
- raise InstallationError('You must give a bundle filename')
- if not options.build_dir:
- options.build_dir = backup_dir(build_prefix, '-bundle')
- if not options.src_dir:
- options.src_dir = backup_dir(src_prefix, '-bundle')
- # We have to get everything when creating a bundle:
- options.ignore_installed = True
- logger.notify('Putting temporary build files in %s and source/develop files in %s'
- % (display_path(options.build_dir), display_path(options.src_dir)))
- bundle_filename = args[0]
- args = args[1:]
- requirement_set = super(BundleCommand, self).run(options, args)
- # FIXME: here it has to do something
- requirement_set.create_bundle(bundle_filename)
- logger.notify('Created bundle in %s' % bundle_filename)
- return requirement_set
- BundleCommand()
- class FreezeCommand(Command):
- name = 'freeze'
- usage = '%prog [OPTIONS]'
- summary = 'Output all currently installed packages (exact versions) to stdout'
- def __init__(self):
- super(FreezeCommand, self).__init__()
- self.parser.add_option(
- '-r', '--requirement',
- dest='requirement',
- action='store',
- default=None,
- metavar='FILENAME',
- help='Use the given requirements file as a hint about how to generate the new frozen requirements')
- self.parser.add_option(
- '-f', '--find-links',
- dest='find_links',
- action='append',
- default=[],
- metavar='URL',
- help='URL for finding packages, which will be added to the frozen requirements file')
- def run(self, options, args):
- requirement = options.requirement
- find_links = options.find_links or []
- ## FIXME: Obviously this should be settable:
- find_tags = False
- skip_match = None
- skip_regex = options.skip_requirements_regex
- if skip_regex:
- skip_match = re.compile(skip_regex)
- logger.move_stdout_to_stderr()
- dependency_links = []
- f = sys.stdout
- for dist in pkg_resources.working_set:
- if dist.has_metadata('dependency_links.txt'):
- dependency_links.extend(dist.get_metadata_lines('dependency_links.txt'))
- for link in find_links:
- if '#egg=' in link:
- dependency_links.append(link)
- for link in find_links:
- f.write('-f %s\n' % link)
- installations = {}
- for dist in pkg_resources.working_set:
- if dist.key in ('setuptools', 'pip', 'python'):
- ## FIXME: also skip virtualenv?
- continue
- req = FrozenRequirement.from_dist(dist, dependency_links, find_tags=find_tags)
- installations[req.name] = req
- if requirement:
- req_f = open(requirement)
- for line in req_f:
- if not line.strip() or line.strip().startswith('#'):
- f.write(line)
- continue
- if skip_match and skip_match.search(line):
- f.write(line)
- continue
- elif line.startswith('-e') or line.startswith('--editable'):
- if line.startswith('-e'):
- line = line[2:].strip()
- else:
- line = line[len('--editable'):].strip().lstrip('=')
- line_req = InstallRequirement.from_editable(line, default_vcs=options.default_vcs)
- elif (line.startswith('-r') or line.startswith('--requirement')
- or line.startswith('-Z') or line.startswith('--always-unzip')):
- logger.debug('Skipping line %r' % line.strip())
- continue
- else:
- line_req = InstallRequirement.from_line(line)
- if not line_req.name:
- logger.notify("Skipping line because it's not clear what it would install: %s"
- % line.strip())
- logger.notify(" (add #egg=PackageName to the URL to avoid this warning)")
- continue
- if line_req.name not in installations:
- logger.warn("Requirement file contains %s, but that package is not installed"
- % line.strip())
- continue
- f.write(str(installations[line_req.name]))
- del installations[line_req.name]
- f.write('## The following requirements were added by pip --freeze:\n')
- for installation in sorted(installations.values(), key=lambda x: x.name):
- f.write(str(installation))
- FreezeCommand()
- class ZipCommand(Command):
- name = 'zip'
- usage = '%prog [OPTIONS] PACKAGE_NAMES...'
- summary = 'Zip individual packages'
- def __init__(self):
- super(ZipCommand, self).__init__()
- if self.name == 'zip':
- self.parser.add_option(
- '--unzip',
- action='store_true',
- dest='unzip',
- help='Unzip (rather than zip) a package')
- else:
- self.parser.add_option(
- '--zip',
- action='store_false',
- dest='unzip',
- default=True,
- help='Zip (rather than unzip) a package')
- self.parser.add_option(
- '--no-pyc',
- action='store_true',
- dest='no_pyc',
- help='Do not include .pyc files in zip files (useful on Google App Engine)')
- self.parser.add_option(
- '-l', '--list',
- action='store_true',
- dest='list',
- help='List the packages available, and their zip status')
- self.parser.add_option(
- '--sort-files',
- action='store_true',
- dest='sort_files',
- help='With --list, sort packages according to how many files they contain')
- self.parser.add_option(
- '--path',
- action='append',
- dest='paths',
- help='Restrict operations to the given paths (may include wildcards)')
- self.parser.add_option(
- '-n', '--simulate',
- action='store_true',
- help='Do not actually perform the zip/unzip operation')
- def paths(self):
- """All the entries of sys.path, possibly restricted by --path"""
- if not self.select_paths:
- return sys.path
- result = []
- match_any = set()
- for path in sys.path:
- path = os.path.normcase(os.path.abspath(path))
- for match in self.select_paths:
- match = os.path.normcase(os.path.abspath(match))
- if '*' in match:
- if re.search(fnmatch.translate(match+'*'), path):
- result.append(path)
- match_any.add(match)
- break
- else:
- if path.startswith(match):
- result.append(path)
- match_any.add(match)
- break
- else:
- logger.debug("Skipping path %s because it doesn't match %s"
- % (path, ', '.join(self.select_paths)))
- for match in self.select_paths:
- if match not in match_any and '*' not in match:
- result.append(match)
- logger.debug("Adding path %s because it doesn't match anything already on sys.path"
- % match)
- return result
- def run(self, options, args):
- self.select_paths = options.paths
- self.simulate = options.simulate
- if options.list:
- return self.list(options, args)
- if not args:
- raise InstallationError(
- 'You must give at least one package to zip or unzip')
- packages = []
- for arg in args:
- module_name, filename = self.find_package(arg)
- if options.unzip and os.path.isdir(filename):
- raise InstallationError(
- 'The module %s (in %s) is not a zip file; cannot be unzipped'
- % (module_name, filename))
- elif not options.unzip and not os.path.isdir(filename):
- raise InstallationError(
- 'The module %s (in %s) is not a directory; cannot be zipped'
- % (module_name, filename))
- packages.append((module_name, filename))
- last_status = None
- for module_name, filename in packages:
- if options.unzip:
- last_status = self.unzip_package(module_name, filename)
- else:
- last_status = self.zip_package(module_name, filename, options.no_pyc)
- return last_status
- def unzip_package(self, module_name, filename):
- zip_filename = os.path.dirname(filename)
- if not os.path.isfile(zip_filename) and zipfile.is_zipfile(zip_filename):
- raise InstallationError(
- 'Module %s (in %s) isn\'t located in a zip file in %s'
- % (module_name, filename, zip_filename))
- package_path = os.path.dirname(zip_filename)
- if not package_path in self.paths():
- logger.warn(
- 'Unpacking %s into %s, but %s is not on sys.path'
- % (display_path(zip_filename), display_path(package_path),
- display_path(package_path)))
- logger.notify('Unzipping %s (in %s)' % (module_name, display_path(zip_filename)))
- if self.simulate:
- logger.notify('Skipping remaining operations because of --simulate')
- return
- logger.indent += 2
- try:
- ## FIXME: this should be undoable:
- zip = zipfile.ZipFile(zip_filename)
- to_save = []
- for name in zip.namelist():
- if name.startswith('%s/' % module_name):
- content = zip.read(name)
- dest = os.path.join(package_path, name)
- if not os.path.exists(os.path.dirname(dest)):
- os.makedirs(os.path.dirname(dest))
- if not content and dest.endswith('/'):
- if not os.path.exists(dest):
- os.makedirs(dest)
- else:
- f = open(dest, 'wb')
- f.write(content)
- f.close()
- else:
- to_save.append((name, zip.read(name)))
- zip.close()
- if not to_save:
- logger.info('Removing now-empty zip file %s' % display_path(zip_filename))
- os.unlink(zip_filename)
- self.remove_filename_from_pth(zip_filename)
- else:
- logger.info('Removing entries in %s/ from zip file %s' % (module_name, display_path(zip_filename)))
- zip = zipfile.ZipFile(zip_filename, 'w')
- for name, content in to_save:
- zip.writestr(name, content)
- zip.close()
- finally:
- logger.indent -= 2
- def zip_package(self, module_name, filename, no_pyc):
- orig_filename = filename
- logger.notify('Zip %s (in %s)' % (module_name, display_path(filename)))
- logger.indent += 2
- if filename.endswith('.egg'):
- dest_filename = filename
- else:
- dest_filename = filename + '.zip'
- try:
- ## FIXME: I think this needs to be undoable:
- if filename == dest_filename:
- filename = backup_dir(orig_filename)
- logger.notify('Moving %s aside to %s' % (orig_filename, filename))
- if not self.simulate:
- shutil.move(orig_filename, filename)
- try:
- logger.info('Creating zip file in %s' % display_path(dest_filename))
- if not self.simulate:
- zip = zipfile.ZipFile(dest_filename, 'w')
- zip.writestr(module_name + '/', '')
- for dirpath, dirnames, filenames in os.walk(filename):
- if no_pyc:
- filenames = [f for f in filenames
- if not f.lower().endswith('.pyc')]
- for fns, is_dir in [(dirnames, True), (filenames, False)]:
- for fn in fns:
- full = os.path.join(dirpath, fn)
- dest = os.path.join(module_name, dirpath[len(filename):].lstrip(os.path.sep), fn)
- if is_dir:
- zip.writestr(dest+'/', '')
- else:
- zip.write(full, dest)
- zip.close()
- logger.info('Removing old directory %s' % display_path(filename))
- if not self.simulate:
- shutil.rmtree(filename)
- except:
- ## FIXME: need to do an undo here
- raise
- ## FIXME: should also be undone:
- self.add_filename_to_pth(dest_filename)
- finally:
- logger.indent -= 2
- def remove_filename_from_pth(self, filename):
- for pth in self.pth_files():
- f = open(pth, 'r')
- lines = f.readlines()
- f.close()
- new_lines = [
- l for l in lines if l.strip() != filename]
- if lines != new_lines:
- logger.info('Removing reference to %s from .pth file %s'
- % (display_path(filename), display_path(pth)))
- if not filter(None, new_lines):
- logger.info('%s file would be empty: deleting' % display_path(pth))
- if not self.simulate:
- os.unlink(pth)
- else:
- if not self.simulate:
- f = open(pth, 'w')
- f.writelines(new_lines)
- f.close()
- return
- logger.warn('Cannot find a reference to %s in any .pth file' % display_path(filename))
- def add_filename_to_pth(self, filename):
- path = os.path.dirname(filename)
- dest = os.path.join(path, filename + '.pth')
- if path not in self.paths():
- logger.warn('Adding .pth file %s, but it is not on sys.path' % display_path(dest))
- if not self.simulate:
- if os.path.exists(dest):
- f = open(dest)
- lines = f.readlines()
- f.close()
- if lines and not lines[-1].endswith('\n'):
- lines[-1] += '\n'
- lines.append(filename+'\n')
- else:
- lines = [filename + '\n']
- f = open(dest, 'w')
- f.writelines(lines)
- f.close()
- def pth_files(self):
- for path in self.paths():
- if not os.path.exists(path) or not os.path.isdir(path):
- continue
- for filename in os.listdir(path):
- if filename.endswith('.pth'):
- yield os.path.join(path, filename)
- def find_package(self, package):
- for path in self.paths():
- full = os.path.join(path, package)
- if os.path.exists(full):
- return package, full
- if not os.path.isdir(path) and zipfile.is_zipfile(path):
- zip = zipfile.ZipFile(path, 'r')
- try:
- zip.read('%s/__init__.py' % package)
- except KeyError:
- pass
- else:
- zip.close()
- return package, full
- zip.close()
- ## FIXME: need special error for package.py case:
- raise InstallationError(
- 'No package with the name %s found' % package)
- def list(self, options, args):
- if args:
- raise InstallationError(
- 'You cannot give an argument with --list')
- for path in sorted(self.paths()):
- if not os.path.exists(path):
- continue
- basename = os.path.basename(path.rstrip(os.path.sep))
- if os.path.isfile(path) and zipfile.is_zipfile(path):
- if os.path.dirname(path) not in self.paths():
- logger.notify('Zipped egg: %s' % display_path(path))
- continue
- if (basename != 'site-packages'
- and not path.replace('\\', '/').endswith('lib/python')):
- continue
- logger.notify('In %s:' % display_path(path))
- logger.indent += 2
- zipped = []
- unzipped = []
- try:
- for filename in sorted(os.listdir(path)):
- ext = os.path.splitext(filename)[1].lower()
- if ext in ('.pth', '.egg-info', '.egg-link'):
- continue
- if ext == '.py':
- logger.info('Not displaying %s: not a package' % display_path(filename))
- continue
- full = os.path.join(path, filename)
- if os.path.isdir(full):
- unzipped.append((filename, self.count_package(full)))
- elif zipfile.is_zipfile(full):
- zipped.append(filename)
- else:
- logger.info('Unknown file: %s' % display_path(filename))
- if zipped:
- logger.notify('Zipped packages:')
- logger.indent += 2
- try:
- for filename in zipped:
- logger.notify(filename)
- finally:
- logger.indent -= 2
- else:
- logger.notify('No zipped packages.')
- if unzipped:
- if options.sort_files:
- unzipped.sort(key=lambda x: -x[1])
- logger.notify('Unzipped packages:')
- logger.indent += 2
- try:
- for filename, count in unzipped:
- logger.notify('%s (%i files)' % (filename, count))
- finally:
- logger.indent -= 2
- else:
- logger.notify('No unzipped packages.')
- finally:
- logger.indent -= 2
- def count_package(self, path):
- total = 0
- for dirpath, dirnames, filenames in os.walk(path):
- filenames = [f for f in filenames
- if not f.lower().endswith('.pyc')]
- total += len(filenames)
- return total
- ZipCommand()
- class UnzipCommand(ZipCommand):
- name = 'unzip'
- summary = 'Unzip individual packages'
- UnzipCommand()
- BASE_COMPLETION = """
- # pip %(shell)s completion start%(script)s# pip %(shell)s completion end
- """
- COMPLETION_SCRIPTS = {
- 'bash': """
- _pip_completion()
- {
- COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]}" \\
- COMP_CWORD=$COMP_CWORD \\
- PIP_AUTO_COMPLETE=1 $1 ) )
- }
- complete -o default -F _pip_completion pip
- """, 'zsh': """
- function _pip_completion {
- local words cword
- read -Ac words
- read -cn cword
- reply=( $( COMP_WORDS="$words[*]" \\
- COMP_CWORD=$(( cword-1 )) \\
- PIP_AUTO_COMPLETE=1 $words[1] ) )
- }
- compctl -K _pip_completion pip
- """
- }
- class CompletionCommand(Command):
- name = 'completion'
- summary = 'A helper command to be used for command completion'
- hidden = True
- def __init__(self):
- super(CompletionCommand, self).__init__()
- self.parser.add_option(
- '--bash', '-b',
- action='store_const',
- const='bash',
- dest='shell',
- help='Emit completion code for bash')
- self.parser.add_option(
- '--zsh', '-z',
- action='store_const',
- const='zsh',
- dest='shell',
- help='Emit completion code for zsh')
- def run(self, options, args):
- """Prints the completion code of the given shell"""
- if options.shell in ('bash', 'zsh'):
- script = COMPLETION_SCRIPTS.get(options.shell, '')
- print BASE_COMPLETION % {'script': script, 'shell': options.shell}
- else:
- print 'ERROR: You must pass --bash or --zsh'
- CompletionCommand()
- def autocomplete():
- """Command and option completion for the main option parser (and options)
- and its subcommands (and options).
- Enable by sourcing one of the completion shell scripts (bash or zsh).
- """
- # Don't complete if user hasn't sourced bash_completion file.
- if not os.environ.has_key('PIP_AUTO_COMPLETE'):
- return
- cwords = os.environ['COMP_WORDS'].split()[1:]
- cword = int(os.environ['COMP_CWORD'])
- try:
- current = cwords[cword-1]
- except IndexError:
- current = ''
- subcommands = [cmd for cmd, cls in _commands.items() if not cls.hidden]
- options = []
- # subcommand
- if cword == 1:
- # show options of main parser only when necessary
- if current.startswith('-') or current.startswith('--'):
- subcommands += [opt.get_opt_string()
- for opt in parser.option_list
- if opt.help != optparse.SUPPRESS_HELP]
- print ' '.join(filter(lambda x: x.startswith(current), subcommands))
- # subcommand options
- # special case: the 'help' subcommand has no options
- elif cwords[0] in subcommands and cwords[0] != 'help':
- subcommand = _commands.get(cwords[0])
- options += [(opt.get_opt_string(), opt.nargs)
- for opt in subcommand.parser.option_list
- if opt.help != optparse.SUPPRESS_HELP]
- # filter out previously specified options from available options
- prev_opts = [x.split('=')[0] for x in cwords[1:cword-1]]
- options = filter(lambda (x, v): x not in prev_opts, options)
- # filter options by current input
- options = [(k, v) for k, v in options if k.startswith(current)]
- for option in options:
- opt_label = option[0]
- # append '=' to options which require args
- if option[1]:
- opt_label += '='
- print opt_label
- sys.exit(1)
- def main(initial_args=None):
- if initial_args is None:
- initial_args = sys.argv[1:]
- autocomplete()
- options, args = parser.parse_args(initial_args)
- if options.help and not args:
- args = ['help']
- if not args:
- parser.error('You must give a command (use "pip help" see a list of commands)')
- command = args[0].lower()
- ## FIXME: search for a command match?
- if command not in _commands:
- parser.error('No command by the name %(script)s %(arg)s\n (maybe you meant "%(script)s install %(arg)s")'
- % dict(script=os.path.basename(sys.argv[0]), arg=command))
- command = _commands[command]
- …
Large files files are truncated, but you can click here to view the full file