/mercurial/util.py
Python | 3556 lines | 3342 code | 84 blank | 130 comment | 95 complexity | 647fc82108eaf3870e819e29536c6de8 MD5 | raw file
Possible License(s): GPL-2.0, BSD-3-Clause
Large files files are truncated, but you can click here to view the full file
- # util.py - Mercurial utility functions and platform specific implementations
- #
- # Copyright 2005 K. Thananchayan <thananck@yahoo.com>
- # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
- # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
- #
- # This software may be used and distributed according to the terms of the
- # GNU General Public License version 2 or any later version.
- """Mercurial utility functions and platform specific implementations.
- This contains helper routines that are independent of the SCM core and
- hide platform-specific details from the core.
- """
- from __future__ import absolute_import
- import bz2
- import calendar
- import collections
- import datetime
- import errno
- import gc
- import hashlib
- import imp
- import os
- import platform as pyplatform
- import re as remod
- import shutil
- import signal
- import socket
- import stat
- import string
- import subprocess
- import sys
- import tempfile
- import textwrap
- import time
- import traceback
- import zlib
- from . import (
- encoding,
- error,
- i18n,
- osutil,
- parsers,
- pycompat,
- )
- empty = pycompat.empty
- httplib = pycompat.httplib
- httpserver = pycompat.httpserver
- pickle = pycompat.pickle
- queue = pycompat.queue
- socketserver = pycompat.socketserver
- stderr = pycompat.stderr
- stdin = pycompat.stdin
- stdout = pycompat.stdout
- stringio = pycompat.stringio
- urlerr = pycompat.urlerr
- urlparse = pycompat.urlparse
- urlreq = pycompat.urlreq
- xmlrpclib = pycompat.xmlrpclib
- def isatty(fp):
- try:
- return fp.isatty()
- except AttributeError:
- return False
- # glibc determines buffering on first write to stdout - if we replace a TTY
- # destined stdout with a pipe destined stdout (e.g. pager), we want line
- # buffering
- if isatty(stdout):
- stdout = os.fdopen(stdout.fileno(), pycompat.sysstr('wb'), 1)
- if pycompat.osname == 'nt':
- from . import windows as platform
- stdout = platform.winstdout(stdout)
- else:
- from . import posix as platform
- _ = i18n._
- bindunixsocket = platform.bindunixsocket
- cachestat = platform.cachestat
- checkexec = platform.checkexec
- checklink = platform.checklink
- copymode = platform.copymode
- executablepath = platform.executablepath
- expandglobs = platform.expandglobs
- explainexit = platform.explainexit
- findexe = platform.findexe
- gethgcmd = platform.gethgcmd
- getuser = platform.getuser
- getpid = os.getpid
- groupmembers = platform.groupmembers
- groupname = platform.groupname
- hidewindow = platform.hidewindow
- isexec = platform.isexec
- isowner = platform.isowner
- localpath = platform.localpath
- lookupreg = platform.lookupreg
- makedir = platform.makedir
- nlinks = platform.nlinks
- normpath = platform.normpath
- normcase = platform.normcase
- normcasespec = platform.normcasespec
- normcasefallback = platform.normcasefallback
- openhardlinks = platform.openhardlinks
- oslink = platform.oslink
- parsepatchoutput = platform.parsepatchoutput
- pconvert = platform.pconvert
- poll = platform.poll
- popen = platform.popen
- posixfile = platform.posixfile
- quotecommand = platform.quotecommand
- readpipe = platform.readpipe
- rename = platform.rename
- removedirs = platform.removedirs
- samedevice = platform.samedevice
- samefile = platform.samefile
- samestat = platform.samestat
- setbinary = platform.setbinary
- setflags = platform.setflags
- setsignalhandler = platform.setsignalhandler
- shellquote = platform.shellquote
- spawndetached = platform.spawndetached
- split = platform.split
- sshargs = platform.sshargs
- statfiles = getattr(osutil, 'statfiles', platform.statfiles)
- statisexec = platform.statisexec
- statislink = platform.statislink
- testpid = platform.testpid
- umask = platform.umask
- unlink = platform.unlink
- unlinkpath = platform.unlinkpath
- username = platform.username
- # Python compatibility
- _notset = object()
- # disable Python's problematic floating point timestamps (issue4836)
- # (Python hypocritically says you shouldn't change this behavior in
- # libraries, and sure enough Mercurial is not a library.)
- os.stat_float_times(False)
- def safehasattr(thing, attr):
- return getattr(thing, attr, _notset) is not _notset
- def bitsfrom(container):
- bits = 0
- for bit in container:
- bits |= bit
- return bits
- DIGESTS = {
- 'md5': hashlib.md5,
- 'sha1': hashlib.sha1,
- 'sha512': hashlib.sha512,
- }
- # List of digest types from strongest to weakest
- DIGESTS_BY_STRENGTH = ['sha512', 'sha1', 'md5']
- for k in DIGESTS_BY_STRENGTH:
- assert k in DIGESTS
- class digester(object):
- """helper to compute digests.
- This helper can be used to compute one or more digests given their name.
- >>> d = digester(['md5', 'sha1'])
- >>> d.update('foo')
- >>> [k for k in sorted(d)]
- ['md5', 'sha1']
- >>> d['md5']
- 'acbd18db4cc2f85cedef654fccc4a4d8'
- >>> d['sha1']
- '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
- >>> digester.preferred(['md5', 'sha1'])
- 'sha1'
- """
- def __init__(self, digests, s=''):
- self._hashes = {}
- for k in digests:
- if k not in DIGESTS:
- raise Abort(_('unknown digest type: %s') % k)
- self._hashes[k] = DIGESTS[k]()
- if s:
- self.update(s)
- def update(self, data):
- for h in self._hashes.values():
- h.update(data)
- def __getitem__(self, key):
- if key not in DIGESTS:
- raise Abort(_('unknown digest type: %s') % k)
- return self._hashes[key].hexdigest()
- def __iter__(self):
- return iter(self._hashes)
- @staticmethod
- def preferred(supported):
- """returns the strongest digest type in both supported and DIGESTS."""
- for k in DIGESTS_BY_STRENGTH:
- if k in supported:
- return k
- return None
- class digestchecker(object):
- """file handle wrapper that additionally checks content against a given
- size and digests.
- d = digestchecker(fh, size, {'md5': '...'})
- When multiple digests are given, all of them are validated.
- """
- def __init__(self, fh, size, digests):
- self._fh = fh
- self._size = size
- self._got = 0
- self._digests = dict(digests)
- self._digester = digester(self._digests.keys())
- def read(self, length=-1):
- content = self._fh.read(length)
- self._digester.update(content)
- self._got += len(content)
- return content
- def validate(self):
- if self._size != self._got:
- raise Abort(_('size mismatch: expected %d, got %d') %
- (self._size, self._got))
- for k, v in self._digests.items():
- if v != self._digester[k]:
- # i18n: first parameter is a digest name
- raise Abort(_('%s mismatch: expected %s, got %s') %
- (k, v, self._digester[k]))
- try:
- buffer = buffer
- except NameError:
- if not pycompat.ispy3:
- def buffer(sliceable, offset=0, length=None):
- if length is not None:
- return sliceable[offset:offset + length]
- return sliceable[offset:]
- else:
- def buffer(sliceable, offset=0, length=None):
- if length is not None:
- return memoryview(sliceable)[offset:offset + length]
- return memoryview(sliceable)[offset:]
- closefds = pycompat.osname == 'posix'
- _chunksize = 4096
- class bufferedinputpipe(object):
- """a manually buffered input pipe
- Python will not let us use buffered IO and lazy reading with 'polling' at
- the same time. We cannot probe the buffer state and select will not detect
- that data are ready to read if they are already buffered.
- This class let us work around that by implementing its own buffering
- (allowing efficient readline) while offering a way to know if the buffer is
- empty from the output (allowing collaboration of the buffer with polling).
- This class lives in the 'util' module because it makes use of the 'os'
- module from the python stdlib.
- """
- def __init__(self, input):
- self._input = input
- self._buffer = []
- self._eof = False
- self._lenbuf = 0
- @property
- def hasbuffer(self):
- """True is any data is currently buffered
- This will be used externally a pre-step for polling IO. If there is
- already data then no polling should be set in place."""
- return bool(self._buffer)
- @property
- def closed(self):
- return self._input.closed
- def fileno(self):
- return self._input.fileno()
- def close(self):
- return self._input.close()
- def read(self, size):
- while (not self._eof) and (self._lenbuf < size):
- self._fillbuffer()
- return self._frombuffer(size)
- def readline(self, *args, **kwargs):
- if 1 < len(self._buffer):
- # this should not happen because both read and readline end with a
- # _frombuffer call that collapse it.
- self._buffer = [''.join(self._buffer)]
- self._lenbuf = len(self._buffer[0])
- lfi = -1
- if self._buffer:
- lfi = self._buffer[-1].find('\n')
- while (not self._eof) and lfi < 0:
- self._fillbuffer()
- if self._buffer:
- lfi = self._buffer[-1].find('\n')
- size = lfi + 1
- if lfi < 0: # end of file
- size = self._lenbuf
- elif 1 < len(self._buffer):
- # we need to take previous chunks into account
- size += self._lenbuf - len(self._buffer[-1])
- return self._frombuffer(size)
- def _frombuffer(self, size):
- """return at most 'size' data from the buffer
- The data are removed from the buffer."""
- if size == 0 or not self._buffer:
- return ''
- buf = self._buffer[0]
- if 1 < len(self._buffer):
- buf = ''.join(self._buffer)
- data = buf[:size]
- buf = buf[len(data):]
- if buf:
- self._buffer = [buf]
- self._lenbuf = len(buf)
- else:
- self._buffer = []
- self._lenbuf = 0
- return data
- def _fillbuffer(self):
- """read data to the buffer"""
- data = os.read(self._input.fileno(), _chunksize)
- if not data:
- self._eof = True
- else:
- self._lenbuf += len(data)
- self._buffer.append(data)
- def popen2(cmd, env=None, newlines=False):
- # Setting bufsize to -1 lets the system decide the buffer size.
- # The default for bufsize is 0, meaning unbuffered. This leads to
- # poor performance on Mac OS X: http://bugs.python.org/issue4194
- p = subprocess.Popen(cmd, shell=True, bufsize=-1,
- close_fds=closefds,
- stdin=subprocess.PIPE, stdout=subprocess.PIPE,
- universal_newlines=newlines,
- env=env)
- return p.stdin, p.stdout
- def popen3(cmd, env=None, newlines=False):
- stdin, stdout, stderr, p = popen4(cmd, env, newlines)
- return stdin, stdout, stderr
- def popen4(cmd, env=None, newlines=False, bufsize=-1):
- p = subprocess.Popen(cmd, shell=True, bufsize=bufsize,
- close_fds=closefds,
- stdin=subprocess.PIPE, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- universal_newlines=newlines,
- env=env)
- return p.stdin, p.stdout, p.stderr, p
- def version():
- """Return version information if available."""
- try:
- from . import __version__
- return __version__.version
- except ImportError:
- return 'unknown'
- def versiontuple(v=None, n=4):
- """Parses a Mercurial version string into an N-tuple.
- The version string to be parsed is specified with the ``v`` argument.
- If it isn't defined, the current Mercurial version string will be parsed.
- ``n`` can be 2, 3, or 4. Here is how some version strings map to
- returned values:
- >>> v = '3.6.1+190-df9b73d2d444'
- >>> versiontuple(v, 2)
- (3, 6)
- >>> versiontuple(v, 3)
- (3, 6, 1)
- >>> versiontuple(v, 4)
- (3, 6, 1, '190-df9b73d2d444')
- >>> versiontuple('3.6.1+190-df9b73d2d444+20151118')
- (3, 6, 1, '190-df9b73d2d444+20151118')
- >>> v = '3.6'
- >>> versiontuple(v, 2)
- (3, 6)
- >>> versiontuple(v, 3)
- (3, 6, None)
- >>> versiontuple(v, 4)
- (3, 6, None, None)
- >>> v = '3.9-rc'
- >>> versiontuple(v, 2)
- (3, 9)
- >>> versiontuple(v, 3)
- (3, 9, None)
- >>> versiontuple(v, 4)
- (3, 9, None, 'rc')
- >>> v = '3.9-rc+2-02a8fea4289b'
- >>> versiontuple(v, 2)
- (3, 9)
- >>> versiontuple(v, 3)
- (3, 9, None)
- >>> versiontuple(v, 4)
- (3, 9, None, 'rc+2-02a8fea4289b')
- """
- if not v:
- v = version()
- parts = remod.split('[\+-]', v, 1)
- if len(parts) == 1:
- vparts, extra = parts[0], None
- else:
- vparts, extra = parts
- vints = []
- for i in vparts.split('.'):
- try:
- vints.append(int(i))
- except ValueError:
- break
- # (3, 6) -> (3, 6, None)
- while len(vints) < 3:
- vints.append(None)
- if n == 2:
- return (vints[0], vints[1])
- if n == 3:
- return (vints[0], vints[1], vints[2])
- if n == 4:
- return (vints[0], vints[1], vints[2], extra)
- # used by parsedate
- defaultdateformats = (
- '%Y-%m-%dT%H:%M:%S', # the 'real' ISO8601
- '%Y-%m-%dT%H:%M', # without seconds
- '%Y-%m-%dT%H%M%S', # another awful but legal variant without :
- '%Y-%m-%dT%H%M', # without seconds
- '%Y-%m-%d %H:%M:%S', # our common legal variant
- '%Y-%m-%d %H:%M', # without seconds
- '%Y-%m-%d %H%M%S', # without :
- '%Y-%m-%d %H%M', # without seconds
- '%Y-%m-%d %I:%M:%S%p',
- '%Y-%m-%d %H:%M',
- '%Y-%m-%d %I:%M%p',
- '%Y-%m-%d',
- '%m-%d',
- '%m/%d',
- '%m/%d/%y',
- '%m/%d/%Y',
- '%a %b %d %H:%M:%S %Y',
- '%a %b %d %I:%M:%S%p %Y',
- '%a, %d %b %Y %H:%M:%S', # GNU coreutils "/bin/date --rfc-2822"
- '%b %d %H:%M:%S %Y',
- '%b %d %I:%M:%S%p %Y',
- '%b %d %H:%M:%S',
- '%b %d %I:%M:%S%p',
- '%b %d %H:%M',
- '%b %d %I:%M%p',
- '%b %d %Y',
- '%b %d',
- '%H:%M:%S',
- '%I:%M:%S%p',
- '%H:%M',
- '%I:%M%p',
- )
- extendeddateformats = defaultdateformats + (
- "%Y",
- "%Y-%m",
- "%b",
- "%b %Y",
- )
- def cachefunc(func):
- '''cache the result of function calls'''
- # XXX doesn't handle keywords args
- if func.__code__.co_argcount == 0:
- cache = []
- def f():
- if len(cache) == 0:
- cache.append(func())
- return cache[0]
- return f
- cache = {}
- if func.__code__.co_argcount == 1:
- # we gain a small amount of time because
- # we don't need to pack/unpack the list
- def f(arg):
- if arg not in cache:
- cache[arg] = func(arg)
- return cache[arg]
- else:
- def f(*args):
- if args not in cache:
- cache[args] = func(*args)
- return cache[args]
- return f
- class sortdict(dict):
- '''a simple sorted dictionary'''
- def __init__(self, data=None):
- self._list = []
- if data:
- self.update(data)
- def copy(self):
- return sortdict(self)
- def __setitem__(self, key, val):
- if key in self:
- self._list.remove(key)
- self._list.append(key)
- dict.__setitem__(self, key, val)
- def __iter__(self):
- return self._list.__iter__()
- def update(self, src):
- if isinstance(src, dict):
- src = src.iteritems()
- for k, v in src:
- self[k] = v
- def clear(self):
- dict.clear(self)
- self._list = []
- def items(self):
- return [(k, self[k]) for k in self._list]
- def __delitem__(self, key):
- dict.__delitem__(self, key)
- self._list.remove(key)
- def pop(self, key, *args, **kwargs):
- dict.pop(self, key, *args, **kwargs)
- try:
- self._list.remove(key)
- except ValueError:
- pass
- def keys(self):
- return self._list[:]
- def iterkeys(self):
- return self._list.__iter__()
- def iteritems(self):
- for k in self._list:
- yield k, self[k]
- def insert(self, index, key, val):
- self._list.insert(index, key)
- dict.__setitem__(self, key, val)
- def __repr__(self):
- if not self:
- return '%s()' % self.__class__.__name__
- return '%s(%r)' % (self.__class__.__name__, self.items())
- class _lrucachenode(object):
- """A node in a doubly linked list.
- Holds a reference to nodes on either side as well as a key-value
- pair for the dictionary entry.
- """
- __slots__ = (u'next', u'prev', u'key', u'value')
- def __init__(self):
- self.next = None
- self.prev = None
- self.key = _notset
- self.value = None
- def markempty(self):
- """Mark the node as emptied."""
- self.key = _notset
- class lrucachedict(object):
- """Dict that caches most recent accesses and sets.
- The dict consists of an actual backing dict - indexed by original
- key - and a doubly linked circular list defining the order of entries in
- the cache.
- The head node is the newest entry in the cache. If the cache is full,
- we recycle head.prev and make it the new head. Cache accesses result in
- the node being moved to before the existing head and being marked as the
- new head node.
- """
- def __init__(self, max):
- self._cache = {}
- self._head = head = _lrucachenode()
- head.prev = head
- head.next = head
- self._size = 1
- self._capacity = max
- def __len__(self):
- return len(self._cache)
- def __contains__(self, k):
- return k in self._cache
- def __iter__(self):
- # We don't have to iterate in cache order, but why not.
- n = self._head
- for i in range(len(self._cache)):
- yield n.key
- n = n.next
- def __getitem__(self, k):
- node = self._cache[k]
- self._movetohead(node)
- return node.value
- def __setitem__(self, k, v):
- node = self._cache.get(k)
- # Replace existing value and mark as newest.
- if node is not None:
- node.value = v
- self._movetohead(node)
- return
- if self._size < self._capacity:
- node = self._addcapacity()
- else:
- # Grab the last/oldest item.
- node = self._head.prev
- # At capacity. Kill the old entry.
- if node.key is not _notset:
- del self._cache[node.key]
- node.key = k
- node.value = v
- self._cache[k] = node
- # And mark it as newest entry. No need to adjust order since it
- # is already self._head.prev.
- self._head = node
- def __delitem__(self, k):
- node = self._cache.pop(k)
- node.markempty()
- # Temporarily mark as newest item before re-adjusting head to make
- # this node the oldest item.
- self._movetohead(node)
- self._head = node.next
- # Additional dict methods.
- def get(self, k, default=None):
- try:
- return self._cache[k].value
- except KeyError:
- return default
- def clear(self):
- n = self._head
- while n.key is not _notset:
- n.markempty()
- n = n.next
- self._cache.clear()
- def copy(self):
- result = lrucachedict(self._capacity)
- n = self._head.prev
- # Iterate in oldest-to-newest order, so the copy has the right ordering
- for i in range(len(self._cache)):
- result[n.key] = n.value
- n = n.prev
- return result
- def _movetohead(self, node):
- """Mark a node as the newest, making it the new head.
- When a node is accessed, it becomes the freshest entry in the LRU
- list, which is denoted by self._head.
- Visually, let's make ``N`` the new head node (* denotes head):
- previous/oldest <-> head <-> next/next newest
- ----<->--- A* ---<->-----
- | |
- E <-> D <-> N <-> C <-> B
- To:
- ----<->--- N* ---<->-----
- | |
- E <-> D <-> C <-> B <-> A
- This requires the following moves:
- C.next = D (node.prev.next = node.next)
- D.prev = C (node.next.prev = node.prev)
- E.next = N (head.prev.next = node)
- N.prev = E (node.prev = head.prev)
- N.next = A (node.next = head)
- A.prev = N (head.prev = node)
- """
- head = self._head
- # C.next = D
- node.prev.next = node.next
- # D.prev = C
- node.next.prev = node.prev
- # N.prev = E
- node.prev = head.prev
- # N.next = A
- # It is tempting to do just "head" here, however if node is
- # adjacent to head, this will do bad things.
- node.next = head.prev.next
- # E.next = N
- node.next.prev = node
- # A.prev = N
- node.prev.next = node
- self._head = node
- def _addcapacity(self):
- """Add a node to the circular linked list.
- The new node is inserted before the head node.
- """
- head = self._head
- node = _lrucachenode()
- head.prev.next = node
- node.prev = head.prev
- node.next = head
- head.prev = node
- self._size += 1
- return node
- def lrucachefunc(func):
- '''cache most recent results of function calls'''
- cache = {}
- order = collections.deque()
- if func.__code__.co_argcount == 1:
- def f(arg):
- if arg not in cache:
- if len(cache) > 20:
- del cache[order.popleft()]
- cache[arg] = func(arg)
- else:
- order.remove(arg)
- order.append(arg)
- return cache[arg]
- else:
- def f(*args):
- if args not in cache:
- if len(cache) > 20:
- del cache[order.popleft()]
- cache[args] = func(*args)
- else:
- order.remove(args)
- order.append(args)
- return cache[args]
- return f
- class propertycache(object):
- def __init__(self, func):
- self.func = func
- self.name = func.__name__
- def __get__(self, obj, type=None):
- result = self.func(obj)
- self.cachevalue(obj, result)
- return result
- def cachevalue(self, obj, value):
- # __dict__ assignment required to bypass __setattr__ (eg: repoview)
- obj.__dict__[self.name] = value
- def pipefilter(s, cmd):
- '''filter string S through command CMD, returning its output'''
- p = subprocess.Popen(cmd, shell=True, close_fds=closefds,
- stdin=subprocess.PIPE, stdout=subprocess.PIPE)
- pout, perr = p.communicate(s)
- return pout
- def tempfilter(s, cmd):
- '''filter string S through a pair of temporary files with CMD.
- CMD is used as a template to create the real command to be run,
- with the strings INFILE and OUTFILE replaced by the real names of
- the temporary files generated.'''
- inname, outname = None, None
- try:
- infd, inname = tempfile.mkstemp(prefix='hg-filter-in-')
- fp = os.fdopen(infd, pycompat.sysstr('wb'))
- fp.write(s)
- fp.close()
- outfd, outname = tempfile.mkstemp(prefix='hg-filter-out-')
- os.close(outfd)
- cmd = cmd.replace('INFILE', inname)
- cmd = cmd.replace('OUTFILE', outname)
- code = os.system(cmd)
- if pycompat.sysplatform == 'OpenVMS' and code & 1:
- code = 0
- if code:
- raise Abort(_("command '%s' failed: %s") %
- (cmd, explainexit(code)))
- return readfile(outname)
- finally:
- try:
- if inname:
- os.unlink(inname)
- except OSError:
- pass
- try:
- if outname:
- os.unlink(outname)
- except OSError:
- pass
- filtertable = {
- 'tempfile:': tempfilter,
- 'pipe:': pipefilter,
- }
- def filter(s, cmd):
- "filter a string through a command that transforms its input to its output"
- for name, fn in filtertable.iteritems():
- if cmd.startswith(name):
- return fn(s, cmd[len(name):].lstrip())
- return pipefilter(s, cmd)
- def binary(s):
- """return true if a string is binary data"""
- return bool(s and '\0' in s)
- def increasingchunks(source, min=1024, max=65536):
- '''return no less than min bytes per chunk while data remains,
- doubling min after each chunk until it reaches max'''
- def log2(x):
- if not x:
- return 0
- i = 0
- while x:
- x >>= 1
- i += 1
- return i - 1
- buf = []
- blen = 0
- for chunk in source:
- buf.append(chunk)
- blen += len(chunk)
- if blen >= min:
- if min < max:
- min = min << 1
- nmin = 1 << log2(blen)
- if nmin > min:
- min = nmin
- if min > max:
- min = max
- yield ''.join(buf)
- blen = 0
- buf = []
- if buf:
- yield ''.join(buf)
- Abort = error.Abort
- def always(fn):
- return True
- def never(fn):
- return False
- def nogc(func):
- """disable garbage collector
- Python's garbage collector triggers a GC each time a certain number of
- container objects (the number being defined by gc.get_threshold()) are
- allocated even when marked not to be tracked by the collector. Tracking has
- no effect on when GCs are triggered, only on what objects the GC looks
- into. As a workaround, disable GC while building complex (huge)
- containers.
- This garbage collector issue have been fixed in 2.7.
- """
- if sys.version_info >= (2, 7):
- return func
- def wrapper(*args, **kwargs):
- gcenabled = gc.isenabled()
- gc.disable()
- try:
- return func(*args, **kwargs)
- finally:
- if gcenabled:
- gc.enable()
- return wrapper
- def pathto(root, n1, n2):
- '''return the relative path from one place to another.
- root should use os.sep to separate directories
- n1 should use os.sep to separate directories
- n2 should use "/" to separate directories
- returns an os.sep-separated path.
- If n1 is a relative path, it's assumed it's
- relative to root.
- n2 should always be relative to root.
- '''
- if not n1:
- return localpath(n2)
- if os.path.isabs(n1):
- if os.path.splitdrive(root)[0] != os.path.splitdrive(n1)[0]:
- return os.path.join(root, localpath(n2))
- n2 = '/'.join((pconvert(root), n2))
- a, b = splitpath(n1), n2.split('/')
- a.reverse()
- b.reverse()
- while a and b and a[-1] == b[-1]:
- a.pop()
- b.pop()
- b.reverse()
- return pycompat.ossep.join((['..'] * len(a)) + b) or '.'
- def mainfrozen():
- """return True if we are a frozen executable.
- The code supports py2exe (most common, Windows only) and tools/freeze
- (portable, not much used).
- """
- return (safehasattr(sys, "frozen") or # new py2exe
- safehasattr(sys, "importers") or # old py2exe
- imp.is_frozen(u"__main__")) # tools/freeze
- # the location of data files matching the source code
- if mainfrozen() and getattr(sys, 'frozen', None) != 'macosx_app':
- # executable version (py2exe) doesn't support __file__
- datapath = os.path.dirname(pycompat.sysexecutable)
- else:
- datapath = os.path.dirname(__file__)
- if not isinstance(datapath, bytes):
- datapath = pycompat.fsencode(datapath)
- i18n.setdatapath(datapath)
- _hgexecutable = None
- def hgexecutable():
- """return location of the 'hg' executable.
- Defaults to $HG or 'hg' in the search path.
- """
- if _hgexecutable is None:
- hg = encoding.environ.get('HG')
- mainmod = sys.modules['__main__']
- if hg:
- _sethgexecutable(hg)
- elif mainfrozen():
- if getattr(sys, 'frozen', None) == 'macosx_app':
- # Env variable set by py2app
- _sethgexecutable(encoding.environ['EXECUTABLEPATH'])
- else:
- _sethgexecutable(pycompat.sysexecutable)
- elif os.path.basename(getattr(mainmod, '__file__', '')) == 'hg':
- _sethgexecutable(mainmod.__file__)
- else:
- exe = findexe('hg') or os.path.basename(sys.argv[0])
- _sethgexecutable(exe)
- return _hgexecutable
- def _sethgexecutable(path):
- """set location of the 'hg' executable"""
- global _hgexecutable
- _hgexecutable = path
- def _isstdout(f):
- fileno = getattr(f, 'fileno', None)
- return fileno and fileno() == sys.__stdout__.fileno()
- def shellenviron(environ=None):
- """return environ with optional override, useful for shelling out"""
- def py2shell(val):
- 'convert python object into string that is useful to shell'
- if val is None or val is False:
- return '0'
- if val is True:
- return '1'
- return str(val)
- env = dict(encoding.environ)
- if environ:
- env.update((k, py2shell(v)) for k, v in environ.iteritems())
- env['HG'] = hgexecutable()
- return env
- def system(cmd, environ=None, cwd=None, onerr=None, errprefix=None, out=None):
- '''enhanced shell command execution.
- run with environment maybe modified, maybe in different dir.
- if command fails and onerr is None, return status, else raise onerr
- object as exception.
- if out is specified, it is assumed to be a file-like object that has a
- write() method. stdout and stderr will be redirected to out.'''
- try:
- stdout.flush()
- except Exception:
- pass
- origcmd = cmd
- cmd = quotecommand(cmd)
- if pycompat.sysplatform == 'plan9' and (sys.version_info[0] == 2
- and sys.version_info[1] < 7):
- # subprocess kludge to work around issues in half-baked Python
- # ports, notably bichued/python:
- if not cwd is None:
- os.chdir(cwd)
- rc = os.system(cmd)
- else:
- env = shellenviron(environ)
- if out is None or _isstdout(out):
- rc = subprocess.call(cmd, shell=True, close_fds=closefds,
- env=env, cwd=cwd)
- else:
- proc = subprocess.Popen(cmd, shell=True, close_fds=closefds,
- env=env, cwd=cwd, stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT)
- for line in iter(proc.stdout.readline, ''):
- out.write(line)
- proc.wait()
- rc = proc.returncode
- if pycompat.sysplatform == 'OpenVMS' and rc & 1:
- rc = 0
- if rc and onerr:
- errmsg = '%s %s' % (os.path.basename(origcmd.split(None, 1)[0]),
- explainexit(rc)[0])
- if errprefix:
- errmsg = '%s: %s' % (errprefix, errmsg)
- raise onerr(errmsg)
- return rc
- def checksignature(func):
- '''wrap a function with code to check for calling errors'''
- def check(*args, **kwargs):
- try:
- return func(*args, **kwargs)
- except TypeError:
- if len(traceback.extract_tb(sys.exc_info()[2])) == 1:
- raise error.SignatureError
- raise
- return check
- def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False):
- '''copy a file, preserving mode and optionally other stat info like
- atime/mtime
- checkambig argument is used with filestat, and is useful only if
- destination file is guarded by any lock (e.g. repo.lock or
- repo.wlock).
- copystat and checkambig should be exclusive.
- '''
- assert not (copystat and checkambig)
- oldstat = None
- if os.path.lexists(dest):
- if checkambig:
- oldstat = checkambig and filestat(dest)
- unlink(dest)
- # hardlinks are problematic on CIFS, quietly ignore this flag
- # until we find a way to work around it cleanly (issue4546)
- if False and hardlink:
- try:
- oslink(src, dest)
- return
- except (IOError, OSError):
- pass # fall back to normal copy
- if os.path.islink(src):
- os.symlink(os.readlink(src), dest)
- # copytime is ignored for symlinks, but in general copytime isn't needed
- # for them anyway
- else:
- try:
- shutil.copyfile(src, dest)
- if copystat:
- # copystat also copies mode
- shutil.copystat(src, dest)
- else:
- shutil.copymode(src, dest)
- if oldstat and oldstat.stat:
- newstat = filestat(dest)
- if newstat.isambig(oldstat):
- # stat of copied file is ambiguous to original one
- advanced = (oldstat.stat.st_mtime + 1) & 0x7fffffff
- os.utime(dest, (advanced, advanced))
- except shutil.Error as inst:
- raise Abort(str(inst))
- def copyfiles(src, dst, hardlink=None, progress=lambda t, pos: None):
- """Copy a directory tree using hardlinks if possible."""
- num = 0
- if hardlink is None:
- hardlink = (os.stat(src).st_dev ==
- os.stat(os.path.dirname(dst)).st_dev)
- if hardlink:
- topic = _('linking')
- else:
- topic = _('copying')
- if os.path.isdir(src):
- os.mkdir(dst)
- for name, kind in osutil.listdir(src):
- srcname = os.path.join(src, name)
- dstname = os.path.join(dst, name)
- def nprog(t, pos):
- if pos is not None:
- return progress(t, pos + num)
- hardlink, n = copyfiles(srcname, dstname, hardlink, progress=nprog)
- num += n
- else:
- if hardlink:
- try:
- oslink(src, dst)
- except (IOError, OSError):
- hardlink = False
- shutil.copy(src, dst)
- else:
- shutil.copy(src, dst)
- num += 1
- progress(topic, num)
- progress(topic, None)
- return hardlink, num
- _winreservednames = '''con prn aux nul
- com1 com2 com3 com4 com5 com6 com7 com8 com9
- lpt1 lpt2 lpt3 lpt4 lpt5 lpt6 lpt7 lpt8 lpt9'''.split()
- _winreservedchars = ':*?"<>|'
- def checkwinfilename(path):
- r'''Check that the base-relative path is a valid filename on Windows.
- Returns None if the path is ok, or a UI string describing the problem.
- >>> checkwinfilename("just/a/normal/path")
- >>> checkwinfilename("foo/bar/con.xml")
- "filename contains 'con', which is reserved on Windows"
- >>> checkwinfilename("foo/con.xml/bar")
- "filename contains 'con', which is reserved on Windows"
- >>> checkwinfilename("foo/bar/xml.con")
- >>> checkwinfilename("foo/bar/AUX/bla.txt")
- "filename contains 'AUX', which is reserved on Windows"
- >>> checkwinfilename("foo/bar/bla:.txt")
- "filename contains ':', which is reserved on Windows"
- >>> checkwinfilename("foo/bar/b\07la.txt")
- "filename contains '\\x07', which is invalid on Windows"
- >>> checkwinfilename("foo/bar/bla ")
- "filename ends with ' ', which is not allowed on Windows"
- >>> checkwinfilename("../bar")
- >>> checkwinfilename("foo\\")
- "filename ends with '\\', which is invalid on Windows"
- >>> checkwinfilename("foo\\/bar")
- "directory name ends with '\\', which is invalid on Windows"
- '''
- if path.endswith('\\'):
- return _("filename ends with '\\', which is invalid on Windows")
- if '\\/' in path:
- return _("directory name ends with '\\', which is invalid on Windows")
- for n in path.replace('\\', '/').split('/'):
- if not n:
- continue
- for c in n:
- if c in _winreservedchars:
- return _("filename contains '%s', which is reserved "
- "on Windows") % c
- if ord(c) <= 31:
- return _("filename contains %r, which is invalid "
- "on Windows") % c
- base = n.split('.')[0]
- if base and base.lower() in _winreservednames:
- return _("filename contains '%s', which is reserved "
- "on Windows") % base
- t = n[-1]
- if t in '. ' and n not in '..':
- return _("filename ends with '%s', which is not allowed "
- "on Windows") % t
- if pycompat.osname == 'nt':
- checkosfilename = checkwinfilename
- timer = time.clock
- else:
- checkosfilename = platform.checkosfilename
- timer = time.time
- if safehasattr(time, "perf_counter"):
- timer = time.perf_counter
- def makelock(info, pathname):
- try:
- return os.symlink(info, pathname)
- except OSError as why:
- if why.errno == errno.EEXIST:
- raise
- except AttributeError: # no symlink in os
- pass
- ld = os.open(pathname, os.O_CREAT | os.O_WRONLY | os.O_EXCL)
- os.write(ld, info)
- os.close(ld)
- def readlock(pathname):
- try:
- return os.readlink(pathname)
- except OSError as why:
- if why.errno not in (errno.EINVAL, errno.ENOSYS):
- raise
- except AttributeError: # no symlink in os
- pass
- fp = posixfile(pathname)
- r = fp.read()
- fp.close()
- return r
- def fstat(fp):
- '''stat file object that may not have fileno method.'''
- try:
- return os.fstat(fp.fileno())
- except AttributeError:
- return os.stat(fp.name)
- # File system features
- def fscasesensitive(path):
- """
- Return true if the given path is on a case-sensitive filesystem
- Requires a path (like /foo/.hg) ending with a foldable final
- directory component.
- """
- s1 = os.lstat(path)
- d, b = os.path.split(path)
- b2 = b.upper()
- if b == b2:
- b2 = b.lower()
- if b == b2:
- return True # no evidence against case sensitivity
- p2 = os.path.join(d, b2)
- try:
- s2 = os.lstat(p2)
- if s2 == s1:
- return False
- return True
- except OSError:
- return True
- try:
- import re2
- _re2 = None
- except ImportError:
- _re2 = False
- class _re(object):
- def _checkre2(self):
- global _re2
- try:
- # check if match works, see issue3964
- _re2 = bool(re2.match(r'\[([^\[]+)\]', '[ui]'))
- except ImportError:
- _re2 = False
- def compile(self, pat, flags=0):
- '''Compile a regular expression, using re2 if possible
- For best performance, use only re2-compatible regexp features. The
- only flags from the re module that are re2-compatible are
- IGNORECASE and MULTILINE.'''
- if _re2 is None:
- self._checkre2()
- if _re2 and (flags & ~(remod.IGNORECASE | remod.MULTILINE)) == 0:
- if flags & remod.IGNORECASE:
- pat = '(?i)' + pat
- if flags & remod.MULTILINE:
- pat = '(?m)' + pat
- try:
- return re2.compile(pat)
- except re2.error:
- pass
- return remod.compile(pat, flags)
- @propertycache
- def escape(self):
- '''Return the version of escape corresponding to self.compile.
- This is imperfect because whether re2 or re is used for a particular
- function depends on the flags, etc, but it's the best we can do.
- '''
- global _re2
- if _re2 is None:
- self._checkre2()
- if _re2:
- return re2.escape
- else:
- return remod.escape
- re = _re()
- _fspathcache = {}
- def fspath(name, root):
- '''Get name in the case stored in the filesystem
- The name should be relative to root, and be normcase-ed for efficiency.
- Note that this function is unnecessary, and should not be
- called, for case-sensitive filesystems (simply because it's expensive).
- The root should be normcase-ed, too.
- '''
- def _makefspathcacheentry(dir):
- return dict((normcase(n), n) for n in os.listdir(dir))
- seps = pycompat.ossep
- if pycompat.osaltsep:
- seps = seps + pycompat.osaltsep
- # Protect backslashes. This gets silly very quickly.
- seps.replace('\\','\\\\')
- pattern = remod.compile(r'([^%s]+)|([%s]+)' % (seps, seps))
- dir = os.path.normpath(root)
- result = []
- for part, sep in pattern.findall(name):
- if sep:
- result.append(sep)
- continue
- if dir not in _fspathcache:
- _fspathcache[dir] = _makefspathcacheentry(dir)
- contents = _fspathcache[dir]
- found = contents.get(part)
- if not found:
- # retry "once per directory" per "dirstate.walk" which
- # may take place for each patches of "hg qpush", for example
- _fspathcache[dir] = contents = _makefspathcacheentry(dir)
- found = contents.get(part)
- result.append(found or part)
- dir = os.path.join(dir, part)
- return ''.join(result)
- def checknlink(testfile):
- '''check whether hardlink count reporting works properly'''
- # testfile may be open, so we need a separate file for checking to
- # work around issue2543 (or testfile may get lost on Samba shares)
- f1 = testfile + ".hgtmp1"
- if os.path.lexists(f1):
- return False
- try:
- posixfile(f1, 'w').close()
- except IOError:
- try:
- os.unlink(f1)
- except OSError:
- pass
- return False
- f2 = testfile + ".hgtmp2"
- fd = None
- try:
- oslink(f1, f2)
- # nlinks() may behave differently for files on Windows shares if
- # the file is open.
- fd = posixfile(f2)
- return nlinks(f2) > 1
- except OSError:
- return False
- finally:
- if fd is not None:
- fd.close()
- for f in (f1, f2):
- try:
- os.unlink(f)
- except OSError:
- pass
- def endswithsep(path):
- '''Check path ends with os.sep or os.altsep.'''
- return (path.endswith(pycompat.ossep)
- or pycompat.osaltsep and path.endswith(pycompat.osaltsep))
- def splitpath(path):
- '''Split path by os.sep.
- Note that this function does not use os.altsep because this is
- an alternative of simple "xxx.split(os.sep)".
- It is recommended to use os.path.normpath() before using this
- function if need.'''
- return path.split(pycompat.ossep)
- def gui():
- '''Are we running in a GUI?'''
- if pycompat.sysplatform == 'darwin':
- if 'SSH_CONNECTION' in encoding.environ:
- # handle SSH access to a box where the user is logged in
- return False
- elif getattr(osutil, 'isgui', None):
- # check if a CoreGraphics session is available
- return osutil.isgui()
- else:
- # pure build; use a safe default
- return True
- else:
- return pycompat.osname == "nt" or encoding.environ.get("DISPLAY")
- def mktempcopy(name, emptyok=False, createmode=None):
- """Create a temporary file with the same contents from name
- The permission bits are copied from the original file.
- If the temporary file is going to be truncated immediately, you
- can use emptyok=True as an optimization.
- Returns the name of the temporary file.
- """
- d, fn = os.path.split(name)
- fd, temp = tempfile.mkstemp(prefix='.%s-' % fn, dir=d)
- os.close(fd)
- # Temporary files are created with mode 0600, which is usually not
- # what we want. If the original file already exists, just copy
- # its mode. Otherwise, manually obey umask.
- copymode(name, temp, createmode)
- if emptyok:
- return temp
- try:
- try:
- ifp = posixfile(name, "rb")
- except IOError as inst:
- if inst.errno == errno.ENOENT:
- return temp
- if not getattr(inst, 'filename', None):
- inst.filename = name
- raise
- ofp = posixfile(temp, "wb")
- for chunk in filechunkiter(ifp):
- ofp.write(chunk)
- ifp.close()
- ofp.close()
- except: # re-raises
- try: os.unlink(temp)
- except OSError: pass
- raise
- return temp
- class filestat(object):
- """help to exactly detect change of a file
- 'stat' attribute is result of 'os.stat()' if specified 'path'
- exists. Otherwise, it is None. This can avoid preparative
- 'exists()' examination on client side of this class.
- """
- def __init__(self, path):
- try:
- self.stat = os.stat(path)
- except OSError as err:
- if err.errno != errno.ENOENT:
- raise
- self.stat = None
- __hash__ = object.__hash__
- def __eq__(self, old):
- try:
- # if ambiguity between stat of new and old file is
- # avoided, comparison of size, ctime and mtime is enough
- # to exactly detect change of a file regardless of platform
- return (self.stat.st_size == old.stat.st_size and
- self.stat.st_ctime == old.stat.st_ctime and
- self.stat.st_mtime == old.stat.st_mtime)
- except AttributeError:
- return False
- def isambig(self, old):
- """Examine whether new (= self) stat is ambiguous against old one
- "S[N]" below means stat of a file at N-th change:
- - S[n-1].ctime < S[n].ctime: can detect change of a file
- - S[n-1].ctime == S[n].ctime
- - S[n-1].ctime < S[n].mtime: means natural advancing (*1)
- - S[n-1].ctime == S[n].mtime: is ambiguous (*2)
- - S[n-1].ctime > S[n].mtime: never occurs naturally (don't care)
- - S[n-1].ctime > S[n].ctime: never occurs naturally (don't care)
- Case (*2) above means that a file was changed twice or more at
- same time in sec (= S[n-1].ctime), and comparison of timestamp
- is ambiguous.
- Base idea to avoid such ambiguity is "advance mtime 1 sec, if
- timestamp is ambiguous".
- But advancing mtime only in case (*2) doesn't work as
- expected, because naturally advanced S[n].mtime in case (*1)
- might be equal to manually advanced S[n-1 or earlier].mtime.
- Therefore, all "S[n-1].ctime == S[n].ctime" cases should be
- treated as ambiguous regardless of mtime, to avoid overlooking
- by confliction between such mtime.
- Advancing mtime "if isambig(oldstat)" ensures "S[n-1].mtime !=
- S[n].mtime", even if size of a file isn't changed.
- """
- try:
- return (self.stat.st_ctime == old.stat.st_ctime)
- except AttributeError:
- return False
- def avoidambig(self, path, old):
- """Change file stat of specified path to avoid ambiguity
- 'old' should be previous filestat of 'path'.
- This skips avoiding ambiguity, if a process doesn't have
- appropriate privileges for 'path'.
- """
- advanced = (old.stat.st_mtime + 1) & 0x7fffffff
- try:
- os.utime(path, (advanced, advanced))
- except OSError as inst:
- if inst.errno == errno.EPERM:
- # utime() on the file created by another user causes EPERM,
- # if a process doesn't have appropriate privileges
- return
- raise
- def __ne__(self, other):
- return not self == other
- class atomictempfile(object):
- '''writable file object that atomically updates a file
- All writes will go to a temporary copy of the original file. Call
- close() when you are done writing, and atomictempfile will rename
- the temporary copy to the original name, making the changes
- visible. If the object is destroyed without being closed, all your
- writes are discarded.
- checkambig argument of constructor is used with filestat, and is
- useful only if target file is guarded by any lock (e.g. repo.lock
- or repo.wlock).
- '''
- def __init__(self, name, mode='w+b', createmode=None, checkambig=False):
- self.__name = name # permanent name
- self._tempname = mktempcopy(name, emptyok=('w' in mode),
- createmode=createmode)
- self._fp = posixfile(self._tempname, mode)
- self._checkambig = checkambig
- # delegated methods
- self.read = self._fp.read
- self.write = self._fp.write
- self.seek = self._fp.seek
- self.tell = self._fp.tell
- self.fileno = self._fp.fileno
- def close(self):
- if not self._fp.closed:
- self._fp.close()
- filename = localpath(self.__name)
- oldstat = self._checkambig and filestat(filename)
- if oldstat and oldstat.stat:
- rename(self._tempname, filename)
- newstat = filestat(filename)
- if newstat.isambig(oldstat):
- # stat of changed file is ambiguous to original one
- advanced = (oldstat.stat.st_mtime + 1) & 0x7fffffff
- os.utime(filename, (advanced, advanced))
- else:
- rename(self._tempname, filename)
- def discard(self):
- if not self._fp.closed:
- try:
- os.unlink(self._tempname)
- except OSError:
- pass
- self._fp.close()
- def __del__(self):
- if safehasattr(self, '_fp'): # constructor actually did something
- self.discard()
- def __enter__(self):
- return self
- def __exit__(self, exctype, excvalue, traceback):
- if exctype is not None:
- self.discard()
- else:
- self.close()
- def makedirs(name, mode=None, notindexed=False):
- """recursive directory creation with parent mode inheritance
- Newly created directories are marked as "not to be indexed by
- the content indexing service", if ``notindexed`` is specified
- for "write" mode access.
- """
- try:
- makedir(name, not…
Large files files are truncated, but you can click here to view the full file