/git-p4.py
Python | 3242 lines | 3134 code | 61 blank | 47 comment | 125 complexity | 9c3354574a939b9049b165a69b54a968 MD5 | raw file
Possible License(s): GPL-2.0, LGPL-2.1, Apache-2.0, BSD-2-Clause
Large files files are truncated, but you can click here to view the full file
- #!/usr/bin/env python
- #
- # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
- #
- # Author: Simon Hausmann <simon@lst.de>
- # Copyright: 2007 Simon Hausmann <simon@lst.de>
- # 2007 Trolltech ASA
- # License: MIT <http://www.opensource.org/licenses/mit-license.php>
- #
- import optparse, sys, os, marshal, subprocess, shelve
- import tempfile, getopt, os.path, time, platform
- import re, shutil
- verbose = False
- # Only labels/tags matching this will be imported/exported
- defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
- def p4_build_cmd(cmd):
- """Build a suitable p4 command line.
- This consolidates building and returning a p4 command line into one
- location. It means that hooking into the environment, or other configuration
- can be done more easily.
- """
- real_cmd = ["p4"]
- user = gitConfig("git-p4.user")
- if len(user) > 0:
- real_cmd += ["-u",user]
- password = gitConfig("git-p4.password")
- if len(password) > 0:
- real_cmd += ["-P", password]
- port = gitConfig("git-p4.port")
- if len(port) > 0:
- real_cmd += ["-p", port]
- host = gitConfig("git-p4.host")
- if len(host) > 0:
- real_cmd += ["-H", host]
- client = gitConfig("git-p4.client")
- if len(client) > 0:
- real_cmd += ["-c", client]
- if isinstance(cmd,basestring):
- real_cmd = ' '.join(real_cmd) + ' ' + cmd
- else:
- real_cmd += cmd
- return real_cmd
- def chdir(dir):
- # P4 uses the PWD environment variable rather than getcwd(). Since we're
- # not using the shell, we have to set it ourselves. This path could
- # be relative, so go there first, then figure out where we ended up.
- os.chdir(dir)
- os.environ['PWD'] = os.getcwd()
- def die(msg):
- if verbose:
- raise Exception(msg)
- else:
- sys.stderr.write(msg + "\n")
- sys.exit(1)
- def write_pipe(c, stdin):
- if verbose:
- sys.stderr.write('Writing pipe: %s\n' % str(c))
- expand = isinstance(c,basestring)
- p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
- pipe = p.stdin
- val = pipe.write(stdin)
- pipe.close()
- if p.wait():
- die('Command failed: %s' % str(c))
- return val
- def p4_write_pipe(c, stdin):
- real_cmd = p4_build_cmd(c)
- return write_pipe(real_cmd, stdin)
- def read_pipe(c, ignore_error=False):
- if verbose:
- sys.stderr.write('Reading pipe: %s\n' % str(c))
- expand = isinstance(c,basestring)
- p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
- pipe = p.stdout
- val = pipe.read()
- if p.wait() and not ignore_error:
- die('Command failed: %s' % str(c))
- return val
- def p4_read_pipe(c, ignore_error=False):
- real_cmd = p4_build_cmd(c)
- return read_pipe(real_cmd, ignore_error)
- def read_pipe_lines(c):
- if verbose:
- sys.stderr.write('Reading pipe: %s\n' % str(c))
- expand = isinstance(c, basestring)
- p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
- pipe = p.stdout
- val = pipe.readlines()
- if pipe.close() or p.wait():
- die('Command failed: %s' % str(c))
- return val
- def p4_read_pipe_lines(c):
- """Specifically invoke p4 on the command supplied. """
- real_cmd = p4_build_cmd(c)
- return read_pipe_lines(real_cmd)
- def p4_has_command(cmd):
- """Ask p4 for help on this command. If it returns an error, the
- command does not exist in this version of p4."""
- real_cmd = p4_build_cmd(["help", cmd])
- p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- p.communicate()
- return p.returncode == 0
- def p4_has_move_command():
- """See if the move command exists, that it supports -k, and that
- it has not been administratively disabled. The arguments
- must be correct, but the filenames do not have to exist. Use
- ones with wildcards so even if they exist, it will fail."""
- if not p4_has_command("move"):
- return False
- cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
- p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- (out, err) = p.communicate()
- # return code will be 1 in either case
- if err.find("Invalid option") >= 0:
- return False
- if err.find("disabled") >= 0:
- return False
- # assume it failed because @... was invalid changelist
- return True
- def system(cmd):
- expand = isinstance(cmd,basestring)
- if verbose:
- sys.stderr.write("executing %s\n" % str(cmd))
- subprocess.check_call(cmd, shell=expand)
- def p4_system(cmd):
- """Specifically invoke p4 as the system command. """
- real_cmd = p4_build_cmd(cmd)
- expand = isinstance(real_cmd, basestring)
- subprocess.check_call(real_cmd, shell=expand)
- def p4_integrate(src, dest):
- p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
- def p4_sync(f, *options):
- p4_system(["sync"] + list(options) + [wildcard_encode(f)])
- def p4_add(f):
- # forcibly add file names with wildcards
- if wildcard_present(f):
- p4_system(["add", "-f", f])
- else:
- p4_system(["add", f])
- def p4_delete(f):
- p4_system(["delete", wildcard_encode(f)])
- def p4_edit(f):
- p4_system(["edit", wildcard_encode(f)])
- def p4_revert(f):
- p4_system(["revert", wildcard_encode(f)])
- def p4_reopen(type, f):
- p4_system(["reopen", "-t", type, wildcard_encode(f)])
- def p4_move(src, dest):
- p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
- def p4_describe(change):
- """Make sure it returns a valid result by checking for
- the presence of field "time". Return a dict of the
- results."""
- ds = p4CmdList(["describe", "-s", str(change)])
- if len(ds) != 1:
- die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
- d = ds[0]
- if "p4ExitCode" in d:
- die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
- str(d)))
- if "code" in d:
- if d["code"] == "error":
- die("p4 describe -s %d returned error code: %s" % (change, str(d)))
- if "time" not in d:
- die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
- return d
- #
- # Canonicalize the p4 type and return a tuple of the
- # base type, plus any modifiers. See "p4 help filetypes"
- # for a list and explanation.
- #
- def split_p4_type(p4type):
- p4_filetypes_historical = {
- "ctempobj": "binary+Sw",
- "ctext": "text+C",
- "cxtext": "text+Cx",
- "ktext": "text+k",
- "kxtext": "text+kx",
- "ltext": "text+F",
- "tempobj": "binary+FSw",
- "ubinary": "binary+F",
- "uresource": "resource+F",
- "uxbinary": "binary+Fx",
- "xbinary": "binary+x",
- "xltext": "text+Fx",
- "xtempobj": "binary+Swx",
- "xtext": "text+x",
- "xunicode": "unicode+x",
- "xutf16": "utf16+x",
- }
- if p4type in p4_filetypes_historical:
- p4type = p4_filetypes_historical[p4type]
- mods = ""
- s = p4type.split("+")
- base = s[0]
- mods = ""
- if len(s) > 1:
- mods = s[1]
- return (base, mods)
- #
- # return the raw p4 type of a file (text, text+ko, etc)
- #
- def p4_type(file):
- results = p4CmdList(["fstat", "-T", "headType", file])
- return results[0]['headType']
- #
- # Given a type base and modifier, return a regexp matching
- # the keywords that can be expanded in the file
- #
- def p4_keywords_regexp_for_type(base, type_mods):
- if base in ("text", "unicode", "binary"):
- kwords = None
- if "ko" in type_mods:
- kwords = 'Id|Header'
- elif "k" in type_mods:
- kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
- else:
- return None
- pattern = r"""
- \$ # Starts with a dollar, followed by...
- (%s) # one of the keywords, followed by...
- (:[^$\n]+)? # possibly an old expansion, followed by...
- \$ # another dollar
- """ % kwords
- return pattern
- else:
- return None
- #
- # Given a file, return a regexp matching the possible
- # RCS keywords that will be expanded, or None for files
- # with kw expansion turned off.
- #
- def p4_keywords_regexp_for_file(file):
- if not os.path.exists(file):
- return None
- else:
- (type_base, type_mods) = split_p4_type(p4_type(file))
- return p4_keywords_regexp_for_type(type_base, type_mods)
- def setP4ExecBit(file, mode):
- # Reopens an already open file and changes the execute bit to match
- # the execute bit setting in the passed in mode.
- p4Type = "+x"
- if not isModeExec(mode):
- p4Type = getP4OpenedType(file)
- p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
- p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
- if p4Type[-1] == "+":
- p4Type = p4Type[0:-1]
- p4_reopen(p4Type, file)
- def getP4OpenedType(file):
- # Returns the perforce file type for the given file.
- result = p4_read_pipe(["opened", wildcard_encode(file)])
- match = re.match(".*\((.+)\)\r?$", result)
- if match:
- return match.group(1)
- else:
- die("Could not determine file type for %s (result: '%s')" % (file, result))
- # Return the set of all p4 labels
- def getP4Labels(depotPaths):
- labels = set()
- if isinstance(depotPaths,basestring):
- depotPaths = [depotPaths]
- for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
- label = l['label']
- labels.add(label)
- return labels
- # Return the set of all git tags
- def getGitTags():
- gitTags = set()
- for line in read_pipe_lines(["git", "tag"]):
- tag = line.strip()
- gitTags.add(tag)
- return gitTags
- def diffTreePattern():
- # This is a simple generator for the diff tree regex pattern. This could be
- # a class variable if this and parseDiffTreeEntry were a part of a class.
- pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
- while True:
- yield pattern
- def parseDiffTreeEntry(entry):
- """Parses a single diff tree entry into its component elements.
- See git-diff-tree(1) manpage for details about the format of the diff
- output. This method returns a dictionary with the following elements:
- src_mode - The mode of the source file
- dst_mode - The mode of the destination file
- src_sha1 - The sha1 for the source file
- dst_sha1 - The sha1 fr the destination file
- status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
- status_score - The score for the status (applicable for 'C' and 'R'
- statuses). This is None if there is no score.
- src - The path for the source file.
- dst - The path for the destination file. This is only present for
- copy or renames. If it is not present, this is None.
- If the pattern is not matched, None is returned."""
- match = diffTreePattern().next().match(entry)
- if match:
- return {
- 'src_mode': match.group(1),
- 'dst_mode': match.group(2),
- 'src_sha1': match.group(3),
- 'dst_sha1': match.group(4),
- 'status': match.group(5),
- 'status_score': match.group(6),
- 'src': match.group(7),
- 'dst': match.group(10)
- }
- return None
- def isModeExec(mode):
- # Returns True if the given git mode represents an executable file,
- # otherwise False.
- return mode[-3:] == "755"
- def isModeExecChanged(src_mode, dst_mode):
- return isModeExec(src_mode) != isModeExec(dst_mode)
- def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
- if isinstance(cmd,basestring):
- cmd = "-G " + cmd
- expand = True
- else:
- cmd = ["-G"] + cmd
- expand = False
- cmd = p4_build_cmd(cmd)
- if verbose:
- sys.stderr.write("Opening pipe: %s\n" % str(cmd))
- # Use a temporary file to avoid deadlocks without
- # subprocess.communicate(), which would put another copy
- # of stdout into memory.
- stdin_file = None
- if stdin is not None:
- stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
- if isinstance(stdin,basestring):
- stdin_file.write(stdin)
- else:
- for i in stdin:
- stdin_file.write(i + '\n')
- stdin_file.flush()
- stdin_file.seek(0)
- p4 = subprocess.Popen(cmd,
- shell=expand,
- stdin=stdin_file,
- stdout=subprocess.PIPE)
- result = []
- try:
- while True:
- entry = marshal.load(p4.stdout)
- if cb is not None:
- cb(entry)
- else:
- result.append(entry)
- except EOFError:
- pass
- exitCode = p4.wait()
- if exitCode != 0:
- entry = {}
- entry["p4ExitCode"] = exitCode
- result.append(entry)
- return result
- def p4Cmd(cmd):
- list = p4CmdList(cmd)
- result = {}
- for entry in list:
- result.update(entry)
- return result;
- def p4Where(depotPath):
- if not depotPath.endswith("/"):
- depotPath += "/"
- depotPath = depotPath + "..."
- outputList = p4CmdList(["where", depotPath])
- output = None
- for entry in outputList:
- if "depotFile" in entry:
- if entry["depotFile"] == depotPath:
- output = entry
- break
- elif "data" in entry:
- data = entry.get("data")
- space = data.find(" ")
- if data[:space] == depotPath:
- output = entry
- break
- if output == None:
- return ""
- if output["code"] == "error":
- return ""
- clientPath = ""
- if "path" in output:
- clientPath = output.get("path")
- elif "data" in output:
- data = output.get("data")
- lastSpace = data.rfind(" ")
- clientPath = data[lastSpace + 1:]
- if clientPath.endswith("..."):
- clientPath = clientPath[:-3]
- return clientPath
- def currentGitBranch():
- return read_pipe("git name-rev HEAD").split(" ")[1].strip()
- def isValidGitDir(path):
- if (os.path.exists(path + "/HEAD")
- and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
- return True;
- return False
- def parseRevision(ref):
- return read_pipe("git rev-parse %s" % ref).strip()
- def branchExists(ref):
- rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
- ignore_error=True)
- return len(rev) > 0
- def extractLogMessageFromGitCommit(commit):
- logMessage = ""
- ## fixme: title is first line of commit, not 1st paragraph.
- foundTitle = False
- for log in read_pipe_lines("git cat-file commit %s" % commit):
- if not foundTitle:
- if len(log) == 1:
- foundTitle = True
- continue
- logMessage += log
- return logMessage
- def extractSettingsGitLog(log):
- values = {}
- for line in log.split("\n"):
- line = line.strip()
- m = re.search (r"^ *\[git-p4: (.*)\]$", line)
- if not m:
- continue
- assignments = m.group(1).split (':')
- for a in assignments:
- vals = a.split ('=')
- key = vals[0].strip()
- val = ('='.join (vals[1:])).strip()
- if val.endswith ('\"') and val.startswith('"'):
- val = val[1:-1]
- values[key] = val
- paths = values.get("depot-paths")
- if not paths:
- paths = values.get("depot-path")
- if paths:
- values['depot-paths'] = paths.split(',')
- return values
- def gitBranchExists(branch):
- proc = subprocess.Popen(["git", "rev-parse", branch],
- stderr=subprocess.PIPE, stdout=subprocess.PIPE);
- return proc.wait() == 0;
- _gitConfig = {}
- def gitConfig(key, args = None): # set args to "--bool", for instance
- if not _gitConfig.has_key(key):
- argsFilter = ""
- if args != None:
- argsFilter = "%s " % args
- cmd = "git config %s%s" % (argsFilter, key)
- _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
- return _gitConfig[key]
- def gitConfigList(key):
- if not _gitConfig.has_key(key):
- _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
- return _gitConfig[key]
- def p4BranchesInGit(branchesAreInRemotes = True):
- branches = {}
- cmdline = "git rev-parse --symbolic "
- if branchesAreInRemotes:
- cmdline += " --remotes"
- else:
- cmdline += " --branches"
- for line in read_pipe_lines(cmdline):
- line = line.strip()
- ## only import to p4/
- if not line.startswith('p4/') or line == "p4/HEAD":
- continue
- branch = line
- # strip off p4
- branch = re.sub ("^p4/", "", line)
- branches[branch] = parseRevision(line)
- return branches
- def findUpstreamBranchPoint(head = "HEAD"):
- branches = p4BranchesInGit()
- # map from depot-path to branch name
- branchByDepotPath = {}
- for branch in branches.keys():
- tip = branches[branch]
- log = extractLogMessageFromGitCommit(tip)
- settings = extractSettingsGitLog(log)
- if settings.has_key("depot-paths"):
- paths = ",".join(settings["depot-paths"])
- branchByDepotPath[paths] = "remotes/p4/" + branch
- settings = None
- parent = 0
- while parent < 65535:
- commit = head + "~%s" % parent
- log = extractLogMessageFromGitCommit(commit)
- settings = extractSettingsGitLog(log)
- if settings.has_key("depot-paths"):
- paths = ",".join(settings["depot-paths"])
- if branchByDepotPath.has_key(paths):
- return [branchByDepotPath[paths], settings]
- parent = parent + 1
- return ["", settings]
- def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
- if not silent:
- print ("Creating/updating branch(es) in %s based on origin branch(es)"
- % localRefPrefix)
- originPrefix = "origin/p4/"
- for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
- line = line.strip()
- if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
- continue
- headName = line[len(originPrefix):]
- remoteHead = localRefPrefix + headName
- originHead = line
- original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
- if (not original.has_key('depot-paths')
- or not original.has_key('change')):
- continue
- update = False
- if not gitBranchExists(remoteHead):
- if verbose:
- print "creating %s" % remoteHead
- update = True
- else:
- settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
- if settings.has_key('change') > 0:
- if settings['depot-paths'] == original['depot-paths']:
- originP4Change = int(original['change'])
- p4Change = int(settings['change'])
- if originP4Change > p4Change:
- print ("%s (%s) is newer than %s (%s). "
- "Updating p4 branch from origin."
- % (originHead, originP4Change,
- remoteHead, p4Change))
- update = True
- else:
- print ("Ignoring: %s was imported from %s while "
- "%s was imported from %s"
- % (originHead, ','.join(original['depot-paths']),
- remoteHead, ','.join(settings['depot-paths'])))
- if update:
- system("git update-ref %s %s" % (remoteHead, originHead))
- def originP4BranchesExist():
- return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
- def p4ChangesForPaths(depotPaths, changeRange):
- assert depotPaths
- cmd = ['changes']
- for p in depotPaths:
- cmd += ["%s...%s" % (p, changeRange)]
- output = p4_read_pipe_lines(cmd)
- changes = {}
- for line in output:
- changeNum = int(line.split(" ")[1])
- changes[changeNum] = True
- changelist = changes.keys()
- changelist.sort()
- return changelist
- def p4PathStartsWith(path, prefix):
- # This method tries to remedy a potential mixed-case issue:
- #
- # If UserA adds //depot/DirA/file1
- # and UserB adds //depot/dira/file2
- #
- # we may or may not have a problem. If you have core.ignorecase=true,
- # we treat DirA and dira as the same directory
- ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
- if ignorecase:
- return path.lower().startswith(prefix.lower())
- return path.startswith(prefix)
- def getClientSpec():
- """Look at the p4 client spec, create a View() object that contains
- all the mappings, and return it."""
- specList = p4CmdList("client -o")
- if len(specList) != 1:
- die('Output from "client -o" is %d lines, expecting 1' %
- len(specList))
- # dictionary of all client parameters
- entry = specList[0]
- # just the keys that start with "View"
- view_keys = [ k for k in entry.keys() if k.startswith("View") ]
- # hold this new View
- view = View()
- # append the lines, in order, to the view
- for view_num in range(len(view_keys)):
- k = "View%d" % view_num
- if k not in view_keys:
- die("Expected view key %s missing" % k)
- view.append(entry[k])
- return view
- def getClientRoot():
- """Grab the client directory."""
- output = p4CmdList("client -o")
- if len(output) != 1:
- die('Output from "client -o" is %d lines, expecting 1' % len(output))
- entry = output[0]
- if "Root" not in entry:
- die('Client has no "Root"')
- return entry["Root"]
- #
- # P4 wildcards are not allowed in filenames. P4 complains
- # if you simply add them, but you can force it with "-f", in
- # which case it translates them into %xx encoding internally.
- #
- def wildcard_decode(path):
- # Search for and fix just these four characters. Do % last so
- # that fixing it does not inadvertently create new %-escapes.
- # Cannot have * in a filename in windows; untested as to
- # what p4 would do in such a case.
- if not platform.system() == "Windows":
- path = path.replace("%2A", "*")
- path = path.replace("%23", "#") \
- .replace("%40", "@") \
- .replace("%25", "%")
- return path
- def wildcard_encode(path):
- # do % first to avoid double-encoding the %s introduced here
- path = path.replace("%", "%25") \
- .replace("*", "%2A") \
- .replace("#", "%23") \
- .replace("@", "%40")
- return path
- def wildcard_present(path):
- return path.translate(None, "*#@%") != path
- class Command:
- def __init__(self):
- self.usage = "usage: %prog [options]"
- self.needsGit = True
- self.verbose = False
- class P4UserMap:
- def __init__(self):
- self.userMapFromPerforceServer = False
- self.myP4UserId = None
- def p4UserId(self):
- if self.myP4UserId:
- return self.myP4UserId
- results = p4CmdList("user -o")
- for r in results:
- if r.has_key('User'):
- self.myP4UserId = r['User']
- return r['User']
- die("Could not find your p4 user id")
- def p4UserIsMe(self, p4User):
- # return True if the given p4 user is actually me
- me = self.p4UserId()
- if not p4User or p4User != me:
- return False
- else:
- return True
- def getUserCacheFilename(self):
- home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
- return home + "/.gitp4-usercache.txt"
- def getUserMapFromPerforceServer(self):
- if self.userMapFromPerforceServer:
- return
- self.users = {}
- self.emails = {}
- for output in p4CmdList("users"):
- if not output.has_key("User"):
- continue
- self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
- self.emails[output["Email"]] = output["User"]
- s = ''
- for (key, val) in self.users.items():
- s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
- open(self.getUserCacheFilename(), "wb").write(s)
- self.userMapFromPerforceServer = True
- def loadUserMapFromCache(self):
- self.users = {}
- self.userMapFromPerforceServer = False
- try:
- cache = open(self.getUserCacheFilename(), "rb")
- lines = cache.readlines()
- cache.close()
- for line in lines:
- entry = line.strip().split("\t")
- self.users[entry[0]] = entry[1]
- except IOError:
- self.getUserMapFromPerforceServer()
- class P4Debug(Command):
- def __init__(self):
- Command.__init__(self)
- self.options = []
- self.description = "A tool to debug the output of p4 -G."
- self.needsGit = False
- def run(self, args):
- j = 0
- for output in p4CmdList(args):
- print 'Element: %d' % j
- j += 1
- print output
- return True
- class P4RollBack(Command):
- def __init__(self):
- Command.__init__(self)
- self.options = [
- optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
- ]
- self.description = "A tool to debug the multi-branch import. Don't use :)"
- self.rollbackLocalBranches = False
- def run(self, args):
- if len(args) != 1:
- return False
- maxChange = int(args[0])
- if "p4ExitCode" in p4Cmd("changes -m 1"):
- die("Problems executing p4");
- if self.rollbackLocalBranches:
- refPrefix = "refs/heads/"
- lines = read_pipe_lines("git rev-parse --symbolic --branches")
- else:
- refPrefix = "refs/remotes/"
- lines = read_pipe_lines("git rev-parse --symbolic --remotes")
- for line in lines:
- if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
- line = line.strip()
- ref = refPrefix + line
- log = extractLogMessageFromGitCommit(ref)
- settings = extractSettingsGitLog(log)
- depotPaths = settings['depot-paths']
- change = settings['change']
- changed = False
- if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
- for p in depotPaths]))) == 0:
- print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
- system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
- continue
- while change and int(change) > maxChange:
- changed = True
- if self.verbose:
- print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
- system("git update-ref %s \"%s^\"" % (ref, ref))
- log = extractLogMessageFromGitCommit(ref)
- settings = extractSettingsGitLog(log)
- depotPaths = settings['depot-paths']
- change = settings['change']
- if changed:
- print "%s rewound to %s" % (ref, change)
- return True
- class P4Submit(Command, P4UserMap):
- conflict_behavior_choices = ("ask", "skip", "quit")
- def __init__(self):
- Command.__init__(self)
- P4UserMap.__init__(self)
- self.options = [
- optparse.make_option("--origin", dest="origin"),
- optparse.make_option("-M", dest="detectRenames", action="store_true"),
- # preserve the user, requires relevant p4 permissions
- optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
- optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
- optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
- optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
- optparse.make_option("--conflict", dest="conflict_behavior",
- choices=self.conflict_behavior_choices)
- ]
- self.description = "Submit changes from git to the perforce depot."
- self.usage += " [name of git branch to submit into perforce depot]"
- self.origin = ""
- self.detectRenames = False
- self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
- self.dry_run = False
- self.prepare_p4_only = False
- self.conflict_behavior = None
- self.isWindows = (platform.system() == "Windows")
- self.exportLabels = False
- self.p4HasMoveCommand = p4_has_move_command()
- def check(self):
- if len(p4CmdList("opened ...")) > 0:
- die("You have files opened with perforce! Close them before starting the sync.")
- def separate_jobs_from_description(self, message):
- """Extract and return a possible Jobs field in the commit
- message. It goes into a separate section in the p4 change
- specification.
- A jobs line starts with "Jobs:" and looks like a new field
- in a form. Values are white-space separated on the same
- line or on following lines that start with a tab.
- This does not parse and extract the full git commit message
- like a p4 form. It just sees the Jobs: line as a marker
- to pass everything from then on directly into the p4 form,
- but outside the description section.
- Return a tuple (stripped log message, jobs string)."""
- m = re.search(r'^Jobs:', message, re.MULTILINE)
- if m is None:
- return (message, None)
- jobtext = message[m.start():]
- stripped_message = message[:m.start()].rstrip()
- return (stripped_message, jobtext)
- def prepareLogMessage(self, template, message, jobs):
- """Edits the template returned from "p4 change -o" to insert
- the message in the Description field, and the jobs text in
- the Jobs field."""
- result = ""
- inDescriptionSection = False
- for line in template.split("\n"):
- if line.startswith("#"):
- result += line + "\n"
- continue
- if inDescriptionSection:
- if line.startswith("Files:") or line.startswith("Jobs:"):
- inDescriptionSection = False
- # insert Jobs section
- if jobs:
- result += jobs + "\n"
- else:
- continue
- else:
- if line.startswith("Description:"):
- inDescriptionSection = True
- line += "\n"
- for messageLine in message.split("\n"):
- line += "\t" + messageLine + "\n"
- result += line + "\n"
- return result
- def patchRCSKeywords(self, file, pattern):
- # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
- (handle, outFileName) = tempfile.mkstemp(dir='.')
- try:
- outFile = os.fdopen(handle, "w+")
- inFile = open(file, "r")
- regexp = re.compile(pattern, re.VERBOSE)
- for line in inFile.readlines():
- line = regexp.sub(r'$\1$', line)
- outFile.write(line)
- inFile.close()
- outFile.close()
- # Forcibly overwrite the original file
- os.unlink(file)
- shutil.move(outFileName, file)
- except:
- # cleanup our temporary file
- os.unlink(outFileName)
- print "Failed to strip RCS keywords in %s" % file
- raise
- print "Patched up RCS keywords in %s" % file
- def p4UserForCommit(self,id):
- # Return the tuple (perforce user,git email) for a given git commit id
- self.getUserMapFromPerforceServer()
- gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
- gitEmail = gitEmail.strip()
- if not self.emails.has_key(gitEmail):
- return (None,gitEmail)
- else:
- return (self.emails[gitEmail],gitEmail)
- def checkValidP4Users(self,commits):
- # check if any git authors cannot be mapped to p4 users
- for id in commits:
- (user,email) = self.p4UserForCommit(id)
- if not user:
- msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
- if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
- print "%s" % msg
- else:
- die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
- def lastP4Changelist(self):
- # Get back the last changelist number submitted in this client spec. This
- # then gets used to patch up the username in the change. If the same
- # client spec is being used by multiple processes then this might go
- # wrong.
- results = p4CmdList("client -o") # find the current client
- client = None
- for r in results:
- if r.has_key('Client'):
- client = r['Client']
- break
- if not client:
- die("could not get client spec")
- results = p4CmdList(["changes", "-c", client, "-m", "1"])
- for r in results:
- if r.has_key('change'):
- return r['change']
- die("Could not get changelist number for last submit - cannot patch up user details")
- def modifyChangelistUser(self, changelist, newUser):
- # fixup the user field of a changelist after it has been submitted.
- changes = p4CmdList("change -o %s" % changelist)
- if len(changes) != 1:
- die("Bad output from p4 change modifying %s to user %s" %
- (changelist, newUser))
- c = changes[0]
- if c['User'] == newUser: return # nothing to do
- c['User'] = newUser
- input = marshal.dumps(c)
- result = p4CmdList("change -f -i", stdin=input)
- for r in result:
- if r.has_key('code'):
- if r['code'] == 'error':
- die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
- if r.has_key('data'):
- print("Updated user field for changelist %s to %s" % (changelist, newUser))
- return
- die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
- def canChangeChangelists(self):
- # check to see if we have p4 admin or super-user permissions, either of
- # which are required to modify changelists.
- results = p4CmdList(["protects", self.depotPath])
- for r in results:
- if r.has_key('perm'):
- if r['perm'] == 'admin':
- return 1
- if r['perm'] == 'super':
- return 1
- return 0
- def prepareSubmitTemplate(self):
- """Run "p4 change -o" to grab a change specification template.
- This does not use "p4 -G", as it is nice to keep the submission
- template in original order, since a human might edit it.
- Remove lines in the Files section that show changes to files
- outside the depot path we're committing into."""
- template = ""
- inFilesSection = False
- for line in p4_read_pipe_lines(['change', '-o']):
- if line.endswith("\r\n"):
- line = line[:-2] + "\n"
- if inFilesSection:
- if line.startswith("\t"):
- # path starts and ends with a tab
- path = line[1:]
- lastTab = path.rfind("\t")
- if lastTab != -1:
- path = path[:lastTab]
- if not p4PathStartsWith(path, self.depotPath):
- continue
- else:
- inFilesSection = False
- else:
- if line.startswith("Files:"):
- inFilesSection = True
- template += line
- return template
- def edit_template(self, template_file):
- """Invoke the editor to let the user change the submission
- message. Return true if okay to continue with the submit."""
- # if configured to skip the editing part, just submit
- if gitConfig("git-p4.skipSubmitEdit") == "true":
- return True
- # look at the modification time, to check later if the user saved
- # the file
- mtime = os.stat(template_file).st_mtime
- # invoke the editor
- if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
- editor = os.environ.get("P4EDITOR")
- else:
- editor = read_pipe("git var GIT_EDITOR").strip()
- system(editor + " " + template_file)
- # If the file was not saved, prompt to see if this patch should
- # be skipped. But skip this verification step if configured so.
- if gitConfig("git-p4.skipSubmitEditCheck") == "true":
- return True
- # modification time updated means user saved the file
- if os.stat(template_file).st_mtime > mtime:
- return True
- while True:
- response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
- if response == 'y':
- return True
- if response == 'n':
- return False
- def applyCommit(self, id):
- """Apply one commit, return True if it succeeded."""
- print "Applying", read_pipe(["git", "show", "-s",
- "--format=format:%h %s", id])
- (p4User, gitEmail) = self.p4UserForCommit(id)
- diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
- filesToAdd = set()
- filesToDelete = set()
- editedFiles = set()
- pureRenameCopy = set()
- filesToChangeExecBit = {}
- for line in diff:
- diff = parseDiffTreeEntry(line)
- modifier = diff['status']
- path = diff['src']
- if modifier == "M":
- p4_edit(path)
- if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
- filesToChangeExecBit[path] = diff['dst_mode']
- editedFiles.add(path)
- elif modifier == "A":
- filesToAdd.add(path)
- filesToChangeExecBit[path] = diff['dst_mode']
- if path in filesToDelete:
- filesToDelete.remove(path)
- elif modifier == "D":
- filesToDelete.add(path)
- if path in filesToAdd:
- filesToAdd.remove(path)
- elif modifier == "C":
- src, dest = diff['src'], diff['dst']
- p4_integrate(src, dest)
- pureRenameCopy.add(dest)
- if diff['src_sha1'] != diff['dst_sha1']:
- p4_edit(dest)
- pureRenameCopy.discard(dest)
- if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
- p4_edit(dest)
- pureRenameCopy.discard(dest)
- filesToChangeExecBit[dest] = diff['dst_mode']
- os.unlink(dest)
- editedFiles.add(dest)
- elif modifier == "R":
- src, dest = diff['src'], diff['dst']
- if self.p4HasMoveCommand:
- p4_edit(src) # src must be open before move
- p4_move(src, dest) # opens for (move/delete, move/add)
- else:
- p4_integrate(src, dest)
- if diff['src_sha1'] != diff['dst_sha1']:
- p4_edit(dest)
- else:
- pureRenameCopy.add(dest)
- if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
- if not self.p4HasMoveCommand:
- p4_edit(dest) # with move: already open, writable
- filesToChangeExecBit[dest] = diff['dst_mode']
- if not self.p4HasMoveCommand:
- os.unlink(dest)
- filesToDelete.add(src)
- editedFiles.add(dest)
- else:
- die("unknown modifier %s for %s" % (modifier, path))
- diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
- patchcmd = diffcmd + " | git apply "
- tryPatchCmd = patchcmd + "--check -"
- applyPatchCmd = patchcmd + "--check --apply -"
- patch_succeeded = True
- if os.system(tryPatchCmd) != 0:
- fixed_rcs_keywords = False
- patch_succeeded = False
- print "Unfortunately applying the change failed!"
- # Patch failed, maybe it's just RCS keyword woes. Look through
- # the patch to see if that's possible.
- if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
- file = None
- pattern = None
- kwfiles = {}
- for file in editedFiles | filesToDelete:
- # did this file's delta contain RCS keywords?
- pattern = p4_keywords_regexp_for_file(file)
- if pattern:
- # this file is a possibility...look for RCS keywords.
- regexp = re.compile(pattern, re.VERBOSE)
- for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
- if regexp.search(line):
- if verbose:
- print "got keyword match on %s in %s in %s" % (pattern, line, file)
- kwfiles[file] = pattern
- break
- for file in kwfiles:
- if verbose:
- print "zapping %s with %s" % (line,pattern)
- self.patchRCSKeywords(file, kwfiles[file])
- fixed_rcs_keywords = True
- if fixed_rcs_keywords:
- print "Retrying the patch with RCS keywords cleaned up"
- if os.system(tryPatchCmd) == 0:
- patch_succeeded = True
- if not patch_succeeded:
- for f in editedFiles:
- p4_revert(f)
- return False
- #
- # Apply the patch for real, and do add/delete/+x handling.
- #
- system(applyPatchCmd)
- for f in filesToAdd:
- p4_add(f)
- for f in filesToDelete:
- p4_revert(f)
- p4_delete(f)
- # Set/clear executable bits
- for f in filesToChangeExecBit.keys():
- mode = filesToChangeExecBit[f]
- setP4ExecBit(f, mode)
- #
- # Build p4 change description, starting with the contents
- # of the git commit message.
- #
- logMessage = extractLogMessageFromGitCommit(id)
- logMessage = logMessage.strip()
- (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
- template = self.prepareSubmitTemplate()
- submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
- if self.preserveUser:
- submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
- if self.checkAuthorship and not self.p4UserIsMe(p4User):
- submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
- submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
- submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
- separatorLine = "######## everything below this line is just the diff #######\n"
- # diff
- if os.environ.has_key("P4DIFF"):
- del(os.environ["P4DIFF"])
- diff = ""
- for editedFile in editedFiles:
- diff += p4_read_pipe(['diff', '-du',
- wildcard_encode(editedFile)])
- # new file diff
- newdiff = ""
- for newFile in filesToAdd:
- newdiff += "==== new file ====\n"
- newdiff += "--- /dev/null\n"
- newdiff += "+++ %s\n" % newFile
- f = open(newFile, "r")
- for line in f.readlines():
- newdiff += "+" + line
- f.close()
- # change description file: submitTemplate, separatorLine, diff, newdiff
- (handle, fileName) = tempfile.mkstemp()
- tmpFile = os.fdopen(handle, "w+")
- if self.isWindows:
- submitTemplate = submitTemplate.replace("\n", "\r\n")
- separatorLine = separatorLine.replace("\n", "\r\n")
- newdiff = newdiff.replace("\n", "\r\n")
- tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
- tmpFile.close()
- if self.prepare_p4_only:
- #
- # Leave the p4 tree prepared, and the submit template around
- # and let the user decide what to do next
- #
- print
- print "P4 workspace prepared for submission."
- print "To submit or revert, go to client workspace"
- print " " + self.clientPath
- print
- print "To submit, use \"p4 submit\" to write a new description,"
- print "or \"p4 submit -i %s\" to use the one prepared by" \
- " \"git p4\"." % fileName
- print "You can delete the file \"%s\" when finished." % fileName
- if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
- print "To preserve change ownership by user %s, you must\n" \
- "do \"p4 change -f <change>\" after submitting and\n" \
- "edit the User field."
- if pureRenameCopy:
- print "After submitting, renamed files must be re-synced."
- print "Invoke \"p4 sync -f\" on each of these files:"
- for f in pureRenameCopy:
- print " " + f
- print
- print "To revert the changes, use \"p4 revert ...\", and delete"
- print "the submit template file \"%s\"" % fileName
- if filesToAdd:
- print "Since the commit adds new files, they must be deleted:"
- for f in filesToAdd:
- print " " + f
- print
- return True
- #
- # Let the user edit the change description, then submit it.
- #
- if self.edit_template(fileName):
- # read the edited message and submit
- ret = True
- tmpFile = open(fileName, "rb")
- message = tmpFile.read()
- tmpFile.close()
- submitTemplate = message[:message.index(separatorLine)]
- if self.isWindows:
- submitTemplate = submitTemplate.replace("\r\n", "\n")
- p4_write_pipe(['submit', '-i'], submitTemplate)
- if self.preserveUser:
- if p4User:
- # Get last changelist number. Cannot easily get it from
- # the submit command output as the output is
- # unmarshalled.
- changelist = self.lastP4Changelist()
- self.modifyChangelistUser(changelist, p4User)
- # The rename/copy happened by applying a patch that created a
- # new file. This leaves it writable, which confuses p4.
- for f in pureRenameCopy:
- p4_sync(f, "-f")
- else:
- # skip this patch
- ret = False
- print "Submission cancelled, undoing p4 changes."
- for f in editedFiles:
- p4_revert(f)
- for f in filesToAdd:
- p4_revert(f)
- os.remove(f)
- for f in filesToDelete:
- p4_revert(f)
- os.remove(fileName)
- return ret
- # Export git tags as p4 labels. Create a p4 label and then tag
- # with that.
- def exportGitTags(self, gitTags):
- validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
- if len(validLabelRegexp) == 0:
- validLabelRegexp = defaultLabelRegexp
- m = re.compile(validLabelRegexp)
- for name in gitTags:
- if not m.match(name):
- if verbose:
- print "tag %s does not match regexp %s" % (name, validLabelRegexp)
- continue
- # Get the p4 commit this corresponds to
- logMessage = extractLogMessageFromGitCommit(name)
- values = extractSettingsGitLog(logMessage)
- if not values.has_key('change'):
- # a tag pointing to something not sent to p4; ignore
- if verbose:
- print "git tag %s does not give a p4 commit" % name
- continue
- else:
- changelist = values['change']
- # Get the tag details.
- inHeader = True
- isAnnotated = False
- body = []
- for l in read_pipe_lines(["git", "cat-file", "-p", name]):
- l = l.strip()
- if inHeader:
- if re.match(r'tag\s+', l):
- …
Large files files are truncated, but you can click here to view the full file