/lib/codereview/codereview.py

   1# coding=utf-8
   2# (The line above is necessary so that I can use 世界 in the
   3# *comment* below without Python getting all bent out of shape.)
   4
   5# Copyright 2007-2009 Google Inc.
   6#
   7# Licensed under the Apache License, Version 2.0 (the "License");
   8# you may not use this file except in compliance with the License.
   9# You may obtain a copy of the License at
  10#
  11#	http://www.apache.org/licenses/LICENSE-2.0
  12#
  13# Unless required by applicable law or agreed to in writing, software
  14# distributed under the License is distributed on an "AS IS" BASIS,
  15# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16# See the License for the specific language governing permissions and
  17# limitations under the License.
  18
  19'''Mercurial interface to codereview.appspot.com.
  20
  21To configure, set the following options in
  22your repository's .hg/hgrc file.
  23
  24	[extensions]
  25	codereview = path/to/codereview.py
  26
  27	[codereview]
  28	server = codereview.appspot.com
  29
  30The server should be running Rietveld; see http://code.google.com/p/rietveld/.
  31
  32In addition to the new commands, this extension introduces
  33the file pattern syntax @nnnnnn, where nnnnnn is a change list
  34number, to mean the files included in that change list, which
  35must be associated with the current client.
  36
  37For example, if change 123456 contains the files x.go and y.go,
  38"hg diff @123456" is equivalent to"hg diff x.go y.go".
  39'''
  40
  41from mercurial import cmdutil, commands, hg, util, error, match
  42from mercurial.node import nullrev, hex, nullid, short
  43import os, re, time
  44import stat
  45import subprocess
  46import threading
  47from HTMLParser import HTMLParser
  48
  49# The standard 'json' package is new in Python 2.6.
  50# Before that it was an external package named simplejson.
  51try:
  52	# Standard location in 2.6 and beyond.
  53	import json
  54except Exception, e:
  55	try:
  56		# Conventional name for earlier package.
  57		import simplejson as json
  58	except:
  59		try:
  60			# Was also bundled with django, which is commonly installed.
  61			from django.utils import simplejson as json
  62		except:
  63			# We give up.
  64			raise e
  65
  66try:
  67	hgversion = util.version()
  68except:
  69	from mercurial.version import version as v
  70	hgversion = v.get_version()
  71
  72try:
  73	from mercurial.discovery import findcommonincoming
  74except:
  75	def findcommonincoming(repo, remote):
  76		return repo.findcommonincoming(remote)
  77
  78# in Mercurial 1.9 the cmdutil.match and cmdutil.revpair moved to scmutil
  79if hgversion >= '1.9':
  80    from mercurial import scmutil
  81else:
  82    scmutil = cmdutil
  83
  84oldMessage = """
  85The code review extension requires Mercurial 1.3 or newer.
  86
  87To install a new Mercurial,
  88
  89	sudo easy_install mercurial
  90
  91works on most systems.
  92"""
  93
  94linuxMessage = """
  95You may need to clear your current Mercurial installation by running:
  96
  97	sudo apt-get remove mercurial mercurial-common
  98	sudo rm -rf /etc/mercurial
  99"""
 100
 101if hgversion < '1.3':
 102	msg = oldMessage
 103	if os.access("/etc/mercurial", 0):
 104		msg += linuxMessage
 105	raise util.Abort(msg)
 106
 107def promptyesno(ui, msg):
 108	# Arguments to ui.prompt changed between 1.3 and 1.3.1.
 109	# Even so, some 1.3.1 distributions seem to have the old prompt!?!?
 110	# What a terrible way to maintain software.
 111	try:
 112		return ui.promptchoice(msg, ["&yes", "&no"], 0) == 0
 113	except AttributeError:
 114		return ui.prompt(msg, ["&yes", "&no"], "y") != "n"
 115
 116# To experiment with Mercurial in the python interpreter:
 117#    >>> repo = hg.repository(ui.ui(), path = ".")
 118
 119#######################################################################
 120# Normally I would split this into multiple files, but it simplifies
 121# import path headaches to keep it all in one file.  Sorry.
 122
 123import sys
 124if __name__ == "__main__":
 125	print >>sys.stderr, "This is a Mercurial extension and should not be invoked directly."
 126	sys.exit(2)
 127
 128server = "codereview.appspot.com"
 129server_url_base = None
 130defaultcc = None
 131contributors = {}
 132missing_codereview = None
 133real_rollback = None
 134releaseBranch = None
 135
 136#######################################################################
 137# RE: UNICODE STRING HANDLING
 138#
 139# Python distinguishes between the str (string of bytes)
 140# and unicode (string of code points) types.  Most operations
 141# work on either one just fine, but some (like regexp matching)
 142# require unicode, and others (like write) require str.
 143#
 144# As befits the language, Python hides the distinction between
 145# unicode and str by converting between them silently, but
 146# *only* if all the bytes/code points involved are 7-bit ASCII.
 147# This means that if you're not careful, your program works
 148# fine on "hello, world" and fails on "hello, 世界".  And of course,
 149# the obvious way to be careful - use static types - is unavailable.
 150# So the only way is trial and error to find where to put explicit
 151# conversions.
 152#
 153# Because more functions do implicit conversion to str (string of bytes)
 154# than do implicit conversion to unicode (string of code points),
 155# the convention in this module is to represent all text as str,
 156# converting to unicode only when calling a unicode-only function
 157# and then converting back to str as soon as possible.
 158
 159def typecheck(s, t):
 160	if type(s) != t:
 161		raise util.Abort("type check failed: %s has type %s != %s" % (repr(s), type(s), t))
 162
 163# If we have to pass unicode instead of str, ustr does that conversion clearly.
 164def ustr(s):
 165	typecheck(s, str)
 166	return s.decode("utf-8")
 167
 168# Even with those, Mercurial still sometimes turns unicode into str
 169# and then tries to use it as ascii.  Change Mercurial's default.
 170def set_mercurial_encoding_to_utf8():
 171	from mercurial import encoding
 172	encoding.encoding = 'utf-8'
 173
 174set_mercurial_encoding_to_utf8()
 175
 176# Even with those we still run into problems.
 177# I tried to do things by the book but could not convince
 178# Mercurial to let me check in a change with UTF-8 in the
 179# CL description or author field, no matter how many conversions
 180# between str and unicode I inserted and despite changing the
 181# default encoding.  I'm tired of this game, so set the default
 182# encoding for all of Python to 'utf-8', not 'ascii'.
 183def default_to_utf8():
 184	import sys
 185	reload(sys)  # site.py deleted setdefaultencoding; get it back
 186	sys.setdefaultencoding('utf-8')
 187
 188default_to_utf8()
 189
 190#######################################################################
 191# Change list parsing.
 192#
 193# Change lists are stored in .hg/codereview/cl.nnnnnn
 194# where nnnnnn is the number assigned by the code review server.
 195# Most data about a change list is stored on the code review server
 196# too: the description, reviewer, and cc list are all stored there.
 197# The only thing in the cl.nnnnnn file is the list of relevant files.
 198# Also, the existence of the cl.nnnnnn file marks this repository
 199# as the one where the change list lives.
 200
 201emptydiff = """Index: ~rietveld~placeholder~
 202===================================================================
 203diff --git a/~rietveld~placeholder~ b/~rietveld~placeholder~
 204new file mode 100644
 205"""
 206
 207class CL(object):
 208	def __init__(self, name):
 209		typecheck(name, str)
 210		self.name = name
 211		self.desc = ''
 212		self.files = []
 213		self.reviewer = []
 214		self.cc = []
 215		self.url = ''
 216		self.local = False
 217		self.web = False
 218		self.copied_from = None	# None means current user
 219		self.mailed = False
 220		self.private = False
 221
 222	def DiskText(self):
 223		cl = self
 224		s = ""
 225		if cl.copied_from:
 226			s += "Author: " + cl.copied_from + "\n\n"
 227		if cl.private:
 228			s += "Private: " + str(self.private) + "\n"
 229		s += "Mailed: " + str(self.mailed) + "\n"
 230		s += "Description:\n"
 231		s += Indent(cl.desc, "\t")
 232		s += "Files:\n"
 233		for f in cl.files:
 234			s += "\t" + f + "\n"
 235		typecheck(s, str)
 236		return s
 237
 238	def EditorText(self):
 239		cl = self
 240		s = _change_prolog
 241		s += "\n"
 242		if cl.copied_from:
 243			s += "Author: " + cl.copied_from + "\n"
 244		if cl.url != '':
 245			s += 'URL: ' + cl.url + '	# cannot edit\n\n'
 246		if cl.private:
 247			s += "Private: True\n"
 248		s += "Reviewer: " + JoinComma(cl.reviewer) + "\n"
 249		s += "CC: " + JoinComma(cl.cc) + "\n"
 250		s += "\n"
 251		s += "Description:\n"
 252		if cl.desc == '':
 253			s += "\t<enter description here>\n"
 254		else:
 255			s += Indent(cl.desc, "\t")
 256		s += "\n"
 257		if cl.local or cl.name == "new":
 258			s += "Files:\n"
 259			for f in cl.files:
 260				s += "\t" + f + "\n"
 261			s += "\n"
 262		typecheck(s, str)
 263		return s
 264
 265	def PendingText(self):
 266		cl = self
 267		s = cl.name + ":" + "\n"
 268		s += Indent(cl.desc, "\t")
 269		s += "\n"
 270		if cl.copied_from:
 271			s += "\tAuthor: " + cl.copied_from + "\n"
 272		s += "\tReviewer: " + JoinComma(cl.reviewer) + "\n"
 273		s += "\tCC: " + JoinComma(cl.cc) + "\n"
 274		s += "\tFiles:\n"
 275		for f in cl.files:
 276			s += "\t\t" + f + "\n"
 277		typecheck(s, str)
 278		return s
 279
 280	def Flush(self, ui, repo):
 281		if self.name == "new":
 282			self.Upload(ui, repo, gofmt_just_warn=True, creating=True)
 283		dir = CodeReviewDir(ui, repo)
 284		path = dir + '/cl.' + self.name
 285		f = open(path+'!', "w")
 286		f.write(self.DiskText())
 287		f.close()
 288		if sys.platform == "win32" and os.path.isfile(path):
 289			os.remove(path)
 290		os.rename(path+'!', path)
 291		if self.web and not self.copied_from:
 292			EditDesc(self.name, desc=self.desc,
 293				reviewers=JoinComma(self.reviewer), cc=JoinComma(self.cc),
 294				private=self.private)
 295
 296	def Delete(self, ui, repo):
 297		dir = CodeReviewDir(ui, repo)
 298		os.unlink(dir + "/cl." + self.name)
 299
 300	def Subject(self):
 301		s = line1(self.desc)
 302		if len(s) > 60:
 303			s = s[0:55] + "..."
 304		if self.name != "new":
 305			s = "code review %s: %s" % (self.name, s)
 306		typecheck(s, str)
 307		return s
 308
 309	def Upload(self, ui, repo, send_mail=False, gofmt=True, gofmt_just_warn=False, creating=False, quiet=False):
 310		if not self.files and not creating:
 311			ui.warn("no files in change list\n")
 312		if ui.configbool("codereview", "force_gofmt", True) and gofmt:
 313			CheckFormat(ui, repo, self.files, just_warn=gofmt_just_warn)
 314		set_status("uploading CL metadata + diffs")
 315		os.chdir(repo.root)
 316		form_fields = [
 317			("content_upload", "1"),
 318			("reviewers", JoinComma(self.reviewer)),
 319			("cc", JoinComma(self.cc)),
 320			("description", self.desc),
 321			("base_hashes", ""),
 322		]
 323
 324		if self.name != "new":
 325			form_fields.append(("issue", self.name))
 326		vcs = None
 327		# We do not include files when creating the issue,
 328		# because we want the patch sets to record the repository
 329		# and base revision they are diffs against.  We use the patch
 330		# set message for that purpose, but there is no message with
 331		# the first patch set.  Instead the message gets used as the
 332		# new CL's overall subject.  So omit the diffs when creating
 333		# and then we'll run an immediate upload.
 334		# This has the effect that every CL begins with an empty "Patch set 1".
 335		if self.files and not creating:
 336			vcs = MercurialVCS(upload_options, ui, repo)
 337			data = vcs.GenerateDiff(self.files)
 338			files = vcs.GetBaseFiles(data)
 339			if len(data) > MAX_UPLOAD_SIZE:
 340				uploaded_diff_file = []
 341				form_fields.append(("separate_patches", "1"))
 342			else:
 343				uploaded_diff_file = [("data", "data.diff", data)]
 344		else:
 345			uploaded_diff_file = [("data", "data.diff", emptydiff)]
 346		
 347		if vcs and self.name != "new":
 348			form_fields.append(("subject", "diff -r " + vcs.base_rev + " " + getremote(ui, repo, {}).path))
 349		else:
 350			# First upload sets the subject for the CL itself.
 351			form_fields.append(("subject", self.Subject()))
 352		ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
 353		response_body = MySend("/upload", body, content_type=ctype)
 354		patchset = None
 355		msg = response_body
 356		lines = msg.splitlines()
 357		if len(lines) >= 2:
 358			msg = lines[0]
 359			patchset = lines[1].strip()
 360			patches = [x.split(" ", 1) for x in lines[2:]]
 361		if response_body.startswith("Issue updated.") and quiet:
 362			pass
 363		else:
 364			ui.status(msg + "\n")
 365		set_status("uploaded CL metadata + diffs")
 366		if not response_body.startswith("Issue created.") and not response_body.startswith("Issue updated."):
 367			raise util.Abort("failed to update issue: " + response_body)
 368		issue = msg[msg.rfind("/")+1:]
 369		self.name = issue
 370		if not self.url:
 371			self.url = server_url_base + self.name
 372		if not uploaded_diff_file:
 373			set_status("uploading patches")
 374			patches = UploadSeparatePatches(issue, rpc, patchset, data, upload_options)
 375		if vcs:
 376			set_status("uploading base files")
 377			vcs.UploadBaseFiles(issue, rpc, patches, patchset, upload_options, files)
 378		if send_mail:
 379			set_status("sending mail")
 380			MySend("/" + issue + "/mail", payload="")
 381		self.web = True
 382		set_status("flushing changes to disk")
 383		self.Flush(ui, repo)
 384		return
 385
 386	def Mail(self, ui, repo):
 387		pmsg = "Hello " + JoinComma(self.reviewer)
 388		if self.cc:
 389			pmsg += " (cc: %s)" % (', '.join(self.cc),)
 390		pmsg += ",\n"
 391		pmsg += "\n"
 392		repourl = getremote(ui, repo, {}).path
 393		if not self.mailed:
 394			pmsg += "I'd like you to review this change to\n" + repourl + "\n"
 395		else:
 396			pmsg += "Please take another look.\n"
 397		typecheck(pmsg, str)
 398		PostMessage(ui, self.name, pmsg, subject=self.Subject())
 399		self.mailed = True
 400		self.Flush(ui, repo)
 401
 402def GoodCLName(name):
 403	typecheck(name, str)
 404	return re.match("^[0-9]+$", name)
 405
 406def ParseCL(text, name):
 407	typecheck(text, str)
 408	typecheck(name, str)
 409	sname = None
 410	lineno = 0
 411	sections = {
 412		'Author': '',
 413		'Description': '',
 414		'Files': '',
 415		'URL': '',
 416		'Reviewer': '',
 417		'CC': '',
 418		'Mailed': '',
 419		'Private': '',
 420	}
 421	for line in text.split('\n'):
 422		lineno += 1
 423		line = line.rstrip()
 424		if line != '' and line[0] == '#':
 425			continue
 426		if line == '' or line[0] == ' ' or line[0] == '\t':
 427			if sname == None and line != '':
 428				return None, lineno, 'text outside section'
 429			if sname != None:
 430				sections[sname] += line + '\n'
 431			continue
 432		p = line.find(':')
 433		if p >= 0:
 434			s, val = line[:p].strip(), line[p+1:].strip()
 435			if s in sections:
 436				sname = s
 437				if val != '':
 438					sections[sname] += val + '\n'
 439				continue
 440		return None, lineno, 'malformed section header'
 441
 442	for k in sections:
 443		sections[k] = StripCommon(sections[k]).rstrip()
 444
 445	cl = CL(name)
 446	if sections['Author']:
 447		cl.copied_from = sections['Author']
 448	cl.desc = sections['Description']
 449	for line in sections['Files'].split('\n'):
 450		i = line.find('#')
 451		if i >= 0:
 452			line = line[0:i].rstrip()
 453		line = line.strip()
 454		if line == '':
 455			continue
 456		cl.files.append(line)
 457	cl.reviewer = SplitCommaSpace(sections['Reviewer'])
 458	cl.cc = SplitCommaSpace(sections['CC'])
 459	cl.url = sections['URL']
 460	if sections['Mailed'] != 'False':
 461		# Odd default, but avoids spurious mailings when
 462		# reading old CLs that do not have a Mailed: line.
 463		# CLs created with this update will always have 
 464		# Mailed: False on disk.
 465		cl.mailed = True
 466	if sections['Private'] in ('True', 'true', 'Yes', 'yes'):
 467		cl.private = True
 468	if cl.desc == '<enter description here>':
 469		cl.desc = ''
 470	return cl, 0, ''
 471
 472def SplitCommaSpace(s):
 473	typecheck(s, str)
 474	s = s.strip()
 475	if s == "":
 476		return []
 477	return re.split(", *", s)
 478
 479def CutDomain(s):
 480	typecheck(s, str)
 481	i = s.find('@')
 482	if i >= 0:
 483		s = s[0:i]
 484	return s
 485
 486def JoinComma(l):
 487	for s in l:
 488		typecheck(s, str)
 489	return ", ".join(l)
 490
 491def ExceptionDetail():
 492	s = str(sys.exc_info()[0])
 493	if s.startswith("<type '") and s.endswith("'>"):
 494		s = s[7:-2]
 495	elif s.startswith("<class '") and s.endswith("'>"):
 496		s = s[8:-2]
 497	arg = str(sys.exc_info()[1])
 498	if len(arg) > 0:
 499		s += ": " + arg
 500	return s
 501
 502def IsLocalCL(ui, repo, name):
 503	return GoodCLName(name) and os.access(CodeReviewDir(ui, repo) + "/cl." + name, 0)
 504
 505# Load CL from disk and/or the web.
 506def LoadCL(ui, repo, name, web=True):
 507	typecheck(name, str)
 508	set_status("loading CL " + name)
 509	if not GoodCLName(name):
 510		return None, "invalid CL name"
 511	dir = CodeReviewDir(ui, repo)
 512	path = dir + "cl." + name
 513	if os.access(path, 0):
 514		ff = open(path)
 515		text = ff.read()
 516		ff.close()
 517		cl, lineno, err = ParseCL(text, name)
 518		if err != "":
 519			return None, "malformed CL data: "+err
 520		cl.local = True
 521	else:
 522		cl = CL(name)
 523	if web:
 524		set_status("getting issue metadata from web")
 525		d = JSONGet(ui, "/api/" + name + "?messages=true")
 526		set_status(None)
 527		if d is None:
 528			return None, "cannot load CL %s from server" % (name,)
 529		if 'owner_email' not in d or 'issue' not in d or str(d['issue']) != name:
 530			return None, "malformed response loading CL data from code review server"
 531		cl.dict = d
 532		cl.reviewer = d.get('reviewers', [])
 533		cl.cc = d.get('cc', [])
 534		if cl.local and cl.copied_from and cl.desc:
 535			# local copy of CL written by someone else
 536			# and we saved a description.  use that one,
 537			# so that committers can edit the description
 538			# before doing hg submit.
 539			pass
 540		else:
 541			cl.desc = d.get('description', "")
 542		cl.url = server_url_base + name
 543		cl.web = True
 544		cl.private = d.get('private', False) != False
 545	set_status("loaded CL " + name)
 546	return cl, ''
 547
 548global_status = None
 549
 550def set_status(s):
 551	# print >>sys.stderr, "\t", time.asctime(), s
 552	global global_status
 553	global_status = s
 554
 555class StatusThread(threading.Thread):
 556	def __init__(self):
 557		threading.Thread.__init__(self)
 558	def run(self):
 559		# pause a reasonable amount of time before
 560		# starting to display status messages, so that
 561		# most hg commands won't ever see them.
 562		time.sleep(30)
 563
 564		# now show status every 15 seconds
 565		while True:
 566			time.sleep(15 - time.time() % 15)
 567			s = global_status
 568			if s is None:
 569				continue
 570			if s == "":
 571				s = "(unknown status)"
 572			print >>sys.stderr, time.asctime(), s
 573
 574def start_status_thread():
 575	t = StatusThread()
 576	t.setDaemon(True)  # allowed to exit if t is still running
 577	t.start()
 578
 579class LoadCLThread(threading.Thread):
 580	def __init__(self, ui, repo, dir, f, web):
 581		threading.Thread.__init__(self)
 582		self.ui = ui
 583		self.repo = repo
 584		self.dir = dir
 585		self.f = f
 586		self.web = web
 587		self.cl = None
 588	def run(self):
 589		cl, err = LoadCL(self.ui, self.repo, self.f[3:], web=self.web)
 590		if err != '':
 591			self.ui.warn("loading "+self.dir+self.f+": " + err + "\n")
 592			return
 593		self.cl = cl
 594
 595# Load all the CLs from this repository.
 596def LoadAllCL(ui, repo, web=True):
 597	dir = CodeReviewDir(ui, repo)
 598	m = {}
 599	files = [f for f in os.listdir(dir) if f.startswith('cl.')]
 600	if not files:
 601		return m
 602	active = []
 603	first = True
 604	for f in files:
 605		t = LoadCLThread(ui, repo, dir, f, web)
 606		t.start()
 607		if web and first:
 608			# first request: wait in case it needs to authenticate
 609			# otherwise we get lots of user/password prompts
 610			# running in parallel.
 611			t.join()
 612			if t.cl:
 613				m[t.cl.name] = t.cl
 614			first = False
 615		else:
 616			active.append(t)
 617	for t in active:
 618		t.join()
 619		if t.cl:
 620			m[t.cl.name] = t.cl
 621	return m
 622
 623# Find repository root.  On error, ui.warn and return None
 624def RepoDir(ui, repo):
 625	url = repo.url();
 626	if not url.startswith('file:'):
 627		ui.warn("repository %s is not in local file system\n" % (url,))
 628		return None
 629	url = url[5:]
 630	if url.endswith('/'):
 631		url = url[:-1]
 632	typecheck(url, str)
 633	return url
 634
 635# Find (or make) code review directory.  On error, ui.warn and return None
 636def CodeReviewDir(ui, repo):
 637	dir = RepoDir(ui, repo)
 638	if dir == None:
 639		return None
 640	dir += '/.hg/codereview/'
 641	if not os.path.isdir(dir):
 642		try:
 643			os.mkdir(dir, 0700)
 644		except:
 645			ui.warn('cannot mkdir %s: %s\n' % (dir, ExceptionDetail()))
 646			return None
 647	typecheck(dir, str)
 648	return dir
 649
 650# Turn leading tabs into spaces, so that the common white space
 651# prefix doesn't get confused when people's editors write out 
 652# some lines with spaces, some with tabs.  Only a heuristic
 653# (some editors don't use 8 spaces either) but a useful one.
 654def TabsToSpaces(line):
 655	i = 0
 656	while i < len(line) and line[i] == '\t':
 657		i += 1
 658	return ' '*(8*i) + line[i:]
 659
 660# Strip maximal common leading white space prefix from text
 661def StripCommon(text):
 662	typecheck(text, str)
 663	ws = None
 664	for line in text.split('\n'):
 665		line = line.rstrip()
 666		if line == '':
 667			continue
 668		line = TabsToSpaces(line)
 669		white = line[:len(line)-len(line.lstrip())]
 670		if ws == None:
 671			ws = white
 672		else:
 673			common = ''
 674			for i in range(min(len(white), len(ws))+1):
 675				if white[0:i] == ws[0:i]:
 676					common = white[0:i]
 677			ws = common
 678		if ws == '':
 679			break
 680	if ws == None:
 681		return text
 682	t = ''
 683	for line in text.split('\n'):
 684		line = line.rstrip()
 685		line = TabsToSpaces(line)
 686		if line.startswith(ws):
 687			line = line[len(ws):]
 688		if line == '' and t == '':
 689			continue
 690		t += line + '\n'
 691	while len(t) >= 2 and t[-2:] == '\n\n':
 692		t = t[:-1]
 693	typecheck(t, str)
 694	return t
 695
 696# Indent text with indent.
 697def Indent(text, indent):
 698	typecheck(text, str)
 699	typecheck(indent, str)
 700	t = ''
 701	for line in text.split('\n'):
 702		t += indent + line + '\n'
 703	typecheck(t, str)
 704	return t
 705
 706# Return the first line of l
 707def line1(text):
 708	typecheck(text, str)
 709	return text.split('\n')[0]
 710
 711_change_prolog = """# Change list.
 712# Lines beginning with # are ignored.
 713# Multi-line values should be indented.
 714"""
 715
 716#######################################################################
 717# Mercurial helper functions
 718
 719# Get effective change nodes taking into account applied MQ patches
 720def effective_revpair(repo):
 721    try:
 722	return scmutil.revpair(repo, ['qparent'])
 723    except:
 724	return scmutil.revpair(repo, None)
 725
 726# Return list of changed files in repository that match pats.
 727# Warn about patterns that did not match.
 728def matchpats(ui, repo, pats, opts):
 729	matcher = scmutil.match(repo, pats, opts)
 730	node1, node2 = effective_revpair(repo)
 731	modified, added, removed, deleted, unknown, ignored, clean = repo.status(node1, node2, matcher, ignored=True, clean=True, unknown=True)
 732	return (modified, added, removed, deleted, unknown, ignored, clean)
 733
 734# Return list of changed files in repository that match pats.
 735# The patterns came from the command line, so we warn
 736# if they have no effect or cannot be understood.
 737def ChangedFiles(ui, repo, pats, opts, taken=None):
 738	taken = taken or {}
 739	# Run each pattern separately so that we can warn about
 740	# patterns that didn't do anything useful.
 741	for p in pats:
 742		modified, added, removed, deleted, unknown, ignored, clean = matchpats(ui, repo, [p], opts)
 743		redo = False
 744		for f in unknown:
 745			promptadd(ui, repo, f)
 746			redo = True
 747		for f in deleted:
 748			promptremove(ui, repo, f)
 749			redo = True
 750		if redo:
 751			modified, added, removed, deleted, unknown, ignored, clean = matchpats(ui, repo, [p], opts)
 752		for f in modified + added + removed:
 753			if f in taken:
 754				ui.warn("warning: %s already in CL %s\n" % (f, taken[f].name))
 755		if not modified and not added and not removed:
 756			ui.warn("warning: %s did not match any modified files\n" % (p,))
 757
 758	# Again, all at once (eliminates duplicates)
 759	modified, added, removed = matchpats(ui, repo, pats, opts)[:3]
 760	l = modified + added + removed
 761	l.sort()
 762	if taken:
 763		l = Sub(l, taken.keys())
 764	return l
 765
 766# Return list of changed files in repository that match pats and still exist.
 767def ChangedExistingFiles(ui, repo, pats, opts):
 768	modified, added = matchpats(ui, repo, pats, opts)[:2]
 769	l = modified + added
 770	l.sort()
 771	return l
 772
 773# Return list of files claimed by existing CLs
 774def Taken(ui, repo):
 775	all = LoadAllCL(ui, repo, web=False)
 776	taken = {}
 777	for _, cl in all.items():
 778		for f in cl.files:
 779			taken[f] = cl
 780	return taken
 781
 782# Return list of changed files that are not claimed by other CLs
 783def DefaultFiles(ui, repo, pats, opts):
 784	return ChangedFiles(ui, repo, pats, opts, taken=Taken(ui, repo))
 785
 786def Sub(l1, l2):
 787	return [l for l in l1 if l not in l2]
 788
 789def Add(l1, l2):
 790	l = l1 + Sub(l2, l1)
 791	l.sort()
 792	return l
 793
 794def Intersect(l1, l2):
 795	return [l for l in l1 if l in l2]
 796
 797def getremote(ui, repo, opts):
 798	# save $http_proxy; creating the HTTP repo object will
 799	# delete it in an attempt to "help"
 800	proxy = os.environ.get('http_proxy')
 801	source = hg.parseurl(ui.expandpath("default"), None)[0]
 802	try:
 803		remoteui = hg.remoteui # hg 1.6
 804	except:
 805		remoteui = cmdutil.remoteui
 806	other = hg.repository(remoteui(repo, opts), source)
 807	if proxy is not None:
 808		os.environ['http_proxy'] = proxy
 809	return other
 810
 811def Incoming(ui, repo, opts):
 812	_, incoming, _ = findcommonincoming(repo, getremote(ui, repo, opts))
 813	return incoming
 814
 815desc_re = '^(.+: |(tag )?(release|weekly)\.|fix build|undo CL)'
 816
 817desc_msg = '''Your CL description appears not to use the standard form.
 818
 819The first line of your change description is conventionally a
 820one-line summary of the change, prefixed by the primary affected package,
 821and is used as the subject for code review mail; the rest of the description
 822elaborates.
 823
 824Examples:
 825
 826	encoding/rot13: new package
 827
 828	math: add IsInf, IsNaN
 829	
 830	net: fix cname in LookupHost
 831
 832	unicode: update to Unicode 5.0.2
 833
 834'''
 835
 836
 837def promptremove(ui, repo, f):
 838	if promptyesno(ui, "hg remove %s (y/n)?" % (f,)):
 839		if commands.remove(ui, repo, 'path:'+f) != 0:
 840			ui.warn("error removing %s" % (f,))
 841
 842def promptadd(ui, repo, f):
 843	if promptyesno(ui, "hg add %s (y/n)?" % (f,)):
 844		if commands.add(ui, repo, 'path:'+f) != 0:
 845			ui.warn("error adding %s" % (f,))
 846
 847def EditCL(ui, repo, cl):
 848	set_status(None)	# do not show status
 849	s = cl.EditorText()
 850	while True:
 851		s = ui.edit(s, ui.username())
 852		clx, line, err = ParseCL(s, cl.name)
 853		if err != '':
 854			if not promptyesno(ui, "error parsing change list: line %d: %s\nre-edit (y/n)?" % (line, err)):
 855				return "change list not modified"
 856			continue
 857		
 858		# Check description.
 859		if clx.desc == '':
 860			if promptyesno(ui, "change list should have a description\nre-edit (y/n)?"):
 861				continue
 862		elif re.search('<enter reason for undo>', clx.desc):
 863			if promptyesno(ui, "change list description omits reason for undo\nre-edit (y/n)?"):
 864				continue
 865		elif not re.match(desc_re, clx.desc.split('\n')[0]):
 866			if promptyesno(ui, desc_msg + "re-edit (y/n)?"):
 867				continue
 868
 869		# Check file list for files that need to be hg added or hg removed
 870		# or simply aren't understood.
 871		pats = ['path:'+f for f in clx.files]
 872		modified, added, removed, deleted, unknown, ignored, clean = matchpats(ui, repo, pats, {})
 873		files = []
 874		for f in clx.files:
 875			if f in modified or f in added or f in removed:
 876				files.append(f)
 877				continue
 878			if f in deleted:
 879				promptremove(ui, repo, f)
 880				files.append(f)
 881				continue
 882			if f in unknown:
 883				promptadd(ui, repo, f)
 884				files.append(f)
 885				continue
 886			if f in ignored:
 887				ui.warn("error: %s is excluded by .hgignore; omitting\n" % (f,))
 888				continue
 889			if f in clean:
 890				ui.warn("warning: %s is listed in the CL but unchanged\n" % (f,))
 891				files.append(f)
 892				continue
 893			p = repo.root + '/' + f
 894			if os.path.isfile(p):
 895				ui.warn("warning: %s is a file but not known to hg\n" % (f,))
 896				files.append(f)
 897				continue
 898			if os.path.isdir(p):
 899				ui.warn("error: %s is a directory, not a file; omitting\n" % (f,))
 900				continue
 901			ui.warn("error: %s does not exist; omitting\n" % (f,))
 902		clx.files = files
 903
 904		cl.desc = clx.desc
 905		cl.reviewer = clx.reviewer
 906		cl.cc = clx.cc
 907		cl.files = clx.files
 908		cl.private = clx.private
 909		break
 910	return ""
 911
 912# For use by submit, etc. (NOT by change)
 913# Get change list number or list of files from command line.
 914# If files are given, make a new change list.
 915def CommandLineCL(ui, repo, pats, opts, defaultcc=None):
 916	if len(pats) > 0 and GoodCLName(pats[0]):
 917		if len(pats) != 1:
 918			return None, "cannot specify change number and file names"
 919		if opts.get('message'):
 920			return None, "cannot use -m with existing CL"
 921		cl, err = LoadCL(ui, repo, pats[0], web=True)
 922		if err != "":
 923			return None, err
 924	else:
 925		cl = CL("new")
 926		cl.local = True
 927		cl.files = ChangedFiles(ui, repo, pats, opts, taken=Taken(ui, repo))
 928		if not cl.files:
 929			return None, "no files changed"
 930	if opts.get('reviewer'):
 931		cl.reviewer = Add(cl.reviewer, SplitCommaSpace(opts.get('reviewer')))
 932	if opts.get('cc'):
 933		cl.cc = Add(cl.cc, SplitCommaSpace(opts.get('cc')))
 934	if defaultcc:
 935		cl.cc = Add(cl.cc, defaultcc)
 936	if cl.name == "new":
 937		if opts.get('message'):
 938			cl.desc = opts.get('message')
 939		else:
 940			err = EditCL(ui, repo, cl)
 941			if err != '':
 942				return None, err
 943	return cl, ""
 944
 945# reposetup replaces cmdutil.match with this wrapper,
 946# which expands the syntax @clnumber to mean the files
 947# in that CL.
 948original_match = None
 949def ReplacementForCmdutilMatch(repo, pats=None, opts=None, globbed=False, default='relpath'):
 950	taken = []
 951	files = []
 952	pats = pats or []
 953	opts = opts or {}
 954	for p in pats:
 955		if p.startswith('@'):
 956			taken.append(p)
 957			clname = p[1:]
 958			if not GoodCLName(clname):
 959				raise util.Abort("invalid CL name " + clname)
 960			cl, err = LoadCL(repo.ui, repo, clname, web=False)
 961			if err != '':
 962				raise util.Abort("loading CL " + clname + ": " + err)
 963			if not cl.files:
 964				raise util.Abort("no files in CL " + clname)
 965			files = Add(files, cl.files)
 966	pats = Sub(pats, taken) + ['path:'+f for f in files]
 967
 968	# work-around for http://selenic.com/hg/rev/785bbc8634f8
 969	if hgversion >= '1.9' and not hasattr(repo, 'match'):
 970		repo = repo[None]
 971
 972	return original_match(repo, pats=pats, opts=opts, globbed=globbed, default=default)
 973
 974def RelativePath(path, cwd):
 975	n = len(cwd)
 976	if path.startswith(cwd) and path[n] == '/':
 977		return path[n+1:]
 978	return path
 979
 980def CheckFormat(ui, repo, files, just_warn=False):
 981	set_status("running gofmt")
 982	CheckGofmt(ui, repo, files, just_warn)
 983	CheckTabfmt(ui, repo, files, just_warn)
 984
 985# Check that gofmt run on the list of files does not change them
 986def CheckGofmt(ui, repo, files, just_warn):
 987	files = [f for f in files if (f.startswith('src/') or f.startswith('test/bench/')) and f.endswith('.go')]
 988	if not files:
 989		return
 990	cwd = os.getcwd()
 991	files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
 992	files = [f for f in files if os.access(f, 0)]
 993	if not files:
 994		return
 995	try:
 996		cmd = subprocess.Popen(["gofmt", "-l"] + files, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=sys.platform != "win32")
 997		cmd.stdin.close()
 998	except:
 999		raise util.Abort("gofmt: " + ExceptionDetail())
1000	data = cmd.stdout.read()
1001	errors = cmd.stderr.read()
1002	cmd.wait()
1003	set_status("done with gofmt")
1004	if len(errors) > 0:
1005		ui.warn("gofmt errors:\n" + errors.rstrip() + "\n")
1006		return
1007	if len(data) > 0:
1008		msg = "gofmt needs to format these files (run hg gofmt):\n" + Indent(data, "\t").rstrip()
1009		if just_warn:
1010			ui.warn("warning: " + msg + "\n")
1011		else:
1012			raise util.Abort(msg)
1013	return
1014
1015# Check that *.[chys] files indent using tabs.
1016def CheckTabfmt(ui, repo, files, just_warn):
1017	files = [f for f in files if f.startswith('src/') and re.search(r"\.[chys]$", f)]
1018	if not files:
1019		return
1020	cwd = os.getcwd()
1021	files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
1022	files = [f for f in files if os.access(f, 0)]
1023	badfiles = []
1024	for f in files:
1025		try:
1026			for line in open(f, 'r'):
1027				# Four leading spaces is enough to complain about,
1028				# except that some Plan 9 code uses four spaces as the label indent,
1029				# so allow that.
1030				if line.startswith('    ') and not re.match('    [A-Za-z0-9_]+:', line):
1031					badfiles.append(f)
1032					break
1033		except:
1034			# ignore cannot open file, etc.
1035			pass
1036	if len(badfiles) > 0:
1037		msg = "these files use spaces for indentation (use tabs instead):\n\t" + "\n\t".join(badfiles)
1038		if just_warn:
1039			ui.warn("warning: " + msg + "\n")
1040		else:
1041			raise util.Abort(msg)
1042	return
1043
1044#######################################################################
1045# Mercurial commands
1046
1047# every command must take a ui and and repo as arguments.
1048# opts is a dict where you can find other command line flags
1049#
1050# Other parameters are taken in order from items on the command line that
1051# don't start with a dash.  If no default value is given in the parameter list,
1052# they are required.
1053#
1054
1055def change(ui, repo, *pats, **opts):
1056	"""create, edit or delete a change list
1057
1058	Create, edit or delete a change list.
1059	A change list is a group of files to be reviewed and submitted together,
1060	plus a textual description of the change.
1061	Change lists are referred to by simple alphanumeric names.
1062
1063	Changes must be reviewed before they can be submitted.
1064
1065	In the absence of options, the change command opens the
1066	change list for editing in the default editor.
1067
1068	Deleting a change with the -d or -D flag does not affect
1069	the contents of the files listed in that change.  To revert
1070	the files listed in a change, use
1071
1072		hg revert @123456
1073
1074	before running hg change -d 123456.
1075	"""
1076
1077	if missing_codereview:
1078		return missing_codereview
1079	
1080	dirty = {}
1081	if len(pats) > 0 and GoodCLName(pats[0]):
1082		name = pats[0]
1083		if len(pats) != 1:
1084			return "cannot specify CL name and file patterns"
1085		pats = pats[1:]
1086		cl, err = LoadCL(ui, repo, name, web=True)
1087		if err != '':
1088			return err
1089		if not cl.local and (opts["stdin"] or not opts["stdout"]):
1090			return "cannot change non-local CL " + name
1091	else:
1092		if repo[None].branch() != "default":
1093			return "cannot run hg change outside default branch"
1094		name = "new"
1095		cl = CL("new")
1096		dirty[cl] = True
1097		files = ChangedFiles(ui, repo, pats, opts, taken=Taken(ui, repo))
1098
1099	if opts["delete"] or opts["deletelocal"]:
1100		if opts["delete"] and opts["deletelocal"]:
1101			return "cannot use -d and -D together"
1102		flag = "-d"
1103		if opts["deletelocal"]:
1104			flag = "-D"
1105		if name == "new":
1106			return "cannot use "+flag+" with file patterns"
1107		if opts["stdin"] or opts["stdout"]:
1108			return "cannot use "+flag+" with -i or -o"
1109		if not cl.local:
1110			return "cannot change non-local CL " + name
1111		if opts["delete"]:
1112			if cl.copied_from:
1113				return "original author must delete CL; hg change -D will remove locally"
1114			PostMessage(ui, cl.name, "*** Abandoned ***", send_mail=cl.mailed)
1115			EditDesc(cl.name, closed=True, private=cl.private)
1116		cl.Delete(ui, repo)
1117		return
1118
1119	if opts["stdin"]:
1120		s = sys.stdin.read()
1121		clx, line, err = ParseCL(s, name)
1122		if err != '':
1123			return "error parsing change list: line %d: %s" % (line, err)
1124		if clx.desc is not None:
1125			cl.desc = clx.desc;
1126			dirty[cl] = True
1127		if clx.reviewer is not None:
1128			cl.reviewer = clx.reviewer
1129			dirty[cl] = True
1130		if clx.cc is not None:
1131			cl.cc = clx.cc
1132			dirty[cl] = True
1133		if clx.files is not None:
1134			cl.files = clx.files
1135			dirty[cl] = True
1136		if clx.private != cl.private:
1137			cl.private = clx.private
1138			dirty[cl] = True
1139
1140	if not opts["stdin"] and not opts["stdout"]:
1141		if name == "new":
1142			cl.files = files
1143		err = EditCL(ui, repo, cl)
1144		if err != "":
1145			return err
1146		dirty[cl] = True
1147
1148	for d, _ in dirty.items():
1149		name = d.name
1150		d.Flush(ui, repo)
1151		if name == "new":
1152			d.Upload(ui, repo, quiet=True)
1153
1154	if opts["stdout"]:
1155		ui.write(cl.EditorText())
1156	elif opts["pending"]:
1157		ui.write(cl.PendingText())
1158	elif name == "new":
1159		if ui.quiet:
1160			ui.write(cl.name)
1161		else:
1162			ui.write("CL created: " + cl.url + "\n")
1163	return
1164
1165def code_login(ui, repo, **opts):
1166	"""log in to code review server
1167
1168	Logs in to the code review server, saving a cookie in
1169	a file in your home directory.
1170	"""
1171	if missing_codereview:
1172		return missing_codereview
1173
1174	MySend(None)
1175
1176def clpatch(ui, repo, clname, **opts):
1177	"""import a patch from the code review server
1178
1179	Imports a patch from the code review server into the local client.
1180	If the local client has already modified any of the files that the
1181	patch modifies, this command will refuse to apply the patch.
1182
1183	Submitting an imported patch will keep the original author's
1184	name as the Author: line but add your own name to a Committer: line.
1185	"""
1186	if repo[None].branch() != "default":
1187		return "cannot run hg clpatch outside default branch"
1188	return clpatch_or_undo(ui, repo, clname, opts, mode="clpatch")
1189
1190def undo(ui, repo, clname, **opts):
1191	"""undo the effect of a CL
1192	
1193	Creates a new CL that undoes an earlier CL.
1194	After creating the CL, opens the CL text for editing so that
1195	you can add the reason for the undo to the description.
1196	"""
1197	if repo[None].branch() != "default":
1198		return "cannot run hg undo outside default branch"
1199	return clpatch_or_undo(ui, repo, clname, opts, mode="undo")
1200
1201def release_apply(ui, repo, clname, **opts):
1202	"""apply a CL to the release branch
1203
1204	Creates a new CL copying a previously committed change
1205	from the main branch to the release branch.
1206	The current client must either be clean or already be in
1207	the release branch.
1208	
1209	The release branch must be created by starting with a
1210	clean client, disabling the code review plugin, and running:
1211	
1212		hg update weekly.YYYY-MM-DD
1213		hg branch release-branch.rNN
1214		hg commit -m 'create release-branch.rNN'
1215		hg push --new-branch
1216	
1217	Then re-enable the code review plugin.
1218	
1219	People can test the release branch by running
1220	
1221		hg update release-branch.rNN
1222	
1223	in a clean client.  To return to the normal tree,
1224	
1225		hg update default
1226	
1227	Move changes since the weekly into the release branch 
1228	using hg release-apply followed by the usual code review
1229	process and hg submit.
1230
1231	When it comes time to tag the release, record the
1232	final long-form tag of the release-branch.rNN
1233	in the *default* branch's .hgtags file.  That is, run
1234	
1235		hg update default
1236	
1237	and then edit .hgtags as you would for a weekly.
1238		
1239	"""
1240	c = repo[None]
1241	if not releaseBranch:
1242		return "no active release branches"
1243	if c.branch() != releaseBranch:
1244		if c.modified() or c.added() or c.removed():
1245			raise util.Abort("uncommitted local changes - cannot switch branches")
1246		err = hg.clean(repo, releaseBranch)
1247		if err:
1248			return err
1249	try:
1250		err = clpatch_or_undo(ui, repo, clname, opts, mode="backport")
1251		if err:
1252			raise util.Abort(err)
1253	except Exception, e:
1254		hg.clean(repo, "default")
1255		raise e
1256	return None
1257
1258def rev2clname(rev):
1259	# Extract CL name from revision description.
1260	# The last line in the description that is a codereview URL is the real one.
1261	# Earlier lines might be part of the user-written description.
1262	all = re.findall('(?m)^http://codereview.appspot.com/([0-9]+)$', rev.description())
1263	if len(all) > 0:
1264		return all[-1]
1265	return ""
1266
1267undoHeader = """undo CL %s / %s
1268
1269<enter reason for undo>
1270
1271««« original CL description
1272"""
1273
1274undoFooter = """
1275»»»
1276"""
1277
1278backportHeader = """[%s] %s
1279
1280««« CL %s / %s
1281"""
1282
1283backportFooter = """
1284»»»
1285"""
1286
1287# Implementation of clpatch/undo.
1288def clpatch_or_undo(ui, repo, clname, opts, mode):
1289	if missing_codereview:
1290		return missing_codereview
1291
1292	if mode == "undo" or mode == "backport":
1293		if hgversion < '1.4':
1294			# Don't have cmdutil.match (see implementation of sync command).
1295			return "hg is too old to run hg %s - update to 1.4 or newer" % mode
1296
1297		# Find revision in Mercurial repository.
1298		# Assume CL number is 7+ decimal digits.
1299		# Otherwise is either change log sequence number (fewer decimal digits),
1300		# hexadecimal hash, or tag name.
1301		# Mercurial will fall over long before the change log
1302		# sequence numbers get to be 7 digits long.
1303		if re.match('^[0-9]{7,}$', clname):
1304			found = False
1305			matchfn = scmutil.match(repo, [], {'rev': None})
1306			def prep(ctx, fns):
1307				pass
1308			for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev': None}, prep):
1309				rev = repo[ctx.rev()]
1310				# Last line with a code review URL is the actual review URL.
1311				# Earlier ones might be part of the CL description.
1312				n = rev2clname(rev)
1313				if n == clname:
1314					found = True
1315					break
1316			if not found:
1317				return "cannot find CL %s in local repository" % clname
1318		else:
1319			rev = repo[clname]
1320			if not rev:
1321				return "unknown revision %s" % clname
1322			clname = rev2clname(rev)
1323			if clname == "":
1324				return "cannot find CL name in revision description"
1325		
1326		# Create fresh CL and start with patch that would reverse the change.
1327		vers = short(rev.node())
1328		cl = CL("new")
1329		desc = str(rev.description())
1330		if mode == "undo":
1331			cl.desc = (undoHeader % (clname, vers)) + desc + undoFooter
1332		else:
1333			cl.desc = (backportHeader % (releaseBranch, line1(desc), clname, vers)) + desc + undoFooter
1334		v1 = vers
1335		v0 = short(rev.parents()[0].node())
1336		if mode == "undo":
1337			arg = v1 + ":" + v0
1338		else:
1339			vers = v0
1340			arg = v0 + ":" + v1
1341		patch = RunShell(["hg", "diff", "--git", "-r", arg])
1342
1343	else:  # clpatch
1344		cl, vers, patch, err = DownloadCL(ui, repo, clname)
1345		if err != "":
1346			return err
1347		if patch == emptydiff:
1348			return "codereview issue %s has no diff" % clname
1349
1350	# find current hg version (hg identify)
1351	ctx = repo[None]
1352	parents = ctx.parents()
1353	id = '+'.join([short(p.node()) for p in parents])
1354
1355	# if version does not match the patch version,
1356	# try to update the patch line numbers.
1357	if vers != "" and id != vers:
1358		# "vers in repo" gives the wrong answer
1359		# on some versions of Mercurial.  Instead, do the actual
1360		# lookup and catch the exception.
1361		try:
1362			repo[vers].description()
1363		except:
1364			return "local repository is out of date; sync to get %s" % (vers)
1365		patch1, err = portPatch(repo, patch, vers, id)
1366		if err != "":
1367			if not opts["ignore_hgpatch_failure"]:
1368				return "codereview issue %s is out of date: %s (%s->%s)" % (clname, err, vers, id)
1369		else:
1370			patch = patch1
1371	argv = ["hgpatch"]
1372	if opts["no_incoming"] or mode == "backport":
1373		argv += ["--checksync=false"]
1374	try:
1375		cmd = subprocess.Popen(argv, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None, close_fds=sys.platform != "win32")
1376	except:
1377		return "hgpatch: " + ExceptionDetail()
1378
1379	out, err = cmd.communicate(patch)
1380	if cmd.returncode != 0 and not opts["ignore_hgpatch_failure"]:
1381		return "hgpatch failed"
1382	cl.local = True
1383	cl.files = out.strip().split()
1384	if not cl.files and not opts["ignore_hgpatch_failure"]:
1385		return "codereview issue %s has no changed files" % clname
1386	files = ChangedFiles(ui, repo, [], opts)
1387	extra = Sub(cl.files, files)
1388	if extra:
1389		ui.warn("warning: these files were listed in the patch but not changed:\n\t" + "\n\t".join(extra) + "\n")
1390	cl.Flush(ui, repo)
1391	if mode == "undo":
1392		err = EditCL(ui, repo, cl)
1393		if err != "":
1394			return "CL created, but error editing: " + err
1395		cl.Flush(ui, repo)
1396	else:
1397		ui.write(cl.PendingText() + "\n")
1398
1399# portPatch rewrites patch from being a patch against
1400# oldver to being a patch against newver.
1401def portPatch(repo, patch, oldver, newver):
1402	lines = patch.splitlines(True) # True = keep \n
1403	delta = None
1404	for i in range(len(lines)):
1405		line = lines[i]
1406		if line.startswith('--- a/'):
1407			file = line[6:-1]
1408			delta = fileDeltas(repo, file, oldver, newver)
1409		if not delta or not line.startswith('@@ '):
1410			continue
1411		# @@ -x,y +z,w @@ means the patch chunk replaces
1412		# the original file's line numbers x up to x+y with the
1413		# line numbers z up to z+w in the new file.
1414		# Find the delta from x in the original to the same
1415		# line in the current version and add that delta to both
1416		# x and z.
1417		m = re.match('@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@', line)
1418		if not m:
1419			return None, "error parsing patch line numbers"
1420		n1, len1, n2, len2 = int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))
1421		d, err = lineDelta(delta, n1, len1)
1422		if err != "":
1423			return "", err
1424		n1 += d
1425		n2 += d
1426		lines[i] = "@@ -%d,%d +%d,%d @@\n" % (n1, len1, n2, len2)
1427		
1428	newpatch = ''.join(lines)
1429	return newpatch, ""
1430
1431# fileDelta returns the line number deltas for the given file's
1432# changes from oldver to newver.
1433# The deltas are a list of (n, len, newdelta) triples that say
1434# lines [n, n+len) were modified, and after that range the
1435# line numbers are +newdelta from what they were before.
1436def fileDeltas(repo, file, oldver, newver):
1437	cmd = ["hg", "diff", "--git", "-r", oldver + ":" + newver, "path:" + file]
1438	data = RunShell(cmd, silent_ok=True)
1439	deltas = []
1440	for line in data.splitlines():
1441		m = re.match('@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@', line)
1442		if not m:
1443			continue
1444		n1, len1, n2, len2 = int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))
1445		deltas.append((n1, len1, n2+len2-(n1+len1)))
1446	return deltas
1447
1448# lineDelta finds the appropriate line number delta to apply to the lines [n, n+len).
1449# It returns an error if those lines were rewritten by the patch.
1450def lineDelta(deltas, n, len):
1451	d = 0
1452	for (old, oldlen, newdelta) in deltas:
1453		if old >= n+len:
1454			break
1455		if old+len > n:
1456			return 0, "patch and recent changes conflict"
1457		d = newdelta
1458	return d, ""
1459
1460def download(ui, repo, clname, **opts):
1461	"""download a change from the code review server
1462
1463	Download prints a description of the given change list
1464	followed by its diff, downloaded from the code review server.
1465	"""
1466	if missing_codereview:
1467		return missing_codereview
1468
1469	cl, vers, patch, err = DownloadCL(ui, repo, clname)
1470	if err != "":
1471		return err
1472	ui.write(cl.EditorText() + "\n")
1473	ui.write(patch + "\n")
1474	return
1475
1476def file(ui, repo, clname, pat, *pats, **opts):
1477	"""assign files to or remove files from a change list
1478
1479	Assign files to or (with -d) remove files from a change list.
1480
1481	The -d option only removes files from the change list.
1482	It does not edit them or remove them from the repository.
1483	"""
1484	if missing_codereview:
1485		return missing_codereview
1486
1487	pats = tuple([pat] + list(pats))
1488	if not GoodCLName(clname):
1489		return "invalid CL name " + clname
1490
1491	dirty = {}
1492	cl, err = LoadCL(ui, repo, clname, web=False)
1493	if err != '':
1494		return err
1495	if not cl.local:
1496		return "cannot change non-local CL " + clname
1497
1498	files = ChangedFiles(ui, repo, pats, opts)
1499
1500	if opts["delete"]:
1501		oldfiles = Intersect(files, cl.files)
1502		if oldfiles:
1503			if not ui.quiet:
1504				ui.status("# Removing files from CL.  To undo:\n")
1505				ui.status("#	cd %s\n" % (repo.root))
1506				for f in oldfiles:
1507					ui.status("#	hg file %s %s\n" % (cl.name, f))
1508			cl.files = Sub(cl.files, oldfiles)
1509			cl.Flush(ui, repo)
1510		else:
1511			ui.status("no such files in CL")
1512		return
1513
1514	if not files:
1515		return "no such modified files"
1516
1517	files = Sub(files, cl.files)
1518	taken = Taken(ui, repo)
1519	warned = False
1520	for f in files:
1521		if f in taken:
1522			if not warned and not ui.quiet:
1523				ui.status("# Taking files from other CLs.  To undo:\n")
1524				ui.status("#	cd %s\n" % (repo.root))
1525				warned = True
1526			ocl = taken[f]
1527			if not ui.quiet:
1528				ui.status("#	hg file %s %s\n" % (ocl.name, f))
1529			if ocl not in dirty:
1530				ocl.files = Sub(ocl.files, files)
1531				dirty[ocl] = True
1532	cl.files = Add(cl.files, files)
1533	dirty[cl] = True
1534	for d, _ in dirty.items():
1535		d.Flush(ui, repo)
1536	return
1537
1538def gofmt(ui, repo, *pats, **opts):
1539	"""apply gofmt to modified files
1540
1541	Applies gofmt to the modified files in the repository that match
1542	the given patterns.
1543	"""
1544	if missing_codereview:
1545		return missing_codereview
1546
1547	files = ChangedExistingFiles(ui, repo, pats, opts)
1548	files = [f for f in files if f.endswith(".go")]
1549	if not files:
1550		return "no modified go files"
1551	cwd = os.getcwd()
1552	files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
1553	try:
1554		cmd = ["gofmt", "-l"]
1555		if not opts["list"]:
1556			cmd += ["-w"]
1557		if os.spawnvp(os.P_WAIT, "gofmt", cmd + files) != 0:
1558			raise util.Abort("gofmt did not exit cleanly")
1559	except error.Abort, e:
1560		raise
1561	except:
1562		raise util.Abort("gofmt: " + ExceptionDetail())
1563	return
1564
1565def mail(ui, repo, *pats, **opts):
1566	"""mail a change for review
1567
1568	Uploads a patch to the code review server and then sends mail
1569	to the reviewer and CC list asking for a review.
1570	"""
1571	if missing_codereview:
1572		return missing_codereview
1573
1574	cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc)
1575	if err != "":
1576		return err
1577	cl.Upload(ui, repo, gofmt_just_warn=True)
1578	if not cl.reviewer:
1579		# If no reviewer is listed, assign the review to defaultcc.
1580		# This makes sure that it appears in the 
1581		# codereview.appspot.com/user/defaultcc
1582		# page, so that it doesn't get dropped on the floor.
1583		if not defaultcc:
1584			return "no reviewers listed in CL"
1585		cl.cc = Sub(cl.cc, defaultcc)
1586		cl.reviewer = defaultcc
1587		cl.Flush(ui, repo)
1588
1589	if cl.files == []:
1590		return "no changed files, not sending mail"
1591
1592	cl.Mail(ui, repo)		
1593
1594def pending(ui, repo, *pats, **opts):
1595	"""show pending changes
1596
1597	Lists pending changes followed by a list of unassigned but modified files.
1598	"""
1599	if missing_codereview:
1600		return missing_codereview
1601
1602	m = LoadAllCL(ui, repo, web=True)
1603	names = m.keys()
1604	names.sort()
1605	for name in names:
1606		cl = m[name]
1607		ui.write(cl.PendingText() + "\n")
1608
1609	files = DefaultFiles(ui, repo, [], opts)
1610	if len(files) > 0:
1611		s = "Changed files not in any CL:\n"
1612		for f in files:
1613			s += "\t" + f + "\n"
1614		ui.write(s)
1615
1616def reposetup(ui, repo):
1617	global original_match
1618	if original_match is None:
1619		start_status_thread()
1620		original_match = scmutil.match
1621		scmutil.match = ReplacementForCmdutilMatch
1622		RietveldSetup(ui, repo)
1623
1624def CheckContributor(ui, repo, user=None):
1625	set_status("checking CONTRIBUTORS file")
1626	user, userline = FindContributor(ui, repo, user, warn=False)
1627	if not userline:
1628		raise util.Abort("cannot find %s in CONTRIBUTORS" % (user,))
1629	return userline
1630
1631def FindContributor(ui, repo, user=None, warn=True):
1632	if not user:
1633		user = ui.config("ui", "username")
1634		if not user:
1635			raise util.Abort("[ui] username is not configured in .hgrc")
1636	user = user.lower()
1637	m = re.match(r".*<(.*)>", user)
1638	if m:
1639		user = m.group(1)
1640
1641	if user not in contributors:
1642		if warn:
1643			ui.warn("warning: cannot find %s in CONTRIBUTORS\n" % (user,))
1644		return user, None
1645	
1646	user, email = contributors[user]
1647	return email, "%s <%s>" % (user, email)
1648
1649def submit(ui, repo, *pats, **opts):
1650	"""submit change to remote repository
1651
1652	Submits change to remote repository.
1653	Bails out if the local repository is not in sync with the remote one.
1654	"""
1655	if missing_codereview:
1656		return missing_codereview
1657
1658	# We already called this on startup but sometimes Mercurial forgets.
1659	set_mercurial_encoding_to_utf8()
1660
1661	repo.ui.quiet = True
1662	if not opts["no_incoming"] and Incoming(ui, repo, opts):
1663		return "local repository out of date; must sync before submit"
1664
1665	cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc)
1666	if err != "":
1667		return err
1668
1669	user = None
1670	if cl.copied_from:
1671		user = cl.copied_from
1672	userline = CheckContributor(ui, repo, user)
1673	typecheck(userline, str)
1674
1675	about = ""
1676	if cl.reviewer:
1677		about += "R=" + JoinComma([CutDomain(s) for s in cl.reviewer]) + "\n"
1678	if opts.get('tbr'):
1679		tbr = SplitCommaSpace(opts.get('tbr'))
1680		cl.reviewer = Add(cl.reviewer, tbr)
1681		about += "TBR=" + JoinComma([CutDomain(s) for s in tbr]) + "\n"
1682	if cl.cc:
1683		about += "CC=" + JoinComma([CutDomain(s) for s in cl.cc]) + "\n"
1684
1685	if not cl.reviewer:
1686		return "no reviewers listed in CL"
1687
1688	if not cl.local:
1689		return "cannot submit non-local CL"
1690
1691	# upload, to sync current patch and also get change number if CL is new.
1692	if not cl.copied_from:
1693		cl.Upload(ui, repo, gofmt_just_warn=True)
1694
1695	# check gofmt for real; allowed upload to warn in order to save CL.
1696	cl.Flush(ui, repo)
1697	CheckFormat(ui, repo, cl.files)
1698
1699	about += "%s%s\n" % (server_url_base, cl.name)
1700
1701	if cl.copied_from:
1702		about += "\nCommitter: " + CheckContributor(ui, repo, None) + "\n"
1703	typecheck(about, str)
1704
1705	if not cl.mailed and not cl.copied_from:		# in case this is TBR
1706		cl.Mail(ui, repo)
1707
1708	# submit changes locally
1709	date = opts.get('date')
1710	if date:
1711		opts['date'] = util.parsedate(date)
1712		typecheck(opts['date'], str)
1713	opts['message'] = cl.desc.rstrip() + "\n\n" + about
1714	typecheck(opts['message'], str)
1715
1716	if opts['dryrun']:
1717		print "NOT SUBMITTING:"
1718		print "User: ", userline
1719		print "Message:"
1720		print Indent(opts['message'], "\t")
1721		print "Files:"
1722		print Indent('\n'.join(cl.files), "\t")
1723		return "dry run; not submitted"
1724
1725	m = match.exact(repo.root, repo.getcwd(), cl.files)
1726	node = repo.commit(ustr(opts['message']), ustr(userline), opts.get('date'), m)
1727	if not node:
1728		return "nothing changed"
1729
1730	# push to remote; if it fails for any reason, roll back
1731	try:
1732		log = repo.changelog
1733		rev = log.rev(node)
1734		parents = log.parentrevs(rev)
1735		if (rev-1 not in parents and
1736				(parents == (nullrev, nullrev) or
1737				len(log.heads(log.node(parents[0]))) > 1 and
1738				(parents[1] == nullrev or len(log.heads(log.node(parents[1]))) > 1))):
1739			# created new head
1740			raise util.Abort("local repository out of date; must sync before submit")
1741
1742		# push changes to remote.
1743		# if it works, we're committed.
1744		# if not, roll back
1745		other = getremote(ui, repo, opts)
1746		r = repo.push(other, False, None)
1747		if r == 0:
1748			raise util.Abort("local repository out of date; must sync before submit")
1749	except:
1750		real_rollback()
1751		raise
1752
1753	# we're committed. upload final patch, close review, add commit message
1754	changeURL = short(node)
1755	url = other.url()
1756	m = re.match("^https?://([^@/]+@)?([^.]+)\.googlecode\.com/hg/?", url)
1757	if m:
1758		changeURL = "http://code.google.com/p/%s/source/detail?r=%s" % (m.group(2), changeURL)
1759	else:
1760		print >>sys.stderr, "URL: ", url
1761	pmsg = "*** Submitted as " + changeURL + " ***\n\n" + opts['message']
1762
1763	# When posting, move reviewers to CC line,
1764	# so that the issue stops showing up in their "My Issues" page.
1765	PostMessage(ui, cl.name, pmsg, reviewers="", cc=JoinComma(cl.reviewer+cl.cc))
1766
1767	if not cl.copied_from:
1768		EditDesc(cl.name, closed=True, private=cl.private)
1769	cl.Delete(ui, repo)
1770	
1771	c = repo[None]
1772	if c.branch() == releaseBranch and not c.modified() and not c.added() and not c.removed():
1773		ui.write("switching from %s to default branch.\n" % releaseBranch)
1774		err = hg.clean(repo, "default")
1775		if err:
1776			return err
1777	return None
1778
1779def sync(ui, repo, **opts):
1780	"""synchronize with remote repository
1781
1782	Incorporates recent changes from the remote repository
1783	into the local repository.
1784	"""
1785	if missing_codereview:
1786		return missing_codereview
1787
1788	if not opts["local"]:
1789		ui.status = sync_note
1790		ui.note = sync_note
1791		other = getremote(ui, repo, opts)
1792		modheads = repo.pull(other)
1793		err = commands.postincoming(ui, repo, modheads, True, "tip")
1794		if err:
1795			return err
1796	commands.update(ui, repo, rev="default")
1797	sync_changes(ui, repo)
1798
1799def sync_note(msg):
1800	# we run sync (pull -u) in verbose mode to get the
1801	# list of files being updated, but that drags along
1802	# a bunch of messages we don't care about.
1803	# omit them.
1804	if msg == 'resolving manifests\n':
1805		return
1806	if msg == 'searching for changes\n':
1807		return
1808	if msg == "couldn't find merge tool hgmerge\n":
1809		return
1810	sys.stdout.write(msg)
1811
1812def sync_changes(ui, repo):
1813	# Look through recent change log descriptions to find
1814	# potential references to http://.*/our-CL-number.
1815	# Double-check them by looking at the Rietveld log.
1816	def Rev(rev):
1817		desc = repo[rev].description().strip()
1818		for clname in re.findall('(?m)^http://(?:[^\n]+)/([0-9]+)$', desc):
1819			if IsLocalCL(ui, repo, clname) and IsRietveldSubmitted(ui, clname, repo[rev].hex()):
1820				ui.warn("CL %s submitted as %s; closing\n" % (clname, repo[rev]))
1821				cl, err = LoadCL(ui, repo, clname, web=False)
1822				if err != "":
1823					ui.warn("loading CL %s: %s\n" % (clname, err))
1824					continue
1825				if not cl.copied_from:
1826					EditDesc(cl.name, closed=True, private=cl.private)
1827				cl.Delete(ui, repo)
1828
1829	if hgversion < '1.4':
1830		get = util.cachefunc(lambda r: repo[r].changeset())
1831		changeiter, matchfn = cmdutil.walkchangerevs(ui, repo, [], get, {'rev': None})
1832		n = 0
1833		for st, rev, fns in changeiter:
1834			if st != 'iter':
1835				continue
1836			n += 1
1837			if n > 100:
1838				break
1839			Rev(rev)
1840	else:
1841		matchfn = scmutil.match(repo, [], {'rev': None})
1842		def prep(ctx, fns):
1843			pass
1844		for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev': None}, prep):
1845			Rev(ctx.rev())
1846
1847	# Remove files that are not modified from the CLs in which they appear.
1848	all = LoadAllCL(ui, repo, web=False)
1849	changed = ChangedFiles(ui, repo, [], {})
1850	for _, cl in all.items():
1851		extra = Sub(cl.files, changed)
1852		if extra:
1853			ui.warn("Removing unmodified files from CL %s:\n" % (cl.name,))
1854			for f in extra:
1855				ui.warn("\t%s\n" % (f,))
1856			cl.files = Sub(cl.files, extra)
1857			cl.Flush(ui, repo)
1858		if not cl.files:
1859			if not cl.copied_from:
1860				ui.warn("CL %s has no files; delete (abandon) with hg change -d %s\n" % (cl.name, cl.name))
1861			else:
1862				ui.warn("CL %s has no files; delete locally with hg change -D %s\n" % (cl.name, cl.name))
1863	return
1864
1865def upload(ui, repo, name, **opts):
1866	"""upload diffs to the code review server
1867
1868	Uploads the current modifications for a given change to the server.
1869	"""
1870	if missing_codereview:
1871		return missing_codereview
1872
1873	repo.ui.quiet = True
1874	cl, err = LoadCL(ui, repo, name, web=True)
1875	if err != "":
1876		return err
1877	if not cl.local:
1878		return "cannot upload non-local change"
1879	cl.Upload(ui, repo)
1880	print "%s%s\n" % (server_url_base, cl.name)
1881	return
1882
1883review_opts = [
1884	('r', 'reviewer', '', 'add reviewer'),
1885	('', 'cc', '', 'add cc'),
1886	('', 'tbr', '', 'add future reviewer'),
1887	('m', 'message', '', 'change description (for new change)'),
1888]
1889
1890cmdtable = {
1891	# The ^ means to show this command in the help text that
1892	# is printed when running hg with no arguments.
1893	"^change": (
1894		change,
1895		[
1896			('d', 'delete', None, 'delete existing change list'),
1897			('D', 'deletelocal', None, 'delete locally, but do not change CL on server'),
1898			('i', 'stdin', None, 'read change list from standard input'),
1899			('o', 'stdout', None, 'print change list to standard output'),
1900			('p', 'pending', None, 'print pending summary to standard output'),
1901		],
1902		"[-d | -D] [-i] [-o] change# or FILE ..."
1903	),
1904	"^clpatch": (
1905		clpatch,
1906		[
1907			('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'),
1908			('', 'no_incoming', None, 'disable check for incoming changes'),
1909		],
1910		"change#"
1911	),
1912	# Would prefer to call this codereview-login, but then
1913	# hg help codereview prints the help for this command
1914	# instead of the help for the extension.
1915	"code-login": (
1916		code_login,
1917		[],
1918		"",
1919	),
1920	"^download": (
1921		download,
1922		[],
1923		"change#"
1924	),
1925	"^file": (
1926		file,
1927		[
1928			('d', 'delete', None, 'delete files from change list (but not repository)'),
1929		],
1930		"[-d] change# FILE ..."
1931	),
1932	"^gofmt": (
1933		gofmt,
1934		[
1935			('l', 'list', None, 'list files that would change, but do not edit them'),
1936		],
1937		"FILE ..."
1938	),
1939	"^pending|p": (
1940		pending,
1941		[],
1942		"[FILE ...]"
1943	),
1944	"^mail": (
1945		mail,
1946		review_opts + [
1947		] + commands.walkopts,
1948		"[-r reviewer] [--cc cc] [change# | file ...]"
1949	),
1950	"^release-apply": (
1951		release_apply,
1952		[
1953			('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'),
1954			('', 'no_incoming', None, 'disable check for incoming changes'),
1955		],
1956		"change#"
1957	),
1958	# TODO: release-start, release-tag, weekly-tag
1959	"^submit": (
1960		submit,
1961		review_opts + [
1962			('', 'no_incoming', None, 'disable initial incoming check (for testing)'),
1963			('n', 'dryrun', None, 'make change only locally (for testing)'),
1964		] + commands.walkopts + commands.commitopts + commands.commitopts2,
1965		"[-r reviewer] [--cc cc] [change# | file ...]"
1966	),
1967	"^sync": (
1968		sync,
1969		[
1970			('', 'local', None, 'do not pull changes from remote repository')
1971		],
1972		"[--local]",
1973	),
1974	"^undo": (
1975		undo,
1976		[
1977			('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'),
1978			('', 'no_incoming', None, 'disable check for incoming changes'),
1979		],
1980		"change#"
1981	),
1982	"^upload": (
1983		upload,
1984		[],
1985		"change#"
1986	),
1987}
1988
1989
1990#######################################################################
1991# Wrappers around upload.py for interacting with Rietveld
1992
1993# HTML form parser
1994class FormParser(HTMLParser):
1995	def __init__(self):
1996		self.map = {}
1997		self.curtag = None
1998		self.curdata = None
1999		HTMLParser.__init__(self)
2000	def handle_starttag(self, tag, attrs):
2001		if tag == "input":
2002			key = None
2003			value = ''
2004			for a in attrs:
2005				if a[0] == 'name':
2006					key = a[1]
2007				if a[0] == 'value':
2008					value = a[1]
2009			if key is not None:
2010				self.map[key] = value
2011		if tag == "textarea":
2012			key = None
2013			for a in attrs:
2014				if a[0] == 'name':
2015					key = a[1]
2016			if key is not None:
2017				self.curtag = key
2018				self.curdata = ''
2019	def handle_endtag(self, tag):
2020		if tag == "textarea" and self.curtag is not None:
2021			self.map[self.curtag] = self.curdata
2022			self.curtag = None
2023			self.curdata = None
2024	def handle_charref(self, name):
2025		self.handle_data(unichr(int(name)))
2026	def handle_entityref(self, name):
2027		import htmlentitydefs
2028		if name in htmlentitydefs.entitydefs:
2029			self.handle_data(htmlentitydefs.entitydefs[name])
2030		else:
2031			self.handle_data("&" + name + ";")
2032	def handle_data(self, data):
2033		if self.curdata is not None:
2034			self.curdata += data
2035
2036def JSONGet(ui, path):
2037	try:
2038		data = MySend(path, force_auth=False)
2039		typecheck(data, str)
2040		d = fix_json(json.loads(data))
2041	except:
2042		ui.warn("JSONGet %s: %s\n" % (path, ExceptionDetail()))
2043		return None
2044	return d
2045
2046# Clean up json parser output to match our expectations:
2047#   * all strings are UTF-8-encoded str, not unicode.
2048#   * missing fields are missing, not None,
2049#     so that d.get("foo", defaultvalue) works.
2050def fix_json(x):
2051	if type(x) in [str, int, float, bool, type(None)]:
2052		pass
2053	elif type(x) is unicode:
2054		x = x.encode("utf-8")
2055	elif type(x) is list:
2056		for i in range(len(x)):
2057			x[i] = fix_json(x[i])
2058	elif type(x) is dict:
2059		todel = []
2060		for k in x:
2061			if x[k] is None:
2062				todel.append(k)
2063			else:
2064				x[k] = fix_json(x[k])
2065		for k in todel:
2066			del x[k]
2067	else:
2068		raise util.Abort("unknown type " + str(type(x)) + " in fix_json")
2069	if type(x) is str:
2070		x = x.replace('\r\n', '\n')
2071	return x
2072
2073def IsRietveldSubmitted(ui, clname, hex):
2074	dict = JSONGet(ui, "/api/" + clname + "?messages=true")
2075	if dict is None:
2076		return False
2077	for msg in dict.get("messages", []):
2078		text = msg.get("text", "")
2079		m = re.match('\*\*\* Submitted as [^*]*?([0-9a-f]+) \*\*\*', text)
2080		if m is not None and len(m.group(1)) >= 8 and hex.startswith(m.group(1)):
2081			return True
2082	return False
2083
2084def IsRietveldMailed(cl):
2085	for msg in cl.dict.get("messages", []):
2086		if msg.get("text", "").find("I'd like you to review this change") >= 0:
2087			return True
2088	return False
2089
2090def DownloadCL(ui, repo, clname):
2091	set_status("downloading CL " + clname)
2092	cl, err = LoadCL(ui, repo, clname, web=True)
2093	if err != "":
2094		return None, None, None, "error loading CL %s: %s" % (clname, err)
2095
2096	# Find most recent diff
2097	diffs = cl.dict.get("patchsets", [])
2098	if not diffs:
2099		return None, None, None, "CL has no patch sets"
2100	patchid = diffs[-1]
2101
2102	patchset = JSONGet(ui, "/api/" + clname + "/" + str(patchid))
2103	if patchset is None:
2104		return None, None, None, "error loading CL patchset %s/%d" % (clname, patchid)
2105	if patchset.get("patchset", 0) != patchid:
2106		return None, None, None, "malformed patchset information"
2107	
2108	vers = ""
2109	msg = patchset.get("message", "").split()
2110	if len(msg) >= 3 and msg[0] == "diff" and msg[1] == "-r":
2111		vers = msg[2]
2112	diff = "/download/issue" + clname + "_" + str(patchid) + ".diff"
2113
2114	diffdata = MySend(diff, force_auth=False)
2115	
2116	# Print warning if email is not in CONTRIBUTORS file.
2117	email = cl.dict.get("owner_email", "")
2118	if not email:
2119		return None, None, None, "cannot find owner for %s" % (clname)
2120	him = FindContributor(ui, repo, email)
2121	me = FindContributor(ui, repo, None)
2122	if him == me:
2123		cl.mailed = IsRietveldMailed(cl)
2124	else:
2125		cl.copied_from = email
2126
2127	return cl, vers, diffdata, ""
2128
2129def MySend(request_path, payload=None,
2130		content_type="application/octet-stream",
2131		timeout=None, force_auth=True,
2132		**kwargs):
2133	"""Run MySend1 maybe twice, because Rietveld is unreliable."""
2134	try:
2135		return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
2136	except Exception, e:
2137		if type(e) != urllib2.HTTPError or e.code != 500:	# only retry on HTTP 500 error
2138			raise
2139		print >>sys.stderr, "Loading "+request_path+": "+ExceptionDetail()+"; trying again in 2 seconds."
2140		time.sleep(2)
2141		return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
2142
2143# Like upload.py Send but only authenticates when the
2144# redirect is to www.google.com/accounts.  This keeps
2145# unnecessary redirects from happening during testing.
2146def MySend1(request_path, payload=None,
2147				content_type="application/octet-stream",
2148				timeout=None, force_auth=True,
2149				**kwargs):
2150	"""Sends an RPC and returns the response.
2151
2152	Args:
2153		request_path: The path to send the request to, eg /api/appversion/create.
2154		payload: The body of the request, or None to send an empty request.
2155		content_type: The Content-Type header to use.
2156		timeout: timeout in seconds; default None i.e. no timeout.
2157			(Note: for large requests on OS X, the timeout doesn't work right.)
2158		kwargs: Any keyword arguments are converted into query string parameters.
2159
2160	Returns:
2161		The response body, as a string.
2162	"""
2163	# TODO: Don't require authentication.  Let the server say
2164	# whether it is necessary.
2165	global rpc
2166	if rpc == None:
2167		rpc = GetRpcServer(upload_options)
2168	self = rpc
2169	if not self.authenticated and force_auth:
2170		self._Authenticate()
2171	if request_path is None:
2172		return
2173
2174	old_timeout = socket.getdefaulttimeout()
2175	socket.setdefaulttimeout(timeout)
2176	try:
2177		tries = 0
2178		while True:
2179			tries += 1
2180			args = dict(kwargs)
2181			url = "http://%s%s" % (self.host, request_path)
2182			if args:
2183				url += "?" + urllib.urlencode(args)
2184			req = self._CreateRequest(url=url, data=payload)
2185			req.add_header("Content-Type", content_type)
2186			try:
2187				f = self.opener.open(req)
2188				response = f.read()
2189				f.close()
2190				# Translate \r\n into \n, because Rietveld doesn't.
2191				response = response.replace('\r\n', '\n')
2192				# who knows what urllib will give us
2193				if type(response) == unicode:
2194					response = response.encode("utf-8")
2195				typecheck(response, str)
2196				return response
2197			except urllib2.HTTPError, e:
2198				if tries > 3:
2199					raise
2200				elif e.code == 401:
2201					self._Authenticate()
2202				elif e.code == 302:
2203					loc = e.info()["location"]
2204					if not loc.startswith('https://www.google.com/a') or loc.find('/ServiceLogin') < 0:
2205						return ''
2206					self._Authenticate()
2207				else:
2208					raise
2209	finally:
2210		socket.setdefaulttimeout(old_timeout)
2211
2212def GetForm(url):
2213	f = FormParser()
2214	f.feed(ustr(MySend(url)))	# f.feed wants unicode
2215	f.close()
2216	# convert back to utf-8 to restore sanity
2217	m = {}
2218	for k,v in f.map.items():
2219		m[k.encode("utf-8")] = v.replace("\r\n", "\n").encode("utf-8")
2220	return m
2221
2222def EditDesc(issue, subject=None, desc=None, reviewers=None, cc=None, closed=False, private=False):
2223	set_status("uploading change to description")
2224	form_fields = GetForm("/" + issue + "/edit")
2225	if subject is not None:
2226		form_fields['subject'] = subject
2227	if desc is not None:
2228		form_fields['description'] = desc
2229	if reviewers is not None:
2230		form_fields['reviewers'] = reviewers
2231	if cc is not None:
2232		form_fields['cc'] = cc
2233	if closed:
2234		form_fields['closed'] = "checked"
2235	if private:
2236		form_fields['private'] = "checked"
2237	ctype, body = EncodeMultipartFormData(form_fields.items(), [])
2238	response = MySend("/" + issue + "/edit", body, content_type=ctype)
2239	if response != "":
2240		print >>sys.stderr, "Error editing description:\n" + "Sent form: \n", form_fields, "\n", response
2241		sys.exit(2)
2242
2243def PostMessage(ui, issue, message, reviewers=None, cc=None, send_mail=True, subject=None):
2244	set_status("uploading message")
2245	form_fields = GetForm("/" + issue + "/publish")
2246	if reviewers is not None:
2247		form_fields['reviewers'] = reviewers
2248	if cc is not None:
2249		form_fields['cc'] = cc
2250	if send_mail:
2251		form_fields['send_mail'] = "checked"
2252	else:
2253		del form_fields['send_mail']
2254	if subject is not None:
2255		form_fields['subject'] = subject
2256	form_fields['message'] = message
2257	
2258	form_fields['message_only'] = '1'	# Don't include draft comments
2259	if reviewers is not None or cc is not None:
2260		form_fields['message_only'] = ''	# Must set '' in order to override cc/reviewer
2261	ctype = "applications/x-www-form-urlencoded"
2262	body = urllib.urlencode(form_fields)
2263	response = MySend("/" + issue + "/publish", body, content_type=ctype)
2264	if response != "":
2265		print response
2266		sys.exit(2)
2267
2268class opt(object):
2269	pass
2270
2271def nocommit(*pats, **opts):
2272	"""(disabled when using this extension)"""
2273	raise util.Abort("codereview extension enabled; use mail, upload, or submit instead of commit")
2274
2275def nobackout(*pats, **opts):
2276	"""(disabled when using this extension)"""
2277	raise util.Abort("codereview extension enabled; use undo instead of backout")
2278
2279def norollback(*pats, **opts):
2280	"""(disabled when using this extension)"""
2281	raise util.Abort("codereview extension enabled; use undo instead of rollback")
2282
2283def RietveldSetup(ui, repo):
2284	global defaultcc, upload_options, rpc, server, server_url_base, force_google_account, verbosity, contributors
2285	global missing_codereview
2286
2287	repo_config_path = ''
2288	# Read repository-specific options from lib/codereview/codereview.cfg
2289	try:
2290		repo_config_path = repo.root + '/lib/codereview/codereview.cfg'
2291		f = open(repo_config_path)
2292		for line in f:
2293			if line.startswith('defaultcc: '):
2294				defaultcc = SplitCommaSpace(line[10:])
2295	except:
2296		# If there are no options, chances are good this is not
2297		# a code review repository; stop now before we foul
2298		# things up even worse.  Might also be that repo doesn't
2299		# even have a root.  See issue 959.
2300		if repo_config_path == '':
2301			missing_codereview = 'codereview disabled: repository has no root'
2302		else:
2303			missing_codereview = 'codereview disabled: cannot open ' + repo_config_path
2304		return
2305
2306	# Should only modify repository with hg submit.
2307	# Disable the built-in Mercurial commands that might
2308	# trip things up.
2309	cmdutil.commit = nocommit
2310	global real_rollback
2311	real_rollback = repo.rollback
2312	repo.rollback = norollback
2313	# would install nobackout if we could; oh well
2314
2315	try:
2316		f = open(repo.root + '/CONTRIBUTORS', 'r')
2317	except:
2318		raise util.Abort("cannot open %s: %s" % (repo.root+'/CONTRIBUTORS', ExceptionDetail()))
2319	for line in f:
2320		# CONTRIBUTORS is a list of lines like:
2321		#	Person <email>
2322		#	Person <email> <alt-email>
2323		# The first email address is the one used in commit logs.
2324		if line.startswith('#'):
2325			continue
2326		m = re.match(r"([^<>]+\S)\s+(<[^<>\s]+>)((\s+<[^<>\s]+>)*)\s*$", line)
2327		if m:
2328			name = m.group(1)
2329			email = m.group(2)[1:-1]
2330			contributors[email.lower()] = (name, email)
2331			for extra in m.group(3).split():
2332				contributors[extra[1:-1].lower()] = (name, email)
2333
2334	if not ui.verbose:
2335		verbosity = 0
2336
2337	# Config options.
2338	x = ui.config("codereview", "server")
2339	if x is not None:
2340		server = x
2341
2342	# TODO(rsc): Take from ui.username?
2343	email = None
2344	x = ui.config("codereview", "email")
2345	if x is not None:
2346		email = x
2347
2348	server_url_base = "http://" + server + "/"
2349
2350	testing = ui.config("codereview", "testing")
2351	force_google_account = ui.configbool("codereview", "force_google_account", False)
2352
2353	upload_options = opt()
2354	upload_options.email = email
2355	upload_options.host = None
2356	upload_options.verbose = 0
2357	upload_options.description = None
2358	upload_options.description_file = None
2359	upload_options.reviewers = None
2360	upload_options.cc = None
2361	upload_options.message = None
2362	upload_options.issue = None
2363	upload_options.download_base = False
2364	upload_options.revision = None
2365	upload_options.send_mail = False
2366	upload_options.vcs = None
2367	upload_options.server = server
2368	upload_options.save_cookies = True
2369
2370	if testing:
2371		upload_options.save_cookies = False
2372		upload_options.email = "test@example.com"
2373
2374	rpc = None
2375	
2376	global releaseBranch
2377	tags = repo.branchtags().keys()
2378	if 'release-branch.r100' in tags:
2379		# NOTE(rsc): This tags.sort is going to get the wrong
2380		# answer when comparing release-branch.r99 with
2381		# release-branch.r100.  If we do ten releases a year
2382		# that gives us 4 years before we have to worry about this.
2383		raise util.Abort('tags.sort needs to be fixed for release-branch.r100')
2384	tags.sort()
2385	for t in tags:
2386		if t.startswith('release-branch.'):
2387			releaseBranch = t			
2388
2389#######################################################################
2390# http://codereview.appspot.com/static/upload.py, heavily edited.
2391
2392#!/usr/bin/env python
2393#
2394# Copyright 2007 Google Inc.
2395#
2396# Licensed under the Apache License, Version 2.0 (the "License");
2397# you may not use this file except in compliance with the License.
2398# You may obtain a copy of the License at
2399#
2400#	http://www.apache.org/licenses/LICENSE-2.0
2401#
2402# Unless required by applicable law or agreed to in writing, software
2403# distributed under the License is distributed on an "AS IS" BASIS,
2404# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2405# See the License for the specific language governing permissions and
2406# limitations under the License.
2407
2408"""Tool for uploading diffs from a version control system to the codereview app.
2409
2410Usage summary: upload.py [options] [-- diff_options]
2411
2412Diff options are passed to the diff command of the underlying system.
2413
2414Supported version control systems:
2415	Git
2416	Mercurial
2417	Subversion
2418
2419It is important for Git/Mercurial users to specify a tree/node/branch to diff
2420against by using the '--rev' option.
2421"""
2422# This code is derived from appcfg.py in the App Engine SDK (open source),
2423# and from ASPN recipe #146306.
2424
2425import cookielib
2426import getpass
2427import logging
2428import mimetypes
2429import optparse
2430import os
2431import re
2432import socket
2433import subprocess
2434import sys
2435import urllib
2436import urllib2
2437import urlparse
2438
2439# The md5 module was deprecated in Python 2.5.
2440try:
2441	from hashlib import md5
2442except ImportError:
2443	from md5 import md5
2444
2445try:
2446	import readline
2447except ImportError:
2448	pass
2449
2450# The logging verbosity:
2451#  0: Errors only.
2452#  1: Status messages.
2453#  2: Info logs.
2454#  3: Debug logs.
2455verbosity = 1
2456
2457# Max size of patch or base file.
2458MAX_UPLOAD_SIZE = 900 * 1024
2459
2460# whitelist for non-binary filetypes which do not start with "text/"
2461# .mm (Objective-C) shows up as application/x-freemind on my Linux box.
2462TEXT_MIMETYPES = [
2463	'application/javascript',
2464	'application/x-javascript',
2465	'application/x-freemind'
2466]
2467
2468def GetEmail(prompt):
2469	"""Prompts the user for their email address and returns it.
2470
2471	The last used email address is saved to a file and offered up as a suggestion
2472	to the user. If the user presses enter without typing in anything the last
2473	used email address is used. If the user enters a new address, it is saved
2474	for next time we prompt.
2475
2476	"""
2477	last_email_file_name = os.path.expanduser("~/.last_codereview_email_address")
2478	last_email = ""
2479	if os.path.exists(last_email_file_name):
2480		try:
2481			last_email_file = open(last_email_file_name, "r")
2482			last_email = last_email_file.readline().strip("\n")
2483			last_email_file.close()
2484			prompt += " [%s]" % last_email
2485		except IOError, e:
2486			pass
2487	email = raw_input(prompt + ": ").strip()
2488	if email:
2489		try:
2490			last_email_file = open(last_email_file_name, "w")
2491			last_email_file.write(email)
2492			last_email_file.close()
2493		except IOError, e:
2494			pass
2495	else:
2496		email = last_email
2497	return email
2498
2499
2500def StatusUpdate(msg):
2501	"""Print a status message to stdout.
2502
2503	If 'verbosity' is greater than 0, print the message.
2504
2505	Args:
2506		msg: The string to print.
2507	"""
2508	if verbosity > 0:
2509		print msg
2510
2511
2512def ErrorExit(msg):
2513	"""Print an error message to stderr and exit."""
2514	print >>sys.stderr, msg
2515	sys.exit(1)
2516
2517
2518class ClientLoginError(urllib2.HTTPError):
2519	"""Raised to indicate there was an error authenticating with ClientLogin."""
2520
2521	def __init__(self, url, code, msg, headers, args):
2522		urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
2523		self.args = args
2524		self.reason = args["Error"]
2525
2526
2527class AbstractRpcServer(object):
2528	"""Provides a common interface for a simple RPC server."""
2529
2530	def __init__(self, host, auth_function, host_override=None, extra_headers={}, save_cookies=False):
2531		"""Creates a new HttpRpcServer.
2532
2533		Args:
2534			host: The host to send requests to.
2535			auth_function: A function that takes no arguments and returns an
2536				(email, password) tuple when called. Will be called if authentication
2537				is required.
2538			host_override: The host header to send to the server (defaults to host).
2539			extra_headers: A dict of extra headers to append to every request.
2540			save_cookies: If True, save the authentication cookies to local disk.
2541				If False, use an in-memory cookiejar instead.  Subclasses must
2542				implement this functionality.  Defaults to False.
2543		"""
2544		self.host = host
2545		self.host_override = host_override
2546		self.auth_function = auth_function
2547		self.authenticated = False
2548		self.extra_headers = extra_headers
2549		self.save_cookies = save_cookies
2550		self.opener = self._GetOpener()
2551		if self.host_override:
2552			logging.info("Server: %s; Host: %s", self.host, self.host_override)
2553		else:
2554			logging.info("Server: %s", self.host)
2555
2556	def _GetOpener(self):
2557		"""Returns an OpenerDirector for making HTTP requests.
2558
2559		Returns:
2560			A urllib2.OpenerDirector object.
2561		"""
2562		raise NotImplementedError()
2563
2564	def _CreateRequest(self, url, data=None):
2565		"""Creates a new urllib request."""
2566		logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
2567		req = urllib2.Request(url, data=data)
2568		if self.host_override:
2569			req.add_header("Host", self.host_override)
2570		for key, value in self.extra_headers.iteritems():
2571			req.add_header(key, value)
2572		return req
2573
2574	def _GetAuthToken(self, email, password):
2575		"""Uses ClientLogin to authenticate the user, returning an auth token.
2576
2577		Args:
2578			email:    The user's email address
2579			password: The user's password
2580
2581		Raises:
2582			ClientLoginError: If there was an error authenticating with ClientLogin.
2583			HTTPError: If there was some other form of HTTP error.
2584
2585		Returns:
2586			The authentication token returned by ClientLogin.
2587		"""
2588		account_type = "GOOGLE"
2589		if self.host.endswith(".google.com") and not force_google_account:
2590			# Needed for use inside Google.
2591			account_type = "HOSTED"
2592		req = self._CreateRequest(
2593				url="https://www.google.com/accounts/ClientLogin",
2594				data=urllib.urlencode({
2595						"Email": email,
2596						"Passwd": password,
2597						"service": "ah",
2598						"source": "rietveld-codereview-upload",
2599						"accountType": account_type,
2600				}),
2601		)
2602		try:
2603			response = self.opener.open(req)
2604			response_body = response.read()
2605			response_dict = dict(x.split("=") for x in response_body.split("\n") if x)
2606			return response_dict["Auth"]
2607		except urllib2.HTTPError, e:
2608			if e.code == 403:
2609				body = e.read()
2610				response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
2611				raise ClientLoginError(req.get_full_url(), e.code, e.msg, e.headers, response_dict)
2612			else:
2613				raise
2614
2615	def _GetAuthCookie(self, auth_token):
2616		"""Fetches authentication cookies for an authentication token.
2617
2618		Args:
2619			auth_token: The authentication token returned by ClientLogin.
2620
2621		Raises:
2622			HTTPError: If there was an error fetching the authentication cookies.
2623		"""
2624		# This is a dummy value to allow us to identify when we're successful.
2625		continue_location = "http://localhost/"
2626		args = {"continue": continue_location, "auth": auth_token}
2627		req = self._CreateRequest("http://%s/_ah/login?%s" % (self.host, urllib.urlencode(args)))
2628		try:
2629			response = self.opener.open(req)
2630		except urllib2.HTTPError, e:
2631			response = e
2632		if (response.code != 302 or
2633				response.info()["location"] != continue_location):
2634			raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg, response.headers, response.fp)
2635		self.authenticated = True
2636
2637	def _Authenticate(self):
2638		"""Authenticates the user.
2639
2640		The authentication process works as follows:
2641		1) We get a username and password from the user
2642		2) We use ClientLogin to obtain an AUTH token for the user
2643				(see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
2644		3) We pass the auth token to /_ah/login on the server to obtain an
2645				authentication cookie. If login was successful, it tries to redirect
2646				us to the URL we provided.
2647
2648		If we attempt to access the upload API without first obtaining an
2649		authentication cookie, it returns a 401 response (or a 302) and
2650		directs us to authenticate ourselves with ClientLogin.
2651		"""
2652		for i in range(3):
2653			credentials = self.auth_function()
2654			try:
2655				auth_token = self._GetAuthToken(credentials[0], credentials[1])
2656			except ClientLoginError, e:
2657				if e.reason == "BadAuthentication":
2658					print >>sys.stderr, "Invalid username or password."
2659					continue
2660				if e.reason == "CaptchaRequired":
2661					print >>sys.stderr, (
2662						"Please go to\n"
2663						"https://www.google.com/accounts/DisplayUnlockCaptcha\n"
2664						"and verify you are a human.  Then try again.")
2665					break
2666				if e.reason == "NotVerified":
2667					print >>sys.stderr, "Account not verified."
2668					break
2669				if e.reason == "TermsNotAgreed":
2670					print >>sys.stderr, "User has not agreed to TOS."
2671					break
2672				if e.reason == "AccountDeleted":
2673					print >>sys.stderr, "The user account has been deleted."
2674					break
2675				if e.reason == "AccountDisabled":
2676					print >>sys.stderr, "The user account has been disabled."
2677					break
2678				if e.reason == "ServiceDisabled":
2679					print >>sys.stderr, "The user's access to the service has been disabled."
2680					break
2681				if e.reason == "ServiceUnavailable":
2682					print >>sys.stderr, "The service is not available; try again later."
2683					break
2684				raise
2685			self._GetAuthCookie(auth_token)
2686			return
2687
2688	def Send(self, request_path, payload=None,
2689					content_type="application/octet-stream",
2690					timeout=None,
2691					**kwargs):
2692		"""Sends an RPC and returns the response.
2693
2694		Args:
2695			request_path: The path to send the request to, eg /api/appversion/create.
2696			payload: The body of the request, or None to send an empty request.
2697			content_type: The Content-Type header to use.
2698			timeout: timeout in seconds; default None i.e. no timeout.
2699				(Note: for large requests on OS X, the timeout doesn't work right.)
2700			kwargs: Any keyword arguments are converted into query string parameters.
2701
2702		Returns:
2703			The response body, as a string.
2704		"""
2705		# TODO: Don't require authentication.  Let the server say
2706		# whether it is necessary.
2707		if not self.authenticated:
2708			self._Authenticate()
2709
2710		old_timeout = socket.getdefaulttimeout()
2711		socket.setdefaulttimeout(timeout)
2712		try:
2713			tries = 0
2714			while True:
2715				tries += 1
2716				args = dict(kwargs)
2717				url = "http://%s%s" % (self.host, request_path)
2718				if args:
2719					url += "?" + urllib.urlencode(args)
2720				req = self._CreateRequest(url=url, data=payload)
2721				req.add_header("Content-Type", content_type)
2722				try:
2723					f = self.opener.open(req)
2724					response = f.read()
2725					f.close()
2726					return response
2727				except urllib2.HTTPError, e:
2728					if tries > 3:
2729						raise
2730					elif e.code == 401 or e.code == 302:
2731						self._Authenticate()
2732					else:
2733						raise
2734		finally:
2735			socket.setdefaulttimeout(old_timeout)
2736
2737
2738class HttpRpcServer(AbstractRpcServer):
2739	"""Provides a simplified RPC-style interface for HTTP requests."""
2740
2741	def _Authenticate(self):
2742		"""Save the cookie jar after authentication."""
2743		super(HttpRpcServer, self)._Authenticate()
2744		if self.save_cookies:
2745			StatusUpdate("Saving authentication cookies to %s" % self.cookie_file)
2746			self.cookie_jar.save()
2747
2748	def _GetOpener(self):
2749		"""Returns an OpenerDirector that supports cookies and ignores redirects.
2750
2751		Returns:
2752			A urllib2.OpenerDirector object.
2753		"""
2754		opener = urllib2.OpenerDirector()
2755		opener.add_handler(urllib2.ProxyHandler())
2756		opener.add_handler(urllib2.UnknownHandler())
2757		opener.add_handler(urllib2.HTTPHandler())
2758		opener.add_handler(urllib2.HTTPDefaultErrorHandler())
2759		opener.add_handler(urllib2.HTTPSHandler())
2760		opener.add_handler(urllib2.HTTPErrorProcessor())
2761		if self.save_cookies:
2762			self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies_" + server)
2763			self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
2764			if os.path.exists(self.cookie_file):
2765				try:
2766					self.cookie_jar.load()
2767					self.authenticated = True
2768					StatusUpdate("Loaded authentication cookies from %s" % self.cookie_file)
2769				except (cookielib.LoadError, IOError):
2770					# Failed to load cookies - just ignore them.
2771					pass
2772			else:
2773				# Create an empty cookie file with mode 600
2774				fd = os.open(self.cookie_file, os.O_CREAT, 0600)
2775				os.close(fd)
2776			# Always chmod the cookie file
2777			os.chmod(self.cookie_file, 0600)
2778		else:
2779			# Don't save cookies across runs of update.py.
2780			self.cookie_jar = cookielib.CookieJar()
2781		opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
2782		return opener
2783
2784
2785def GetRpcServer(options):
2786	"""Returns an instance of an AbstractRpcServer.
2787
2788	Returns:
2789		A new AbstractRpcServer, on which RPC calls can be made.
2790	"""
2791
2792	rpc_server_class = HttpRpcServer
2793
2794	def GetUserCredentials():
2795		"""Prompts the user for a username and password."""
2796		# Disable status prints so they don't obscure the password prompt.
2797		global global_status
2798		st = global_status
2799		global_status = None
2800
2801		email = options.email
2802		if email is None:
2803			email = GetEmail("Email (login for uploading to %s)" % options.server)
2804		password = getpass.getpass("Password for %s: " % email)
2805
2806		# Put status back.
2807		global_status = st
2808		return (email, password)
2809
2810	# If this is the dev_appserver, use fake authentication.
2811	host = (options.host or options.server).lower()
2812	if host == "localhost" or host.startswith("localhost:"):
2813		email = options.email
2814		if email is None:
2815			email = "test@example.com"
2816			logging.info("Using debug user %s.  Override with --email" % email)
2817		server = rpc_server_class(
2818				options.server,
2819				lambda: (email, "password"),
2820				host_override=options.host,
2821				extra_headers={"Cookie": 'dev_appserver_login="%s:False"' % email},
2822				save_cookies=options.save_cookies)
2823		# Don't try to talk to ClientLogin.
2824		server.authenticated = True
2825		return server
2826
2827	return rpc_server_class(options.server, GetUserCredentials,
2828		host_override=options.host, save_cookies=options.save_cookies)
2829
2830
2831def EncodeMultipartFormData(fields, files):
2832	"""Encode form fields for multipart/form-data.
2833
2834	Args:
2835		fields: A sequence of (name, value) elements for regular form fields.
2836		files: A sequence of (name, filename, value) elements for data to be
2837					uploaded as files.
2838	Returns:
2839		(content_type, body) ready for httplib.HTTP instance.
2840
2841	Source:
2842		http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
2843	"""
2844	BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
2845	CRLF = '\r\n'
2846	lines = []
2847	for (key, value) in fields:
2848		typecheck(key, str)
2849		typecheck(value, str)
2850		lines.append('--' + BOUNDARY)
2851		lines.append('Content-Disposition: form-data; name="%s"' % key)
2852		lines.append('')
2853		lines.append(value)
2854	for (key, filename, value) in files:
2855		typecheck(key, str)
2856		typecheck(filename, str)
2857		typecheck(value, str)
2858		lines.append('--' + BOUNDARY)
2859		lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
2860		lines.append('Content-Type: %s' % GetContentType(filename))
2861		lines.append('')
2862		lines.append(value)
2863	lines.append('--' + BOUNDARY + '--')
2864	lines.append('')
2865	body = CRLF.join(lines)
2866	content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
2867	return content_type, body
2868
2869
2870def GetContentType(filename):
2871	"""Helper to guess the content-type from the filename."""
2872	return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
2873
2874
2875# Use a shell for subcommands on Windows to get a PATH search.
2876use_shell = sys.platform.startswith("win")
2877
2878def RunShellWithReturnCode(command, print_output=False,
2879		universal_newlines=True, env=os.environ):
2880	"""Executes a command and returns the output from stdout and the return code.
2881
2882	Args:
2883		command: Command to execute.
2884		print_output: If True, the output is printed to stdout.
2885			If False, both stdout and stderr are ignored.
2886		universal_newlines: Use universal_newlines flag (default: True).
2887
2888	Returns:
2889		Tuple (output, return code)
2890	"""
2891	logging.info("Running %s", command)
2892	p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2893		shell=use_shell, universal_newlines=universal_newlines, env=env)
2894	if print_output:
2895		output_array = []
2896		while True:
2897			line = p.stdout.readline()
2898			if not line:
2899				break
2900			print line.strip("\n")
2901			output_array.append(line)
2902		output = "".join(output_array)
2903	else:
2904		output = p.stdout.read()
2905	p.wait()
2906	errout = p.stderr.read()
2907	if print_output and errout:
2908		print >>sys.stderr, errout
2909	p.stdout.close()
2910	p.stderr.close()
2911	return output, p.returncode
2912
2913
2914def RunShell(command, silent_ok=False, universal_newlines=True,
2915		print_output=False, env=os.environ):
2916	data, retcode = RunShellWithReturnCode(command, print_output, universal_newlines, env)
2917	if retcode:
2918		ErrorExit("Got error status from %s:\n%s" % (command, data))
2919	if not silent_ok and not data:
2920		ErrorExit("No output from %s" % command)
2921	return data
2922
2923
2924class VersionControlSystem(object):
2925	"""Abstract base class providing an interface to the VCS."""
2926
2927	def __init__(self, options):
2928		"""Constructor.
2929
2930		Args:
2931			options: Command line options.
2932		"""
2933		self.options = options
2934
2935	def GenerateDiff(self, args):
2936		"""Return the current diff as a string.
2937
2938		Args:
2939			args: Extra arguments to pass to the diff command.
2940		"""
2941		raise NotImplementedError(
2942				"abstract method -- subclass %s must override" % self.__class__)
2943
2944	def GetUnknownFiles(self):
2945		"""Return a list of files unknown to the VCS."""
2946		raise NotImplementedError(
2947				"abstract method -- subclass %s must override" % self.__class__)
2948
2949	def CheckForUnknownFiles(self):
2950		"""Show an "are you sure?" prompt if there are unknown files."""
2951		unknown_files = self.GetUnknownFiles()
2952		if unknown_files:
2953			print "The following files are not added to version control:"
2954			for line in unknown_files:
2955				print line
2956			prompt = "Are you sure to continue?(y/N) "
2957			answer = raw_input(prompt).strip()
2958			if answer != "y":
2959				ErrorExit("User aborted")
2960
2961	def GetBaseFile(self, filename):
2962		"""Get the content of the upstream version of a file.
2963
2964		Returns:
2965			A tuple (base_content, new_content, is_binary, status)
2966				base_content: The contents of the base file.
2967				new_content: For text files, this is empty.  For binary files, this is
2968					the contents of the new file, since the diff output won't contain
2969					information to reconstruct the current file.
2970				is_binary: True iff the file is binary.
2971				status: The status of the file.
2972		"""
2973
2974		raise NotImplementedError(
2975				"abstract method -- subclass %s must override" % self.__class__)
2976
2977
2978	def GetBaseFiles(self, diff):
2979		"""Helper that calls GetBase file for each file in the patch.
2980
2981		Returns:
2982			A dictionary that maps from filename to GetBaseFile's tuple.  Filenames
2983			are retrieved based on lines that start with "Index:" or
2984			"Property changes on:".
2985		"""
2986		files = {}
2987		for line in diff.splitlines(True):
2988			if line.startswith('Index:') or line.startswith('Property changes on:'):
2989				unused, filename = line.split(':', 1)
2990				# On Windows if a file has property changes its filename uses '\'
2991				# instead of '/'.
2992				filename = filename.strip().replace('\\', '/')
2993				files[filename] = self.GetBaseFile(filename)
2994		return files
2995
2996
2997	def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options,
2998											files):
2999		"""Uploads the base files (and if necessary, the current ones as well)."""
3000
3001		def UploadFile(filename, file_id, content, is_binary, status, is_base):
3002			"""Uploads a file to the server."""
3003			set_status("uploading " + filename)
3004			file_too_large = False
3005			if is_base:
3006				type = "base"
3007			else:
3008				type = "current"
3009			if len(content) > MAX_UPLOAD_SIZE:
3010				print ("Not uploading the %s file for %s because it's too large." %
3011							(type, filename))
3012				file_too_large = True
3013				content = ""
3014			checksum = md5(content).hexdigest()
3015			if options.verbose > 0 and not file_too_large:
3016				print "Uploading %s file for %s" % (type, filename)
3017			url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id)
3018			form_fields = [
3019				("filename", filename),
3020				("status", status),
3021				("checksum", checksum),
3022				("is_binary", str(is_binary)),
3023				("is_current", str(not is_base)),
3024			]
3025			if file_too_large:
3026				form_fields.append(("file_too_large", "1"))
3027			if options.email:
3028				form_fields.append(("user", options.email))
3029			ctype, body = EncodeMultipartFormData(form_fields, [("data", filename, content)])
3030			response_body = rpc_server.Send(url, body, content_type=ctype)
3031			if not response_body.startswith("OK"):
3032				StatusUpdate("  --> %s" % response_body)
3033				sys.exit(1)
3034
3035		# Don't want to spawn too many threads, nor do we want to
3036		# hit Rietveld too hard, or it will start serving 500 errors.
3037		# When 8 works, it's no better than 4, and sometimes 8 is
3038		# too many for Rietveld to handle.
3039		MAX_PARALLEL_UPLOADS = 4
3040
3041		sema = threading.BoundedSemaphore(MAX_PARALLEL_UPLOADS)
3042		upload_threads = []
3043		finished_upload_threads = []
3044		
3045		class UploadFileThread(threading.Thread):
3046			def __init__(self, args):
3047				threading.Thread.__init__(self)
3048				self.args = args
3049			def run(self):
3050				UploadFile(*self.args)
3051				finished_upload_threads.append(self)
3052				sema.release()
3053
3054		def StartUploadFile(*args):
3055			sema.acquire()
3056			while len(finished_upload_threads) > 0:
3057				t = finished_upload_threads.pop()
3058				upload_threads.remove(t)
3059				t.join()
3060			t = UploadFileThread(args)
3061			upload_threads.append(t)
3062			t.start()
3063
3064		def WaitForUploads():			
3065			for t in upload_threads:
3066				t.join()
3067
3068		patches = dict()
3069		[patches.setdefault(v, k) for k, v in patch_list]
3070		for filename in patches.keys():
3071			base_content, new_content, is_binary, status = files[filename]
3072			file_id_str = patches.get(filename)
3073			if file_id_str.find("nobase") != -1:
3074				base_content = None
3075				file_id_str = file_id_str[file_id_str.rfind("_") + 1:]
3076			file_id = int(file_id_str)
3077			if base_content != None:
3078				StartUploadFile(filename, file_id, base_content, is_binary, status, True)
3079			if new_content != None:
3080				StartUploadFile(filename, file_id, new_content, is_binary, status, False)
3081		WaitForUploads()
3082
3083	def IsImage(self, filename):
3084		"""Returns true if the filename has an image extension."""
3085		mimetype =  mimetypes.guess_type(filename)[0]
3086		if not mimetype:
3087			return False
3088		return mimetype.startswith("image/")
3089
3090	def IsBinary(self, filename):
3091		"""Returns true if the guessed mimetyped isnt't in text group."""
3092		mimetype = mimetypes.guess_type(filename)[0]
3093		if not mimetype:
3094			return False  # e.g. README, "real" binaries usually have an extension
3095		# special case for text files which don't start with text/
3096		if mimetype in TEXT_MIMETYPES:
3097			return False
3098		return not mimetype.startswith("text/")
3099
3100class FakeMercurialUI(object):
3101	def __init__(self):
3102		self.quiet = True
3103		self.output = ''
3104	
3105	def write(self, *args, **opts):
3106		self.output += ' '.join(args)
3107
3108use_hg_shell = False	# set to True to shell out to hg always; slower
3109
3110class MercurialVCS(VersionControlSystem):
3111	"""Implementation of the VersionControlSystem interface for Mercurial."""
3112
3113	def __init__(self, options, ui, repo):
3114		super(MercurialVCS, self).__init__(options)
3115		self.ui = ui
3116		self.repo = repo
3117		# Absolute path to repository (we can be in a subdir)
3118		self.repo_dir = os.path.normpath(repo.root)
3119		# Compute the subdir
3120		cwd = os.path.normpath(os.getcwd())
3121		assert cwd.startswith(self.repo_dir)
3122		self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/")
3123		if self.options.revision:
3124			self.base_rev = self.options.revision
3125		else:
3126			mqparent, err = RunShellWithReturnCode(['hg', 'log', '--rev', 'qparent', '--template={node}'])
3127			if not err and mqparent != "":
3128				self.base_rev = mqparent
3129			else:
3130				self.base_rev = RunShell(["hg", "parents", "-q"]).split(':')[1].strip()
3131	def _GetRelPath(self, filename):
3132		"""Get relative path of a file according to the current directory,
3133		given its logical path in the repo."""
3134		assert filename.startswith(self.subdir), (filename, self.subdir)
3135		return filename[len(self.subdir):].lstrip(r"\/")
3136
3137	def GenerateDiff(self, extra_args):
3138		# If no file specified, restrict to the current subdir
3139		extra_args = extra_args or ["."]
3140		cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args
3141		data = RunShell(cmd, silent_ok=True)
3142		svndiff = []
3143		filecount = 0
3144		for line in data.splitlines():
3145			m = re.match("diff --git a/(\S+) b/(\S+)", line)
3146			if m:
3147				# Modify line to make it look like as it comes from svn diff.
3148				# With this modification no changes on the server side are required
3149				# to make upload.py work with Mercurial repos.
3150				# NOTE: for proper handling of moved/copied files, we have to use
3151				# the second filename.
3152				filename = m.group(2)
3153				svndiff.append("Index: %s" % filename)
3154				svndiff.append("=" * 67)
3155				filecount += 1
3156				logging.info(line)
3157			else:
3158				svndiff.append(line)
3159		if not filecount:
3160			ErrorExit("No valid patches found in output from hg diff")
3161		return "\n".join(svndiff) + "\n"
3162
3163	def GetUnknownFiles(self):
3164		"""Return a list of files unknown to the VCS."""
3165		args = []
3166		status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."],
3167				silent_ok=True)
3168		unknown_files = []
3169		for line in status.splitlines():
3170			st, fn = line.split(" ", 1)
3171			if st == "?":
3172				unknown_files.append(fn)
3173		return unknown_files
3174
3175	def GetBaseFile(self, filename):
3176		set_status("inspecting " + filename)
3177		# "hg status" and "hg cat" both take a path relative to the current subdir
3178		# rather than to the repo root, but "hg diff" has given us the full path
3179		# to the repo root.
3180		base_content = ""
3181		new_content = None
3182		is_binary = False
3183		oldrelpath = relpath = self._GetRelPath(filename)
3184		# "hg status -C" returns two lines for moved/copied files, one otherwise
3185		if use_hg_shell:
3186			out = RunShell(["hg", "status", "-C", "--rev", self.base_rev, relpath])
3187		else:
3188			fui = FakeMercurialUI()
3189			ret = commands.status(fui, self.repo, *[relpath], **{'rev': [self.base_rev], 'copies': True})
3190			if ret:
3191				raise util.Abort(ret)
3192			out = fui.output
3193		out = out.splitlines()
3194		# HACK: strip error message about missing file/directory if it isn't in
3195		# the working copy
3196		if out[0].startswith('%s: ' % relpath):
3197			out = out[1:]
3198		status, what = out[0].split(' ', 1)
3199		if len(out) > 1 and status == "A" and what == relpath:
3200			oldrelpath = out[1].strip()
3201			status = "M"
3202		if ":" in self.base_rev:
3203			base_rev = self.base_rev.split(":", 1)[0]
3204		else:
3205			base_rev = self.base_rev
3206		if status != "A":
3207			if use_hg_shell:
3208				base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath], silent_ok=True)
3209			else:
3210				base_content = str(self.repo[base_rev][oldrelpath].data())
3211			is_binary = "\0" in base_content  # Mercurial's heuristic
3212		if status != "R":
3213			new_content = open(relpath, "rb").read()
3214			is_binary = is_binary or "\0" in new_content
3215		if is_binary and base_content and use_hg_shell:
3216			# Fetch again without converting newlines
3217			base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath],
3218				silent_ok=True, universal_newlines=False)
3219		if not is_binary or not self.IsImage(relpath):
3220			new_content = None
3221		return base_content, new_content, is_binary, status
3222
3223
3224# NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync.
3225def SplitPatch(data):
3226	"""Splits a patch into separate pieces for each file.
3227
3228	Args:
3229		data: A string containing the output of svn diff.
3230
3231	Returns:
3232		A list of 2-tuple (filename, text) where text is the svn diff output
3233			pertaining to filename.
3234	"""
3235	patches = []
3236	filename = None
3237	diff = []
3238	for line in data.splitlines(True):
3239		new_filename = None
3240		if line.startswith('Index:'):
3241			unused, new_filename = line.split(':', 1)
3242			new_filename = new_filename.strip()
3243		elif line.startswith('Property changes on:'):
3244			unused, temp_filename = line.split(':', 1)
3245			# When a file is modified, paths use '/' between directories, however
3246			# when a property is modified '\' is used on Windows.  Make them the same
3247			# otherwise the file shows up twice.
3248			temp_filename = temp_filename.strip().replace('\\', '/')
3249			if temp_filename != filename:
3250				# File has property changes but no modifications, create a new diff.
3251				new_filename = temp_filename
3252		if new_filename:
3253			if filename and diff:
3254				patches.append((filename, ''.join(diff)))
3255			filename = new_filename
3256			diff = [line]
3257			continue
3258		if diff is not None:
3259			diff.append(line)
3260	if filename and diff:
3261		patches.append((filename, ''.join(diff)))
3262	return patches
3263
3264
3265def UploadSeparatePatches(issue, rpc_server, patchset, data, options):
3266	"""Uploads a separate patch for each file in the diff output.
3267
3268	Returns a list of [patch_key, filename] for each file.
3269	"""
3270	patches = SplitPatch(data)
3271	rv = []
3272	for patch in patches:
3273		set_status("uploading patch for " + patch[0])
3274		if len(patch[1]) > MAX_UPLOAD_SIZE:
3275			print ("Not uploading the patch for " + patch[0] +
3276				" because the file is too large.")
3277			continue
3278		form_fields = [("filename", patch[0])]
3279		if not options.download_base:
3280			form_fields.append(("content_upload", "1"))
3281		files = [("data", "data.diff", patch[1])]
3282		ctype, body = EncodeMultipartFormData(form_fields, files)
3283		url = "/%d/upload_patch/%d" % (int(issue), int(patchset))
3284		print "Uploading patch for " + patch[0]
3285		response_body = rpc_server.Send(url, body, content_type=ctype)
3286		lines = response_body.splitlines()
3287		if not lines or lines[0] != "OK":
3288			StatusUpdate("  --> %s" % response_body)
3289			sys.exit(1)
3290		rv.append([lines[1], patch[0]])
3291	return rv