/lib/codereview/codereview.py
Python | 3291 lines | 3134 code | 45 blank | 112 comment | 128 complexity | cb8a7871ced764bda93ec1e72f7c9f48 MD5 | raw file
Possible License(s): BSD-3-Clause
1# coding=utf-8 2# (The line above is necessary so that I can use 世界 in the 3# *comment* below without Python getting all bent out of shape.) 4 5# Copyright 2007-2009 Google Inc. 6# 7# Licensed under the Apache License, Version 2.0 (the "License"); 8# you may not use this file except in compliance with the License. 9# You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, software 14# distributed under the License is distributed on an "AS IS" BASIS, 15# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16# See the License for the specific language governing permissions and 17# limitations under the License. 18 19'''Mercurial interface to codereview.appspot.com. 20 21To configure, set the following options in 22your repository's .hg/hgrc file. 23 24 [extensions] 25 codereview = path/to/codereview.py 26 27 [codereview] 28 server = codereview.appspot.com 29 30The server should be running Rietveld; see http://code.google.com/p/rietveld/. 31 32In addition to the new commands, this extension introduces 33the file pattern syntax @nnnnnn, where nnnnnn is a change list 34number, to mean the files included in that change list, which 35must be associated with the current client. 36 37For example, if change 123456 contains the files x.go and y.go, 38"hg diff @123456" is equivalent to"hg diff x.go y.go". 39''' 40 41from mercurial import cmdutil, commands, hg, util, error, match 42from mercurial.node import nullrev, hex, nullid, short 43import os, re, time 44import stat 45import subprocess 46import threading 47from HTMLParser import HTMLParser 48 49# The standard 'json' package is new in Python 2.6. 50# Before that it was an external package named simplejson. 51try: 52 # Standard location in 2.6 and beyond. 53 import json 54except Exception, e: 55 try: 56 # Conventional name for earlier package. 57 import simplejson as json 58 except: 59 try: 60 # Was also bundled with django, which is commonly installed. 61 from django.utils import simplejson as json 62 except: 63 # We give up. 64 raise e 65 66try: 67 hgversion = util.version() 68except: 69 from mercurial.version import version as v 70 hgversion = v.get_version() 71 72try: 73 from mercurial.discovery import findcommonincoming 74except: 75 def findcommonincoming(repo, remote): 76 return repo.findcommonincoming(remote) 77 78# in Mercurial 1.9 the cmdutil.match and cmdutil.revpair moved to scmutil 79if hgversion >= '1.9': 80 from mercurial import scmutil 81else: 82 scmutil = cmdutil 83 84oldMessage = """ 85The code review extension requires Mercurial 1.3 or newer. 86 87To install a new Mercurial, 88 89 sudo easy_install mercurial 90 91works on most systems. 92""" 93 94linuxMessage = """ 95You may need to clear your current Mercurial installation by running: 96 97 sudo apt-get remove mercurial mercurial-common 98 sudo rm -rf /etc/mercurial 99""" 100 101if hgversion < '1.3': 102 msg = oldMessage 103 if os.access("/etc/mercurial", 0): 104 msg += linuxMessage 105 raise util.Abort(msg) 106 107def promptyesno(ui, msg): 108 # Arguments to ui.prompt changed between 1.3 and 1.3.1. 109 # Even so, some 1.3.1 distributions seem to have the old prompt!?!? 110 # What a terrible way to maintain software. 111 try: 112 return ui.promptchoice(msg, ["&yes", "&no"], 0) == 0 113 except AttributeError: 114 return ui.prompt(msg, ["&yes", "&no"], "y") != "n" 115 116# To experiment with Mercurial in the python interpreter: 117# >>> repo = hg.repository(ui.ui(), path = ".") 118 119####################################################################### 120# Normally I would split this into multiple files, but it simplifies 121# import path headaches to keep it all in one file. Sorry. 122 123import sys 124if __name__ == "__main__": 125 print >>sys.stderr, "This is a Mercurial extension and should not be invoked directly." 126 sys.exit(2) 127 128server = "codereview.appspot.com" 129server_url_base = None 130defaultcc = None 131contributors = {} 132missing_codereview = None 133real_rollback = None 134releaseBranch = None 135 136####################################################################### 137# RE: UNICODE STRING HANDLING 138# 139# Python distinguishes between the str (string of bytes) 140# and unicode (string of code points) types. Most operations 141# work on either one just fine, but some (like regexp matching) 142# require unicode, and others (like write) require str. 143# 144# As befits the language, Python hides the distinction between 145# unicode and str by converting between them silently, but 146# *only* if all the bytes/code points involved are 7-bit ASCII. 147# This means that if you're not careful, your program works 148# fine on "hello, world" and fails on "hello, 世界". And of course, 149# the obvious way to be careful - use static types - is unavailable. 150# So the only way is trial and error to find where to put explicit 151# conversions. 152# 153# Because more functions do implicit conversion to str (string of bytes) 154# than do implicit conversion to unicode (string of code points), 155# the convention in this module is to represent all text as str, 156# converting to unicode only when calling a unicode-only function 157# and then converting back to str as soon as possible. 158 159def typecheck(s, t): 160 if type(s) != t: 161 raise util.Abort("type check failed: %s has type %s != %s" % (repr(s), type(s), t)) 162 163# If we have to pass unicode instead of str, ustr does that conversion clearly. 164def ustr(s): 165 typecheck(s, str) 166 return s.decode("utf-8") 167 168# Even with those, Mercurial still sometimes turns unicode into str 169# and then tries to use it as ascii. Change Mercurial's default. 170def set_mercurial_encoding_to_utf8(): 171 from mercurial import encoding 172 encoding.encoding = 'utf-8' 173 174set_mercurial_encoding_to_utf8() 175 176# Even with those we still run into problems. 177# I tried to do things by the book but could not convince 178# Mercurial to let me check in a change with UTF-8 in the 179# CL description or author field, no matter how many conversions 180# between str and unicode I inserted and despite changing the 181# default encoding. I'm tired of this game, so set the default 182# encoding for all of Python to 'utf-8', not 'ascii'. 183def default_to_utf8(): 184 import sys 185 reload(sys) # site.py deleted setdefaultencoding; get it back 186 sys.setdefaultencoding('utf-8') 187 188default_to_utf8() 189 190####################################################################### 191# Change list parsing. 192# 193# Change lists are stored in .hg/codereview/cl.nnnnnn 194# where nnnnnn is the number assigned by the code review server. 195# Most data about a change list is stored on the code review server 196# too: the description, reviewer, and cc list are all stored there. 197# The only thing in the cl.nnnnnn file is the list of relevant files. 198# Also, the existence of the cl.nnnnnn file marks this repository 199# as the one where the change list lives. 200 201emptydiff = """Index: ~rietveld~placeholder~ 202=================================================================== 203diff --git a/~rietveld~placeholder~ b/~rietveld~placeholder~ 204new file mode 100644 205""" 206 207class CL(object): 208 def __init__(self, name): 209 typecheck(name, str) 210 self.name = name 211 self.desc = '' 212 self.files = [] 213 self.reviewer = [] 214 self.cc = [] 215 self.url = '' 216 self.local = False 217 self.web = False 218 self.copied_from = None # None means current user 219 self.mailed = False 220 self.private = False 221 222 def DiskText(self): 223 cl = self 224 s = "" 225 if cl.copied_from: 226 s += "Author: " + cl.copied_from + "\n\n" 227 if cl.private: 228 s += "Private: " + str(self.private) + "\n" 229 s += "Mailed: " + str(self.mailed) + "\n" 230 s += "Description:\n" 231 s += Indent(cl.desc, "\t") 232 s += "Files:\n" 233 for f in cl.files: 234 s += "\t" + f + "\n" 235 typecheck(s, str) 236 return s 237 238 def EditorText(self): 239 cl = self 240 s = _change_prolog 241 s += "\n" 242 if cl.copied_from: 243 s += "Author: " + cl.copied_from + "\n" 244 if cl.url != '': 245 s += 'URL: ' + cl.url + ' # cannot edit\n\n' 246 if cl.private: 247 s += "Private: True\n" 248 s += "Reviewer: " + JoinComma(cl.reviewer) + "\n" 249 s += "CC: " + JoinComma(cl.cc) + "\n" 250 s += "\n" 251 s += "Description:\n" 252 if cl.desc == '': 253 s += "\t<enter description here>\n" 254 else: 255 s += Indent(cl.desc, "\t") 256 s += "\n" 257 if cl.local or cl.name == "new": 258 s += "Files:\n" 259 for f in cl.files: 260 s += "\t" + f + "\n" 261 s += "\n" 262 typecheck(s, str) 263 return s 264 265 def PendingText(self): 266 cl = self 267 s = cl.name + ":" + "\n" 268 s += Indent(cl.desc, "\t") 269 s += "\n" 270 if cl.copied_from: 271 s += "\tAuthor: " + cl.copied_from + "\n" 272 s += "\tReviewer: " + JoinComma(cl.reviewer) + "\n" 273 s += "\tCC: " + JoinComma(cl.cc) + "\n" 274 s += "\tFiles:\n" 275 for f in cl.files: 276 s += "\t\t" + f + "\n" 277 typecheck(s, str) 278 return s 279 280 def Flush(self, ui, repo): 281 if self.name == "new": 282 self.Upload(ui, repo, gofmt_just_warn=True, creating=True) 283 dir = CodeReviewDir(ui, repo) 284 path = dir + '/cl.' + self.name 285 f = open(path+'!', "w") 286 f.write(self.DiskText()) 287 f.close() 288 if sys.platform == "win32" and os.path.isfile(path): 289 os.remove(path) 290 os.rename(path+'!', path) 291 if self.web and not self.copied_from: 292 EditDesc(self.name, desc=self.desc, 293 reviewers=JoinComma(self.reviewer), cc=JoinComma(self.cc), 294 private=self.private) 295 296 def Delete(self, ui, repo): 297 dir = CodeReviewDir(ui, repo) 298 os.unlink(dir + "/cl." + self.name) 299 300 def Subject(self): 301 s = line1(self.desc) 302 if len(s) > 60: 303 s = s[0:55] + "..." 304 if self.name != "new": 305 s = "code review %s: %s" % (self.name, s) 306 typecheck(s, str) 307 return s 308 309 def Upload(self, ui, repo, send_mail=False, gofmt=True, gofmt_just_warn=False, creating=False, quiet=False): 310 if not self.files and not creating: 311 ui.warn("no files in change list\n") 312 if ui.configbool("codereview", "force_gofmt", True) and gofmt: 313 CheckFormat(ui, repo, self.files, just_warn=gofmt_just_warn) 314 set_status("uploading CL metadata + diffs") 315 os.chdir(repo.root) 316 form_fields = [ 317 ("content_upload", "1"), 318 ("reviewers", JoinComma(self.reviewer)), 319 ("cc", JoinComma(self.cc)), 320 ("description", self.desc), 321 ("base_hashes", ""), 322 ] 323 324 if self.name != "new": 325 form_fields.append(("issue", self.name)) 326 vcs = None 327 # We do not include files when creating the issue, 328 # because we want the patch sets to record the repository 329 # and base revision they are diffs against. We use the patch 330 # set message for that purpose, but there is no message with 331 # the first patch set. Instead the message gets used as the 332 # new CL's overall subject. So omit the diffs when creating 333 # and then we'll run an immediate upload. 334 # This has the effect that every CL begins with an empty "Patch set 1". 335 if self.files and not creating: 336 vcs = MercurialVCS(upload_options, ui, repo) 337 data = vcs.GenerateDiff(self.files) 338 files = vcs.GetBaseFiles(data) 339 if len(data) > MAX_UPLOAD_SIZE: 340 uploaded_diff_file = [] 341 form_fields.append(("separate_patches", "1")) 342 else: 343 uploaded_diff_file = [("data", "data.diff", data)] 344 else: 345 uploaded_diff_file = [("data", "data.diff", emptydiff)] 346 347 if vcs and self.name != "new": 348 form_fields.append(("subject", "diff -r " + vcs.base_rev + " " + getremote(ui, repo, {}).path)) 349 else: 350 # First upload sets the subject for the CL itself. 351 form_fields.append(("subject", self.Subject())) 352 ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file) 353 response_body = MySend("/upload", body, content_type=ctype) 354 patchset = None 355 msg = response_body 356 lines = msg.splitlines() 357 if len(lines) >= 2: 358 msg = lines[0] 359 patchset = lines[1].strip() 360 patches = [x.split(" ", 1) for x in lines[2:]] 361 if response_body.startswith("Issue updated.") and quiet: 362 pass 363 else: 364 ui.status(msg + "\n") 365 set_status("uploaded CL metadata + diffs") 366 if not response_body.startswith("Issue created.") and not response_body.startswith("Issue updated."): 367 raise util.Abort("failed to update issue: " + response_body) 368 issue = msg[msg.rfind("/")+1:] 369 self.name = issue 370 if not self.url: 371 self.url = server_url_base + self.name 372 if not uploaded_diff_file: 373 set_status("uploading patches") 374 patches = UploadSeparatePatches(issue, rpc, patchset, data, upload_options) 375 if vcs: 376 set_status("uploading base files") 377 vcs.UploadBaseFiles(issue, rpc, patches, patchset, upload_options, files) 378 if send_mail: 379 set_status("sending mail") 380 MySend("/" + issue + "/mail", payload="") 381 self.web = True 382 set_status("flushing changes to disk") 383 self.Flush(ui, repo) 384 return 385 386 def Mail(self, ui, repo): 387 pmsg = "Hello " + JoinComma(self.reviewer) 388 if self.cc: 389 pmsg += " (cc: %s)" % (', '.join(self.cc),) 390 pmsg += ",\n" 391 pmsg += "\n" 392 repourl = getremote(ui, repo, {}).path 393 if not self.mailed: 394 pmsg += "I'd like you to review this change to\n" + repourl + "\n" 395 else: 396 pmsg += "Please take another look.\n" 397 typecheck(pmsg, str) 398 PostMessage(ui, self.name, pmsg, subject=self.Subject()) 399 self.mailed = True 400 self.Flush(ui, repo) 401 402def GoodCLName(name): 403 typecheck(name, str) 404 return re.match("^[0-9]+$", name) 405 406def ParseCL(text, name): 407 typecheck(text, str) 408 typecheck(name, str) 409 sname = None 410 lineno = 0 411 sections = { 412 'Author': '', 413 'Description': '', 414 'Files': '', 415 'URL': '', 416 'Reviewer': '', 417 'CC': '', 418 'Mailed': '', 419 'Private': '', 420 } 421 for line in text.split('\n'): 422 lineno += 1 423 line = line.rstrip() 424 if line != '' and line[0] == '#': 425 continue 426 if line == '' or line[0] == ' ' or line[0] == '\t': 427 if sname == None and line != '': 428 return None, lineno, 'text outside section' 429 if sname != None: 430 sections[sname] += line + '\n' 431 continue 432 p = line.find(':') 433 if p >= 0: 434 s, val = line[:p].strip(), line[p+1:].strip() 435 if s in sections: 436 sname = s 437 if val != '': 438 sections[sname] += val + '\n' 439 continue 440 return None, lineno, 'malformed section header' 441 442 for k in sections: 443 sections[k] = StripCommon(sections[k]).rstrip() 444 445 cl = CL(name) 446 if sections['Author']: 447 cl.copied_from = sections['Author'] 448 cl.desc = sections['Description'] 449 for line in sections['Files'].split('\n'): 450 i = line.find('#') 451 if i >= 0: 452 line = line[0:i].rstrip() 453 line = line.strip() 454 if line == '': 455 continue 456 cl.files.append(line) 457 cl.reviewer = SplitCommaSpace(sections['Reviewer']) 458 cl.cc = SplitCommaSpace(sections['CC']) 459 cl.url = sections['URL'] 460 if sections['Mailed'] != 'False': 461 # Odd default, but avoids spurious mailings when 462 # reading old CLs that do not have a Mailed: line. 463 # CLs created with this update will always have 464 # Mailed: False on disk. 465 cl.mailed = True 466 if sections['Private'] in ('True', 'true', 'Yes', 'yes'): 467 cl.private = True 468 if cl.desc == '<enter description here>': 469 cl.desc = '' 470 return cl, 0, '' 471 472def SplitCommaSpace(s): 473 typecheck(s, str) 474 s = s.strip() 475 if s == "": 476 return [] 477 return re.split(", *", s) 478 479def CutDomain(s): 480 typecheck(s, str) 481 i = s.find('@') 482 if i >= 0: 483 s = s[0:i] 484 return s 485 486def JoinComma(l): 487 for s in l: 488 typecheck(s, str) 489 return ", ".join(l) 490 491def ExceptionDetail(): 492 s = str(sys.exc_info()[0]) 493 if s.startswith("<type '") and s.endswith("'>"): 494 s = s[7:-2] 495 elif s.startswith("<class '") and s.endswith("'>"): 496 s = s[8:-2] 497 arg = str(sys.exc_info()[1]) 498 if len(arg) > 0: 499 s += ": " + arg 500 return s 501 502def IsLocalCL(ui, repo, name): 503 return GoodCLName(name) and os.access(CodeReviewDir(ui, repo) + "/cl." + name, 0) 504 505# Load CL from disk and/or the web. 506def LoadCL(ui, repo, name, web=True): 507 typecheck(name, str) 508 set_status("loading CL " + name) 509 if not GoodCLName(name): 510 return None, "invalid CL name" 511 dir = CodeReviewDir(ui, repo) 512 path = dir + "cl." + name 513 if os.access(path, 0): 514 ff = open(path) 515 text = ff.read() 516 ff.close() 517 cl, lineno, err = ParseCL(text, name) 518 if err != "": 519 return None, "malformed CL data: "+err 520 cl.local = True 521 else: 522 cl = CL(name) 523 if web: 524 set_status("getting issue metadata from web") 525 d = JSONGet(ui, "/api/" + name + "?messages=true") 526 set_status(None) 527 if d is None: 528 return None, "cannot load CL %s from server" % (name,) 529 if 'owner_email' not in d or 'issue' not in d or str(d['issue']) != name: 530 return None, "malformed response loading CL data from code review server" 531 cl.dict = d 532 cl.reviewer = d.get('reviewers', []) 533 cl.cc = d.get('cc', []) 534 if cl.local and cl.copied_from and cl.desc: 535 # local copy of CL written by someone else 536 # and we saved a description. use that one, 537 # so that committers can edit the description 538 # before doing hg submit. 539 pass 540 else: 541 cl.desc = d.get('description', "") 542 cl.url = server_url_base + name 543 cl.web = True 544 cl.private = d.get('private', False) != False 545 set_status("loaded CL " + name) 546 return cl, '' 547 548global_status = None 549 550def set_status(s): 551 # print >>sys.stderr, "\t", time.asctime(), s 552 global global_status 553 global_status = s 554 555class StatusThread(threading.Thread): 556 def __init__(self): 557 threading.Thread.__init__(self) 558 def run(self): 559 # pause a reasonable amount of time before 560 # starting to display status messages, so that 561 # most hg commands won't ever see them. 562 time.sleep(30) 563 564 # now show status every 15 seconds 565 while True: 566 time.sleep(15 - time.time() % 15) 567 s = global_status 568 if s is None: 569 continue 570 if s == "": 571 s = "(unknown status)" 572 print >>sys.stderr, time.asctime(), s 573 574def start_status_thread(): 575 t = StatusThread() 576 t.setDaemon(True) # allowed to exit if t is still running 577 t.start() 578 579class LoadCLThread(threading.Thread): 580 def __init__(self, ui, repo, dir, f, web): 581 threading.Thread.__init__(self) 582 self.ui = ui 583 self.repo = repo 584 self.dir = dir 585 self.f = f 586 self.web = web 587 self.cl = None 588 def run(self): 589 cl, err = LoadCL(self.ui, self.repo, self.f[3:], web=self.web) 590 if err != '': 591 self.ui.warn("loading "+self.dir+self.f+": " + err + "\n") 592 return 593 self.cl = cl 594 595# Load all the CLs from this repository. 596def LoadAllCL(ui, repo, web=True): 597 dir = CodeReviewDir(ui, repo) 598 m = {} 599 files = [f for f in os.listdir(dir) if f.startswith('cl.')] 600 if not files: 601 return m 602 active = [] 603 first = True 604 for f in files: 605 t = LoadCLThread(ui, repo, dir, f, web) 606 t.start() 607 if web and first: 608 # first request: wait in case it needs to authenticate 609 # otherwise we get lots of user/password prompts 610 # running in parallel. 611 t.join() 612 if t.cl: 613 m[t.cl.name] = t.cl 614 first = False 615 else: 616 active.append(t) 617 for t in active: 618 t.join() 619 if t.cl: 620 m[t.cl.name] = t.cl 621 return m 622 623# Find repository root. On error, ui.warn and return None 624def RepoDir(ui, repo): 625 url = repo.url(); 626 if not url.startswith('file:'): 627 ui.warn("repository %s is not in local file system\n" % (url,)) 628 return None 629 url = url[5:] 630 if url.endswith('/'): 631 url = url[:-1] 632 typecheck(url, str) 633 return url 634 635# Find (or make) code review directory. On error, ui.warn and return None 636def CodeReviewDir(ui, repo): 637 dir = RepoDir(ui, repo) 638 if dir == None: 639 return None 640 dir += '/.hg/codereview/' 641 if not os.path.isdir(dir): 642 try: 643 os.mkdir(dir, 0700) 644 except: 645 ui.warn('cannot mkdir %s: %s\n' % (dir, ExceptionDetail())) 646 return None 647 typecheck(dir, str) 648 return dir 649 650# Turn leading tabs into spaces, so that the common white space 651# prefix doesn't get confused when people's editors write out 652# some lines with spaces, some with tabs. Only a heuristic 653# (some editors don't use 8 spaces either) but a useful one. 654def TabsToSpaces(line): 655 i = 0 656 while i < len(line) and line[i] == '\t': 657 i += 1 658 return ' '*(8*i) + line[i:] 659 660# Strip maximal common leading white space prefix from text 661def StripCommon(text): 662 typecheck(text, str) 663 ws = None 664 for line in text.split('\n'): 665 line = line.rstrip() 666 if line == '': 667 continue 668 line = TabsToSpaces(line) 669 white = line[:len(line)-len(line.lstrip())] 670 if ws == None: 671 ws = white 672 else: 673 common = '' 674 for i in range(min(len(white), len(ws))+1): 675 if white[0:i] == ws[0:i]: 676 common = white[0:i] 677 ws = common 678 if ws == '': 679 break 680 if ws == None: 681 return text 682 t = '' 683 for line in text.split('\n'): 684 line = line.rstrip() 685 line = TabsToSpaces(line) 686 if line.startswith(ws): 687 line = line[len(ws):] 688 if line == '' and t == '': 689 continue 690 t += line + '\n' 691 while len(t) >= 2 and t[-2:] == '\n\n': 692 t = t[:-1] 693 typecheck(t, str) 694 return t 695 696# Indent text with indent. 697def Indent(text, indent): 698 typecheck(text, str) 699 typecheck(indent, str) 700 t = '' 701 for line in text.split('\n'): 702 t += indent + line + '\n' 703 typecheck(t, str) 704 return t 705 706# Return the first line of l 707def line1(text): 708 typecheck(text, str) 709 return text.split('\n')[0] 710 711_change_prolog = """# Change list. 712# Lines beginning with # are ignored. 713# Multi-line values should be indented. 714""" 715 716####################################################################### 717# Mercurial helper functions 718 719# Get effective change nodes taking into account applied MQ patches 720def effective_revpair(repo): 721 try: 722 return scmutil.revpair(repo, ['qparent']) 723 except: 724 return scmutil.revpair(repo, None) 725 726# Return list of changed files in repository that match pats. 727# Warn about patterns that did not match. 728def matchpats(ui, repo, pats, opts): 729 matcher = scmutil.match(repo, pats, opts) 730 node1, node2 = effective_revpair(repo) 731 modified, added, removed, deleted, unknown, ignored, clean = repo.status(node1, node2, matcher, ignored=True, clean=True, unknown=True) 732 return (modified, added, removed, deleted, unknown, ignored, clean) 733 734# Return list of changed files in repository that match pats. 735# The patterns came from the command line, so we warn 736# if they have no effect or cannot be understood. 737def ChangedFiles(ui, repo, pats, opts, taken=None): 738 taken = taken or {} 739 # Run each pattern separately so that we can warn about 740 # patterns that didn't do anything useful. 741 for p in pats: 742 modified, added, removed, deleted, unknown, ignored, clean = matchpats(ui, repo, [p], opts) 743 redo = False 744 for f in unknown: 745 promptadd(ui, repo, f) 746 redo = True 747 for f in deleted: 748 promptremove(ui, repo, f) 749 redo = True 750 if redo: 751 modified, added, removed, deleted, unknown, ignored, clean = matchpats(ui, repo, [p], opts) 752 for f in modified + added + removed: 753 if f in taken: 754 ui.warn("warning: %s already in CL %s\n" % (f, taken[f].name)) 755 if not modified and not added and not removed: 756 ui.warn("warning: %s did not match any modified files\n" % (p,)) 757 758 # Again, all at once (eliminates duplicates) 759 modified, added, removed = matchpats(ui, repo, pats, opts)[:3] 760 l = modified + added + removed 761 l.sort() 762 if taken: 763 l = Sub(l, taken.keys()) 764 return l 765 766# Return list of changed files in repository that match pats and still exist. 767def ChangedExistingFiles(ui, repo, pats, opts): 768 modified, added = matchpats(ui, repo, pats, opts)[:2] 769 l = modified + added 770 l.sort() 771 return l 772 773# Return list of files claimed by existing CLs 774def Taken(ui, repo): 775 all = LoadAllCL(ui, repo, web=False) 776 taken = {} 777 for _, cl in all.items(): 778 for f in cl.files: 779 taken[f] = cl 780 return taken 781 782# Return list of changed files that are not claimed by other CLs 783def DefaultFiles(ui, repo, pats, opts): 784 return ChangedFiles(ui, repo, pats, opts, taken=Taken(ui, repo)) 785 786def Sub(l1, l2): 787 return [l for l in l1 if l not in l2] 788 789def Add(l1, l2): 790 l = l1 + Sub(l2, l1) 791 l.sort() 792 return l 793 794def Intersect(l1, l2): 795 return [l for l in l1 if l in l2] 796 797def getremote(ui, repo, opts): 798 # save $http_proxy; creating the HTTP repo object will 799 # delete it in an attempt to "help" 800 proxy = os.environ.get('http_proxy') 801 source = hg.parseurl(ui.expandpath("default"), None)[0] 802 try: 803 remoteui = hg.remoteui # hg 1.6 804 except: 805 remoteui = cmdutil.remoteui 806 other = hg.repository(remoteui(repo, opts), source) 807 if proxy is not None: 808 os.environ['http_proxy'] = proxy 809 return other 810 811def Incoming(ui, repo, opts): 812 _, incoming, _ = findcommonincoming(repo, getremote(ui, repo, opts)) 813 return incoming 814 815desc_re = '^(.+: |(tag )?(release|weekly)\.|fix build|undo CL)' 816 817desc_msg = '''Your CL description appears not to use the standard form. 818 819The first line of your change description is conventionally a 820one-line summary of the change, prefixed by the primary affected package, 821and is used as the subject for code review mail; the rest of the description 822elaborates. 823 824Examples: 825 826 encoding/rot13: new package 827 828 math: add IsInf, IsNaN 829 830 net: fix cname in LookupHost 831 832 unicode: update to Unicode 5.0.2 833 834''' 835 836 837def promptremove(ui, repo, f): 838 if promptyesno(ui, "hg remove %s (y/n)?" % (f,)): 839 if commands.remove(ui, repo, 'path:'+f) != 0: 840 ui.warn("error removing %s" % (f,)) 841 842def promptadd(ui, repo, f): 843 if promptyesno(ui, "hg add %s (y/n)?" % (f,)): 844 if commands.add(ui, repo, 'path:'+f) != 0: 845 ui.warn("error adding %s" % (f,)) 846 847def EditCL(ui, repo, cl): 848 set_status(None) # do not show status 849 s = cl.EditorText() 850 while True: 851 s = ui.edit(s, ui.username()) 852 clx, line, err = ParseCL(s, cl.name) 853 if err != '': 854 if not promptyesno(ui, "error parsing change list: line %d: %s\nre-edit (y/n)?" % (line, err)): 855 return "change list not modified" 856 continue 857 858 # Check description. 859 if clx.desc == '': 860 if promptyesno(ui, "change list should have a description\nre-edit (y/n)?"): 861 continue 862 elif re.search('<enter reason for undo>', clx.desc): 863 if promptyesno(ui, "change list description omits reason for undo\nre-edit (y/n)?"): 864 continue 865 elif not re.match(desc_re, clx.desc.split('\n')[0]): 866 if promptyesno(ui, desc_msg + "re-edit (y/n)?"): 867 continue 868 869 # Check file list for files that need to be hg added or hg removed 870 # or simply aren't understood. 871 pats = ['path:'+f for f in clx.files] 872 modified, added, removed, deleted, unknown, ignored, clean = matchpats(ui, repo, pats, {}) 873 files = [] 874 for f in clx.files: 875 if f in modified or f in added or f in removed: 876 files.append(f) 877 continue 878 if f in deleted: 879 promptremove(ui, repo, f) 880 files.append(f) 881 continue 882 if f in unknown: 883 promptadd(ui, repo, f) 884 files.append(f) 885 continue 886 if f in ignored: 887 ui.warn("error: %s is excluded by .hgignore; omitting\n" % (f,)) 888 continue 889 if f in clean: 890 ui.warn("warning: %s is listed in the CL but unchanged\n" % (f,)) 891 files.append(f) 892 continue 893 p = repo.root + '/' + f 894 if os.path.isfile(p): 895 ui.warn("warning: %s is a file but not known to hg\n" % (f,)) 896 files.append(f) 897 continue 898 if os.path.isdir(p): 899 ui.warn("error: %s is a directory, not a file; omitting\n" % (f,)) 900 continue 901 ui.warn("error: %s does not exist; omitting\n" % (f,)) 902 clx.files = files 903 904 cl.desc = clx.desc 905 cl.reviewer = clx.reviewer 906 cl.cc = clx.cc 907 cl.files = clx.files 908 cl.private = clx.private 909 break 910 return "" 911 912# For use by submit, etc. (NOT by change) 913# Get change list number or list of files from command line. 914# If files are given, make a new change list. 915def CommandLineCL(ui, repo, pats, opts, defaultcc=None): 916 if len(pats) > 0 and GoodCLName(pats[0]): 917 if len(pats) != 1: 918 return None, "cannot specify change number and file names" 919 if opts.get('message'): 920 return None, "cannot use -m with existing CL" 921 cl, err = LoadCL(ui, repo, pats[0], web=True) 922 if err != "": 923 return None, err 924 else: 925 cl = CL("new") 926 cl.local = True 927 cl.files = ChangedFiles(ui, repo, pats, opts, taken=Taken(ui, repo)) 928 if not cl.files: 929 return None, "no files changed" 930 if opts.get('reviewer'): 931 cl.reviewer = Add(cl.reviewer, SplitCommaSpace(opts.get('reviewer'))) 932 if opts.get('cc'): 933 cl.cc = Add(cl.cc, SplitCommaSpace(opts.get('cc'))) 934 if defaultcc: 935 cl.cc = Add(cl.cc, defaultcc) 936 if cl.name == "new": 937 if opts.get('message'): 938 cl.desc = opts.get('message') 939 else: 940 err = EditCL(ui, repo, cl) 941 if err != '': 942 return None, err 943 return cl, "" 944 945# reposetup replaces cmdutil.match with this wrapper, 946# which expands the syntax @clnumber to mean the files 947# in that CL. 948original_match = None 949def ReplacementForCmdutilMatch(repo, pats=None, opts=None, globbed=False, default='relpath'): 950 taken = [] 951 files = [] 952 pats = pats or [] 953 opts = opts or {} 954 for p in pats: 955 if p.startswith('@'): 956 taken.append(p) 957 clname = p[1:] 958 if not GoodCLName(clname): 959 raise util.Abort("invalid CL name " + clname) 960 cl, err = LoadCL(repo.ui, repo, clname, web=False) 961 if err != '': 962 raise util.Abort("loading CL " + clname + ": " + err) 963 if not cl.files: 964 raise util.Abort("no files in CL " + clname) 965 files = Add(files, cl.files) 966 pats = Sub(pats, taken) + ['path:'+f for f in files] 967 968 # work-around for http://selenic.com/hg/rev/785bbc8634f8 969 if hgversion >= '1.9' and not hasattr(repo, 'match'): 970 repo = repo[None] 971 972 return original_match(repo, pats=pats, opts=opts, globbed=globbed, default=default) 973 974def RelativePath(path, cwd): 975 n = len(cwd) 976 if path.startswith(cwd) and path[n] == '/': 977 return path[n+1:] 978 return path 979 980def CheckFormat(ui, repo, files, just_warn=False): 981 set_status("running gofmt") 982 CheckGofmt(ui, repo, files, just_warn) 983 CheckTabfmt(ui, repo, files, just_warn) 984 985# Check that gofmt run on the list of files does not change them 986def CheckGofmt(ui, repo, files, just_warn): 987 files = [f for f in files if (f.startswith('src/') or f.startswith('test/bench/')) and f.endswith('.go')] 988 if not files: 989 return 990 cwd = os.getcwd() 991 files = [RelativePath(repo.root + '/' + f, cwd) for f in files] 992 files = [f for f in files if os.access(f, 0)] 993 if not files: 994 return 995 try: 996 cmd = subprocess.Popen(["gofmt", "-l"] + files, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=sys.platform != "win32") 997 cmd.stdin.close() 998 except: 999 raise util.Abort("gofmt: " + ExceptionDetail()) 1000 data = cmd.stdout.read() 1001 errors = cmd.stderr.read() 1002 cmd.wait() 1003 set_status("done with gofmt") 1004 if len(errors) > 0: 1005 ui.warn("gofmt errors:\n" + errors.rstrip() + "\n") 1006 return 1007 if len(data) > 0: 1008 msg = "gofmt needs to format these files (run hg gofmt):\n" + Indent(data, "\t").rstrip() 1009 if just_warn: 1010 ui.warn("warning: " + msg + "\n") 1011 else: 1012 raise util.Abort(msg) 1013 return 1014 1015# Check that *.[chys] files indent using tabs. 1016def CheckTabfmt(ui, repo, files, just_warn): 1017 files = [f for f in files if f.startswith('src/') and re.search(r"\.[chys]$", f)] 1018 if not files: 1019 return 1020 cwd = os.getcwd() 1021 files = [RelativePath(repo.root + '/' + f, cwd) for f in files] 1022 files = [f for f in files if os.access(f, 0)] 1023 badfiles = [] 1024 for f in files: 1025 try: 1026 for line in open(f, 'r'): 1027 # Four leading spaces is enough to complain about, 1028 # except that some Plan 9 code uses four spaces as the label indent, 1029 # so allow that. 1030 if line.startswith(' ') and not re.match(' [A-Za-z0-9_]+:', line): 1031 badfiles.append(f) 1032 break 1033 except: 1034 # ignore cannot open file, etc. 1035 pass 1036 if len(badfiles) > 0: 1037 msg = "these files use spaces for indentation (use tabs instead):\n\t" + "\n\t".join(badfiles) 1038 if just_warn: 1039 ui.warn("warning: " + msg + "\n") 1040 else: 1041 raise util.Abort(msg) 1042 return 1043 1044####################################################################### 1045# Mercurial commands 1046 1047# every command must take a ui and and repo as arguments. 1048# opts is a dict where you can find other command line flags 1049# 1050# Other parameters are taken in order from items on the command line that 1051# don't start with a dash. If no default value is given in the parameter list, 1052# they are required. 1053# 1054 1055def change(ui, repo, *pats, **opts): 1056 """create, edit or delete a change list 1057 1058 Create, edit or delete a change list. 1059 A change list is a group of files to be reviewed and submitted together, 1060 plus a textual description of the change. 1061 Change lists are referred to by simple alphanumeric names. 1062 1063 Changes must be reviewed before they can be submitted. 1064 1065 In the absence of options, the change command opens the 1066 change list for editing in the default editor. 1067 1068 Deleting a change with the -d or -D flag does not affect 1069 the contents of the files listed in that change. To revert 1070 the files listed in a change, use 1071 1072 hg revert @123456 1073 1074 before running hg change -d 123456. 1075 """ 1076 1077 if missing_codereview: 1078 return missing_codereview 1079 1080 dirty = {} 1081 if len(pats) > 0 and GoodCLName(pats[0]): 1082 name = pats[0] 1083 if len(pats) != 1: 1084 return "cannot specify CL name and file patterns" 1085 pats = pats[1:] 1086 cl, err = LoadCL(ui, repo, name, web=True) 1087 if err != '': 1088 return err 1089 if not cl.local and (opts["stdin"] or not opts["stdout"]): 1090 return "cannot change non-local CL " + name 1091 else: 1092 if repo[None].branch() != "default": 1093 return "cannot run hg change outside default branch" 1094 name = "new" 1095 cl = CL("new") 1096 dirty[cl] = True 1097 files = ChangedFiles(ui, repo, pats, opts, taken=Taken(ui, repo)) 1098 1099 if opts["delete"] or opts["deletelocal"]: 1100 if opts["delete"] and opts["deletelocal"]: 1101 return "cannot use -d and -D together" 1102 flag = "-d" 1103 if opts["deletelocal"]: 1104 flag = "-D" 1105 if name == "new": 1106 return "cannot use "+flag+" with file patterns" 1107 if opts["stdin"] or opts["stdout"]: 1108 return "cannot use "+flag+" with -i or -o" 1109 if not cl.local: 1110 return "cannot change non-local CL " + name 1111 if opts["delete"]: 1112 if cl.copied_from: 1113 return "original author must delete CL; hg change -D will remove locally" 1114 PostMessage(ui, cl.name, "*** Abandoned ***", send_mail=cl.mailed) 1115 EditDesc(cl.name, closed=True, private=cl.private) 1116 cl.Delete(ui, repo) 1117 return 1118 1119 if opts["stdin"]: 1120 s = sys.stdin.read() 1121 clx, line, err = ParseCL(s, name) 1122 if err != '': 1123 return "error parsing change list: line %d: %s" % (line, err) 1124 if clx.desc is not None: 1125 cl.desc = clx.desc; 1126 dirty[cl] = True 1127 if clx.reviewer is not None: 1128 cl.reviewer = clx.reviewer 1129 dirty[cl] = True 1130 if clx.cc is not None: 1131 cl.cc = clx.cc 1132 dirty[cl] = True 1133 if clx.files is not None: 1134 cl.files = clx.files 1135 dirty[cl] = True 1136 if clx.private != cl.private: 1137 cl.private = clx.private 1138 dirty[cl] = True 1139 1140 if not opts["stdin"] and not opts["stdout"]: 1141 if name == "new": 1142 cl.files = files 1143 err = EditCL(ui, repo, cl) 1144 if err != "": 1145 return err 1146 dirty[cl] = True 1147 1148 for d, _ in dirty.items(): 1149 name = d.name 1150 d.Flush(ui, repo) 1151 if name == "new": 1152 d.Upload(ui, repo, quiet=True) 1153 1154 if opts["stdout"]: 1155 ui.write(cl.EditorText()) 1156 elif opts["pending"]: 1157 ui.write(cl.PendingText()) 1158 elif name == "new": 1159 if ui.quiet: 1160 ui.write(cl.name) 1161 else: 1162 ui.write("CL created: " + cl.url + "\n") 1163 return 1164 1165def code_login(ui, repo, **opts): 1166 """log in to code review server 1167 1168 Logs in to the code review server, saving a cookie in 1169 a file in your home directory. 1170 """ 1171 if missing_codereview: 1172 return missing_codereview 1173 1174 MySend(None) 1175 1176def clpatch(ui, repo, clname, **opts): 1177 """import a patch from the code review server 1178 1179 Imports a patch from the code review server into the local client. 1180 If the local client has already modified any of the files that the 1181 patch modifies, this command will refuse to apply the patch. 1182 1183 Submitting an imported patch will keep the original author's 1184 name as the Author: line but add your own name to a Committer: line. 1185 """ 1186 if repo[None].branch() != "default": 1187 return "cannot run hg clpatch outside default branch" 1188 return clpatch_or_undo(ui, repo, clname, opts, mode="clpatch") 1189 1190def undo(ui, repo, clname, **opts): 1191 """undo the effect of a CL 1192 1193 Creates a new CL that undoes an earlier CL. 1194 After creating the CL, opens the CL text for editing so that 1195 you can add the reason for the undo to the description. 1196 """ 1197 if repo[None].branch() != "default": 1198 return "cannot run hg undo outside default branch" 1199 return clpatch_or_undo(ui, repo, clname, opts, mode="undo") 1200 1201def release_apply(ui, repo, clname, **opts): 1202 """apply a CL to the release branch 1203 1204 Creates a new CL copying a previously committed change 1205 from the main branch to the release branch. 1206 The current client must either be clean or already be in 1207 the release branch. 1208 1209 The release branch must be created by starting with a 1210 clean client, disabling the code review plugin, and running: 1211 1212 hg update weekly.YYYY-MM-DD 1213 hg branch release-branch.rNN 1214 hg commit -m 'create release-branch.rNN' 1215 hg push --new-branch 1216 1217 Then re-enable the code review plugin. 1218 1219 People can test the release branch by running 1220 1221 hg update release-branch.rNN 1222 1223 in a clean client. To return to the normal tree, 1224 1225 hg update default 1226 1227 Move changes since the weekly into the release branch 1228 using hg release-apply followed by the usual code review 1229 process and hg submit. 1230 1231 When it comes time to tag the release, record the 1232 final long-form tag of the release-branch.rNN 1233 in the *default* branch's .hgtags file. That is, run 1234 1235 hg update default 1236 1237 and then edit .hgtags as you would for a weekly. 1238 1239 """ 1240 c = repo[None] 1241 if not releaseBranch: 1242 return "no active release branches" 1243 if c.branch() != releaseBranch: 1244 if c.modified() or c.added() or c.removed(): 1245 raise util.Abort("uncommitted local changes - cannot switch branches") 1246 err = hg.clean(repo, releaseBranch) 1247 if err: 1248 return err 1249 try: 1250 err = clpatch_or_undo(ui, repo, clname, opts, mode="backport") 1251 if err: 1252 raise util.Abort(err) 1253 except Exception, e: 1254 hg.clean(repo, "default") 1255 raise e 1256 return None 1257 1258def rev2clname(rev): 1259 # Extract CL name from revision description. 1260 # The last line in the description that is a codereview URL is the real one. 1261 # Earlier lines might be part of the user-written description. 1262 all = re.findall('(?m)^http://codereview.appspot.com/([0-9]+)$', rev.description()) 1263 if len(all) > 0: 1264 return all[-1] 1265 return "" 1266 1267undoHeader = """undo CL %s / %s 1268 1269<enter reason for undo> 1270 1271««« original CL description 1272""" 1273 1274undoFooter = """ 1275»»» 1276""" 1277 1278backportHeader = """[%s] %s 1279 1280««« CL %s / %s 1281""" 1282 1283backportFooter = """ 1284»»» 1285""" 1286 1287# Implementation of clpatch/undo. 1288def clpatch_or_undo(ui, repo, clname, opts, mode): 1289 if missing_codereview: 1290 return missing_codereview 1291 1292 if mode == "undo" or mode == "backport": 1293 if hgversion < '1.4': 1294 # Don't have cmdutil.match (see implementation of sync command). 1295 return "hg is too old to run hg %s - update to 1.4 or newer" % mode 1296 1297 # Find revision in Mercurial repository. 1298 # Assume CL number is 7+ decimal digits. 1299 # Otherwise is either change log sequence number (fewer decimal digits), 1300 # hexadecimal hash, or tag name. 1301 # Mercurial will fall over long before the change log 1302 # sequence numbers get to be 7 digits long. 1303 if re.match('^[0-9]{7,}$', clname): 1304 found = False 1305 matchfn = scmutil.match(repo, [], {'rev': None}) 1306 def prep(ctx, fns): 1307 pass 1308 for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev': None}, prep): 1309 rev = repo[ctx.rev()] 1310 # Last line with a code review URL is the actual review URL. 1311 # Earlier ones might be part of the CL description. 1312 n = rev2clname(rev) 1313 if n == clname: 1314 found = True 1315 break 1316 if not found: 1317 return "cannot find CL %s in local repository" % clname 1318 else: 1319 rev = repo[clname] 1320 if not rev: 1321 return "unknown revision %s" % clname 1322 clname = rev2clname(rev) 1323 if clname == "": 1324 return "cannot find CL name in revision description" 1325 1326 # Create fresh CL and start with patch that would reverse the change. 1327 vers = short(rev.node()) 1328 cl = CL("new") 1329 desc = str(rev.description()) 1330 if mode == "undo": 1331 cl.desc = (undoHeader % (clname, vers)) + desc + undoFooter 1332 else: 1333 cl.desc = (backportHeader % (releaseBranch, line1(desc), clname, vers)) + desc + undoFooter 1334 v1 = vers 1335 v0 = short(rev.parents()[0].node()) 1336 if mode == "undo": 1337 arg = v1 + ":" + v0 1338 else: 1339 vers = v0 1340 arg = v0 + ":" + v1 1341 patch = RunShell(["hg", "diff", "--git", "-r", arg]) 1342 1343 else: # clpatch 1344 cl, vers, patch, err = DownloadCL(ui, repo, clname) 1345 if err != "": 1346 return err 1347 if patch == emptydiff: 1348 return "codereview issue %s has no diff" % clname 1349 1350 # find current hg version (hg identify) 1351 ctx = repo[None] 1352 parents = ctx.parents() 1353 id = '+'.join([short(p.node()) for p in parents]) 1354 1355 # if version does not match the patch version, 1356 # try to update the patch line numbers. 1357 if vers != "" and id != vers: 1358 # "vers in repo" gives the wrong answer 1359 # on some versions of Mercurial. Instead, do the actual 1360 # lookup and catch the exception. 1361 try: 1362 repo[vers].description() 1363 except: 1364 return "local repository is out of date; sync to get %s" % (vers) 1365 patch1, err = portPatch(repo, patch, vers, id) 1366 if err != "": 1367 if not opts["ignore_hgpatch_failure"]: 1368 return "codereview issue %s is out of date: %s (%s->%s)" % (clname, err, vers, id) 1369 else: 1370 patch = patch1 1371 argv = ["hgpatch"] 1372 if opts["no_incoming"] or mode == "backport": 1373 argv += ["--checksync=false"] 1374 try: 1375 cmd = subprocess.Popen(argv, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None, close_fds=sys.platform != "win32") 1376 except: 1377 return "hgpatch: " + ExceptionDetail() 1378 1379 out, err = cmd.communicate(patch) 1380 if cmd.returncode != 0 and not opts["ignore_hgpatch_failure"]: 1381 return "hgpatch failed" 1382 cl.local = True 1383 cl.files = out.strip().split() 1384 if not cl.files and not opts["ignore_hgpatch_failure"]: 1385 return "codereview issue %s has no changed files" % clname 1386 files = ChangedFiles(ui, repo, [], opts) 1387 extra = Sub(cl.files, files) 1388 if extra: 1389 ui.warn("warning: these files were listed in the patch but not changed:\n\t" + "\n\t".join(extra) + "\n") 1390 cl.Flush(ui, repo) 1391 if mode == "undo": 1392 err = EditCL(ui, repo, cl) 1393 if err != "": 1394 return "CL created, but error editing: " + err 1395 cl.Flush(ui, repo) 1396 else: 1397 ui.write(cl.PendingText() + "\n") 1398 1399# portPatch rewrites patch from being a patch against 1400# oldver to being a patch against newver. 1401def portPatch(repo, patch, oldver, newver): 1402 lines = patch.splitlines(True) # True = keep \n 1403 delta = None 1404 for i in range(len(lines)): 1405 line = lines[i] 1406 if line.startswith('--- a/'): 1407 file = line[6:-1] 1408 delta = fileDeltas(repo, file, oldver, newver) 1409 if not delta or not line.startswith('@@ '): 1410 continue 1411 # @@ -x,y +z,w @@ means the patch chunk replaces 1412 # the original file's line numbers x up to x+y with the 1413 # line numbers z up to z+w in the new file. 1414 # Find the delta from x in the original to the same 1415 # line in the current version and add that delta to both 1416 # x and z. 1417 m = re.match('@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@', line) 1418 if not m: 1419 return None, "error parsing patch line numbers" 1420 n1, len1, n2, len2 = int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)) 1421 d, err = lineDelta(delta, n1, len1) 1422 if err != "": 1423 return "", err 1424 n1 += d 1425 n2 += d 1426 lines[i] = "@@ -%d,%d +%d,%d @@\n" % (n1, len1, n2, len2) 1427 1428 newpatch = ''.join(lines) 1429 return newpatch, "" 1430 1431# fileDelta returns the line number deltas for the given file's 1432# changes from oldver to newver. 1433# The deltas are a list of (n, len, newdelta) triples that say 1434# lines [n, n+len) were modified, and after that range the 1435# line numbers are +newdelta from what they were before. 1436def fileDeltas(repo, file, oldver, newver): 1437 cmd = ["hg", "diff", "--git", "-r", oldver + ":" + newver, "path:" + file] 1438 data = RunShell(cmd, silent_ok=True) 1439 deltas = [] 1440 for line in data.splitlines(): 1441 m = re.match('@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@', line) 1442 if not m: 1443 continue 1444 n1, len1, n2, len2 = int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)) 1445 deltas.append((n1, len1, n2+len2-(n1+len1))) 1446 return deltas 1447 1448# lineDelta finds the appropriate line number delta to apply to the lines [n, n+len). 1449# It returns an error if those lines were rewritten by the patch. 1450def lineDelta(deltas, n, len): 1451 d = 0 1452 for (old, oldlen, newdelta) in deltas: 1453 if old >= n+len: 1454 break 1455 if old+len > n: 1456 return 0, "patch and recent changes conflict" 1457 d = newdelta 1458 return d, "" 1459 1460def download(ui, repo, clname, **opts): 1461 """download a change from the code review server 1462 1463 Download prints a description of the given change list 1464 followed by its diff, downloaded from the code review server. 1465 """ 1466 if missing_codereview: 1467 return missing_codereview 1468 1469 cl, vers, patch, err = DownloadCL(ui, repo, clname) 1470 if err != "": 1471 return err 1472 ui.write(cl.EditorText() + "\n") 1473 ui.write(patch + "\n") 1474 return 1475 1476def file(ui, repo, clname, pat, *pats, **opts): 1477 """assign files to or remove files from a change list 1478 1479 Assign files to or (with -d) remove files from a change list. 1480 1481 The -d option only removes files from the change list. 1482 It does not edit them or remove them from the repository. 1483 """ 1484 if missing_codereview: 1485 return missing_codereview 1486 1487 pats = tuple([pat] + list(pats)) 1488 if not GoodCLName(clname): 1489 return "invalid CL name " + clname 1490 1491 dirty = {} 1492 cl, err = LoadCL(ui, repo, clname, web=False) 1493 if err != '': 1494 return err 1495 if not cl.local: 1496 return "cannot change non-local CL " + clname 1497 1498 files = ChangedFiles(ui, repo, pats, opts) 1499 1500 if opts["delete"]: 1501 oldfiles = Intersect(files, cl.files) 1502 if oldfiles: 1503 if not ui.quiet: 1504 ui.status("# Removing files from CL. To undo:\n") 1505 ui.status("# cd %s\n" % (repo.root)) 1506 for f in oldfiles: 1507 ui.status("# hg file %s %s\n" % (cl.name, f)) 1508 cl.files = Sub(cl.files, oldfiles) 1509 cl.Flush(ui, repo) 1510 else: 1511 ui.status("no such files in CL") 1512 return 1513 1514 if not files: 1515 return "no such modified files" 1516 1517 files = Sub(files, cl.files) 1518 taken = Taken(ui, repo) 1519 warned = False 1520 for f in files: 1521 if f in taken: 1522 if not warned and not ui.quiet: 1523 ui.status("# Taking files from other CLs. To undo:\n") 1524 ui.status("# cd %s\n" % (repo.root)) 1525 warned = True 1526 ocl = taken[f] 1527 if not ui.quiet: 1528 ui.status("# hg file %s %s\n" % (ocl.name, f)) 1529 if ocl not in dirty: 1530 ocl.files = Sub(ocl.files, files) 1531 dirty[ocl] = True 1532 cl.files = Add(cl.files, files) 1533 dirty[cl] = True 1534 for d, _ in dirty.items(): 1535 d.Flush(ui, repo) 1536 return 1537 1538def gofmt(ui, repo, *pats, **opts): 1539 """apply gofmt to modified files 1540 1541 Applies gofmt to the modified files in the repository that match 1542 the given patterns. 1543 """ 1544 if missing_codereview: 1545 return missing_codereview 1546 1547 files = ChangedExistingFiles(ui, repo, pats, opts) 1548 files = [f for f in files if f.endswith(".go")] 1549 if not files: 1550 return "no modified go files" 1551 cwd = os.getcwd() 1552 files = [RelativePath(repo.root + '/' + f, cwd) for f in files] 1553 try: 1554 cmd = ["gofmt", "-l"] 1555 if not opts["list"]: 1556 cmd += ["-w"] 1557 if os.spawnvp(os.P_WAIT, "gofmt", cmd + files) != 0: 1558 raise util.Abort("gofmt did not exit cleanly") 1559 except error.Abort, e: 1560 raise 1561 except: 1562 raise util.Abort("gofmt: " + ExceptionDetail()) 1563 return 1564 1565def mail(ui, repo, *pats, **opts): 1566 """mail a change for review 1567 1568 Uploads a patch to the code review server and then sends mail 1569 to the reviewer and CC list asking for a review. 1570 """ 1571 if missing_codereview: 1572 return missing_codereview 1573 1574 cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc) 1575 if err != "": 1576 return err 1577 cl.Upload(ui, repo, gofmt_just_warn=True) 1578 if not cl.reviewer: 1579 # If no reviewer is listed, assign the review to defaultcc. 1580 # This makes sure that it appears in the 1581 # codereview.appspot.com/user/defaultcc 1582 # page, so that it doesn't get dropped on the floor. 1583 if not defaultcc: 1584 return "no reviewers listed in CL" 1585 cl.cc = Sub(cl.cc, defaultcc) 1586 cl.reviewer = defaultcc 1587 cl.Flush(ui, repo) 1588 1589 if cl.files == []: 1590 return "no changed files, not sending mail" 1591 1592 cl.Mail(ui, repo) 1593 1594def pending(ui, repo, *pats, **opts): 1595 """show pending changes 1596 1597 Lists pending changes followed by a list of unassigned but modified files. 1598 """ 1599 if missing_codereview: 1600 return missing_codereview 1601 1602 m = LoadAllCL(ui, repo, web=True) 1603 names = m.keys() 1604 names.sort() 1605 for name in names: 1606 cl = m[name] 1607 ui.write(cl.PendingText() + "\n") 1608 1609 files = DefaultFiles(ui, repo, [], opts) 1610 if len(files) > 0: 1611 s = "Changed files not in any CL:\n" 1612 for f in files: 1613 s += "\t" + f + "\n" 1614 ui.write(s) 1615 1616def reposetup(ui, repo): 1617 global original_match 1618 if original_match is None: 1619 start_status_thread() 1620 original_match = scmutil.match 1621 scmutil.match = ReplacementForCmdutilMatch 1622 RietveldSetup(ui, repo) 1623 1624def CheckContributor(ui, repo, user=None): 1625 set_status("checking CONTRIBUTORS file") 1626 user, userline = FindContributor(ui, repo, user, warn=False) 1627 if not userline: 1628 raise util.Abort("cannot find %s in CONTRIBUTORS" % (user,)) 1629 return userline 1630 1631def FindContributor(ui, repo, user=None, warn=True): 1632 if not user: 1633 user = ui.config("ui", "username") 1634 if not user: 1635 raise util.Abort("[ui] username is not configured in .hgrc") 1636 user = user.lower() 1637 m = re.match(r".*<(.*)>", user) 1638 if m: 1639 user = m.group(1) 1640 1641 if user not in contributors: 1642 if warn: 1643 ui.warn("warning: cannot find %s in CONTRIBUTORS\n" % (user,)) 1644 return user, None 1645 1646 user, email = contributors[user] 1647 return email, "%s <%s>" % (user, email) 1648 1649def submit(ui, repo, *pats, **opts): 1650 """submit change to remote repository 1651 1652 Submits change to remote repository. 1653 Bails out if the local repository is not in sync with the remote one. 1654 """ 1655 if missing_codereview: 1656 return missing_codereview 1657 1658 # We already called this on startup but sometimes Mercurial forgets. 1659 set_mercurial_encoding_to_utf8() 1660 1661 repo.ui.quiet = True 1662 if not opts["no_incoming"] and Incoming(ui, repo, opts): 1663 return "local repository out of date; must sync before submit" 1664 1665 cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc) 1666 if err != "": 1667 return err 1668 1669 user = None 1670 if cl.copied_from: 1671 user = cl.copied_from 1672 userline = CheckContributor(ui, repo, user) 1673 typecheck(userline, str) 1674 1675 about = "" 1676 if cl.reviewer: 1677 about += "R=" + JoinComma([CutDomain(s) for s in cl.reviewer]) + "\n" 1678 if opts.get('tbr'): 1679 tbr = SplitCommaSpace(opts.get('tbr')) 1680 cl.reviewer = Add(cl.reviewer, tbr) 1681 about += "TBR=" + JoinComma([CutDomain(s) for s in tbr]) + "\n" 1682 if cl.cc: 1683 about += "CC=" + JoinComma([CutDomain(s) for s in cl.cc]) + "\n" 1684 1685 if not cl.reviewer: 1686 return "no reviewers listed in CL" 1687 1688 if not cl.local: 1689 return "cannot submit non-local CL" 1690 1691 # upload, to sync current patch and also get change number if CL is new. 1692 if not cl.copied_from: 1693 cl.Upload(ui, repo, gofmt_just_warn=True) 1694 1695 # check gofmt for real; allowed upload to warn in order to save CL. 1696 cl.Flush(ui, repo) 1697 CheckFormat(ui, repo, cl.files) 1698 1699 about += "%s%s\n" % (server_url_base, cl.name) 1700 1701 if cl.copied_from: 1702 about += "\nCommitter: " + CheckContributor(ui, repo, None) + "\n" 1703 typecheck(about, str) 1704 1705 if not cl.mailed and not cl.copied_from: # in case this is TBR 1706 cl.Mail(ui, repo) 1707 1708 # submit changes locally 1709 date = opts.get('date') 1710 if date: 1711 opts['date'] = util.parsedate(date) 1712 typecheck(opts['date'], str) 1713 opts['message'] = cl.desc.rstrip() + "\n\n" + about 1714 typecheck(opts['message'], str) 1715 1716 if opts['dryrun']: 1717 print "NOT SUBMITTING:" 1718 print "User: ", userline 1719 print "Message:" 1720 print Indent(opts['message'], "\t") 1721 print "Files:" 1722 print Indent('\n'.join(cl.files), "\t") 1723 return "dry run; not submitted" 1724 1725 m = match.exact(repo.root, repo.getcwd(), cl.files) 1726 node = repo.commit(ustr(opts['message']), ustr(userline), opts.get('date'), m) 1727 if not node: 1728 return "nothing changed" 1729 1730 # push to remote; if it fails for any reason, roll back 1731 try: 1732 log = repo.changelog 1733 rev = log.rev(node) 1734 parents = log.parentrevs(rev) 1735 if (rev-1 not in parents and 1736 (parents == (nullrev, nullrev) or 1737 len(log.heads(log.node(parents[0]))) > 1 and 1738 (parents[1] == nullrev or len(log.heads(log.node(parents[1]))) > 1))): 1739 # created new head 1740 raise util.Abort("local repository out of date; must sync before submit") 1741 1742 # push changes to remote. 1743 # if it works, we're committed. 1744 # if not, roll back 1745 other = getremote(ui, repo, opts) 1746 r = repo.push(other, False, None) 1747 if r == 0: 1748 raise util.Abort("local repository out of date; must sync before submit") 1749 except: 1750 real_rollback() 1751 raise 1752 1753 # we're committed. upload final patch, close review, add commit message 1754 changeURL = short(node) 1755 url = other.url() 1756 m = re.match("^https?://([^@/]+@)?([^.]+)\.googlecode\.com/hg/?", url) 1757 if m: 1758 changeURL = "http://code.google.com/p/%s/source/detail?r=%s" % (m.group(2), changeURL) 1759 else: 1760 print >>sys.stderr, "URL: ", url 1761 pmsg = "*** Submitted as " + changeURL + " ***\n\n" + opts['message'] 1762 1763 # When posting, move reviewers to CC line, 1764 # so that the issue stops showing up in their "My Issues" page. 1765 PostMessage(ui, cl.name, pmsg, reviewers="", cc=JoinComma(cl.reviewer+cl.cc)) 1766 1767 if not cl.copied_from: 1768 EditDesc(cl.name, closed=True, private=cl.private) 1769 cl.Delete(ui, repo) 1770 1771 c = repo[None] 1772 if c.branch() == releaseBranch and not c.modified() and not c.added() and not c.removed(): 1773 ui.write("switching from %s to default branch.\n" % releaseBranch) 1774 err = hg.clean(repo, "default") 1775 if err: 1776 return err 1777 return None 1778 1779def sync(ui, repo, **opts): 1780 """synchronize with remote repository 1781 1782 Incorporates recent changes from the remote repository 1783 into the local repository. 1784 """ 1785 if missing_codereview: 1786 return missing_codereview 1787 1788 if not opts["local"]: 1789 ui.status = sync_note 1790 ui.note = sync_note 1791 other = getremote(ui, repo, opts) 1792 modheads = repo.pull(other) 1793 err = commands.postincoming(ui, repo, modheads, True, "tip") 1794 if err: 1795 return err 1796 commands.update(ui, repo, rev="default") 1797 sync_changes(ui, repo) 1798 1799def sync_note(msg): 1800 # we run sync (pull -u) in verbose mode to get the 1801 # list of files being updated, but that drags along 1802 # a bunch of messages we don't care about. 1803 # omit them. 1804 if msg == 'resolving manifests\n': 1805 return 1806 if msg == 'searching for changes\n': 1807 return 1808 if msg == "couldn't find merge tool hgmerge\n": 1809 return 1810 sys.stdout.write(msg) 1811 1812def sync_changes(ui, repo): 1813 # Look through recent change log descriptions to find 1814 # potential references to http://.*/our-CL-number. 1815 # Double-check them by looking at the Rietveld log. 1816 def Rev(rev): 1817 desc = repo[rev].description().strip() 1818 for clname in re.findall('(?m)^http://(?:[^\n]+)/([0-9]+)$', desc): 1819 if IsLocalCL(ui, repo, clname) and IsRietveldSubmitted(ui, clname, repo[rev].hex()): 1820 ui.warn("CL %s submitted as %s; closing\n" % (clname, repo[rev])) 1821 cl, err = LoadCL(ui, repo, clname, web=False) 1822 if err != "": 1823 ui.warn("loading CL %s: %s\n" % (clname, err)) 1824 continue 1825 if not cl.copied_from: 1826 EditDesc(cl.name, closed=True, private=cl.private) 1827 cl.Delete(ui, repo) 1828 1829 if hgversion < '1.4': 1830 get = util.cachefunc(lambda r: repo[r].changeset()) 1831 changeiter, matchfn = cmdutil.walkchangerevs(ui, repo, [], get, {'rev': None}) 1832 n = 0 1833 for st, rev, fns in changeiter: 1834 if st != 'iter': 1835 continue 1836 n += 1 1837 if n > 100: 1838 break 1839 Rev(rev) 1840 else: 1841 matchfn = scmutil.match(repo, [], {'rev': None}) 1842 def prep(ctx, fns): 1843 pass 1844 for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev': None}, prep): 1845 Rev(ctx.rev()) 1846 1847 # Remove files that are not modified from the CLs in which they appear. 1848 all = LoadAllCL(ui, repo, web=False) 1849 changed = ChangedFiles(ui, repo, [], {}) 1850 for _, cl in all.items(): 1851 extra = Sub(cl.files, changed) 1852 if extra: 1853 ui.warn("Removing unmodified files from CL %s:\n" % (cl.name,)) 1854 for f in extra: 1855 ui.warn("\t%s\n" % (f,)) 1856 cl.files = Sub(cl.files, extra) 1857 cl.Flush(ui, repo) 1858 if not cl.files: 1859 if not cl.copied_from: 1860 ui.warn("CL %s has no files; delete (abandon) with hg change -d %s\n" % (cl.name, cl.name)) 1861 else: 1862 ui.warn("CL %s has no files; delete locally with hg change -D %s\n" % (cl.name, cl.name)) 1863 return 1864 1865def upload(ui, repo, name, **opts): 1866 """upload diffs to the code review server 1867 1868 Uploads the current modifications for a given change to the server. 1869 """ 1870 if missing_codereview: 1871 return missing_codereview 1872 1873 repo.ui.quiet = True 1874 cl, err = LoadCL(ui, repo, name, web=True) 1875 if err != "": 1876 return err 1877 if not cl.local: 1878 return "cannot upload non-local change" 1879 cl.Upload(ui, repo) 1880 print "%s%s\n" % (server_url_base, cl.name) 1881 return 1882 1883review_opts = [ 1884 ('r', 'reviewer', '', 'add reviewer'), 1885 ('', 'cc', '', 'add cc'), 1886 ('', 'tbr', '', 'add future reviewer'), 1887 ('m', 'message', '', 'change description (for new change)'), 1888] 1889 1890cmdtable = { 1891 # The ^ means to show this command in the help text that 1892 # is printed when running hg with no arguments. 1893 "^change": ( 1894 change, 1895 [ 1896 ('d', 'delete', None, 'delete existing change list'), 1897 ('D', 'deletelocal', None, 'delete locally, but do not change CL on server'), 1898 ('i', 'stdin', None, 'read change list from standard input'), 1899 ('o', 'stdout', None, 'print change list to standard output'), 1900 ('p', 'pending', None, 'print pending summary to standard output'), 1901 ], 1902 "[-d | -D] [-i] [-o] change# or FILE ..." 1903 ), 1904 "^clpatch": ( 1905 clpatch, 1906 [ 1907 ('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'), 1908 ('', 'no_incoming', None, 'disable check for incoming changes'), 1909 ], 1910 "change#" 1911 ), 1912 # Would prefer to call this codereview-login, but then 1913 # hg help codereview prints the help for this command 1914 # instead of the help for the extension. 1915 "code-login": ( 1916 code_login, 1917 [], 1918 "", 1919 ), 1920 "^download": ( 1921 download, 1922 [], 1923 "change#" 1924 ), 1925 "^file": ( 1926 file, 1927 [ 1928 ('d', 'delete', None, 'delete files from change list (but not repository)'), 1929 ], 1930 "[-d] change# FILE ..." 1931 ), 1932 "^gofmt": ( 1933 gofmt, 1934 [ 1935 ('l', 'list', None, 'list files that would change, but do not edit them'), 1936 ], 1937 "FILE ..." 1938 ), 1939 "^pending|p": ( 1940 pending, 1941 [], 1942 "[FILE ...]" 1943 ), 1944 "^mail": ( 1945 mail, 1946 review_opts + [ 1947 ] + commands.walkopts, 1948 "[-r reviewer] [--cc cc] [change# | file ...]" 1949 ), 1950 "^release-apply": ( 1951 release_apply, 1952 [ 1953 ('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'), 1954 ('', 'no_incoming', None, 'disable check for incoming changes'), 1955 ], 1956 "change#" 1957 ), 1958 # TODO: release-start, release-tag, weekly-tag 1959 "^submit": ( 1960 submit, 1961 review_opts + [ 1962 ('', 'no_incoming', None, 'disable initial incoming check (for testing)'), 1963 ('n', 'dryrun', None, 'make change only locally (for testing)'), 1964 ] + commands.walkopts + commands.commitopts + commands.commitopts2, 1965 "[-r reviewer] [--cc cc] [change# | file ...]" 1966 ), 1967 "^sync": ( 1968 sync, 1969 [ 1970 ('', 'local', None, 'do not pull changes from remote repository') 1971 ], 1972 "[--local]", 1973 ), 1974 "^undo": ( 1975 undo, 1976 [ 1977 ('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'), 1978 ('', 'no_incoming', None, 'disable check for incoming changes'), 1979 ], 1980 "change#" 1981 ), 1982 "^upload": ( 1983 upload, 1984 [], 1985 "change#" 1986 ), 1987} 1988 1989 1990####################################################################### 1991# Wrappers around upload.py for interacting with Rietveld 1992 1993# HTML form parser 1994class FormParser(HTMLParser): 1995 def __init__(self): 1996 self.map = {} 1997 self.curtag = None 1998 self.curdata = None 1999 HTMLParser.__init__(self) 2000 def handle_starttag(self, tag, attrs): 2001 if tag == "input": 2002 key = None 2003 value = '' 2004 for a in attrs: 2005 if a[0] == 'name': 2006 key = a[1] 2007 if a[0] == 'value': 2008 value = a[1] 2009 if key is not None: 2010 self.map[key] = value 2011 if tag == "textarea": 2012 key = None 2013 for a in attrs: 2014 if a[0] == 'name': 2015 key = a[1] 2016 if key is not None: 2017 self.curtag = key 2018 self.curdata = '' 2019 def handle_endtag(self, tag): 2020 if tag == "textarea" and self.curtag is not None: 2021 self.map[self.curtag] = self.curdata 2022 self.curtag = None 2023 self.curdata = None 2024 def handle_charref(self, name): 2025 self.handle_data(unichr(int(name))) 2026 def handle_entityref(self, name): 2027 import htmlentitydefs 2028 if name in htmlentitydefs.entitydefs: 2029 self.handle_data(htmlentitydefs.entitydefs[name]) 2030 else: 2031 self.handle_data("&" + name + ";") 2032 def handle_data(self, data): 2033 if self.curdata is not None: 2034 self.curdata += data 2035 2036def JSONGet(ui, path): 2037 try: 2038 data = MySend(path, force_auth=False) 2039 typecheck(data, str) 2040 d = fix_json(json.loads(data)) 2041 except: 2042 ui.warn("JSONGet %s: %s\n" % (path, ExceptionDetail())) 2043 return None 2044 return d 2045 2046# Clean up json parser output to match our expectations: 2047# * all strings are UTF-8-encoded str, not unicode. 2048# * missing fields are missing, not None, 2049# so that d.get("foo", defaultvalue) works. 2050def fix_json(x): 2051 if type(x) in [str, int, float, bool, type(None)]: 2052 pass 2053 elif type(x) is unicode: 2054 x = x.encode("utf-8") 2055 elif type(x) is list: 2056 for i in range(len(x)): 2057 x[i] = fix_json(x[i]) 2058 elif type(x) is dict: 2059 todel = [] 2060 for k in x: 2061 if x[k] is None: 2062 todel.append(k) 2063 else: 2064 x[k] = fix_json(x[k]) 2065 for k in todel: 2066 del x[k] 2067 else: 2068 raise util.Abort("unknown type " + str(type(x)) + " in fix_json") 2069 if type(x) is str: 2070 x = x.replace('\r\n', '\n') 2071 return x 2072 2073def IsRietveldSubmitted(ui, clname, hex): 2074 dict = JSONGet(ui, "/api/" + clname + "?messages=true") 2075 if dict is None: 2076 return False 2077 for msg in dict.get("messages", []): 2078 text = msg.get("text", "") 2079 m = re.match('\*\*\* Submitted as [^*]*?([0-9a-f]+) \*\*\*', text) 2080 if m is not None and len(m.group(1)) >= 8 and hex.startswith(m.group(1)): 2081 return True 2082 return False 2083 2084def IsRietveldMailed(cl): 2085 for msg in cl.dict.get("messages", []): 2086 if msg.get("text", "").find("I'd like you to review this change") >= 0: 2087 return True 2088 return False 2089 2090def DownloadCL(ui, repo, clname): 2091 set_status("downloading CL " + clname) 2092 cl, err = LoadCL(ui, repo, clname, web=True) 2093 if err != "": 2094 return None, None, None, "error loading CL %s: %s" % (clname, err) 2095 2096 # Find most recent diff 2097 diffs = cl.dict.get("patchsets", []) 2098 if not diffs: 2099 return None, None, None, "CL has no patch sets" 2100 patchid = diffs[-1] 2101 2102 patchset = JSONGet(ui, "/api/" + clname + "/" + str(patchid)) 2103 if patchset is None: 2104 return None, None, None, "error loading CL patchset %s/%d" % (clname, patchid) 2105 if patchset.get("patchset", 0) != patchid: 2106 return None, None, None, "malformed patchset information" 2107 2108 vers = "" 2109 msg = patchset.get("message", "").split() 2110 if len(msg) >= 3 and msg[0] == "diff" and msg[1] == "-r": 2111 vers = msg[2] 2112 diff = "/download/issue" + clname + "_" + str(patchid) + ".diff" 2113 2114 diffdata = MySend(diff, force_auth=False) 2115 2116 # Print warning if email is not in CONTRIBUTORS file. 2117 email = cl.dict.get("owner_email", "") 2118 if not email: 2119 return None, None, None, "cannot find owner for %s" % (clname) 2120 him = FindContributor(ui, repo, email) 2121 me = FindContributor(ui, repo, None) 2122 if him == me: 2123 cl.mailed = IsRietveldMailed(cl) 2124 else: 2125 cl.copied_from = email 2126 2127 return cl, vers, diffdata, "" 2128 2129def MySend(request_path, payload=None, 2130 content_type="application/octet-stream", 2131 timeout=None, force_auth=True, 2132 **kwargs): 2133 """Run MySend1 maybe twice, because Rietveld is unreliable.""" 2134 try: 2135 return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs) 2136 except Exception, e: 2137 if type(e) != urllib2.HTTPError or e.code != 500: # only retry on HTTP 500 error 2138 raise 2139 print >>sys.stderr, "Loading "+request_path+": "+ExceptionDetail()+"; trying again in 2 seconds." 2140 time.sleep(2) 2141 return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs) 2142 2143# Like upload.py Send but only authenticates when the 2144# redirect is to www.google.com/accounts. This keeps 2145# unnecessary redirects from happening during testing. 2146def MySend1(request_path, payload=None, 2147 content_type="application/octet-stream", 2148 timeout=None, force_auth=True, 2149 **kwargs): 2150 """Sends an RPC and returns the response. 2151 2152 Args: 2153 request_path: The path to send the request to, eg /api/appversion/create. 2154 payload: The body of the request, or None to send an empty request. 2155 content_type: The Content-Type header to use. 2156 timeout: timeout in seconds; default None i.e. no timeout. 2157 (Note: for large requests on OS X, the timeout doesn't work right.) 2158 kwargs: Any keyword arguments are converted into query string parameters. 2159 2160 Returns: 2161 The response body, as a string. 2162 """ 2163 # TODO: Don't require authentication. Let the server say 2164 # whether it is necessary. 2165 global rpc 2166 if rpc == None: 2167 rpc = GetRpcServer(upload_options) 2168 self = rpc 2169 if not self.authenticated and force_auth: 2170 self._Authenticate() 2171 if request_path is None: 2172 return 2173 2174 old_timeout = socket.getdefaulttimeout() 2175 socket.setdefaulttimeout(timeout) 2176 try: 2177 tries = 0 2178 while True: 2179 tries += 1 2180 args = dict(kwargs) 2181 url = "http://%s%s" % (self.host, request_path) 2182 if args: 2183 url += "?" + urllib.urlencode(args) 2184 req = self._CreateRequest(url=url, data=payload) 2185 req.add_header("Content-Type", content_type) 2186 try: 2187 f = self.opener.open(req) 2188 response = f.read() 2189 f.close() 2190 # Translate \r\n into \n, because Rietveld doesn't. 2191 response = response.replace('\r\n', '\n') 2192 # who knows what urllib will give us 2193 if type(response) == unicode: 2194 response = response.encode("utf-8") 2195 typecheck(response, str) 2196 return response 2197 except urllib2.HTTPError, e: 2198 if tries > 3: 2199 raise 2200 elif e.code == 401: 2201 self._Authenticate() 2202 elif e.code == 302: 2203 loc = e.info()["location"] 2204 if not loc.startswith('https://www.google.com/a') or loc.find('/ServiceLogin') < 0: 2205 return '' 2206 self._Authenticate() 2207 else: 2208 raise 2209 finally: 2210 socket.setdefaulttimeout(old_timeout) 2211 2212def GetForm(url): 2213 f = FormParser() 2214 f.feed(ustr(MySend(url))) # f.feed wants unicode 2215 f.close() 2216 # convert back to utf-8 to restore sanity 2217 m = {} 2218 for k,v in f.map.items(): 2219 m[k.encode("utf-8")] = v.replace("\r\n", "\n").encode("utf-8") 2220 return m 2221 2222def EditDesc(issue, subject=None, desc=None, reviewers=None, cc=None, closed=False, private=False): 2223 set_status("uploading change to description") 2224 form_fields = GetForm("/" + issue + "/edit") 2225 if subject is not None: 2226 form_fields['subject'] = subject 2227 if desc is not None: 2228 form_fields['description'] = desc 2229 if reviewers is not None: 2230 form_fields['reviewers'] = reviewers 2231 if cc is not None: 2232 form_fields['cc'] = cc 2233 if closed: 2234 form_fields['closed'] = "checked" 2235 if private: 2236 form_fields['private'] = "checked" 2237 ctype, body = EncodeMultipartFormData(form_fields.items(), []) 2238 response = MySend("/" + issue + "/edit", body, content_type=ctype) 2239 if response != "": 2240 print >>sys.stderr, "Error editing description:\n" + "Sent form: \n", form_fields, "\n", response 2241 sys.exit(2) 2242 2243def PostMessage(ui, issue, message, reviewers=None, cc=None, send_mail=True, subject=None): 2244 set_status("uploading message") 2245 form_fields = GetForm("/" + issue + "/publish") 2246 if reviewers is not None: 2247 form_fields['reviewers'] = reviewers 2248 if cc is not None: 2249 form_fields['cc'] = cc 2250 if send_mail: 2251 form_fields['send_mail'] = "checked" 2252 else: 2253 del form_fields['send_mail'] 2254 if subject is not None: 2255 form_fields['subject'] = subject 2256 form_fields['message'] = message 2257 2258 form_fields['message_only'] = '1' # Don't include draft comments 2259 if reviewers is not None or cc is not None: 2260 form_fields['message_only'] = '' # Must set '' in order to override cc/reviewer 2261 ctype = "applications/x-www-form-urlencoded" 2262 body = urllib.urlencode(form_fields) 2263 response = MySend("/" + issue + "/publish", body, content_type=ctype) 2264 if response != "": 2265 print response 2266 sys.exit(2) 2267 2268class opt(object): 2269 pass 2270 2271def nocommit(*pats, **opts): 2272 """(disabled when using this extension)""" 2273 raise util.Abort("codereview extension enabled; use mail, upload, or submit instead of commit") 2274 2275def nobackout(*pats, **opts): 2276 """(disabled when using this extension)""" 2277 raise util.Abort("codereview extension enabled; use undo instead of backout") 2278 2279def norollback(*pats, **opts): 2280 """(disabled when using this extension)""" 2281 raise util.Abort("codereview extension enabled; use undo instead of rollback") 2282 2283def RietveldSetup(ui, repo): 2284 global defaultcc, upload_options, rpc, server, server_url_base, force_google_account, verbosity, contributors 2285 global missing_codereview 2286 2287 repo_config_path = '' 2288 # Read repository-specific options from lib/codereview/codereview.cfg 2289 try: 2290 repo_config_path = repo.root + '/lib/codereview/codereview.cfg' 2291 f = open(repo_config_path) 2292 for line in f: 2293 if line.startswith('defaultcc: '): 2294 defaultcc = SplitCommaSpace(line[10:]) 2295 except: 2296 # If there are no options, chances are good this is not 2297 # a code review repository; stop now before we foul 2298 # things up even worse. Might also be that repo doesn't 2299 # even have a root. See issue 959. 2300 if repo_config_path == '': 2301 missing_codereview = 'codereview disabled: repository has no root' 2302 else: 2303 missing_codereview = 'codereview disabled: cannot open ' + repo_config_path 2304 return 2305 2306 # Should only modify repository with hg submit. 2307 # Disable the built-in Mercurial commands that might 2308 # trip things up. 2309 cmdutil.commit = nocommit 2310 global real_rollback 2311 real_rollback = repo.rollback 2312 repo.rollback = norollback 2313 # would install nobackout if we could; oh well 2314 2315 try: 2316 f = open(repo.root + '/CONTRIBUTORS', 'r') 2317 except: 2318 raise util.Abort("cannot open %s: %s" % (repo.root+'/CONTRIBUTORS', ExceptionDetail())) 2319 for line in f: 2320 # CONTRIBUTORS is a list of lines like: 2321 # Person <email> 2322 # Person <email> <alt-email> 2323 # The first email address is the one used in commit logs. 2324 if line.startswith('#'): 2325 continue 2326 m = re.match(r"([^<>]+\S)\s+(<[^<>\s]+>)((\s+<[^<>\s]+>)*)\s*$", line) 2327 if m: 2328 name = m.group(1) 2329 email = m.group(2)[1:-1] 2330 contributors[email.lower()] = (name, email) 2331 for extra in m.group(3).split(): 2332 contributors[extra[1:-1].lower()] = (name, email) 2333 2334 if not ui.verbose: 2335 verbosity = 0 2336 2337 # Config options. 2338 x = ui.config("codereview", "server") 2339 if x is not None: 2340 server = x 2341 2342 # TODO(rsc): Take from ui.username? 2343 email = None 2344 x = ui.config("codereview", "email") 2345 if x is not None: 2346 email = x 2347 2348 server_url_base = "http://" + server + "/" 2349 2350 testing = ui.config("codereview", "testing") 2351 force_google_account = ui.configbool("codereview", "force_google_account", False) 2352 2353 upload_options = opt() 2354 upload_options.email = email 2355 upload_options.host = None 2356 upload_options.verbose = 0 2357 upload_options.description = None 2358 upload_options.description_file = None 2359 upload_options.reviewers = None 2360 upload_options.cc = None 2361 upload_options.message = None 2362 upload_options.issue = None 2363 upload_options.download_base = False 2364 upload_options.revision = None 2365 upload_options.send_mail = False 2366 upload_options.vcs = None 2367 upload_options.server = server 2368 upload_options.save_cookies = True 2369 2370 if testing: 2371 upload_options.save_cookies = False 2372 upload_options.email = "test@example.com" 2373 2374 rpc = None 2375 2376 global releaseBranch 2377 tags = repo.branchtags().keys() 2378 if 'release-branch.r100' in tags: 2379 # NOTE(rsc): This tags.sort is going to get the wrong 2380 # answer when comparing release-branch.r99 with 2381 # release-branch.r100. If we do ten releases a year 2382 # that gives us 4 years before we have to worry about this. 2383 raise util.Abort('tags.sort needs to be fixed for release-branch.r100') 2384 tags.sort() 2385 for t in tags: 2386 if t.startswith('release-branch.'): 2387 releaseBranch = t 2388 2389####################################################################### 2390# http://codereview.appspot.com/static/upload.py, heavily edited. 2391 2392#!/usr/bin/env python 2393# 2394# Copyright 2007 Google Inc. 2395# 2396# Licensed under the Apache License, Version 2.0 (the "License"); 2397# you may not use this file except in compliance with the License. 2398# You may obtain a copy of the License at 2399# 2400# http://www.apache.org/licenses/LICENSE-2.0 2401# 2402# Unless required by applicable law or agreed to in writing, software 2403# distributed under the License is distributed on an "AS IS" BASIS, 2404# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 2405# See the License for the specific language governing permissions and 2406# limitations under the License. 2407 2408"""Tool for uploading diffs from a version control system to the codereview app. 2409 2410Usage summary: upload.py [options] [-- diff_options] 2411 2412Diff options are passed to the diff command of the underlying system. 2413 2414Supported version control systems: 2415 Git 2416 Mercurial 2417 Subversion 2418 2419It is important for Git/Mercurial users to specify a tree/node/branch to diff 2420against by using the '--rev' option. 2421""" 2422# This code is derived from appcfg.py in the App Engine SDK (open source), 2423# and from ASPN recipe #146306. 2424 2425import cookielib 2426import getpass 2427import logging 2428import mimetypes 2429import optparse 2430import os 2431import re 2432import socket 2433import subprocess 2434import sys 2435import urllib 2436import urllib2 2437import urlparse 2438 2439# The md5 module was deprecated in Python 2.5. 2440try: 2441 from hashlib import md5 2442except ImportError: 2443 from md5 import md5 2444 2445try: 2446 import readline 2447except ImportError: 2448 pass 2449 2450# The logging verbosity: 2451# 0: Errors only. 2452# 1: Status messages. 2453# 2: Info logs. 2454# 3: Debug logs. 2455verbosity = 1 2456 2457# Max size of patch or base file. 2458MAX_UPLOAD_SIZE = 900 * 1024 2459 2460# whitelist for non-binary filetypes which do not start with "text/" 2461# .mm (Objective-C) shows up as application/x-freemind on my Linux box. 2462TEXT_MIMETYPES = [ 2463 'application/javascript', 2464 'application/x-javascript', 2465 'application/x-freemind' 2466] 2467 2468def GetEmail(prompt): 2469 """Prompts the user for their email address and returns it. 2470 2471 The last used email address is saved to a file and offered up as a suggestion 2472 to the user. If the user presses enter without typing in anything the last 2473 used email address is used. If the user enters a new address, it is saved 2474 for next time we prompt. 2475 2476 """ 2477 last_email_file_name = os.path.expanduser("~/.last_codereview_email_address") 2478 last_email = "" 2479 if os.path.exists(last_email_file_name): 2480 try: 2481 last_email_file = open(last_email_file_name, "r") 2482 last_email = last_email_file.readline().strip("\n") 2483 last_email_file.close() 2484 prompt += " [%s]" % last_email 2485 except IOError, e: 2486 pass 2487 email = raw_input(prompt + ": ").strip() 2488 if email: 2489 try: 2490 last_email_file = open(last_email_file_name, "w") 2491 last_email_file.write(email) 2492 last_email_file.close() 2493 except IOError, e: 2494 pass 2495 else: 2496 email = last_email 2497 return email 2498 2499 2500def StatusUpdate(msg): 2501 """Print a status message to stdout. 2502 2503 If 'verbosity' is greater than 0, print the message. 2504 2505 Args: 2506 msg: The string to print. 2507 """ 2508 if verbosity > 0: 2509 print msg 2510 2511 2512def ErrorExit(msg): 2513 """Print an error message to stderr and exit.""" 2514 print >>sys.stderr, msg 2515 sys.exit(1) 2516 2517 2518class ClientLoginError(urllib2.HTTPError): 2519 """Raised to indicate there was an error authenticating with ClientLogin.""" 2520 2521 def __init__(self, url, code, msg, headers, args): 2522 urllib2.HTTPError.__init__(self, url, code, msg, headers, None) 2523 self.args = args 2524 self.reason = args["Error"] 2525 2526 2527class AbstractRpcServer(object): 2528 """Provides a common interface for a simple RPC server.""" 2529 2530 def __init__(self, host, auth_function, host_override=None, extra_headers={}, save_cookies=False): 2531 """Creates a new HttpRpcServer. 2532 2533 Args: 2534 host: The host to send requests to. 2535 auth_function: A function that takes no arguments and returns an 2536 (email, password) tuple when called. Will be called if authentication 2537 is required. 2538 host_override: The host header to send to the server (defaults to host). 2539 extra_headers: A dict of extra headers to append to every request. 2540 save_cookies: If True, save the authentication cookies to local disk. 2541 If False, use an in-memory cookiejar instead. Subclasses must 2542 implement this functionality. Defaults to False. 2543 """ 2544 self.host = host 2545 self.host_override = host_override 2546 self.auth_function = auth_function 2547 self.authenticated = False 2548 self.extra_headers = extra_headers 2549 self.save_cookies = save_cookies 2550 self.opener = self._GetOpener() 2551 if self.host_override: 2552 logging.info("Server: %s; Host: %s", self.host, self.host_override) 2553 else: 2554 logging.info("Server: %s", self.host) 2555 2556 def _GetOpener(self): 2557 """Returns an OpenerDirector for making HTTP requests. 2558 2559 Returns: 2560 A urllib2.OpenerDirector object. 2561 """ 2562 raise NotImplementedError() 2563 2564 def _CreateRequest(self, url, data=None): 2565 """Creates a new urllib request.""" 2566 logging.debug("Creating request for: '%s' with payload:\n%s", url, data) 2567 req = urllib2.Request(url, data=data) 2568 if self.host_override: 2569 req.add_header("Host", self.host_override) 2570 for key, value in self.extra_headers.iteritems(): 2571 req.add_header(key, value) 2572 return req 2573 2574 def _GetAuthToken(self, email, password): 2575 """Uses ClientLogin to authenticate the user, returning an auth token. 2576 2577 Args: 2578 email: The user's email address 2579 password: The user's password 2580 2581 Raises: 2582 ClientLoginError: If there was an error authenticating with ClientLogin. 2583 HTTPError: If there was some other form of HTTP error. 2584 2585 Returns: 2586 The authentication token returned by ClientLogin. 2587 """ 2588 account_type = "GOOGLE" 2589 if self.host.endswith(".google.com") and not force_google_account: 2590 # Needed for use inside Google. 2591 account_type = "HOSTED" 2592 req = self._CreateRequest( 2593 url="https://www.google.com/accounts/ClientLogin", 2594 data=urllib.urlencode({ 2595 "Email": email, 2596 "Passwd": password, 2597 "service": "ah", 2598 "source": "rietveld-codereview-upload", 2599 "accountType": account_type, 2600 }), 2601 ) 2602 try: 2603 response = self.opener.open(req) 2604 response_body = response.read() 2605 response_dict = dict(x.split("=") for x in response_body.split("\n") if x) 2606 return response_dict["Auth"] 2607 except urllib2.HTTPError, e: 2608 if e.code == 403: 2609 body = e.read() 2610 response_dict = dict(x.split("=", 1) for x in body.split("\n") if x) 2611 raise ClientLoginError(req.get_full_url(), e.code, e.msg, e.headers, response_dict) 2612 else: 2613 raise 2614 2615 def _GetAuthCookie(self, auth_token): 2616 """Fetches authentication cookies for an authentication token. 2617 2618 Args: 2619 auth_token: The authentication token returned by ClientLogin. 2620 2621 Raises: 2622 HTTPError: If there was an error fetching the authentication cookies. 2623 """ 2624 # This is a dummy value to allow us to identify when we're successful. 2625 continue_location = "http://localhost/" 2626 args = {"continue": continue_location, "auth": auth_token} 2627 req = self._CreateRequest("http://%s/_ah/login?%s" % (self.host, urllib.urlencode(args))) 2628 try: 2629 response = self.opener.open(req) 2630 except urllib2.HTTPError, e: 2631 response = e 2632 if (response.code != 302 or 2633 response.info()["location"] != continue_location): 2634 raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg, response.headers, response.fp) 2635 self.authenticated = True 2636 2637 def _Authenticate(self): 2638 """Authenticates the user. 2639 2640 The authentication process works as follows: 2641 1) We get a username and password from the user 2642 2) We use ClientLogin to obtain an AUTH token for the user 2643 (see http://code.google.com/apis/accounts/AuthForInstalledApps.html). 2644 3) We pass the auth token to /_ah/login on the server to obtain an 2645 authentication cookie. If login was successful, it tries to redirect 2646 us to the URL we provided. 2647 2648 If we attempt to access the upload API without first obtaining an 2649 authentication cookie, it returns a 401 response (or a 302) and 2650 directs us to authenticate ourselves with ClientLogin. 2651 """ 2652 for i in range(3): 2653 credentials = self.auth_function() 2654 try: 2655 auth_token = self._GetAuthToken(credentials[0], credentials[1]) 2656 except ClientLoginError, e: 2657 if e.reason == "BadAuthentication": 2658 print >>sys.stderr, "Invalid username or password." 2659 continue 2660 if e.reason == "CaptchaRequired": 2661 print >>sys.stderr, ( 2662 "Please go to\n" 2663 "https://www.google.com/accounts/DisplayUnlockCaptcha\n" 2664 "and verify you are a human. Then try again.") 2665 break 2666 if e.reason == "NotVerified": 2667 print >>sys.stderr, "Account not verified." 2668 break 2669 if e.reason == "TermsNotAgreed": 2670 print >>sys.stderr, "User has not agreed to TOS." 2671 break 2672 if e.reason == "AccountDeleted": 2673 print >>sys.stderr, "The user account has been deleted." 2674 break 2675 if e.reason == "AccountDisabled": 2676 print >>sys.stderr, "The user account has been disabled." 2677 break 2678 if e.reason == "ServiceDisabled": 2679 print >>sys.stderr, "The user's access to the service has been disabled." 2680 break 2681 if e.reason == "ServiceUnavailable": 2682 print >>sys.stderr, "The service is not available; try again later." 2683 break 2684 raise 2685 self._GetAuthCookie(auth_token) 2686 return 2687 2688 def Send(self, request_path, payload=None, 2689 content_type="application/octet-stream", 2690 timeout=None, 2691 **kwargs): 2692 """Sends an RPC and returns the response. 2693 2694 Args: 2695 request_path: The path to send the request to, eg /api/appversion/create. 2696 payload: The body of the request, or None to send an empty request. 2697 content_type: The Content-Type header to use. 2698 timeout: timeout in seconds; default None i.e. no timeout. 2699 (Note: for large requests on OS X, the timeout doesn't work right.) 2700 kwargs: Any keyword arguments are converted into query string parameters. 2701 2702 Returns: 2703 The response body, as a string. 2704 """ 2705 # TODO: Don't require authentication. Let the server say 2706 # whether it is necessary. 2707 if not self.authenticated: 2708 self._Authenticate() 2709 2710 old_timeout = socket.getdefaulttimeout() 2711 socket.setdefaulttimeout(timeout) 2712 try: 2713 tries = 0 2714 while True: 2715 tries += 1 2716 args = dict(kwargs) 2717 url = "http://%s%s" % (self.host, request_path) 2718 if args: 2719 url += "?" + urllib.urlencode(args) 2720 req = self._CreateRequest(url=url, data=payload) 2721 req.add_header("Content-Type", content_type) 2722 try: 2723 f = self.opener.open(req) 2724 response = f.read() 2725 f.close() 2726 return response 2727 except urllib2.HTTPError, e: 2728 if tries > 3: 2729 raise 2730 elif e.code == 401 or e.code == 302: 2731 self._Authenticate() 2732 else: 2733 raise 2734 finally: 2735 socket.setdefaulttimeout(old_timeout) 2736 2737 2738class HttpRpcServer(AbstractRpcServer): 2739 """Provides a simplified RPC-style interface for HTTP requests.""" 2740 2741 def _Authenticate(self): 2742 """Save the cookie jar after authentication.""" 2743 super(HttpRpcServer, self)._Authenticate() 2744 if self.save_cookies: 2745 StatusUpdate("Saving authentication cookies to %s" % self.cookie_file) 2746 self.cookie_jar.save() 2747 2748 def _GetOpener(self): 2749 """Returns an OpenerDirector that supports cookies and ignores redirects. 2750 2751 Returns: 2752 A urllib2.OpenerDirector object. 2753 """ 2754 opener = urllib2.OpenerDirector() 2755 opener.add_handler(urllib2.ProxyHandler()) 2756 opener.add_handler(urllib2.UnknownHandler()) 2757 opener.add_handler(urllib2.HTTPHandler()) 2758 opener.add_handler(urllib2.HTTPDefaultErrorHandler()) 2759 opener.add_handler(urllib2.HTTPSHandler()) 2760 opener.add_handler(urllib2.HTTPErrorProcessor()) 2761 if self.save_cookies: 2762 self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies_" + server) 2763 self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file) 2764 if os.path.exists(self.cookie_file): 2765 try: 2766 self.cookie_jar.load() 2767 self.authenticated = True 2768 StatusUpdate("Loaded authentication cookies from %s" % self.cookie_file) 2769 except (cookielib.LoadError, IOError): 2770 # Failed to load cookies - just ignore them. 2771 pass 2772 else: 2773 # Create an empty cookie file with mode 600 2774 fd = os.open(self.cookie_file, os.O_CREAT, 0600) 2775 os.close(fd) 2776 # Always chmod the cookie file 2777 os.chmod(self.cookie_file, 0600) 2778 else: 2779 # Don't save cookies across runs of update.py. 2780 self.cookie_jar = cookielib.CookieJar() 2781 opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar)) 2782 return opener 2783 2784 2785def GetRpcServer(options): 2786 """Returns an instance of an AbstractRpcServer. 2787 2788 Returns: 2789 A new AbstractRpcServer, on which RPC calls can be made. 2790 """ 2791 2792 rpc_server_class = HttpRpcServer 2793 2794 def GetUserCredentials(): 2795 """Prompts the user for a username and password.""" 2796 # Disable status prints so they don't obscure the password prompt. 2797 global global_status 2798 st = global_status 2799 global_status = None 2800 2801 email = options.email 2802 if email is None: 2803 email = GetEmail("Email (login for uploading to %s)" % options.server) 2804 password = getpass.getpass("Password for %s: " % email) 2805 2806 # Put status back. 2807 global_status = st 2808 return (email, password) 2809 2810 # If this is the dev_appserver, use fake authentication. 2811 host = (options.host or options.server).lower() 2812 if host == "localhost" or host.startswith("localhost:"): 2813 email = options.email 2814 if email is None: 2815 email = "test@example.com" 2816 logging.info("Using debug user %s. Override with --email" % email) 2817 server = rpc_server_class( 2818 options.server, 2819 lambda: (email, "password"), 2820 host_override=options.host, 2821 extra_headers={"Cookie": 'dev_appserver_login="%s:False"' % email}, 2822 save_cookies=options.save_cookies) 2823 # Don't try to talk to ClientLogin. 2824 server.authenticated = True 2825 return server 2826 2827 return rpc_server_class(options.server, GetUserCredentials, 2828 host_override=options.host, save_cookies=options.save_cookies) 2829 2830 2831def EncodeMultipartFormData(fields, files): 2832 """Encode form fields for multipart/form-data. 2833 2834 Args: 2835 fields: A sequence of (name, value) elements for regular form fields. 2836 files: A sequence of (name, filename, value) elements for data to be 2837 uploaded as files. 2838 Returns: 2839 (content_type, body) ready for httplib.HTTP instance. 2840 2841 Source: 2842 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306 2843 """ 2844 BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-' 2845 CRLF = '\r\n' 2846 lines = [] 2847 for (key, value) in fields: 2848 typecheck(key, str) 2849 typecheck(value, str) 2850 lines.append('--' + BOUNDARY) 2851 lines.append('Content-Disposition: form-data; name="%s"' % key) 2852 lines.append('') 2853 lines.append(value) 2854 for (key, filename, value) in files: 2855 typecheck(key, str) 2856 typecheck(filename, str) 2857 typecheck(value, str) 2858 lines.append('--' + BOUNDARY) 2859 lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename)) 2860 lines.append('Content-Type: %s' % GetContentType(filename)) 2861 lines.append('') 2862 lines.append(value) 2863 lines.append('--' + BOUNDARY + '--') 2864 lines.append('') 2865 body = CRLF.join(lines) 2866 content_type = 'multipart/form-data; boundary=%s' % BOUNDARY 2867 return content_type, body 2868 2869 2870def GetContentType(filename): 2871 """Helper to guess the content-type from the filename.""" 2872 return mimetypes.guess_type(filename)[0] or 'application/octet-stream' 2873 2874 2875# Use a shell for subcommands on Windows to get a PATH search. 2876use_shell = sys.platform.startswith("win") 2877 2878def RunShellWithReturnCode(command, print_output=False, 2879 universal_newlines=True, env=os.environ): 2880 """Executes a command and returns the output from stdout and the return code. 2881 2882 Args: 2883 command: Command to execute. 2884 print_output: If True, the output is printed to stdout. 2885 If False, both stdout and stderr are ignored. 2886 universal_newlines: Use universal_newlines flag (default: True). 2887 2888 Returns: 2889 Tuple (output, return code) 2890 """ 2891 logging.info("Running %s", command) 2892 p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, 2893 shell=use_shell, universal_newlines=universal_newlines, env=env) 2894 if print_output: 2895 output_array = [] 2896 while True: 2897 line = p.stdout.readline() 2898 if not line: 2899 break 2900 print line.strip("\n") 2901 output_array.append(line) 2902 output = "".join(output_array) 2903 else: 2904 output = p.stdout.read() 2905 p.wait() 2906 errout = p.stderr.read() 2907 if print_output and errout: 2908 print >>sys.stderr, errout 2909 p.stdout.close() 2910 p.stderr.close() 2911 return output, p.returncode 2912 2913 2914def RunShell(command, silent_ok=False, universal_newlines=True, 2915 print_output=False, env=os.environ): 2916 data, retcode = RunShellWithReturnCode(command, print_output, universal_newlines, env) 2917 if retcode: 2918 ErrorExit("Got error status from %s:\n%s" % (command, data)) 2919 if not silent_ok and not data: 2920 ErrorExit("No output from %s" % command) 2921 return data 2922 2923 2924class VersionControlSystem(object): 2925 """Abstract base class providing an interface to the VCS.""" 2926 2927 def __init__(self, options): 2928 """Constructor. 2929 2930 Args: 2931 options: Command line options. 2932 """ 2933 self.options = options 2934 2935 def GenerateDiff(self, args): 2936 """Return the current diff as a string. 2937 2938 Args: 2939 args: Extra arguments to pass to the diff command. 2940 """ 2941 raise NotImplementedError( 2942 "abstract method -- subclass %s must override" % self.__class__) 2943 2944 def GetUnknownFiles(self): 2945 """Return a list of files unknown to the VCS.""" 2946 raise NotImplementedError( 2947 "abstract method -- subclass %s must override" % self.__class__) 2948 2949 def CheckForUnknownFiles(self): 2950 """Show an "are you sure?" prompt if there are unknown files.""" 2951 unknown_files = self.GetUnknownFiles() 2952 if unknown_files: 2953 print "The following files are not added to version control:" 2954 for line in unknown_files: 2955 print line 2956 prompt = "Are you sure to continue?(y/N) " 2957 answer = raw_input(prompt).strip() 2958 if answer != "y": 2959 ErrorExit("User aborted") 2960 2961 def GetBaseFile(self, filename): 2962 """Get the content of the upstream version of a file. 2963 2964 Returns: 2965 A tuple (base_content, new_content, is_binary, status) 2966 base_content: The contents of the base file. 2967 new_content: For text files, this is empty. For binary files, this is 2968 the contents of the new file, since the diff output won't contain 2969 information to reconstruct the current file. 2970 is_binary: True iff the file is binary. 2971 status: The status of the file. 2972 """ 2973 2974 raise NotImplementedError( 2975 "abstract method -- subclass %s must override" % self.__class__) 2976 2977 2978 def GetBaseFiles(self, diff): 2979 """Helper that calls GetBase file for each file in the patch. 2980 2981 Returns: 2982 A dictionary that maps from filename to GetBaseFile's tuple. Filenames 2983 are retrieved based on lines that start with "Index:" or 2984 "Property changes on:". 2985 """ 2986 files = {} 2987 for line in diff.splitlines(True): 2988 if line.startswith('Index:') or line.startswith('Property changes on:'): 2989 unused, filename = line.split(':', 1) 2990 # On Windows if a file has property changes its filename uses '\' 2991 # instead of '/'. 2992 filename = filename.strip().replace('\\', '/') 2993 files[filename] = self.GetBaseFile(filename) 2994 return files 2995 2996 2997 def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options, 2998 files): 2999 """Uploads the base files (and if necessary, the current ones as well).""" 3000 3001 def UploadFile(filename, file_id, content, is_binary, status, is_base): 3002 """Uploads a file to the server.""" 3003 set_status("uploading " + filename) 3004 file_too_large = False 3005 if is_base: 3006 type = "base" 3007 else: 3008 type = "current" 3009 if len(content) > MAX_UPLOAD_SIZE: 3010 print ("Not uploading the %s file for %s because it's too large." % 3011 (type, filename)) 3012 file_too_large = True 3013 content = "" 3014 checksum = md5(content).hexdigest() 3015 if options.verbose > 0 and not file_too_large: 3016 print "Uploading %s file for %s" % (type, filename) 3017 url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id) 3018 form_fields = [ 3019 ("filename", filename), 3020 ("status", status), 3021 ("checksum", checksum), 3022 ("is_binary", str(is_binary)), 3023 ("is_current", str(not is_base)), 3024 ] 3025 if file_too_large: 3026 form_fields.append(("file_too_large", "1")) 3027 if options.email: 3028 form_fields.append(("user", options.email)) 3029 ctype, body = EncodeMultipartFormData(form_fields, [("data", filename, content)]) 3030 response_body = rpc_server.Send(url, body, content_type=ctype) 3031 if not response_body.startswith("OK"): 3032 StatusUpdate(" --> %s" % response_body) 3033 sys.exit(1) 3034 3035 # Don't want to spawn too many threads, nor do we want to 3036 # hit Rietveld too hard, or it will start serving 500 errors. 3037 # When 8 works, it's no better than 4, and sometimes 8 is 3038 # too many for Rietveld to handle. 3039 MAX_PARALLEL_UPLOADS = 4 3040 3041 sema = threading.BoundedSemaphore(MAX_PARALLEL_UPLOADS) 3042 upload_threads = [] 3043 finished_upload_threads = [] 3044 3045 class UploadFileThread(threading.Thread): 3046 def __init__(self, args): 3047 threading.Thread.__init__(self) 3048 self.args = args 3049 def run(self): 3050 UploadFile(*self.args) 3051 finished_upload_threads.append(self) 3052 sema.release() 3053 3054 def StartUploadFile(*args): 3055 sema.acquire() 3056 while len(finished_upload_threads) > 0: 3057 t = finished_upload_threads.pop() 3058 upload_threads.remove(t) 3059 t.join() 3060 t = UploadFileThread(args) 3061 upload_threads.append(t) 3062 t.start() 3063 3064 def WaitForUploads(): 3065 for t in upload_threads: 3066 t.join() 3067 3068 patches = dict() 3069 [patches.setdefault(v, k) for k, v in patch_list] 3070 for filename in patches.keys(): 3071 base_content, new_content, is_binary, status = files[filename] 3072 file_id_str = patches.get(filename) 3073 if file_id_str.find("nobase") != -1: 3074 base_content = None 3075 file_id_str = file_id_str[file_id_str.rfind("_") + 1:] 3076 file_id = int(file_id_str) 3077 if base_content != None: 3078 StartUploadFile(filename, file_id, base_content, is_binary, status, True) 3079 if new_content != None: 3080 StartUploadFile(filename, file_id, new_content, is_binary, status, False) 3081 WaitForUploads() 3082 3083 def IsImage(self, filename): 3084 """Returns true if the filename has an image extension.""" 3085 mimetype = mimetypes.guess_type(filename)[0] 3086 if not mimetype: 3087 return False 3088 return mimetype.startswith("image/") 3089 3090 def IsBinary(self, filename): 3091 """Returns true if the guessed mimetyped isnt't in text group.""" 3092 mimetype = mimetypes.guess_type(filename)[0] 3093 if not mimetype: 3094 return False # e.g. README, "real" binaries usually have an extension 3095 # special case for text files which don't start with text/ 3096 if mimetype in TEXT_MIMETYPES: 3097 return False 3098 return not mimetype.startswith("text/") 3099 3100class FakeMercurialUI(object): 3101 def __init__(self): 3102 self.quiet = True 3103 self.output = '' 3104 3105 def write(self, *args, **opts): 3106 self.output += ' '.join(args) 3107 3108use_hg_shell = False # set to True to shell out to hg always; slower 3109 3110class MercurialVCS(VersionControlSystem): 3111 """Implementation of the VersionControlSystem interface for Mercurial.""" 3112 3113 def __init__(self, options, ui, repo): 3114 super(MercurialVCS, self).__init__(options) 3115 self.ui = ui 3116 self.repo = repo 3117 # Absolute path to repository (we can be in a subdir) 3118 self.repo_dir = os.path.normpath(repo.root) 3119 # Compute the subdir 3120 cwd = os.path.normpath(os.getcwd()) 3121 assert cwd.startswith(self.repo_dir) 3122 self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/") 3123 if self.options.revision: 3124 self.base_rev = self.options.revision 3125 else: 3126 mqparent, err = RunShellWithReturnCode(['hg', 'log', '--rev', 'qparent', '--template={node}']) 3127 if not err and mqparent != "": 3128 self.base_rev = mqparent 3129 else: 3130 self.base_rev = RunShell(["hg", "parents", "-q"]).split(':')[1].strip() 3131 def _GetRelPath(self, filename): 3132 """Get relative path of a file according to the current directory, 3133 given its logical path in the repo.""" 3134 assert filename.startswith(self.subdir), (filename, self.subdir) 3135 return filename[len(self.subdir):].lstrip(r"\/") 3136 3137 def GenerateDiff(self, extra_args): 3138 # If no file specified, restrict to the current subdir 3139 extra_args = extra_args or ["."] 3140 cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args 3141 data = RunShell(cmd, silent_ok=True) 3142 svndiff = [] 3143 filecount = 0 3144 for line in data.splitlines(): 3145 m = re.match("diff --git a/(\S+) b/(\S+)", line) 3146 if m: 3147 # Modify line to make it look like as it comes from svn diff. 3148 # With this modification no changes on the server side are required 3149 # to make upload.py work with Mercurial repos. 3150 # NOTE: for proper handling of moved/copied files, we have to use 3151 # the second filename. 3152 filename = m.group(2) 3153 svndiff.append("Index: %s" % filename) 3154 svndiff.append("=" * 67) 3155 filecount += 1 3156 logging.info(line) 3157 else: 3158 svndiff.append(line) 3159 if not filecount: 3160 ErrorExit("No valid patches found in output from hg diff") 3161 return "\n".join(svndiff) + "\n" 3162 3163 def GetUnknownFiles(self): 3164 """Return a list of files unknown to the VCS.""" 3165 args = [] 3166 status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."], 3167 silent_ok=True) 3168 unknown_files = [] 3169 for line in status.splitlines(): 3170 st, fn = line.split(" ", 1) 3171 if st == "?": 3172 unknown_files.append(fn) 3173 return unknown_files 3174 3175 def GetBaseFile(self, filename): 3176 set_status("inspecting " + filename) 3177 # "hg status" and "hg cat" both take a path relative to the current subdir 3178 # rather than to the repo root, but "hg diff" has given us the full path 3179 # to the repo root. 3180 base_content = "" 3181 new_content = None 3182 is_binary = False 3183 oldrelpath = relpath = self._GetRelPath(filename) 3184 # "hg status -C" returns two lines for moved/copied files, one otherwise 3185 if use_hg_shell: 3186 out = RunShell(["hg", "status", "-C", "--rev", self.base_rev, relpath]) 3187 else: 3188 fui = FakeMercurialUI() 3189 ret = commands.status(fui, self.repo, *[relpath], **{'rev': [self.base_rev], 'copies': True}) 3190 if ret: 3191 raise util.Abort(ret) 3192 out = fui.output 3193 out = out.splitlines() 3194 # HACK: strip error message about missing file/directory if it isn't in 3195 # the working copy 3196 if out[0].startswith('%s: ' % relpath): 3197 out = out[1:] 3198 status, what = out[0].split(' ', 1) 3199 if len(out) > 1 and status == "A" and what == relpath: 3200 oldrelpath = out[1].strip() 3201 status = "M" 3202 if ":" in self.base_rev: 3203 base_rev = self.base_rev.split(":", 1)[0] 3204 else: 3205 base_rev = self.base_rev 3206 if status != "A": 3207 if use_hg_shell: 3208 base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath], silent_ok=True) 3209 else: 3210 base_content = str(self.repo[base_rev][oldrelpath].data()) 3211 is_binary = "\0" in base_content # Mercurial's heuristic 3212 if status != "R": 3213 new_content = open(relpath, "rb").read() 3214 is_binary = is_binary or "\0" in new_content 3215 if is_binary and base_content and use_hg_shell: 3216 # Fetch again without converting newlines 3217 base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath], 3218 silent_ok=True, universal_newlines=False) 3219 if not is_binary or not self.IsImage(relpath): 3220 new_content = None 3221 return base_content, new_content, is_binary, status 3222 3223 3224# NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync. 3225def SplitPatch(data): 3226 """Splits a patch into separate pieces for each file. 3227 3228 Args: 3229 data: A string containing the output of svn diff. 3230 3231 Returns: 3232 A list of 2-tuple (filename, text) where text is the svn diff output 3233 pertaining to filename. 3234 """ 3235 patches = [] 3236 filename = None 3237 diff = [] 3238 for line in data.splitlines(True): 3239 new_filename = None 3240 if line.startswith('Index:'): 3241 unused, new_filename = line.split(':', 1) 3242 new_filename = new_filename.strip() 3243 elif line.startswith('Property changes on:'): 3244 unused, temp_filename = line.split(':', 1) 3245 # When a file is modified, paths use '/' between directories, however 3246 # when a property is modified '\' is used on Windows. Make them the same 3247 # otherwise the file shows up twice. 3248 temp_filename = temp_filename.strip().replace('\\', '/') 3249 if temp_filename != filename: 3250 # File has property changes but no modifications, create a new diff. 3251 new_filename = temp_filename 3252 if new_filename: 3253 if filename and diff: 3254 patches.append((filename, ''.join(diff))) 3255 filename = new_filename 3256 diff = [line] 3257 continue 3258 if diff is not None: 3259 diff.append(line) 3260 if filename and diff: 3261 patches.append((filename, ''.join(diff))) 3262 return patches 3263 3264 3265def UploadSeparatePatches(issue, rpc_server, patchset, data, options): 3266 """Uploads a separate patch for each file in the diff output. 3267 3268 Returns a list of [patch_key, filename] for each file. 3269 """ 3270 patches = SplitPatch(data) 3271 rv = [] 3272 for patch in patches: 3273 set_status("uploading patch for " + patch[0]) 3274 if len(patch[1]) > MAX_UPLOAD_SIZE: 3275 print ("Not uploading the patch for " + patch[0] + 3276 " because the file is too large.") 3277 continue 3278 form_fields = [("filename", patch[0])] 3279 if not options.download_base: 3280 form_fields.append(("content_upload", "1")) 3281 files = [("data", "data.diff", patch[1])] 3282 ctype, body = EncodeMultipartFormData(form_fields, files) 3283 url = "/%d/upload_patch/%d" % (int(issue), int(patchset)) 3284 print "Uploading patch for " + patch[0] 3285 response_body = rpc_server.Send(url, body, content_type=ctype) 3286 lines = response_body.splitlines() 3287 if not lines or lines[0] != "OK": 3288 StatusUpdate(" --> %s" % response_body) 3289 sys.exit(1) 3290 rv.append([lines[1], patch[0]]) 3291 return rv