PageRenderTime 94ms CodeModel.GetById 15ms app.highlight 67ms RepoModel.GetById 1ms app.codeStats 0ms

/upload.py

http://testability-explorer.googlecode.com/
Python | 1389 lines | 1267 code | 23 blank | 99 comment | 44 complexity | dc3d71fd2b2a0afce7cc961fc85fe1b9 MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1#!/usr/bin/env python
   2#
   3# Copyright 2007 Google Inc.
   4#
   5# Licensed under the Apache License, Version 2.0 (the "License");
   6# you may not use this file except in compliance with the License.
   7# You may obtain a copy of the License at
   8#
   9#     http://www.apache.org/licenses/LICENSE-2.0
  10#
  11# Unless required by applicable law or agreed to in writing, software
  12# distributed under the License is distributed on an "AS IS" BASIS,
  13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14# See the License for the specific language governing permissions and
  15# limitations under the License.
  16
  17"""Tool for uploading diffs from a version control system to the codereview app.
  18
  19Usage summary: upload.py [options] [-- diff_options]
  20
  21Diff options are passed to the diff command of the underlying system.
  22
  23Supported version control systems:
  24  Git
  25  Mercurial
  26  Subversion
  27
  28It is important for Git/Mercurial users to specify a tree/node/branch to diff
  29against by using the '--rev' option.
  30"""
  31# This code is derived from appcfg.py in the App Engine SDK (open source),
  32# and from ASPN recipe #146306.
  33
  34# Defaults edited by alexeagle@google.com (Alex Eagle)
  35
  36import cookielib
  37import getpass
  38import logging
  39import md5
  40import mimetypes
  41import optparse
  42import os
  43import re
  44import socket
  45import subprocess
  46import sys
  47import urllib
  48import urllib2
  49import urlparse
  50
  51try:
  52  import readline
  53except ImportError:
  54  pass
  55
  56# The logging verbosity:
  57#  0: Errors only.
  58#  1: Status messages.
  59#  2: Info logs.
  60#  3: Debug logs.
  61verbosity = 1
  62
  63# Max size of patch or base file.
  64MAX_UPLOAD_SIZE = 900 * 1024
  65
  66
  67def GetEmail(prompt):
  68  """Prompts the user for their email address and returns it.
  69
  70  The last used email address is saved to a file and offered up as a suggestion
  71  to the user. If the user presses enter without typing in anything the last
  72  used email address is used. If the user enters a new address, it is saved
  73  for next time we prompt.
  74
  75  """
  76  last_email_file_name = os.path.expanduser("~/.last_codereview_email_address")
  77  last_email = ""
  78  if os.path.exists(last_email_file_name):
  79    try:
  80      last_email_file = open(last_email_file_name, "r")
  81      last_email = last_email_file.readline().strip("\n")
  82      last_email_file.close()
  83      prompt += " [%s]" % last_email
  84    except IOError, e:
  85      pass
  86  email = raw_input(prompt + ": ").strip()
  87  if email:
  88    try:
  89      last_email_file = open(last_email_file_name, "w")
  90      last_email_file.write(email)
  91      last_email_file.close()
  92    except IOError, e:
  93      pass
  94  else:
  95    email = last_email
  96  return email
  97
  98
  99def StatusUpdate(msg):
 100  """Print a status message to stdout.
 101
 102  If 'verbosity' is greater than 0, print the message.
 103
 104  Args:
 105    msg: The string to print.
 106  """
 107  if verbosity > 0:
 108    print msg
 109
 110
 111def ErrorExit(msg):
 112  """Print an error message to stderr and exit."""
 113  print >>sys.stderr, msg
 114  sys.exit(1)
 115
 116
 117class ClientLoginError(urllib2.HTTPError):
 118  """Raised to indicate there was an error authenticating with ClientLogin."""
 119
 120  def __init__(self, url, code, msg, headers, args):
 121    urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
 122    self.args = args
 123    self.reason = args["Error"]
 124
 125
 126class AbstractRpcServer(object):
 127  """Provides a common interface for a simple RPC server."""
 128
 129  def __init__(self, host, auth_function, host_override=None, extra_headers={},
 130               save_cookies=False):
 131    """Creates a new HttpRpcServer.
 132
 133    Args:
 134      host: The host to send requests to.
 135      auth_function: A function that takes no arguments and returns an
 136        (email, password) tuple when called. Will be called if authentication
 137        is required.
 138      host_override: The host header to send to the server (defaults to host).
 139      extra_headers: A dict of extra headers to append to every request.
 140      save_cookies: If True, save the authentication cookies to local disk.
 141        If False, use an in-memory cookiejar instead.  Subclasses must
 142        implement this functionality.  Defaults to False.
 143    """
 144    self.host = host
 145    self.host_override = host_override
 146    self.auth_function = auth_function
 147    self.authenticated = False
 148    self.extra_headers = extra_headers
 149    self.save_cookies = save_cookies
 150    self.opener = self._GetOpener()
 151    if self.host_override:
 152      logging.info("Server: %s; Host: %s", self.host, self.host_override)
 153    else:
 154      logging.info("Server: %s", self.host)
 155
 156  def _GetOpener(self):
 157    """Returns an OpenerDirector for making HTTP requests.
 158
 159    Returns:
 160      A urllib2.OpenerDirector object.
 161    """
 162    raise NotImplementedError()
 163
 164  def _CreateRequest(self, url, data=None):
 165    """Creates a new urllib request."""
 166    logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
 167    req = urllib2.Request(url, data=data)
 168    if self.host_override:
 169      req.add_header("Host", self.host_override)
 170    for key, value in self.extra_headers.iteritems():
 171      req.add_header(key, value)
 172    return req
 173
 174  def _GetAuthToken(self, email, password):
 175    """Uses ClientLogin to authenticate the user, returning an auth token.
 176
 177    Args:
 178      email:    The user's email address
 179      password: The user's password
 180
 181    Raises:
 182      ClientLoginError: If there was an error authenticating with ClientLogin.
 183      HTTPError: If there was some other form of HTTP error.
 184
 185    Returns:
 186      The authentication token returned by ClientLogin.
 187    """
 188    account_type = "GOOGLE"
 189    if self.host.endswith(".google.com"):
 190      # Needed for use inside Google.
 191      account_type = "HOSTED"
 192    req = self._CreateRequest(
 193        url="https://www.google.com/accounts/ClientLogin",
 194        data=urllib.urlencode({
 195            "Email": email,
 196            "Passwd": password,
 197            "service": "ah",
 198            "source": "rietveld-codereview-upload",
 199            "accountType": account_type,
 200        }),
 201    )
 202    try:
 203      response = self.opener.open(req)
 204      response_body = response.read()
 205      response_dict = dict(x.split("=")
 206                           for x in response_body.split("\n") if x)
 207      return response_dict["Auth"]
 208    except urllib2.HTTPError, e:
 209      if e.code == 403:
 210        body = e.read()
 211        response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
 212        raise ClientLoginError(req.get_full_url(), e.code, e.msg,
 213                               e.headers, response_dict)
 214      else:
 215        raise
 216
 217  def _GetAuthCookie(self, auth_token):
 218    """Fetches authentication cookies for an authentication token.
 219
 220    Args:
 221      auth_token: The authentication token returned by ClientLogin.
 222
 223    Raises:
 224      HTTPError: If there was an error fetching the authentication cookies.
 225    """
 226    # This is a dummy value to allow us to identify when we're successful.
 227    continue_location = "http://localhost/"
 228    args = {"continue": continue_location, "auth": auth_token}
 229    req = self._CreateRequest("http://%s/_ah/login?%s" %
 230                              (self.host, urllib.urlencode(args)))
 231    try:
 232      response = self.opener.open(req)
 233    except urllib2.HTTPError, e:
 234      response = e
 235    if (response.code != 302 or
 236        response.info()["location"] != continue_location):
 237      raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg,
 238                              response.headers, response.fp)
 239    self.authenticated = True
 240
 241  def _Authenticate(self):
 242    """Authenticates the user.
 243
 244    The authentication process works as follows:
 245     1) We get a username and password from the user
 246     2) We use ClientLogin to obtain an AUTH token for the user
 247        (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
 248     3) We pass the auth token to /_ah/login on the server to obtain an
 249        authentication cookie. If login was successful, it tries to redirect
 250        us to the URL we provided.
 251
 252    If we attempt to access the upload API without first obtaining an
 253    authentication cookie, it returns a 401 response and directs us to
 254    authenticate ourselves with ClientLogin.
 255    """
 256    for i in range(3):
 257      credentials = self.auth_function()
 258      try:
 259        auth_token = self._GetAuthToken(credentials[0], credentials[1])
 260      except ClientLoginError, e:
 261        if e.reason == "BadAuthentication":
 262          print >>sys.stderr, "Invalid username or password."
 263          continue
 264        if e.reason == "CaptchaRequired":
 265          print >>sys.stderr, (
 266              "Please go to\n"
 267              "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
 268              "and verify you are a human.  Then try again.")
 269          break
 270        if e.reason == "NotVerified":
 271          print >>sys.stderr, "Account not verified."
 272          break
 273        if e.reason == "TermsNotAgreed":
 274          print >>sys.stderr, "User has not agreed to TOS."
 275          break
 276        if e.reason == "AccountDeleted":
 277          print >>sys.stderr, "The user account has been deleted."
 278          break
 279        if e.reason == "AccountDisabled":
 280          print >>sys.stderr, "The user account has been disabled."
 281          break
 282        if e.reason == "ServiceDisabled":
 283          print >>sys.stderr, ("The user's access to the service has been "
 284                               "disabled.")
 285          break
 286        if e.reason == "ServiceUnavailable":
 287          print >>sys.stderr, "The service is not available; try again later."
 288          break
 289        raise
 290      self._GetAuthCookie(auth_token)
 291      return
 292
 293  def Send(self, request_path, payload=None,
 294           content_type="application/octet-stream",
 295           timeout=None,
 296           **kwargs):
 297    """Sends an RPC and returns the response.
 298
 299    Args:
 300      request_path: The path to send the request to, eg /api/appversion/create.
 301      payload: The body of the request, or None to send an empty request.
 302      content_type: The Content-Type header to use.
 303      timeout: timeout in seconds; default None i.e. no timeout.
 304        (Note: for large requests on OS X, the timeout doesn't work right.)
 305      kwargs: Any keyword arguments are converted into query string parameters.
 306
 307    Returns:
 308      The response body, as a string.
 309    """
 310    # TODO: Don't require authentication.  Let the server say
 311    # whether it is necessary.
 312    if not self.authenticated:
 313      self._Authenticate()
 314
 315    old_timeout = socket.getdefaulttimeout()
 316    socket.setdefaulttimeout(timeout)
 317    try:
 318      tries = 0
 319      while True:
 320        tries += 1
 321        args = dict(kwargs)
 322        url = "http://%s%s" % (self.host, request_path)
 323        if args:
 324          url += "?" + urllib.urlencode(args)
 325        req = self._CreateRequest(url=url, data=payload)
 326        req.add_header("Content-Type", content_type)
 327        try:
 328          f = self.opener.open(req)
 329          response = f.read()
 330          f.close()
 331          return response
 332        except urllib2.HTTPError, e:
 333          if tries > 3:
 334            raise
 335          elif e.code == 401:
 336            self._Authenticate()
 337##           elif e.code >= 500 and e.code < 600:
 338##             # Server Error - try again.
 339##             continue
 340          else:
 341            raise
 342    finally:
 343      socket.setdefaulttimeout(old_timeout)
 344
 345
 346class HttpRpcServer(AbstractRpcServer):
 347  """Provides a simplified RPC-style interface for HTTP requests."""
 348
 349  def _Authenticate(self):
 350    """Save the cookie jar after authentication."""
 351    super(HttpRpcServer, self)._Authenticate()
 352    if self.save_cookies:
 353      StatusUpdate("Saving authentication cookies to %s" % self.cookie_file)
 354      self.cookie_jar.save()
 355
 356  def _GetOpener(self):
 357    """Returns an OpenerDirector that supports cookies and ignores redirects.
 358
 359    Returns:
 360      A urllib2.OpenerDirector object.
 361    """
 362    opener = urllib2.OpenerDirector()
 363    opener.add_handler(urllib2.ProxyHandler())
 364    opener.add_handler(urllib2.UnknownHandler())
 365    opener.add_handler(urllib2.HTTPHandler())
 366    opener.add_handler(urllib2.HTTPDefaultErrorHandler())
 367    opener.add_handler(urllib2.HTTPSHandler())
 368    opener.add_handler(urllib2.HTTPErrorProcessor())
 369    if self.save_cookies:
 370      self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies")
 371      self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
 372      if os.path.exists(self.cookie_file):
 373        try:
 374          self.cookie_jar.load()
 375          self.authenticated = True
 376          StatusUpdate("Loaded authentication cookies from %s" %
 377                       self.cookie_file)
 378        except (cookielib.LoadError, IOError):
 379          # Failed to load cookies - just ignore them.
 380          pass
 381      else:
 382        # Create an empty cookie file with mode 600
 383        fd = os.open(self.cookie_file, os.O_CREAT, 0600)
 384        os.close(fd)
 385      # Always chmod the cookie file
 386      os.chmod(self.cookie_file, 0600)
 387    else:
 388      # Don't save cookies across runs of update.py.
 389      self.cookie_jar = cookielib.CookieJar()
 390    opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
 391    return opener
 392
 393
 394parser = optparse.OptionParser(usage="%prog [options] [-- diff_options]")
 395parser.add_option("-y", "--assume_yes", action="store_true",
 396                  dest="assume_yes", default=False,
 397                  help="Assume that the answer to yes/no questions is 'yes'.")
 398# Logging
 399group = parser.add_option_group("Logging options")
 400group.add_option("-q", "--quiet", action="store_const", const=0,
 401                 dest="verbose", help="Print errors only.")
 402group.add_option("-v", "--verbose", action="store_const", const=2,
 403                 dest="verbose", default=1,
 404                 help="Print info level logs (default).")
 405group.add_option("--noisy", action="store_const", const=3,
 406                 dest="verbose", help="Print all logs.")
 407# Review server
 408group = parser.add_option_group("Review server options")
 409group.add_option("-s", "--server", action="store", dest="server",
 410                 default="codereview.appspot.com",
 411                 metavar="SERVER",
 412                 help=("The server to upload to. The format is host[:port]. "
 413                       "Defaults to '%default'."))
 414group.add_option("-e", "--email", action="store", dest="email",
 415                 metavar="EMAIL", default=None,
 416                 help="The username to use. Will prompt if omitted.")
 417group.add_option("-H", "--host", action="store", dest="host",
 418                 metavar="HOST", default=None,
 419                 help="Overrides the Host header sent with all RPCs.")
 420group.add_option("--no_cookies", action="store_false",
 421                 dest="save_cookies", default=True,
 422                 help="Do not save authentication cookies to local disk.")
 423# Issue
 424group = parser.add_option_group("Issue options")
 425group.add_option("-d", "--description", action="store", dest="description",
 426                 metavar="DESCRIPTION", default=None,
 427                 help="Optional description when creating an issue.")
 428group.add_option("-f", "--description_file", action="store",
 429                 dest="description_file", metavar="DESCRIPTION_FILE",
 430                 default=None,
 431                 help="Optional path of a file that contains "
 432                      "the description when creating an issue.")
 433group.add_option("-r", "--reviewers", action="store", dest="reviewers",
 434                 metavar="REVIEWERS", default="aeagle22206@gmail.com",
 435                 help="Add reviewers (comma separated email addresses).")
 436group.add_option("--cc", action="store", dest="cc",
 437                 metavar="CC", default="testability-explorer-dev@googlegroups.com",
 438                 help="Add CC (comma separated email addresses).")
 439# Upload options
 440group = parser.add_option_group("Patch options")
 441group.add_option("-m", "--message", action="store", dest="message",
 442                 metavar="MESSAGE", default=None,
 443                 help="A message to identify the patch. "
 444                      "Will prompt if omitted.")
 445group.add_option("-i", "--issue", type="int", action="store",
 446                 metavar="ISSUE", default=None,
 447                 help="Issue number to which to add. Defaults to new issue.")
 448group.add_option("--download_base", action="store_true",
 449                 dest="download_base", default=False,
 450                 help="Base files will be downloaded by the server "
 451                 "(side-by-side diffs may not work on files with CRs).")
 452group.add_option("--rev", action="store", dest="revision",
 453                 metavar="REV", default=None,
 454                 help="Branch/tree/revision to diff against (used by DVCS).")
 455group.add_option("--send_mail", action="store_true",
 456                 dest="send_mail", default=False,
 457                 help="Send notification email to reviewers.")
 458
 459
 460def GetRpcServer(options):
 461  """Returns an instance of an AbstractRpcServer.
 462
 463  Returns:
 464    A new AbstractRpcServer, on which RPC calls can be made.
 465  """
 466
 467  rpc_server_class = HttpRpcServer
 468
 469  def GetUserCredentials():
 470    """Prompts the user for a username and password."""
 471    email = options.email
 472    if email is None:
 473      email = GetEmail("Email (login for uploading to %s)" % options.server)
 474    password = getpass.getpass("Password for %s: " % email)
 475    return (email, password)
 476
 477  # If this is the dev_appserver, use fake authentication.
 478  host = (options.host or options.server).lower()
 479  if host == "localhost" or host.startswith("localhost:"):
 480    email = options.email
 481    if email is None:
 482      email = "test@example.com"
 483      logging.info("Using debug user %s.  Override with --email" % email)
 484    server = rpc_server_class(
 485        options.server,
 486        lambda: (email, "password"),
 487        host_override=options.host,
 488        extra_headers={"Cookie":
 489                       'dev_appserver_login="%s:False"' % email},
 490        save_cookies=options.save_cookies)
 491    # Don't try to talk to ClientLogin.
 492    server.authenticated = True
 493    return server
 494
 495  return rpc_server_class(options.server, GetUserCredentials,
 496                          host_override=options.host,
 497                          save_cookies=options.save_cookies)
 498
 499
 500def EncodeMultipartFormData(fields, files):
 501  """Encode form fields for multipart/form-data.
 502
 503  Args:
 504    fields: A sequence of (name, value) elements for regular form fields.
 505    files: A sequence of (name, filename, value) elements for data to be
 506           uploaded as files.
 507  Returns:
 508    (content_type, body) ready for httplib.HTTP instance.
 509
 510  Source:
 511    http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
 512  """
 513  BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
 514  CRLF = '\r\n'
 515  lines = []
 516  for (key, value) in fields:
 517    lines.append('--' + BOUNDARY)
 518    lines.append('Content-Disposition: form-data; name="%s"' % key)
 519    lines.append('')
 520    lines.append(value)
 521  for (key, filename, value) in files:
 522    lines.append('--' + BOUNDARY)
 523    lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' %
 524             (key, filename))
 525    lines.append('Content-Type: %s' % GetContentType(filename))
 526    lines.append('')
 527    lines.append(value)
 528  lines.append('--' + BOUNDARY + '--')
 529  lines.append('')
 530  body = CRLF.join(lines)
 531  content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
 532  return content_type, body
 533
 534
 535def GetContentType(filename):
 536  """Helper to guess the content-type from the filename."""
 537  return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
 538
 539
 540# Use a shell for subcommands on Windows to get a PATH search.
 541use_shell = sys.platform.startswith("win")
 542
 543def RunShellWithReturnCode(command, print_output=False,
 544                           universal_newlines=True):
 545  """Executes a command and returns the output from stdout and the return code.
 546
 547  Args:
 548    command: Command to execute.
 549    print_output: If True, the output is printed to stdout.
 550                  If False, both stdout and stderr are ignored.
 551    universal_newlines: Use universal_newlines flag (default: True).
 552
 553  Returns:
 554    Tuple (output, return code)
 555  """
 556  logging.info("Running %s", command)
 557  p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 558                       shell=use_shell, universal_newlines=universal_newlines)
 559  if print_output:
 560    output_array = []
 561    while True:
 562      line = p.stdout.readline()
 563      if not line:
 564        break
 565      print line.strip("\n")
 566      output_array.append(line)
 567    output = "".join(output_array)
 568  else:
 569    output = p.stdout.read()
 570  p.wait()
 571  errout = p.stderr.read()
 572  if print_output and errout:
 573    print >>sys.stderr, errout
 574  p.stdout.close()
 575  p.stderr.close()
 576  return output, p.returncode
 577
 578
 579def RunShell(command, silent_ok=False, universal_newlines=True,
 580             print_output=False):
 581  data, retcode = RunShellWithReturnCode(command, print_output,
 582                                         universal_newlines)
 583  if retcode:
 584    ErrorExit("Got error status from %s:\n%s" % (command, data))
 585  if not silent_ok and not data:
 586    ErrorExit("No output from %s" % command)
 587  return data
 588
 589
 590class VersionControlSystem(object):
 591  """Abstract base class providing an interface to the VCS."""
 592
 593  def __init__(self, options):
 594    """Constructor.
 595
 596    Args:
 597      options: Command line options.
 598    """
 599    self.options = options
 600
 601  def GenerateDiff(self, args):
 602    """Return the current diff as a string.
 603
 604    Args:
 605      args: Extra arguments to pass to the diff command.
 606    """
 607    raise NotImplementedError(
 608        "abstract method -- subclass %s must override" % self.__class__)
 609
 610  def GetUnknownFiles(self):
 611    """Return a list of files unknown to the VCS."""
 612    raise NotImplementedError(
 613        "abstract method -- subclass %s must override" % self.__class__)
 614
 615  def CheckForUnknownFiles(self):
 616    """Show an "are you sure?" prompt if there are unknown files."""
 617    unknown_files = self.GetUnknownFiles()
 618    if unknown_files:
 619      print "The following files are not added to version control:"
 620      for line in unknown_files:
 621        print line
 622      prompt = "Are you sure to continue?(y/N) "
 623      answer = raw_input(prompt).strip()
 624      if answer != "y":
 625        ErrorExit("User aborted")
 626
 627  def GetBaseFile(self, filename):
 628    """Get the content of the upstream version of a file.
 629
 630    Returns:
 631      A tuple (base_content, new_content, is_binary, status)
 632        base_content: The contents of the base file.
 633        new_content: For text files, this is empty.  For binary files, this is
 634          the contents of the new file, since the diff output won't contain
 635          information to reconstruct the current file.
 636        is_binary: True iff the file is binary.
 637        status: The status of the file.
 638    """
 639
 640    raise NotImplementedError(
 641        "abstract method -- subclass %s must override" % self.__class__)
 642
 643
 644  def GetBaseFiles(self, diff):
 645    """Helper that calls GetBase file for each file in the patch.
 646
 647    Returns:
 648      A dictionary that maps from filename to GetBaseFile's tuple.  Filenames
 649      are retrieved based on lines that start with "Index:" or
 650      "Property changes on:".
 651    """
 652    files = {}
 653    for line in diff.splitlines(True):
 654      if line.startswith('Index:') or line.startswith('Property changes on:'):
 655        unused, filename = line.split(':', 1)
 656        # On Windows if a file has property changes its filename uses '\'
 657        # instead of '/'.
 658        filename = filename.strip().replace('\\', '/')
 659        files[filename] = self.GetBaseFile(filename)
 660    return files
 661
 662
 663  def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options,
 664                      files):
 665    """Uploads the base files (and if necessary, the current ones as well)."""
 666
 667    def UploadFile(filename, file_id, content, is_binary, status, is_base):
 668      """Uploads a file to the server."""
 669      file_too_large = False
 670      if is_base:
 671        type = "base"
 672      else:
 673        type = "current"
 674      if len(content) > MAX_UPLOAD_SIZE:
 675        print ("Not uploading the %s file for %s because it's too large." %
 676               (type, filename))
 677        file_too_large = True
 678        content = ""
 679      checksum = md5.new(content).hexdigest()
 680      if options.verbose > 0 and not file_too_large:
 681        print "Uploading %s file for %s" % (type, filename)
 682      url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id)
 683      form_fields = [("filename", filename),
 684                     ("status", status),
 685                     ("checksum", checksum),
 686                     ("is_binary", str(is_binary)),
 687                     ("is_current", str(not is_base)),
 688                    ]
 689      if file_too_large:
 690        form_fields.append(("file_too_large", "1"))
 691      if options.email:
 692        form_fields.append(("user", options.email))
 693      ctype, body = EncodeMultipartFormData(form_fields,
 694                                            [("data", filename, content)])
 695      response_body = rpc_server.Send(url, body,
 696                                      content_type=ctype)
 697      if not response_body.startswith("OK"):
 698        StatusUpdate("  --> %s" % response_body)
 699        sys.exit(1)
 700
 701    patches = dict()
 702    [patches.setdefault(v, k) for k, v in patch_list]
 703    for filename in patches.keys():
 704      base_content, new_content, is_binary, status = files[filename]
 705      file_id_str = patches.get(filename)
 706      if file_id_str.find("nobase") != -1:
 707        base_content = None
 708        file_id_str = file_id_str[file_id_str.rfind("_") + 1:]
 709      file_id = int(file_id_str)
 710      if base_content != None:
 711        UploadFile(filename, file_id, base_content, is_binary, status, True)
 712      if new_content != None:
 713        UploadFile(filename, file_id, new_content, is_binary, status, False)
 714
 715  def IsImage(self, filename):
 716    """Returns true if the filename has an image extension."""
 717    mimetype =  mimetypes.guess_type(filename)[0]
 718    if not mimetype:
 719      return False
 720    return mimetype.startswith("image/")
 721
 722
 723class SubversionVCS(VersionControlSystem):
 724  """Implementation of the VersionControlSystem interface for Subversion."""
 725
 726  def __init__(self, options):
 727    super(SubversionVCS, self).__init__(options)
 728    if self.options.revision:
 729      match = re.match(r"(\d+)(:(\d+))?", self.options.revision)
 730      if not match:
 731        ErrorExit("Invalid Subversion revision %s." % self.options.revision)
 732      self.rev_start = match.group(1)
 733      self.rev_end = match.group(3)
 734    else:
 735      self.rev_start = self.rev_end = None
 736    # Cache output from "svn list -r REVNO dirname".
 737    # Keys: dirname, Values: 2-tuple (ouput for start rev and end rev).
 738    self.svnls_cache = {}
 739    # SVN base URL is required to fetch files deleted in an older revision.
 740    # Result is cached to not guess it over and over again in GetBaseFile().
 741    required = self.options.download_base or self.options.revision is not None
 742    self.svn_base = self._GuessBase(required)
 743
 744  def GuessBase(self, required):
 745    """Wrapper for _GuessBase."""
 746    return self.svn_base
 747
 748  def _GuessBase(self, required):
 749    """Returns the SVN base URL.
 750
 751    Args:
 752      required: If true, exits if the url can't be guessed, otherwise None is
 753        returned.
 754    """
 755    info = RunShell(["svn", "info"])
 756    for line in info.splitlines():
 757      words = line.split()
 758      if len(words) == 2 and words[0] == "URL:":
 759        url = words[1]
 760        scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
 761        username, netloc = urllib.splituser(netloc)
 762        if username:
 763          logging.info("Removed username from base URL")
 764        if netloc.endswith("svn.python.org"):
 765          if netloc == "svn.python.org":
 766            if path.startswith("/projects/"):
 767              path = path[9:]
 768          elif netloc != "pythondev@svn.python.org":
 769            ErrorExit("Unrecognized Python URL: %s" % url)
 770          base = "http://svn.python.org/view/*checkout*%s/" % path
 771          logging.info("Guessed Python base = %s", base)
 772        elif netloc.endswith("svn.collab.net"):
 773          if path.startswith("/repos/"):
 774            path = path[6:]
 775          base = "http://svn.collab.net/viewvc/*checkout*%s/" % path
 776          logging.info("Guessed CollabNet base = %s", base)
 777        elif netloc.endswith(".googlecode.com"):
 778          path = path + "/"
 779          base = urlparse.urlunparse(("http", netloc, path, params,
 780                                      query, fragment))
 781          logging.info("Guessed Google Code base = %s", base)
 782        else:
 783          path = path + "/"
 784          base = urlparse.urlunparse((scheme, netloc, path, params,
 785                                      query, fragment))
 786          logging.info("Guessed base = %s", base)
 787        return base
 788    if required:
 789      ErrorExit("Can't find URL in output from svn info")
 790    return None
 791
 792  def GenerateDiff(self, args):
 793    cmd = ["svn", "diff"]
 794    if self.options.revision:
 795      cmd += ["-r", self.options.revision]
 796    cmd.extend(args)
 797    data = RunShell(cmd)
 798    count = 0
 799    for line in data.splitlines():
 800      if line.startswith("Index:") or line.startswith("Property changes on:"):
 801        count += 1
 802        logging.info(line)
 803    if not count:
 804      ErrorExit("No valid patches found in output from svn diff")
 805    return data
 806
 807  def _CollapseKeywords(self, content, keyword_str):
 808    """Collapses SVN keywords."""
 809    # svn cat translates keywords but svn diff doesn't. As a result of this
 810    # behavior patching.PatchChunks() fails with a chunk mismatch error.
 811    # This part was originally written by the Review Board development team
 812    # who had the same problem (http://reviews.review-board.org/r/276/).
 813    # Mapping of keywords to known aliases
 814    svn_keywords = {
 815      # Standard keywords
 816      'Date':                ['Date', 'LastChangedDate'],
 817      'Revision':            ['Revision', 'LastChangedRevision', 'Rev'],
 818      'Author':              ['Author', 'LastChangedBy'],
 819      'HeadURL':             ['HeadURL', 'URL'],
 820      'Id':                  ['Id'],
 821
 822      # Aliases
 823      'LastChangedDate':     ['LastChangedDate', 'Date'],
 824      'LastChangedRevision': ['LastChangedRevision', 'Rev', 'Revision'],
 825      'LastChangedBy':       ['LastChangedBy', 'Author'],
 826      'URL':                 ['URL', 'HeadURL'],
 827    }
 828
 829    def repl(m):
 830       if m.group(2):
 831         return "$%s::%s$" % (m.group(1), " " * len(m.group(3)))
 832       return "$%s$" % m.group(1)
 833    keywords = [keyword
 834                for name in keyword_str.split(" ")
 835                for keyword in svn_keywords.get(name, [])]
 836    return re.sub(r"\$(%s):(:?)([^\$]+)\$" % '|'.join(keywords), repl, content)
 837
 838  def GetUnknownFiles(self):
 839    status = RunShell(["svn", "status", "--ignore-externals"], silent_ok=True)
 840    unknown_files = []
 841    for line in status.split("\n"):
 842      if line and line[0] == "?":
 843        unknown_files.append(line)
 844    return unknown_files
 845
 846  def ReadFile(self, filename):
 847    """Returns the contents of a file."""
 848    file = open(filename, 'rb')
 849    result = ""
 850    try:
 851      result = file.read()
 852    finally:
 853      file.close()
 854    return result
 855
 856  def GetStatus(self, filename):
 857    """Returns the status of a file."""
 858    if not self.options.revision:
 859      status = RunShell(["svn", "status", "--ignore-externals", filename])
 860      if not status:
 861        ErrorExit("svn status returned no output for %s" % filename)
 862      status_lines = status.splitlines()
 863      # If file is in a cl, the output will begin with
 864      # "\n--- Changelist 'cl_name':\n".  See
 865      # http://svn.collab.net/repos/svn/trunk/notes/changelist-design.txt
 866      if (len(status_lines) == 3 and
 867          not status_lines[0] and
 868          status_lines[1].startswith("--- Changelist")):
 869        status = status_lines[2]
 870      else:
 871        status = status_lines[0]
 872    # If we have a revision to diff against we need to run "svn list"
 873    # for the old and the new revision and compare the results to get
 874    # the correct status for a file.
 875    else:
 876      dirname, relfilename = os.path.split(filename)
 877      if dirname not in self.svnls_cache:
 878        cmd = ["svn", "list", "-r", self.rev_start, dirname or "."]
 879        out, returncode = RunShellWithReturnCode(cmd)
 880        if returncode:
 881          ErrorExit("Failed to get status for %s." % filename)
 882        old_files = out.splitlines()
 883        args = ["svn", "list"]
 884        if self.rev_end:
 885          args += ["-r", self.rev_end]
 886        cmd = args + [dirname or "."]
 887        out, returncode = RunShellWithReturnCode(cmd)
 888        if returncode:
 889          ErrorExit("Failed to run command %s" % cmd)
 890        self.svnls_cache[dirname] = (old_files, out.splitlines())
 891      old_files, new_files = self.svnls_cache[dirname]
 892      if relfilename in old_files and relfilename not in new_files:
 893        status = "D   "
 894      elif relfilename in old_files and relfilename in new_files:
 895        status = "M   "
 896      else:
 897        status = "A   "
 898    return status
 899
 900  def GetBaseFile(self, filename):
 901    status = self.GetStatus(filename)
 902    base_content = None
 903    new_content = None
 904
 905    # If a file is copied its status will be "A  +", which signifies
 906    # "addition-with-history".  See "svn st" for more information.  We need to
 907    # upload the original file or else diff parsing will fail if the file was
 908    # edited.
 909    if status[0] == "A" and status[3] != "+":
 910      # We'll need to upload the new content if we're adding a binary file
 911      # since diff's output won't contain it.
 912      mimetype = RunShell(["svn", "propget", "svn:mime-type", filename],
 913                          silent_ok=True)
 914      base_content = ""
 915      is_binary = bool(mimetype) and not mimetype.startswith("text/")
 916      if is_binary and self.IsImage(filename):
 917        new_content = self.ReadFile(filename)
 918    elif (status[0] in ("M", "D", "R") or
 919          (status[0] == "A" and status[3] == "+") or  # Copied file.
 920          (status[0] == " " and status[1] == "M")):  # Property change.
 921      args = []
 922      if self.options.revision:
 923        url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
 924      else:
 925        # Don't change filename, it's needed later.
 926        url = filename
 927        args += ["-r", "BASE"]
 928      cmd = ["svn"] + args + ["propget", "svn:mime-type", url]
 929      mimetype, returncode = RunShellWithReturnCode(cmd)
 930      if returncode:
 931        # File does not exist in the requested revision.
 932        # Reset mimetype, it contains an error message.
 933        mimetype = ""
 934      get_base = False
 935      is_binary = bool(mimetype) and not mimetype.startswith("text/")
 936      if status[0] == " ":
 937        # Empty base content just to force an upload.
 938        base_content = ""
 939      elif is_binary:
 940        if self.IsImage(filename):
 941          get_base = True
 942          if status[0] == "M":
 943            if not self.rev_end:
 944              new_content = self.ReadFile(filename)
 945            else:
 946              url = "%s/%s@%s" % (self.svn_base, filename, self.rev_end)
 947              new_content = RunShell(["svn", "cat", url],
 948                                     universal_newlines=True, silent_ok=True)
 949        else:
 950          base_content = ""
 951      else:
 952        get_base = True
 953
 954      if get_base:
 955        if is_binary:
 956          universal_newlines = False
 957        else:
 958          universal_newlines = True
 959        if self.rev_start:
 960          # "svn cat -r REV delete_file.txt" doesn't work. cat requires
 961          # the full URL with "@REV" appended instead of using "-r" option.
 962          url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
 963          base_content = RunShell(["svn", "cat", url],
 964                                  universal_newlines=universal_newlines,
 965                                  silent_ok=True)
 966        else:
 967          base_content = RunShell(["svn", "cat", filename],
 968                                  universal_newlines=universal_newlines,
 969                                  silent_ok=True)
 970        if not is_binary:
 971          args = []
 972          if self.rev_start:
 973            url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
 974          else:
 975            url = filename
 976            args += ["-r", "BASE"]
 977          cmd = ["svn"] + args + ["propget", "svn:keywords", url]
 978          keywords, returncode = RunShellWithReturnCode(cmd)
 979          if keywords and not returncode:
 980            base_content = self._CollapseKeywords(base_content, keywords)
 981    else:
 982      StatusUpdate("svn status returned unexpected output: %s" % status)
 983      sys.exit(1)
 984    return base_content, new_content, is_binary, status[0:5]
 985
 986
 987class GitVCS(VersionControlSystem):
 988  """Implementation of the VersionControlSystem interface for Git."""
 989
 990  def __init__(self, options):
 991    super(GitVCS, self).__init__(options)
 992    # Map of filename -> hash of base file.
 993    self.base_hashes = {}
 994
 995  def GenerateDiff(self, extra_args):
 996    # This is more complicated than svn's GenerateDiff because we must convert
 997    # the diff output to include an svn-style "Index:" line as well as record
 998    # the hashes of the base files, so we can upload them along with our diff.
 999    if self.options.revision:
1000      extra_args = [self.options.revision] + extra_args
1001    gitdiff = RunShell(["git", "diff", "--full-index"] + extra_args)
1002    svndiff = []
1003    filecount = 0
1004    filename = None
1005    for line in gitdiff.splitlines():
1006      match = re.match(r"diff --git a/(.*) b/.*$", line)
1007      if match:
1008        filecount += 1
1009        filename = match.group(1)
1010        svndiff.append("Index: %s\n" % filename)
1011      else:
1012        # The "index" line in a git diff looks like this (long hashes elided):
1013        #   index 82c0d44..b2cee3f 100755
1014        # We want to save the left hash, as that identifies the base file.
1015        match = re.match(r"index (\w+)\.\.", line)
1016        if match:
1017          self.base_hashes[filename] = match.group(1)
1018      svndiff.append(line + "\n")
1019    if not filecount:
1020      ErrorExit("No valid patches found in output from git diff")
1021    return "".join(svndiff)
1022
1023  def GetUnknownFiles(self):
1024    status = RunShell(["git", "ls-files", "--exclude-standard", "--others"],
1025                      silent_ok=True)
1026    return status.splitlines()
1027
1028  def GetBaseFile(self, filename):
1029    hash = self.base_hashes[filename]
1030    base_content = None
1031    new_content = None
1032    is_binary = False
1033    if hash == "0" * 40:  # All-zero hash indicates no base file.
1034      status = "A"
1035      base_content = ""
1036    else:
1037      status = "M"
1038      base_content, returncode = RunShellWithReturnCode(["git", "show", hash])
1039      if returncode:
1040        ErrorExit("Got error status from 'git show %s'" % hash)
1041    return (base_content, new_content, is_binary, status)
1042
1043
1044class MercurialVCS(VersionControlSystem):
1045  """Implementation of the VersionControlSystem interface for Mercurial."""
1046
1047  def __init__(self, options, repo_dir):
1048    super(MercurialVCS, self).__init__(options)
1049    # Absolute path to repository (we can be in a subdir)
1050    self.repo_dir = os.path.normpath(repo_dir)
1051    # Compute the subdir
1052    cwd = os.path.normpath(os.getcwd())
1053    assert cwd.startswith(self.repo_dir)
1054    self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/")
1055    if self.options.revision:
1056      self.base_rev = self.options.revision
1057    else:
1058      self.base_rev = RunShell(["hg", "parent", "-q"]).split(':')[1].strip()
1059
1060  def _GetRelPath(self, filename):
1061    """Get relative path of a file according to the current directory,
1062    given its logical path in the repo."""
1063    assert filename.startswith(self.subdir), filename
1064    return filename[len(self.subdir):].lstrip(r"\/")
1065
1066  def GenerateDiff(self, extra_args):
1067    # If no file specified, restrict to the current subdir
1068    extra_args = extra_args or ["."]
1069    cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args
1070    data = RunShell(cmd, silent_ok=True)
1071    svndiff = []
1072    filecount = 0
1073    for line in data.splitlines():
1074      m = re.match("diff --git a/(\S+) b/(\S+)", line)
1075      if m:
1076        # Modify line to make it look like as it comes from svn diff.
1077        # With this modification no changes on the server side are required
1078        # to make upload.py work with Mercurial repos.
1079        # NOTE: for proper handling of moved/copied files, we have to use
1080        # the second filename.
1081        filename = m.group(2)
1082        svndiff.append("Index: %s" % filename)
1083        svndiff.append("=" * 67)
1084        filecount += 1
1085        logging.info(line)
1086      else:
1087        svndiff.append(line)
1088    if not filecount:
1089      ErrorExit("No valid patches found in output from hg diff")
1090    return "\n".join(svndiff) + "\n"
1091
1092  def GetUnknownFiles(self):
1093    """Return a list of files unknown to the VCS."""
1094    args = []
1095    status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."],
1096        silent_ok=True)
1097    unknown_files = []
1098    for line in status.splitlines():
1099      st, fn = line.split(" ", 1)
1100      if st == "?":
1101        unknown_files.append(fn)
1102    return unknown_files
1103
1104  def GetBaseFile(self, filename):
1105    # "hg status" and "hg cat" both take a path relative to the current subdir
1106    # rather than to the repo root, but "hg diff" has given us the full path
1107    # to the repo root.
1108    base_content = ""
1109    new_content = None
1110    is_binary = False
1111    oldrelpath = relpath = self._GetRelPath(filename)
1112    # "hg status -C" returns two lines for moved/copied files, one otherwise
1113    out = RunShell(["hg", "status", "-C", "--rev", self.base_rev, relpath])
1114    out = out.splitlines()
1115    # HACK: strip error message about missing file/directory if it isn't in
1116    # the working copy
1117    if out[0].startswith('%s: ' % relpath):
1118      out = out[1:]
1119    if len(out) > 1:
1120      # Moved/copied => considered as modified, use old filename to
1121      # retrieve base contents
1122      oldrelpath = out[1].strip()
1123      status = "M"
1124    else:
1125      status, _ = out[0].split(' ', 1)
1126    if status != "A":
1127      base_content = RunShell(["hg", "cat", "-r", self.base_rev, oldrelpath],
1128        silent_ok=True)
1129      is_binary = "\0" in base_content  # Mercurial's heuristic
1130    if status != "R":
1131      new_content = open(relpath, "rb").read()
1132      is_binary = is_binary or "\0" in new_content
1133    if is_binary and base_content:
1134      # Fetch again without converting newlines
1135      base_content = RunShell(["hg", "cat", "-r", self.base_rev, oldrelpath],
1136        silent_ok=True, universal_newlines=False)
1137    if not is_binary or not self.IsImage(relpath):
1138      new_content = None
1139    return base_content, new_content, is_binary, status
1140
1141
1142# NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync.
1143def SplitPatch(data):
1144  """Splits a patch into separate pieces for each file.
1145
1146  Args:
1147    data: A string containing the output of svn diff.
1148
1149  Returns:
1150    A list of 2-tuple (filename, text) where text is the svn diff output
1151      pertaining to filename.
1152  """
1153  patches = []
1154  filename = None
1155  diff = []
1156  for line in data.splitlines(True):
1157    new_filename = None
1158    if line.startswith('Index:'):
1159      unused, new_filename = line.split(':', 1)
1160      new_filename = new_filename.strip()
1161    elif line.startswith('Property changes on:'):
1162      unused, temp_filename = line.split(':', 1)
1163      # When a file is modified, paths use '/' between directories, however
1164      # when a property is modified '\' is used on Windows.  Make them the same
1165      # otherwise the file shows up twice.
1166      temp_filename = temp_filename.strip().replace('\\', '/')
1167      if temp_filename != filename:
1168        # File has property changes but no modifications, create a new diff.
1169        new_filename = temp_filename
1170    if new_filename:
1171      if filename and diff:
1172        patches.append((filename, ''.join(diff)))
1173      filename = new_filename
1174      diff = [line]
1175      continue
1176    if diff is not None:
1177      diff.append(line)
1178  if filename and diff:
1179    patches.append((filename, ''.join(diff)))
1180  return patches
1181
1182
1183def UploadSeparatePatches(issue, rpc_server, patchset, data, options):
1184  """Uploads a separate patch for each file in the diff output.
1185
1186  Returns a list of [patch_key, filename] for each file.
1187  """
1188  patches = SplitPatch(data)
1189  rv = []
1190  for patch in patches:
1191    if len(patch[1]) > MAX_UPLOAD_SIZE:
1192      print ("Not uploading the patch for " + patch[0] +
1193             " because the file is too large.")
1194      continue
1195    form_fields = [("filename", patch[0])]
1196    if not options.download_base:
1197      form_fields.append(("content_upload", "1"))
1198    files = [("data", "data.diff", patch[1])]
1199    ctype, body = EncodeMultipartFormData(form_fields, files)
1200    url = "/%d/upload_patch/%d" % (int(issue), int(patchset))
1201    print "Uploading patch for " + patch[0]
1202    response_body = rpc_server.Send(url, body, content_type=ctype)
1203    lines = response_body.splitlines()
1204    if not lines or lines[0] != "OK":
1205      StatusUpdate("  --> %s" % response_body)
1206      sys.exit(1)
1207    rv.append([lines[1], patch[0]])
1208  return rv
1209
1210
1211def GuessVCS(options):
1212  """Helper to guess the version control system.
1213
1214  This examines the current directory, guesses which VersionControlSystem
1215  we're using, and returns an instance of the appropriate class.  Exit with an
1216  error if we can't figure it out.
1217
1218  Returns:
1219    A VersionControlSystem instance. Exits if the VCS can't be guessed.
1220  """
1221  # Mercurial has a command to get the base directory of a repository
1222  # Try running it, but don't die if we don't have hg installed.
1223  # NOTE: we try Mercurial first as it can sit on top of an SVN working copy.
1224  try:
1225    out, returncode = RunShellWithReturnCode(["hg", "root"])
1226    if returncode == 0:
1227      return MercurialVCS(options, out.strip())
1228  except OSError, (errno, message):
1229    if errno != 2:  # ENOENT -- they don't have hg installed.
1230      raise
1231
1232  # Subversion has a .svn in all working directories.
1233  if os.path.isdir('.svn'):
1234    logging.info("Guessed VCS = Subversion")
1235    return SubversionVCS(options)
1236
1237  # Git has a command to test if you're in a git tree.
1238  # Try running it, but don't die if we don't have git installed.
1239  try:
1240    out, returncode = RunShellWithReturnCode(["git", "rev-parse",
1241                                              "--is-inside-work-tree"])
1242    if returncode == 0:
1243      return GitVCS(options)
1244  except OSError, (errno, message):
1245    if errno != 2:  # ENOENT -- they don't have git installed.
1246      raise
1247
1248  ErrorExit(("Could not guess version control system. "
1249             "Are you in a working copy directory?"))
1250
1251
1252def RealMain(argv, data=None):
1253  """The real main function.
1254
1255  Args:
1256    argv: Command line arguments.
1257    data: Diff contents. If None (default) the diff is generated by
1258      the VersionControlSystem implementation returned by GuessVCS().
1259
1260  Returns:
1261    A 2-tuple (issue id, patchset id).
1262    The patchset id is None if the base files are not uploaded by this
1263    script (applies only to SVN checkouts).
1264  """
1265  logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:"
1266                              "%(lineno)s %(message)s "))
1267  os.environ['LC_ALL'] = 'C'
1268  options, args = parser.parse_args(argv[1:])
1269  global verbosity
1270  verbosity = options.verbose
1271  if verbosity >= 3:
1272    logging.getLogger().setLevel(logging.DEBUG)
1273  elif verbosity >= 2:
1274    logging.getLogger().setLevel(logging.INFO)
1275  vcs = GuessVCS(options)
1276  if isinstance(vcs, SubversionVCS):
1277    # base field is only allowed for Subversion.
1278    # Note: Fetching base files may become deprecated in future releases.
1279    base = vcs.GuessBase(options.download_base)
1280  else:
1281    base = None
1282  if not base and options.download_base:
1283    options.download_base = True
1284    logging.info("Enabled upload of base file")
1285  if not options.assume_yes:
1286    vcs.CheckForUnknownFiles()
1287  if data is None:
1288    data = vcs.GenerateDiff(args)
1289  files = vcs.GetBaseFiles(data)
1290  if verbosity >= 1:
1291    print "Upload server:", options.server, "(change with -s/--server)"
1292  if options.issue:
1293    prompt = "Message describing this patch set: "
1294  else:
1295    prompt = "New issue subject: "
1296  message = options.message or raw_input(prompt).strip()
1297  if not message:
1298    ErrorExit("A non-empty message is required")
1299  rpc_server = GetRpcServer(options)
1300  form_fields = [("subject", message)]
1301  if base:
1302    form_fields.append(("base", base))
1303  if options.issue:
1304    form_fields.append(("issue", str(options.issue)))
1305  if options.email:
1306    form_fields.append(("user", options.email))
1307  if options.reviewers:
1308    for reviewer in options.reviewers.split(','):
1309      if "@" in reviewer and not reviewer.split("@")[1].count(".") == 1:
1310        ErrorExit("Invalid email address: %s" % reviewer)
1311    form_fields.append(("reviewers", options.reviewers))
1312  if options.cc:
1313    for cc in options.cc.split(','):
1314      if "@" in cc and not cc.split("@")[1].count(".") == 1:
1315        ErrorExit("Invalid email address: %s" % cc)
1316    form_fields.append(("cc", options.cc))
1317  description = options.description
1318  if options.description_file:
1319    if options.description:
1320      ErrorExit("Can't specify description and description_file")
1321    file = open(options.description_file, 'r')
1322    description = file.read()
1323    file.close()
1324  if description:
1325    form_fields.append(("description", description))
1326  # Send a hash of all the base file so the server can determine if a copy
1327  # already exists in an earlier patchset.
1328  base_hashes = ""
1329  for file, info in files.iteritems():
1330    if not info[0] is None:
1331      checksum = md5.new(info[0]).hexdigest()
1332      if base_hashes:
1333        base_hashes += "|"
1334      base_hashes += checksum + ":" + file
1335  form_fields.append(("base_hashes", base_hashes))
1336  # If we're uploading base files, don't send the email before the uploads, so
1337  # that it contains the file status.
1338  if options.send_mail and options.download_base:
1339    form_fields.append(("send_mail", "1"))
1340  if not options.download_base:
1341    form_fields.append(("content_upload", "1"))
1342  if len(data) > MAX_UPLOAD_SIZE:
1343    print "Patch is large, so uploading file patches separately."
1344    uploaded_diff_file = []
1345    form_fields.append(("separate_patches", "1"))
1346  else:
1347    uploaded_diff_file = [("data", "data.diff", data)]
1348  ctype, body = EncodeMultipartFormData(form_fields, u

Large files files are truncated, but you can click here to view the full file