PageRenderTime 57ms CodeModel.GetById 23ms RepoModel.GetById 0ms app.codeStats 0ms

/src/qt/qtwebkit/Tools/Scripts/webkitpy/common/checkout/changelog.py

https://gitlab.com/x33n/phantomjs
Python | 459 lines | 409 code | 11 blank | 39 comment | 6 complexity | 560eb23aaf476e0c7b242b49b2cff6f8 MD5 | raw file
  1. # Copyright (C) 2009, Google Inc. All rights reserved.
  2. #
  3. # Redistribution and use in source and binary forms, with or without
  4. # modification, are permitted provided that the following conditions are
  5. # met:
  6. #
  7. # * Redistributions of source code must retain the above copyright
  8. # notice, this list of conditions and the following disclaimer.
  9. # * Redistributions in binary form must reproduce the above
  10. # copyright notice, this list of conditions and the following disclaimer
  11. # in the documentation and/or other materials provided with the
  12. # distribution.
  13. # * Neither the name of Google Inc. nor the names of its
  14. # contributors may be used to endorse or promote products derived from
  15. # this software without specific prior written permission.
  16. #
  17. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18. # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19. # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20. # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21. # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22. # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23. # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24. # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25. # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28. #
  29. # WebKit's Python module for parsing and modifying ChangeLog files
  30. import logging
  31. import re
  32. from StringIO import StringIO
  33. import textwrap
  34. from webkitpy.common.config.committers import CommitterList
  35. from webkitpy.common.system.filesystem import FileSystem
  36. import webkitpy.common.config.urls as config_urls
  37. _log = logging.getLogger(__name__)
  38. # FIXME: parse_bug_id_from_changelog should not be a free function.
  39. # Parse the bug ID out of a Changelog message based on the format that is
  40. # used by prepare-ChangeLog
  41. def parse_bug_id_from_changelog(message):
  42. if not message:
  43. return None
  44. match = re.search("^\s*" + config_urls.bug_url_short + "$", message, re.MULTILINE)
  45. if match:
  46. return int(match.group('bug_id'))
  47. match = re.search("^\s*" + config_urls.bug_url_long + "$", message, re.MULTILINE)
  48. if match:
  49. return int(match.group('bug_id'))
  50. # We weren't able to find a bug URL in the format used by prepare-ChangeLog. Fall back to the
  51. # first bug URL found anywhere in the message.
  52. return config_urls.parse_bug_id(message)
  53. class ChangeLogEntry(object):
  54. # e.g. 2009-06-03 Eric Seidel <eric@webkit.org>
  55. date_line_regexp = r'^(?P<date>\d{4}-\d{2}-\d{2})\s+(?P<authors>(?P<name>[^<]+?)\s+<(?P<email>[^<>]+)>.*?)$'
  56. # e.g. * Source/WebCore/page/EventHandler.cpp: Implement FooBarQuux.
  57. touched_files_regexp = r'^\s*\*\s*(?P<file>[A-Za-z0-9_\-\./\\]+)\s*\:'
  58. # e.g. (ChangeLogEntry.touched_functions): Added.
  59. touched_functions_regexp = r'^\s*\((?P<function>[^)]*)\):'
  60. # e.g. Reviewed by Darin Adler.
  61. # (Discard everything after the first period to match more invalid lines.)
  62. reviewed_by_regexp = r'^\s*((\w+\s+)+and\s+)?(Review|Rubber(\s*|-)stamp)(s|ed)?\s+([a-z]+\s+)*?by\s+(?P<reviewer>.*?)[\.,]?\s*$'
  63. reviewed_byless_regexp = r'^\s*((Review|Rubber(\s*|-)stamp)(s|ed)?|RS)(\s+|\s*=\s*)(?P<reviewer>([A-Z]\w+\s*)+)[\.,]?\s*$'
  64. reviewer_name_noise_regexp = re.compile(r"""
  65. (\s+((tweaked\s+)?and\s+)?(landed|committed|okayed)\s+by.+) # "landed by", "commented by", etc...
  66. |(^(Reviewed\s+)?by\s+) # extra "Reviewed by" or "by"
  67. |([(<]\s*[\w_\-\.]+@[\w_\-\.]+[>)]) # email addresses
  68. |([(<](https?://?bugs.)webkit.org[^>)]+[>)]) # bug url
  69. |("[^"]+") # wresler names like 'Sean/Shawn/Shaun' in 'Geoffrey "Sean/Shawn/Shaun" Garen'
  70. |('[^']+') # wresler names like "The Belly" in "Sam 'The Belly' Weinig"
  71. |((Mr|Ms|Dr|Mrs|Prof)\.(\s+|$))
  72. """, re.IGNORECASE | re.VERBOSE)
  73. reviewer_name_casesensitive_noise_regexp = re.compile(r"""
  74. ((\s+|^)(and\s+)?([a-z-]+\s+){5,}by\s+) # e.g. "and given a good once-over by"
  75. |(\(\s*(?!(and|[A-Z])).+\)) # any parenthesis that doesn't start with "and" or a capital letter
  76. |(with(\s+[a-z-]+)+) # phrases with "with no hesitation" in "Sam Weinig with no hesitation"
  77. """, re.VERBOSE)
  78. reviewer_name_noise_needing_a_backreference_regexp = re.compile(r"""
  79. (\S\S)\.(?:(\s.+|$)) # Text after the two word characters (don't match initials) and a period followed by a space.
  80. """, re.IGNORECASE | re.VERBOSE)
  81. nobody_regexp = re.compile(r"""(\s+|^)nobody(
  82. ((,|\s+-)?\s+(\w+\s+)+fix.*) # e.g. nobody, build fix...
  83. |(\s*\([^)]+\).*) # NOBODY (..)...
  84. |$)""", re.IGNORECASE | re.VERBOSE)
  85. # e.g. == Rolled over to ChangeLog-2011-02-16 ==
  86. rolled_over_regexp = r'^== Rolled over to ChangeLog-\d{4}-\d{2}-\d{2} ==$'
  87. # e.g. git-svn-id: http://svn.webkit.org/repository/webkit/trunk@96161 268f45cc-cd09-0410-ab3c-d52691b4dbfc
  88. svn_id_regexp = r'git-svn-id: http://svn.webkit.org/repository/webkit/trunk@(?P<svnid>\d+) '
  89. split_names_regexp = r'\s*(?:,(?:\s+and\s+|&)?|(?:^|\s+)and\s+|&&|[/+&])\s*'
  90. def __init__(self, contents, committer_list=CommitterList(), revision=None):
  91. self._contents = contents
  92. self._committer_list = committer_list
  93. self._revision = revision
  94. self._parse_entry()
  95. @classmethod
  96. def _parse_reviewer_text(cls, text):
  97. match = re.search(ChangeLogEntry.reviewed_by_regexp, text, re.MULTILINE | re.IGNORECASE)
  98. if not match:
  99. # There are cases where people omit "by". We match it only if reviewer part looked nice
  100. # in order to avoid matching random lines that start with Reviewed
  101. match = re.search(ChangeLogEntry.reviewed_byless_regexp, text, re.MULTILINE | re.IGNORECASE)
  102. if not match:
  103. return None, None
  104. reviewer_text = match.group("reviewer")
  105. reviewer_text = ChangeLogEntry.nobody_regexp.sub('', reviewer_text)
  106. reviewer_text = ChangeLogEntry.reviewer_name_noise_regexp.sub('', reviewer_text)
  107. reviewer_text = ChangeLogEntry.reviewer_name_casesensitive_noise_regexp.sub('', reviewer_text)
  108. reviewer_text = ChangeLogEntry.reviewer_name_noise_needing_a_backreference_regexp.sub(r'\1', reviewer_text)
  109. reviewer_text = reviewer_text.replace('(', '').replace(')', '')
  110. reviewer_text = re.sub(r'\s\s+|[,.]\s*$', ' ', reviewer_text).strip()
  111. if not len(reviewer_text):
  112. return None, None
  113. reviewer_list = ChangeLogEntry._split_reviewer_names(reviewer_text)
  114. # Get rid of "reviewers" like "even though this is just a..." in "Reviewed by Sam Weinig, even though this is just a..."
  115. # and "who wrote the original code" in "Noam Rosenthal, who wrote the original code"
  116. reviewer_list = [reviewer for reviewer in reviewer_list if not re.match('^who\s|^([a-z]+(\s+|\.|$)){6,}$', reviewer)]
  117. return reviewer_text, reviewer_list
  118. @classmethod
  119. def _split_reviewer_names(cls, text):
  120. return re.split(ChangeLogEntry.split_names_regexp, text)
  121. @classmethod
  122. def _split_author_names_with_emails(cls, text):
  123. regex = '>' + ChangeLogEntry.split_names_regexp
  124. names = re.split(regex, text)
  125. if len(names) > 1:
  126. names = [name + ">" for name in names[:-1]] + [names[-1]]
  127. return names
  128. def _fuzz_match_reviewers(self, reviewers_text_list):
  129. if not reviewers_text_list:
  130. return []
  131. list_of_reviewers = [self._committer_list.contributors_by_fuzzy_match(reviewer)[0] for reviewer in reviewers_text_list]
  132. # Flatten lists and get rid of any reviewers with more than one candidate.
  133. return [reviewers[0] for reviewers in list_of_reviewers if len(reviewers) == 1]
  134. @classmethod
  135. def _parse_author_name_and_email(cls, author_name_and_email):
  136. match = re.match(r'(?P<name>.+?)\s+<(?P<email>[^>]+)>', author_name_and_email)
  137. return {'name': match.group("name"), 'email': match.group("email")}
  138. @classmethod
  139. def _parse_author_text(cls, text):
  140. if not text:
  141. return []
  142. authors = cls._split_author_names_with_emails(text)
  143. assert(authors and len(authors) >= 1)
  144. return [cls._parse_author_name_and_email(author) for author in authors]
  145. @classmethod
  146. def _parse_touched_functions(cls, text):
  147. result = {}
  148. cur_file = None
  149. for line in text.splitlines():
  150. file_match = re.match(cls.touched_files_regexp, line)
  151. if file_match:
  152. cur_file = file_match.group("file")
  153. result[cur_file] = []
  154. func_match = re.match(cls.touched_functions_regexp, line)
  155. if func_match and cur_file:
  156. result[cur_file].append(func_match.group("function"))
  157. return result
  158. @classmethod
  159. def _parse_bug_description(cls, text):
  160. # If line 4 is a bug url, line 3 is the bug description.
  161. # It's too hard to guess in other cases, so we return None.
  162. lines = text.splitlines()
  163. if len(lines) < 4:
  164. return None
  165. for bug_url in (config_urls.bug_url_short, config_urls.bug_url_long):
  166. if re.match("^\s*" + bug_url + "$", lines[3]):
  167. return lines[2].strip()
  168. return None
  169. def _parse_entry(self):
  170. match = re.match(self.date_line_regexp, self._contents, re.MULTILINE)
  171. if not match:
  172. _log.warning("Creating invalid ChangeLogEntry:\n%s" % self._contents)
  173. self._date_line = match.group()
  174. self._date = match.group("date")
  175. self._bug_description = self._parse_bug_description(self._contents)
  176. # FIXME: group("name") does not seem to be Unicode? Probably due to self._contents not being unicode.
  177. self._author_text = match.group("authors") if match else None
  178. self._authors = ChangeLogEntry._parse_author_text(self._author_text)
  179. self._reviewer_text, self._reviewers_text_list = ChangeLogEntry._parse_reviewer_text(self._contents)
  180. self._reviewers = self._fuzz_match_reviewers(self._reviewers_text_list)
  181. self._author = self._committer_list.contributor_by_email(self.author_email()) or self._committer_list.contributor_by_name(self.author_name())
  182. self._touched_files = re.findall(self.touched_files_regexp, self._contents, re.MULTILINE)
  183. self._touched_functions = self._parse_touched_functions(self._contents)
  184. def date_line(self):
  185. return self._date_line
  186. def date(self):
  187. return self._date
  188. def author_text(self):
  189. return self._author_text
  190. def revision(self):
  191. return self._revision
  192. def author_name(self):
  193. return self._authors[0]['name']
  194. def author_email(self):
  195. return self._authors[0]['email']
  196. def author(self):
  197. return self._author # Might be None
  198. def authors(self):
  199. return self._authors
  200. # FIXME: Eventually we would like to map reviwer names to reviewer objects.
  201. # See https://bugs.webkit.org/show_bug.cgi?id=26533
  202. def reviewer_text(self):
  203. return self._reviewer_text
  204. # Might be None, might also not be a Reviewer!
  205. def reviewer(self):
  206. return self._reviewers[0] if len(self._reviewers) > 0 else None
  207. def reviewers(self):
  208. return self._reviewers
  209. def has_valid_reviewer(self):
  210. if self._reviewers_text_list:
  211. for reviewer in self._reviewers_text_list:
  212. reviewer = self._committer_list.committer_by_name(reviewer)
  213. if reviewer:
  214. return True
  215. return bool(re.search("unreviewed", self._contents, re.IGNORECASE))
  216. def contents(self):
  217. return self._contents
  218. def bug_id(self):
  219. return parse_bug_id_from_changelog(self._contents)
  220. def bug_description(self):
  221. return self._bug_description
  222. def touched_files(self):
  223. return self._touched_files
  224. # Returns a dict from file name to lists of function names.
  225. def touched_functions(self):
  226. return self._touched_functions
  227. def touched_files_text(self):
  228. match = re.search(self.touched_files_regexp, self._contents, re.MULTILINE)
  229. return self._contents[match.start():].lstrip("\n\r") if match else ""
  230. # Determine if any text has been added to the section on touched files
  231. def is_touched_files_text_clean(self):
  232. file_line_end = r"( (Added|Removed|(Copied|Renamed) from [A-Za-z0-9_\-./\\]+).)?$"
  233. for line in self.touched_files_text().splitlines():
  234. if re.match(self.touched_files_regexp + file_line_end, line):
  235. continue
  236. if re.match(self.touched_functions_regexp + "$", line):
  237. continue
  238. return False
  239. return True
  240. # FIXME: Various methods on ChangeLog should move into ChangeLogEntry instead.
  241. class ChangeLog(object):
  242. def __init__(self, path, filesystem=None):
  243. self.path = path
  244. self._filesystem = filesystem or FileSystem()
  245. _changelog_indent = " " * 8
  246. @classmethod
  247. def parse_latest_entry_from_file(cls, changelog_file):
  248. try:
  249. return next(cls.parse_entries_from_file(changelog_file))
  250. except StopIteration, e:
  251. return None
  252. svn_blame_regexp = re.compile(r'^(\s*(?P<revision>\d+) [^ ]+)\s*(?P<line>.*?\n)')
  253. @classmethod
  254. def _separate_revision_and_line(cls, line):
  255. match = cls.svn_blame_regexp.match(line)
  256. if not match:
  257. return None, line
  258. return int(match.group('revision')), match.group('line')
  259. @classmethod
  260. def parse_entries_from_file(cls, changelog_file):
  261. """changelog_file must be a file-like object which returns
  262. unicode strings, e.g. from StringIO(unicode()) or
  263. fs.open_text_file_for_reading()"""
  264. date_line_regexp = re.compile(ChangeLogEntry.date_line_regexp)
  265. rolled_over_regexp = re.compile(ChangeLogEntry.rolled_over_regexp)
  266. # The first line should be a date line.
  267. revision, first_line = cls._separate_revision_and_line(changelog_file.readline())
  268. assert(isinstance(first_line, unicode))
  269. if not date_line_regexp.match(cls.svn_blame_regexp.sub('', first_line)):
  270. raise StopIteration
  271. entry_lines = [first_line]
  272. revisions_in_entry = {revision: 1} if revision != None else None
  273. for line in changelog_file:
  274. if revisions_in_entry:
  275. revision, line = cls._separate_revision_and_line(line)
  276. if rolled_over_regexp.match(line):
  277. break
  278. if date_line_regexp.match(line):
  279. most_probable_revision = max(revisions_in_entry, key=revisions_in_entry.__getitem__) if revisions_in_entry else None
  280. # Remove the extra newline at the end
  281. yield ChangeLogEntry(''.join(entry_lines[:-1]), revision=most_probable_revision)
  282. entry_lines = []
  283. revisions_in_entry = {revision: 0}
  284. entry_lines.append(line)
  285. if revisions_in_entry:
  286. revisions_in_entry[revision] = revisions_in_entry.get(revision, 0) + 1
  287. most_probable_revision = max(revisions_in_entry, key=revisions_in_entry.__getitem__) if revisions_in_entry else None
  288. yield ChangeLogEntry(''.join(entry_lines[:-1]), revision=most_probable_revision)
  289. def latest_entry(self):
  290. # ChangeLog files are always UTF-8, we read them in as such to support Reviewers with unicode in their names.
  291. changelog_file = self._filesystem.open_text_file_for_reading(self.path)
  292. try:
  293. return self.parse_latest_entry_from_file(changelog_file)
  294. finally:
  295. changelog_file.close()
  296. # _wrap_line and _wrap_lines exist to work around
  297. # http://bugs.python.org/issue1859
  298. def _wrap_line(self, line):
  299. return textwrap.fill(line,
  300. width=70,
  301. initial_indent=self._changelog_indent,
  302. # Don't break urls which may be longer than width.
  303. break_long_words=False,
  304. subsequent_indent=self._changelog_indent)
  305. # Workaround as suggested by guido in
  306. # http://bugs.python.org/issue1859#msg60040
  307. def _wrap_lines(self, message):
  308. lines = [self._wrap_line(line) for line in message.splitlines()]
  309. return "\n".join(lines)
  310. def update_with_unreviewed_message(self, message):
  311. first_boilerplate_line_regexp = re.compile(
  312. "%sNeed a short description \(OOPS!\)\." % self._changelog_indent)
  313. removing_boilerplate = False
  314. result = StringIO()
  315. with self._filesystem.open_text_file_for_reading(self.path) as file:
  316. for line in file:
  317. if first_boilerplate_line_regexp.search(line):
  318. message_lines = self._wrap_lines(message)
  319. result.write(first_boilerplate_line_regexp.sub(message_lines, line))
  320. # Remove all the ChangeLog boilerplate before the first changed
  321. # file.
  322. removing_boilerplate = True
  323. elif removing_boilerplate:
  324. if line.find('*') >= 0: # each changed file is preceded by a *
  325. removing_boilerplate = False
  326. if not removing_boilerplate:
  327. result.write(line)
  328. self._filesystem.write_text_file(self.path, result.getvalue())
  329. def set_reviewer(self, reviewer):
  330. latest_entry = self.latest_entry()
  331. latest_entry_contents = latest_entry.contents()
  332. reviewer_text = latest_entry.reviewer()
  333. found_nobody = re.search("NOBODY\s*\(OOPS!\)", latest_entry_contents, re.MULTILINE)
  334. if not found_nobody and not reviewer_text:
  335. bug_url_number_of_items = len(re.findall(config_urls.bug_url_long, latest_entry_contents, re.MULTILINE))
  336. bug_url_number_of_items += len(re.findall(config_urls.bug_url_short, latest_entry_contents, re.MULTILINE))
  337. result = StringIO()
  338. with self._filesystem.open_text_file_for_reading(self.path) as file:
  339. for line in file:
  340. found_bug_url = re.search(config_urls.bug_url_long, line)
  341. if not found_bug_url:
  342. found_bug_url = re.search(config_urls.bug_url_short, line)
  343. result.write(line)
  344. if found_bug_url:
  345. if bug_url_number_of_items == 1:
  346. result.write("\n Reviewed by %s.\n" % reviewer)
  347. bug_url_number_of_items -= 1
  348. self._filesystem.write_text_file(self.path, result.getvalue())
  349. else:
  350. data = self._filesystem.read_text_file(self.path)
  351. newdata = data.replace("NOBODY (OOPS!)", reviewer)
  352. self._filesystem.write_text_file(self.path, newdata)
  353. def set_short_description_and_bug_url(self, short_description, bug_url):
  354. message = "%s\n%s%s" % (short_description, self._changelog_indent, bug_url)
  355. bug_boilerplate = "%sNeed the bug URL (OOPS!).\n" % self._changelog_indent
  356. result = StringIO()
  357. with self._filesystem.open_text_file_for_reading(self.path) as file:
  358. for line in file:
  359. line = line.replace("Need a short description (OOPS!).", message)
  360. if line != bug_boilerplate:
  361. result.write(line)
  362. self._filesystem.write_text_file(self.path, result.getvalue())
  363. def delete_entries(self, num_entries):
  364. date_line_regexp = re.compile(ChangeLogEntry.date_line_regexp)
  365. rolled_over_regexp = re.compile(ChangeLogEntry.rolled_over_regexp)
  366. entries = 0
  367. result = StringIO()
  368. with self._filesystem.open_text_file_for_reading(self.path) as file:
  369. for line in file:
  370. if date_line_regexp.match(line):
  371. entries += 1
  372. elif rolled_over_regexp.match(line):
  373. entries = num_entries + 1
  374. if entries > num_entries:
  375. result.write(line)
  376. self._filesystem.write_text_file(self.path, result.getvalue())
  377. def prepend_text(self, text):
  378. data = self._filesystem.read_text_file(self.path)
  379. self._filesystem.write_text_file(self.path, text + data)