PageRenderTime 61ms CodeModel.GetById 30ms RepoModel.GetById 0ms app.codeStats 1ms

/tests/DumpRenderTree/assets/run_layout_tests.py

https://bitbucket.org/Lloir/miragebase
Python | 320 lines | 217 code | 44 blank | 59 comment | 33 complexity | 2045facdbf2c345787ca7ef131d6e756 MD5 | raw file
  1. #!/usr/bin/python
  2. """Run layout tests using Android emulator and instrumentation.
  3. First, you need to get an SD card or sdcard image that has layout tests on it.
  4. Layout tests are in following directory:
  5. /sdcard/webkit/layout_tests
  6. For example, /sdcard/webkit/layout_tests/fast
  7. Usage:
  8. Run all tests under fast/ directory:
  9. run_layout_tests.py, or
  10. run_layout_tests.py fast
  11. Run all tests under a sub directory:
  12. run_layout_tests.py fast/dom
  13. Run a single test:
  14. run_layout_tests.py fast/dom/
  15. After a merge, if there are changes of layout tests in SD card, you need to
  16. use --refresh-test-list option *once* to re-generate test list on the card.
  17. Some other options are:
  18. --rebaseline generates expected layout tests results under /sdcard/webkit/expected_result/
  19. --time-out-ms (default is 8000 millis) for each test
  20. --adb-options="-e" passes option string to adb
  21. --results-directory=..., (default is ./layout-test-results) directory name under which results are stored.
  22. --js-engine the JavaScript engine currently in use, determines which set of Android-specific expected results we should use, should be 'jsc' or 'v8'
  23. """
  24. import logging
  25. import optparse
  26. import os
  27. import subprocess
  28. import sys
  29. import time
  30. def CountLineNumber(filename):
  31. """Compute the number of lines in a given file.
  32. Args:
  33. filename: a file name related to the current directory.
  34. """
  35. fp = open(os.path.abspath(filename), "r");
  36. lines = 0
  37. for line in fp.readlines():
  38. lines = lines + 1
  39. fp.close()
  40. return lines
  41. def DumpRenderTreeFinished(adb_cmd):
  42. """ Check if DumpRenderTree finished running tests
  43. Args:
  44. output: adb_cmd string
  45. """
  46. # pull /sdcard/webkit/running_test.txt, if the content is "#DONE", it's done
  47. shell_cmd_str = adb_cmd + " shell cat /sdcard/webkit/running_test.txt"
  48. adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
  49. return adb_output.strip() == "#DONE"
  50. def DiffResults(marker, new_results, old_results, diff_results, strip_reason,
  51. new_count_first=True):
  52. """ Given two result files, generate diff and
  53. write to diff_results file. All arguments are absolute paths
  54. to files.
  55. """
  56. old_file = open(old_results, "r")
  57. new_file = open(new_results, "r")
  58. diff_file = open(diff_results, "a")
  59. # Read lines from each file
  60. ndict = new_file.readlines()
  61. cdict = old_file.readlines()
  62. # Write marker to diff file
  63. diff_file.writelines(marker + "\n")
  64. diff_file.writelines("###############\n")
  65. # Strip reason from result lines
  66. if strip_reason is True:
  67. for i in range(0, len(ndict)):
  68. ndict[i] = ndict[i].split(' ')[0] + "\n"
  69. for i in range(0, len(cdict)):
  70. cdict[i] = cdict[i].split(' ')[0] + "\n"
  71. params = {
  72. "new": [0, ndict, cdict, "+"],
  73. "miss": [0, cdict, ndict, "-"]
  74. }
  75. if new_count_first:
  76. order = ["new", "miss"]
  77. else:
  78. order = ["miss", "new"]
  79. for key in order:
  80. for line in params[key][1]:
  81. if line not in params[key][2]:
  82. if line[-1] != "\n":
  83. line += "\n";
  84. diff_file.writelines(params[key][3] + line)
  85. params[key][0] += 1
  86. logging.info(marker + " >>> " + str(params["new"][0]) + " new, " +
  87. str(params["miss"][0]) + " misses")
  88. diff_file.writelines("\n\n")
  89. old_file.close()
  90. new_file.close()
  91. diff_file.close()
  92. return
  93. def CompareResults(ref_dir, results_dir):
  94. """Compare results in two directories
  95. Args:
  96. ref_dir: the reference directory having layout results as references
  97. results_dir: the results directory
  98. """
  99. logging.info("Comparing results to " + ref_dir)
  100. diff_result = os.path.join(results_dir, "layout_tests_diff.txt")
  101. if os.path.exists(diff_result):
  102. os.remove(diff_result)
  103. files=["crashed", "failed", "passed", "nontext"]
  104. for f in files:
  105. result_file_name = "layout_tests_" + f + ".txt"
  106. DiffResults(f, os.path.join(results_dir, result_file_name),
  107. os.path.join(ref_dir, result_file_name), diff_result,
  108. False, f != "passed")
  109. logging.info("Detailed diffs are in " + diff_result)
  110. def main(options, args):
  111. """Run the tests. Will call sys.exit when complete.
  112. Args:
  113. options: a dictionary of command line options
  114. args: a list of sub directories or files to test
  115. """
  116. # Set up logging format.
  117. log_level = logging.INFO
  118. if options.verbose:
  119. log_level = logging.DEBUG
  120. logging.basicConfig(level=log_level,
  121. format='%(message)s')
  122. # Include all tests if none are specified.
  123. if not args:
  124. path = '/';
  125. else:
  126. path = ' '.join(args);
  127. adb_cmd = "adb ";
  128. if options.adb_options:
  129. adb_cmd += options.adb_options
  130. # Re-generate the test list if --refresh-test-list is on
  131. if options.refresh_test_list:
  132. logging.info("Generating test list.");
  133. generate_test_list_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#generateTestList -e path \"" + path + "\" -w com.android.dumprendertree/.LayoutTestsAutoRunner"
  134. adb_output = subprocess.Popen(generate_test_list_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
  135. if adb_output.find('Process crashed') != -1:
  136. logging.info("Aborting because cannot generate test list.\n" + adb_output)
  137. sys.exit(1)
  138. logging.info("Running tests")
  139. # Count crashed tests.
  140. crashed_tests = []
  141. timeout_ms = '15000'
  142. if options.time_out_ms:
  143. timeout_ms = options.time_out_ms
  144. # Run test until it's done
  145. run_layout_test_cmd_prefix = adb_cmd + " shell am instrument"
  146. run_layout_test_cmd_postfix = " -e path \"" + path + "\" -e timeout " + timeout_ms
  147. if options.rebaseline:
  148. run_layout_test_cmd_postfix += " -e rebaseline true"
  149. # If the JS engine is not specified on the command line, try reading the
  150. # JS_ENGINE environment variable, which is used by the build system in
  151. # external/webkit/Android.mk.
  152. js_engine = options.js_engine
  153. if not js_engine and os.environ.has_key('JS_ENGINE'):
  154. js_engine = os.environ['JS_ENGINE']
  155. if js_engine:
  156. run_layout_test_cmd_postfix += " -e jsengine " + js_engine
  157. run_layout_test_cmd_postfix += " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
  158. # Call LayoutTestsAutoTest::startLayoutTests.
  159. run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#startLayoutTests" + run_layout_test_cmd_postfix
  160. adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
  161. while not DumpRenderTreeFinished(adb_cmd):
  162. # Get the running_test.txt
  163. logging.error("DumpRenderTree crashed, output:\n" + adb_output)
  164. shell_cmd_str = adb_cmd + " shell cat /sdcard/webkit/running_test.txt"
  165. crashed_test = ""
  166. while not crashed_test:
  167. (crashed_test, err) = subprocess.Popen(
  168. shell_cmd_str, shell=True, stdout=subprocess.PIPE,
  169. stderr=subprocess.PIPE).communicate()
  170. crashed_test = crashed_test.strip()
  171. if not crashed_test:
  172. logging.error('Cannot get crashed test name, device offline?')
  173. logging.error('stderr: ' + err)
  174. logging.error('retrying in 10s...')
  175. time.sleep(10)
  176. logging.info(crashed_test + " CRASHED");
  177. crashed_tests.append(crashed_test);
  178. logging.info("Resuming layout test runner...");
  179. # Call LayoutTestsAutoTest::resumeLayoutTests
  180. run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#resumeLayoutTests" + run_layout_test_cmd_postfix
  181. adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
  182. if adb_output.find('INSTRUMENTATION_FAILED') != -1:
  183. logging.error("Error happened : " + adb_output)
  184. sys.exit(1)
  185. logging.debug(adb_output);
  186. logging.info("Done\n");
  187. # Pull results from /sdcard
  188. results_dir = options.results_directory
  189. if not os.path.exists(results_dir):
  190. os.makedirs(results_dir)
  191. if not os.path.isdir(results_dir):
  192. logging.error("Cannot create results dir: " + results_dir);
  193. sys.exit(1);
  194. result_files = ["/sdcard/layout_tests_passed.txt",
  195. "/sdcard/layout_tests_failed.txt",
  196. "/sdcard/layout_tests_ignored.txt",
  197. "/sdcard/layout_tests_nontext.txt"]
  198. for file in result_files:
  199. shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
  200. adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
  201. logging.debug(adb_output)
  202. # Create the crash list.
  203. fp = open(results_dir + "/layout_tests_crashed.txt", "w");
  204. for crashed_test in crashed_tests:
  205. fp.writelines(crashed_test + '\n')
  206. fp.close()
  207. # Count the number of tests in each category.
  208. passed_tests = CountLineNumber(results_dir + "/layout_tests_passed.txt")
  209. logging.info(str(passed_tests) + " passed")
  210. failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
  211. logging.info(str(failed_tests) + " failed")
  212. ignored_tests = CountLineNumber(results_dir + "/layout_tests_ignored.txt")
  213. logging.info(str(ignored_tests) + " ignored results")
  214. crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
  215. logging.info(str(crashed_tests) + " crashed")
  216. nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
  217. logging.info(str(nontext_tests) + " no dumpAsText")
  218. logging.info(str(passed_tests + failed_tests + ignored_tests + crashed_tests + nontext_tests) + " TOTAL")
  219. logging.info("Results are stored under: " + results_dir + "\n")
  220. # Comparing results to references to find new fixes and regressions.
  221. results_dir = os.path.abspath(options.results_directory)
  222. ref_dir = options.ref_directory
  223. # if ref_dir is null, cannonify ref_dir to the script dir.
  224. if not ref_dir:
  225. script_self = sys.argv[0]
  226. script_dir = os.path.dirname(script_self)
  227. ref_dir = os.path.join(script_dir, "results")
  228. ref_dir = os.path.abspath(ref_dir)
  229. CompareResults(ref_dir, results_dir)
  230. if '__main__' == __name__:
  231. option_parser = optparse.OptionParser()
  232. option_parser.add_option("", "--rebaseline", action="store_true",
  233. default=False,
  234. help="generate expected results for those tests not having one")
  235. option_parser.add_option("", "--time-out-ms",
  236. default=None,
  237. help="set the timeout for each test")
  238. option_parser.add_option("", "--verbose", action="store_true",
  239. default=False,
  240. help="include debug-level logging")
  241. option_parser.add_option("", "--refresh-test-list", action="store_true",
  242. default=False,
  243. help="re-generate test list, it may take some time.")
  244. option_parser.add_option("", "--adb-options",
  245. default=None,
  246. help="pass options to adb, such as -d -e, etc");
  247. option_parser.add_option("", "--results-directory",
  248. default="layout-test-results",
  249. help="directory which results are stored.")
  250. option_parser.add_option("", "--ref-directory",
  251. default=None,
  252. dest="ref_directory",
  253. help="directory where reference results are stored.")
  254. option_parser.add_option("", "--js-engine",
  255. default=None,
  256. help="The JavaScript engine currently in use, which determines which set of Android-specific expected results we should use. Should be 'jsc' or 'v8'.");
  257. options, args = option_parser.parse_args();
  258. main(options, args)