/tests/DumpRenderTree/assets/run_layout_tests.py
Python | 320 lines | 217 code | 44 blank | 59 comment | 33 complexity | 2045facdbf2c345787ca7ef131d6e756 MD5 | raw file
Possible License(s): MPL-2.0-no-copyleft-exception, CC0-1.0
- #!/usr/bin/python
- """Run layout tests using Android emulator and instrumentation.
- First, you need to get an SD card or sdcard image that has layout tests on it.
- Layout tests are in following directory:
- /sdcard/webkit/layout_tests
- For example, /sdcard/webkit/layout_tests/fast
- Usage:
- Run all tests under fast/ directory:
- run_layout_tests.py, or
- run_layout_tests.py fast
- Run all tests under a sub directory:
- run_layout_tests.py fast/dom
- Run a single test:
- run_layout_tests.py fast/dom/
- After a merge, if there are changes of layout tests in SD card, you need to
- use --refresh-test-list option *once* to re-generate test list on the card.
- Some other options are:
- --rebaseline generates expected layout tests results under /sdcard/webkit/expected_result/
- --time-out-ms (default is 8000 millis) for each test
- --adb-options="-e" passes option string to adb
- --results-directory=..., (default is ./layout-test-results) directory name under which results are stored.
- --js-engine the JavaScript engine currently in use, determines which set of Android-specific expected results we should use, should be 'jsc' or 'v8'
- """
- import logging
- import optparse
- import os
- import subprocess
- import sys
- import time
- def CountLineNumber(filename):
- """Compute the number of lines in a given file.
- Args:
- filename: a file name related to the current directory.
- """
- fp = open(os.path.abspath(filename), "r");
- lines = 0
- for line in fp.readlines():
- lines = lines + 1
- fp.close()
- return lines
- def DumpRenderTreeFinished(adb_cmd):
- """ Check if DumpRenderTree finished running tests
- Args:
- output: adb_cmd string
- """
- # pull /sdcard/webkit/running_test.txt, if the content is "#DONE", it's done
- shell_cmd_str = adb_cmd + " shell cat /sdcard/webkit/running_test.txt"
- adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
- return adb_output.strip() == "#DONE"
- def DiffResults(marker, new_results, old_results, diff_results, strip_reason,
- new_count_first=True):
- """ Given two result files, generate diff and
- write to diff_results file. All arguments are absolute paths
- to files.
- """
- old_file = open(old_results, "r")
- new_file = open(new_results, "r")
- diff_file = open(diff_results, "a")
- # Read lines from each file
- ndict = new_file.readlines()
- cdict = old_file.readlines()
- # Write marker to diff file
- diff_file.writelines(marker + "\n")
- diff_file.writelines("###############\n")
- # Strip reason from result lines
- if strip_reason is True:
- for i in range(0, len(ndict)):
- ndict[i] = ndict[i].split(' ')[0] + "\n"
- for i in range(0, len(cdict)):
- cdict[i] = cdict[i].split(' ')[0] + "\n"
- params = {
- "new": [0, ndict, cdict, "+"],
- "miss": [0, cdict, ndict, "-"]
- }
- if new_count_first:
- order = ["new", "miss"]
- else:
- order = ["miss", "new"]
- for key in order:
- for line in params[key][1]:
- if line not in params[key][2]:
- if line[-1] != "\n":
- line += "\n";
- diff_file.writelines(params[key][3] + line)
- params[key][0] += 1
- logging.info(marker + " >>> " + str(params["new"][0]) + " new, " +
- str(params["miss"][0]) + " misses")
- diff_file.writelines("\n\n")
- old_file.close()
- new_file.close()
- diff_file.close()
- return
- def CompareResults(ref_dir, results_dir):
- """Compare results in two directories
- Args:
- ref_dir: the reference directory having layout results as references
- results_dir: the results directory
- """
- logging.info("Comparing results to " + ref_dir)
- diff_result = os.path.join(results_dir, "layout_tests_diff.txt")
- if os.path.exists(diff_result):
- os.remove(diff_result)
- files=["crashed", "failed", "passed", "nontext"]
- for f in files:
- result_file_name = "layout_tests_" + f + ".txt"
- DiffResults(f, os.path.join(results_dir, result_file_name),
- os.path.join(ref_dir, result_file_name), diff_result,
- False, f != "passed")
- logging.info("Detailed diffs are in " + diff_result)
- def main(options, args):
- """Run the tests. Will call sys.exit when complete.
- Args:
- options: a dictionary of command line options
- args: a list of sub directories or files to test
- """
- # Set up logging format.
- log_level = logging.INFO
- if options.verbose:
- log_level = logging.DEBUG
- logging.basicConfig(level=log_level,
- format='%(message)s')
- # Include all tests if none are specified.
- if not args:
- path = '/';
- else:
- path = ' '.join(args);
- adb_cmd = "adb ";
- if options.adb_options:
- adb_cmd += options.adb_options
- # Re-generate the test list if --refresh-test-list is on
- if options.refresh_test_list:
- logging.info("Generating test list.");
- generate_test_list_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#generateTestList -e path \"" + path + "\" -w com.android.dumprendertree/.LayoutTestsAutoRunner"
- adb_output = subprocess.Popen(generate_test_list_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
- if adb_output.find('Process crashed') != -1:
- logging.info("Aborting because cannot generate test list.\n" + adb_output)
- sys.exit(1)
- logging.info("Running tests")
- # Count crashed tests.
- crashed_tests = []
- timeout_ms = '15000'
- if options.time_out_ms:
- timeout_ms = options.time_out_ms
- # Run test until it's done
- run_layout_test_cmd_prefix = adb_cmd + " shell am instrument"
- run_layout_test_cmd_postfix = " -e path \"" + path + "\" -e timeout " + timeout_ms
- if options.rebaseline:
- run_layout_test_cmd_postfix += " -e rebaseline true"
- # If the JS engine is not specified on the command line, try reading the
- # JS_ENGINE environment variable, which is used by the build system in
- # external/webkit/Android.mk.
- js_engine = options.js_engine
- if not js_engine and os.environ.has_key('JS_ENGINE'):
- js_engine = os.environ['JS_ENGINE']
- if js_engine:
- run_layout_test_cmd_postfix += " -e jsengine " + js_engine
- run_layout_test_cmd_postfix += " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
- # Call LayoutTestsAutoTest::startLayoutTests.
- run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#startLayoutTests" + run_layout_test_cmd_postfix
- adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
- while not DumpRenderTreeFinished(adb_cmd):
- # Get the running_test.txt
- logging.error("DumpRenderTree crashed, output:\n" + adb_output)
- shell_cmd_str = adb_cmd + " shell cat /sdcard/webkit/running_test.txt"
- crashed_test = ""
- while not crashed_test:
- (crashed_test, err) = subprocess.Popen(
- shell_cmd_str, shell=True, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE).communicate()
- crashed_test = crashed_test.strip()
- if not crashed_test:
- logging.error('Cannot get crashed test name, device offline?')
- logging.error('stderr: ' + err)
- logging.error('retrying in 10s...')
- time.sleep(10)
- logging.info(crashed_test + " CRASHED");
- crashed_tests.append(crashed_test);
- logging.info("Resuming layout test runner...");
- # Call LayoutTestsAutoTest::resumeLayoutTests
- run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#resumeLayoutTests" + run_layout_test_cmd_postfix
- adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
- if adb_output.find('INSTRUMENTATION_FAILED') != -1:
- logging.error("Error happened : " + adb_output)
- sys.exit(1)
- logging.debug(adb_output);
- logging.info("Done\n");
- # Pull results from /sdcard
- results_dir = options.results_directory
- if not os.path.exists(results_dir):
- os.makedirs(results_dir)
- if not os.path.isdir(results_dir):
- logging.error("Cannot create results dir: " + results_dir);
- sys.exit(1);
- result_files = ["/sdcard/layout_tests_passed.txt",
- "/sdcard/layout_tests_failed.txt",
- "/sdcard/layout_tests_ignored.txt",
- "/sdcard/layout_tests_nontext.txt"]
- for file in result_files:
- shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
- adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
- logging.debug(adb_output)
- # Create the crash list.
- fp = open(results_dir + "/layout_tests_crashed.txt", "w");
- for crashed_test in crashed_tests:
- fp.writelines(crashed_test + '\n')
- fp.close()
- # Count the number of tests in each category.
- passed_tests = CountLineNumber(results_dir + "/layout_tests_passed.txt")
- logging.info(str(passed_tests) + " passed")
- failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
- logging.info(str(failed_tests) + " failed")
- ignored_tests = CountLineNumber(results_dir + "/layout_tests_ignored.txt")
- logging.info(str(ignored_tests) + " ignored results")
- crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
- logging.info(str(crashed_tests) + " crashed")
- nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
- logging.info(str(nontext_tests) + " no dumpAsText")
- logging.info(str(passed_tests + failed_tests + ignored_tests + crashed_tests + nontext_tests) + " TOTAL")
- logging.info("Results are stored under: " + results_dir + "\n")
- # Comparing results to references to find new fixes and regressions.
- results_dir = os.path.abspath(options.results_directory)
- ref_dir = options.ref_directory
- # if ref_dir is null, cannonify ref_dir to the script dir.
- if not ref_dir:
- script_self = sys.argv[0]
- script_dir = os.path.dirname(script_self)
- ref_dir = os.path.join(script_dir, "results")
- ref_dir = os.path.abspath(ref_dir)
- CompareResults(ref_dir, results_dir)
- if '__main__' == __name__:
- option_parser = optparse.OptionParser()
- option_parser.add_option("", "--rebaseline", action="store_true",
- default=False,
- help="generate expected results for those tests not having one")
- option_parser.add_option("", "--time-out-ms",
- default=None,
- help="set the timeout for each test")
- option_parser.add_option("", "--verbose", action="store_true",
- default=False,
- help="include debug-level logging")
- option_parser.add_option("", "--refresh-test-list", action="store_true",
- default=False,
- help="re-generate test list, it may take some time.")
- option_parser.add_option("", "--adb-options",
- default=None,
- help="pass options to adb, such as -d -e, etc");
- option_parser.add_option("", "--results-directory",
- default="layout-test-results",
- help="directory which results are stored.")
- option_parser.add_option("", "--ref-directory",
- default=None,
- dest="ref_directory",
- help="directory where reference results are stored.")
- option_parser.add_option("", "--js-engine",
- default=None,
- help="The JavaScript engine currently in use, which determines which set of Android-specific expected results we should use. Should be 'jsc' or 'v8'.");
- options, args = option_parser.parse_args();
- main(options, args)