PageRenderTime 65ms CodeModel.GetById 37ms RepoModel.GetById 1ms app.codeStats 0ms

/o3d/tests/test_driver.py

https://github.com/draco003/chromium
Python | 433 lines | 330 code | 16 blank | 87 comment | 6 complexity | 760a4636ad492ac015319d43b1d59b94 MD5 | raw file
  1. #!/usr/bin/python2.4
  2. # Copyright 2009, Google Inc.
  3. # All rights reserved.
  4. #
  5. # Redistribution and use in source and binary forms, with or without
  6. # modification, are permitted provided that the following conditions are
  7. # met:
  8. #
  9. # * Redistributions of source code must retain the above copyright
  10. # notice, this list of conditions and the following disclaimer.
  11. # * Redistributions in binary form must reproduce the above
  12. # copyright notice, this list of conditions and the following disclaimer
  13. # in the documentation and/or other materials provided with the
  14. # distribution.
  15. # * Neither the name of Google Inc. nor the names of its
  16. # contributors may be used to endorse or promote products derived from
  17. # this software without specific prior written permission.
  18. #
  19. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20. # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21. # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22. # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23. # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24. # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25. # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26. # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27. # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30. """Test harness script that invokes O3D system-tests.
  31. This script constructs the necessary testing environment required for the
  32. DirectX-based O3D system-tests. It invokes each test within the PIX
  33. debugging tool to capture the graphics command stream and framebuffer
  34. contents.
  35. See (http://wiki.corp.google.com/twiki/bin/view/Main/ClientThreeDTestingPlan)
  36. for the o3d testing plan.
  37. Usage:
  38. test_driver.py [-capture_only] [-hardware]
  39. The script will scan all of the directories, and invoke system-tests for all
  40. <name>_test directories present. Each of the system-tests in system_tests.exe
  41. will be invoked one at a time through PIXWin.exe. PIX has been configured
  42. to interact with the testing framework to capture graphics command streams
  43. and framebuffer contents.
  44. After the test has executed, the script will parse the persisted log files
  45. and compare the generated results.
  46. The framework assumes that each test-case is named with a camel-caps
  47. translation of the directory name. So in the case of the directory
  48. tests\import_test, the system invokes the gUnit test ImportTest.
  49. Command Arguments:
  50. -capture_only: The script will only capture results. All verification with
  51. reference-data will be bypassed. This flag should be used when updating the
  52. reference-data.
  53. - hardware: The the system tests will execute using the graphics hardware, as
  54. opposed to the software reference rasterizer. Use this mode to look for
  55. problems related to a specific hardware platform.
  56. """
  57. import os
  58. import re
  59. from stat import ST_MODE
  60. from stat import S_ISDIR
  61. import sys
  62. _PIX_EXECUTABLE = '.\PIXWin.exe'
  63. # Template for the command line required to invoke a system-test within PIX.
  64. _PIX_EXPERIMENT_INVOKE = ('%s %s -start -runfile %s -targetstartfolder %s '
  65. '-targetargs %s')
  66. # Template for the command line required to invoke PIX when constructing a
  67. # .csv file from a .PIXRun file.
  68. _PIX_RUNFILE_INVOKE = '%s %s -exporttocsv %s'
  69. # Template command line argument for the gunit system that selectively
  70. # executes individual groups of tests.
  71. _GUNIT_INVOKE = '--gunit_filter=%s.*'
  72. _SYSTEM_TEST = '.\system_test.exe'
  73. _EXPERIMENT_FILE_REFERENCE = 'testing_framework_reference.PIXExp'
  74. _EXPERIMENT_FILE_HARDWARE = 'testing_framework_hardware.PIXExp'
  75. # Command line required to invoke the perceptual diff utility.
  76. _PDIFF = ('perceptualdiff.exe %s %s -verbose -fov 45 -threshold 40000')
  77. # Regular expression used to search for hexadecimal pointer values in
  78. # generated log files
  79. _HEX_EXPRESSION = re.compile('0x[0-9ABCDEF]+')
  80. # Constant command argument string to enable capture-only mode.
  81. _CAPTURE_FLAG = '-capture_only'
  82. # Command flags to specify usage of the hardware rasterizer for the tests.
  83. _HARDWARE_RASTERIZER = '-hardware'
  84. # Boolen set to True if test results should be captured and not validated.
  85. _GENERATE_ONLY = False
  86. # Boolean set to True if hardware rasterizer is to be used instead of the DX
  87. # reference rasterizer.
  88. _HARDWARE_DEVICE = False
  89. def ConstructTestName(filesystem_name):
  90. """Returns a camel-caps translation of the input string.
  91. Args:
  92. filesystem_name: The name of the test, as found on the file-system.
  93. Typically this name is_of_this_form.
  94. Returns:
  95. A camel-caps version of the input string. _ delimiters are removed, and
  96. the first letters of words are capitalized.
  97. """
  98. names = filesystem_name.split('_')
  99. test_name = ''
  100. for word in names:
  101. test_name += word.upper()[0] + word[1:]
  102. return test_name
  103. def AssertCapturedFrames(test_name):
  104. """Performs the image validation for test-case frame captures.
  105. Args:
  106. test_name: The name of the test, as found on the file-system.
  107. Returns:
  108. True if all images match within tolerance, false otherwise.
  109. """
  110. print 'Status: Validating captured frames against reference data.'
  111. reference_files = os.listdir('./' + test_name + '/reference_frames')
  112. generated_files = os.listdir('./' + test_name)
  113. generated_files = [(file_name) for file_name in generated_files
  114. if '.png' in file_name]
  115. if set(reference_files) != set(generated_files):
  116. print 'Error: Reference file set does not match generated file set.'
  117. print 'Reference files: ' + ', '.join(reference_files)
  118. print 'Generated files: ' + ', '.join(generated_files)
  119. return False
  120. # Assuming both the result and reference image sets are the same size,
  121. # verify that corresponding images are similar within tolerance.
  122. for file in reference_files:
  123. reference_file = test_name + '/reference_frames/' + file
  124. generated_file = test_name + '/' + file
  125. if os.system(_PDIFF % (reference_file, generated_file)):
  126. error = ('Error: Reference framebuffer (%s) does not match generated '
  127. 'file (%s).')
  128. print error % (reference_file, generated_file)
  129. return False
  130. return True
  131. def GenerateCSVStream(test_name):
  132. """Invokes PIX to convert a PIXRun file to a .csv file.
  133. Args:
  134. test_name: The name of the test for which the .csv file is
  135. to be generated.
  136. """
  137. runfile_invoke = _PIX_RUNFILE_INVOKE % (
  138. _PIX_EXECUTABLE,
  139. test_name + '/' + test_name + '.PIXRun',
  140. test_name + '/' + test_name + '.csv')
  141. os.system(runfile_invoke)
  142. def RemovePointerReferences(call_log):
  143. """Returns the input argument with all pointer values removed.
  144. Args:
  145. call_log: String containing an dx call-log entry.
  146. Returns:
  147. The input string with all substrings matching '0x[0-9A-F]+', which is the
  148. pointer syntax used for the log files, removed.
  149. """
  150. return _HEX_EXPRESSION.sub('', call_log)
  151. def PartitionCSVLine(line):
  152. """Returns a list of all column values in a comma-separated-value line.
  153. Args:
  154. line: The input line from a comma-separated-value source.
  155. Returns:
  156. CSV files may include quoted text segments containing commas. This
  157. routine will return an array of all of the columns present in the line,
  158. respecting quoted regions.
  159. """
  160. # This routine is more complicated than expected due to the presence of
  161. # quoted lists within colums. The algorithm proceeds by first splitting
  162. # the line by quoted sub-strings. Sub-strings not matching a quoted
  163. # expression are then split on ','.
  164. quoted_column = re.compile('(\"[^\"]*\")')
  165. block_partitions = quoted_column.split(line)
  166. output_columns = []
  167. for chunk in block_partitions:
  168. # If this chunk is a quoted string, append it to the output verbatim.
  169. if quoted_column.match(chunk):
  170. output_columns += [chunk]
  171. else:
  172. output_columns += chunk.split(',')
  173. # Return the output set of columns with all empty entries removed.
  174. # Empty columns will be present due to the interaction of the split by
  175. # quoted string, and split by ','.
  176. # For example: here, is, "a (tuple, example)", to, try
  177. # After the first split: ['here, is, ', 'a (tuple, example)', 'to, try']
  178. # After the second split:
  179. # ['here', 'is', '', 'a (tuple, example)', 'to', 'try']
  180. # Since these empty columns are not present in the original list, we remove
  181. # them here.
  182. return [(column) for column in output_columns if column != ""]
  183. def AssertCapturedStreams(test_name):
  184. """Performes graphics command stream verification for test with name
  185. test_name.
  186. Args:
  187. test_name: The name of the test, as found on the file-system.
  188. Returns:
  189. Returns true if the generated testing streams match the reference streams.
  190. """
  191. print 'Status: Validating generated command streams.'
  192. generated_stream = open(test_name + '/' + test_name + '.csv')
  193. reference_stream = open(test_name + '/reference_stream.csv')
  194. reference_lines = reference_stream.readlines()
  195. generated_lines = generated_stream.readlines()
  196. if len(reference_lines) != len(generated_lines):
  197. print 'Error: Reference and generated logs differ in length.'
  198. return False
  199. # Compare each of the log lines from both files.
  200. for index in range(0, len(reference_lines)):
  201. generated_line = generated_lines[index]
  202. reference_line = reference_lines[index]
  203. # Partition each csv line correctly wrt quoted blocks
  204. reference_columns = PartitionCSVLine(reference_line)
  205. generated_columns = PartitionCSVLine(generated_line)
  206. # Only perform deep-validation on 'Call' commands.
  207. if reference_columns[0] != 'Call':
  208. continue
  209. generated_log = RemovePointerReferences(generated_columns[2])
  210. reference_log = RemovePointerReferences(reference_columns[2])
  211. if (generated_log != reference_log or
  212. reference_columns[0] != generated_columns[0] or
  213. reference_columns[1] != generated_columns[1]):
  214. print 'Error: Log file mis-match. Line: %i' % index
  215. print 'Reference = %s' % reference_line
  216. print 'Generated = %s' % generated_line
  217. return False
  218. return True
  219. def InvokeTest(test_name):
  220. """Invoke the system-test with name test_name.
  221. Args:
  222. test_name: The name of the test, as found on the file-system.
  223. Returns:
  224. True if the test succeeds, false otherwise.
  225. """
  226. global _GENERATE_ONLY
  227. if _HARDWARE_DEVICE:
  228. pix_experiment_file = _EXPERIMENT_FILE_HARDWARE
  229. else:
  230. pix_experiment_file = _EXPERIMENT_FILE_REFERENCE
  231. print 'Status: Executing test : %s\n' % test_name
  232. gunit_invoke = _GUNIT_INVOKE % ConstructTestName(test_name)
  233. pix_invoke = _PIX_EXPERIMENT_INVOKE % (_PIX_EXECUTABLE,
  234. pix_experiment_file,
  235. test_name + '.PIXRun',
  236. test_name,
  237. gunit_invoke)
  238. os.system(pix_invoke)
  239. # Invoke PIX to translate the just created .PIXRun file to a .csv suitable
  240. # for parsing.
  241. GenerateCSVStream(test_name)
  242. # If invoked for capture-only, then exit here before validation.
  243. if _GENERATE_ONLY:
  244. return True
  245. if not(AssertCapturedFrames(test_name)):
  246. return False
  247. if not(AssertCapturedStreams(test_name)):
  248. return False
  249. return True
  250. def BuildTestList():
  251. """Returns a list of all available system tests.
  252. Returns:
  253. A list of test_names, constructed from the set of directory names
  254. in the current directory. All directories containing the substring
  255. 'test' are included in the returned set.
  256. """
  257. testing_directory = os.listdir('.')
  258. test_list = []
  259. for test in testing_directory:
  260. mode = os.stat(test)[ST_MODE]
  261. if ('test' in test and test != 'unittest_data' and
  262. test != 'bitmap_test' and
  263. test != 'conditioner_test_data' and S_ISDIR(mode)):
  264. test_list += [test]
  265. return test_list
  266. def ValidateArgs(argv):
  267. """Validates the script arguments, and displays a help message,
  268. if necessary.
  269. Args:
  270. argv: Array of script arguments, in argv format.
  271. Returns:
  272. True if the arguments are valid. See the usage description for
  273. valid arguments.
  274. """
  275. global _GENERATE_ONLY
  276. global _HARDWARE_DEVICE
  277. # TODO : Make use of the gflags library to ease parsing of command line
  278. # arguments.
  279. argument_set = [_CAPTURE_FLAG, _HARDWARE_RASTERIZER]
  280. for arg in argv[1:]:
  281. if arg not in argument_set:
  282. print 'O3D System-Test Harness - Usage:'
  283. print ('test_driver.py [' + _CAPTURE_FLAG + '] [' +
  284. _HARDWARE_RASTERIZER + ']')
  285. print 'Arguments:'
  286. print _CAPTURE_FLAG + ' : Force generation of reference data.'
  287. print (_HARDWARE_RASTERIZER +
  288. ' : Force usage of hardware for test data generation')
  289. return False
  290. if _CAPTURE_FLAG in argv[1:]:
  291. _GENERATE_ONLY = True
  292. if _HARDWARE_RASTERIZER in argv[1:]:
  293. print 'Rendering with local graphics hardware.'
  294. _HARDWARE_DEVICE = True
  295. else:
  296. print 'Rendering with DX9 reference rasterizer.'
  297. return True
  298. def main(argv):
  299. """Main entry point of the script.
  300. Args:
  301. argv: The c-like arguments to the script.
  302. Returns:
  303. True if all tests pass, false othwerwise.
  304. """
  305. print 'Running O3D system tests.'
  306. if not ValidateArgs(argv):
  307. return False
  308. os.chdir(os.path.dirname(argv[0]))
  309. test_set = BuildTestList()
  310. # Invoke each test, tracking failures.
  311. test_failures = []
  312. for test in test_set:
  313. if not InvokeTest(test):
  314. test_failures += [test]
  315. if len(test_failures) == 0:
  316. print 'Success: All tests passed.'
  317. return True
  318. else:
  319. print 'Error Summary: The following tests failed:'
  320. print test_failures
  321. return False
  322. if __name__ == '__main__':
  323. if main(sys.argv):
  324. # Return with a 0 for success (per unix convention).
  325. sys.exit(0)
  326. else:
  327. # Return with a 1 for failure (per unix convention).
  328. sys.exit(1)