/Unittests/googletest/test/gtest_shuffle_test.py

http://unladen-swallow.googlecode.com/ · Python · 331 lines · 255 code · 27 blank · 49 comment · 8 complexity · 78c01e0182a2518b127d4b6444adef15 MD5 · raw file

  1. #!/usr/bin/env python
  2. #
  3. # Copyright 2009 Google Inc. All Rights Reserved.
  4. #
  5. # Redistribution and use in source and binary forms, with or without
  6. # modification, are permitted provided that the following conditions are
  7. # met:
  8. #
  9. # * Redistributions of source code must retain the above copyright
  10. # notice, this list of conditions and the following disclaimer.
  11. # * Redistributions in binary form must reproduce the above
  12. # copyright notice, this list of conditions and the following disclaimer
  13. # in the documentation and/or other materials provided with the
  14. # distribution.
  15. # * Neither the name of Google Inc. nor the names of its
  16. # contributors may be used to endorse or promote products derived from
  17. # this software without specific prior written permission.
  18. #
  19. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20. # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21. # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22. # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23. # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24. # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25. # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26. # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27. # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30. """Verifies that test shuffling works."""
  31. __author__ = 'wan@google.com (Zhanyong Wan)'
  32. import os
  33. import gtest_test_utils
  34. # Command to run the gtest_shuffle_test_ program.
  35. COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_shuffle_test_')
  36. # The environment variables for test sharding.
  37. TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS'
  38. SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX'
  39. TEST_FILTER = 'A*.A:A*.B:C*'
  40. ALL_TESTS = []
  41. ACTIVE_TESTS = []
  42. FILTERED_TESTS = []
  43. SHARDED_TESTS = []
  44. SHUFFLED_ALL_TESTS = []
  45. SHUFFLED_ACTIVE_TESTS = []
  46. SHUFFLED_FILTERED_TESTS = []
  47. SHUFFLED_SHARDED_TESTS = []
  48. def AlsoRunDisabledTestsFlag():
  49. return '--gtest_also_run_disabled_tests'
  50. def FilterFlag(test_filter):
  51. return '--gtest_filter=%s' % (test_filter,)
  52. def RepeatFlag(n):
  53. return '--gtest_repeat=%s' % (n,)
  54. def ShuffleFlag():
  55. return '--gtest_shuffle'
  56. def RandomSeedFlag(n):
  57. return '--gtest_random_seed=%s' % (n,)
  58. def RunAndReturnOutput(extra_env, args):
  59. """Runs the test program and returns its output."""
  60. try:
  61. original_env = os.environ.copy()
  62. os.environ.update(extra_env)
  63. return gtest_test_utils.Subprocess([COMMAND] + args).output
  64. finally:
  65. for key in extra_env.iterkeys():
  66. if key in original_env:
  67. os.environ[key] = original_env[key]
  68. else:
  69. del os.environ[key]
  70. def GetTestsForAllIterations(extra_env, args):
  71. """Runs the test program and returns a list of test lists.
  72. Args:
  73. extra_env: a map from environment variables to their values
  74. args: command line flags to pass to gtest_shuffle_test_
  75. Returns:
  76. A list where the i-th element is the list of tests run in the i-th
  77. test iteration.
  78. """
  79. test_iterations = []
  80. for line in RunAndReturnOutput(extra_env, args).split('\n'):
  81. if line.startswith('----'):
  82. tests = []
  83. test_iterations.append(tests)
  84. elif line.strip():
  85. tests.append(line.strip()) # 'TestCaseName.TestName'
  86. return test_iterations
  87. def GetTestCases(tests):
  88. """Returns a list of test cases in the given full test names.
  89. Args:
  90. tests: a list of full test names
  91. Returns:
  92. A list of test cases from 'tests', in their original order.
  93. Consecutive duplicates are removed.
  94. """
  95. test_cases = []
  96. for test in tests:
  97. test_case = test.split('.')[0]
  98. if not test_case in test_cases:
  99. test_cases.append(test_case)
  100. return test_cases
  101. def CalculateTestLists():
  102. """Calculates the list of tests run under different flags."""
  103. if not ALL_TESTS:
  104. ALL_TESTS.extend(
  105. GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0])
  106. if not ACTIVE_TESTS:
  107. ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0])
  108. if not FILTERED_TESTS:
  109. FILTERED_TESTS.extend(
  110. GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0])
  111. if not SHARDED_TESTS:
  112. SHARDED_TESTS.extend(
  113. GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
  114. SHARD_INDEX_ENV_VAR: '1'},
  115. [])[0])
  116. if not SHUFFLED_ALL_TESTS:
  117. SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations(
  118. {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0])
  119. if not SHUFFLED_ACTIVE_TESTS:
  120. SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations(
  121. {}, [ShuffleFlag(), RandomSeedFlag(1)])[0])
  122. if not SHUFFLED_FILTERED_TESTS:
  123. SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations(
  124. {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0])
  125. if not SHUFFLED_SHARDED_TESTS:
  126. SHUFFLED_SHARDED_TESTS.extend(
  127. GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
  128. SHARD_INDEX_ENV_VAR: '1'},
  129. [ShuffleFlag(), RandomSeedFlag(1)])[0])
  130. class GTestShuffleUnitTest(gtest_test_utils.TestCase):
  131. """Tests test shuffling."""
  132. def setUp(self):
  133. CalculateTestLists()
  134. def testShufflePreservesNumberOfTests(self):
  135. self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS))
  136. self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS))
  137. self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS))
  138. self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS))
  139. def testShuffleChangesTestOrder(self):
  140. self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS)
  141. self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS)
  142. self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS,
  143. SHUFFLED_FILTERED_TESTS)
  144. self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS,
  145. SHUFFLED_SHARDED_TESTS)
  146. def testShuffleChangesTestCaseOrder(self):
  147. self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS),
  148. GetTestCases(SHUFFLED_ALL_TESTS))
  149. self.assert_(
  150. GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS),
  151. GetTestCases(SHUFFLED_ACTIVE_TESTS))
  152. self.assert_(
  153. GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS),
  154. GetTestCases(SHUFFLED_FILTERED_TESTS))
  155. self.assert_(
  156. GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS),
  157. GetTestCases(SHUFFLED_SHARDED_TESTS))
  158. def testShuffleDoesNotRepeatTest(self):
  159. for test in SHUFFLED_ALL_TESTS:
  160. self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test),
  161. '%s appears more than once' % (test,))
  162. for test in SHUFFLED_ACTIVE_TESTS:
  163. self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test),
  164. '%s appears more than once' % (test,))
  165. for test in SHUFFLED_FILTERED_TESTS:
  166. self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test),
  167. '%s appears more than once' % (test,))
  168. for test in SHUFFLED_SHARDED_TESTS:
  169. self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test),
  170. '%s appears more than once' % (test,))
  171. def testShuffleDoesNotCreateNewTest(self):
  172. for test in SHUFFLED_ALL_TESTS:
  173. self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,))
  174. for test in SHUFFLED_ACTIVE_TESTS:
  175. self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,))
  176. for test in SHUFFLED_FILTERED_TESTS:
  177. self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,))
  178. for test in SHUFFLED_SHARDED_TESTS:
  179. self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,))
  180. def testShuffleIncludesAllTests(self):
  181. for test in ALL_TESTS:
  182. self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,))
  183. for test in ACTIVE_TESTS:
  184. self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,))
  185. for test in FILTERED_TESTS:
  186. self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,))
  187. for test in SHARDED_TESTS:
  188. self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,))
  189. def testShuffleLeavesDeathTestsAtFront(self):
  190. non_death_test_found = False
  191. for test in SHUFFLED_ACTIVE_TESTS:
  192. if 'DeathTest.' in test:
  193. self.assert_(not non_death_test_found,
  194. '%s appears after a non-death test' % (test,))
  195. else:
  196. non_death_test_found = True
  197. def _VerifyTestCasesDoNotInterleave(self, tests):
  198. test_cases = []
  199. for test in tests:
  200. [test_case, _] = test.split('.')
  201. if test_cases and test_cases[-1] != test_case:
  202. test_cases.append(test_case)
  203. self.assertEqual(1, test_cases.count(test_case),
  204. 'Test case %s is not grouped together in %s' %
  205. (test_case, tests))
  206. def testShuffleDoesNotInterleaveTestCases(self):
  207. self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS)
  208. self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS)
  209. self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS)
  210. self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS)
  211. def testShuffleRestoresOrderAfterEachIteration(self):
  212. # Get the test lists in all 3 iterations, using random seed 1, 2,
  213. # and 3 respectively. Google Test picks a different seed in each
  214. # iteration, and this test depends on the current implementation
  215. # picking successive numbers. This dependency is not ideal, but
  216. # makes the test much easier to write.
  217. [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
  218. GetTestsForAllIterations(
  219. {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))
  220. # Make sure running the tests with random seed 1 gets the same
  221. # order as in iteration 1 above.
  222. [tests_with_seed1] = GetTestsForAllIterations(
  223. {}, [ShuffleFlag(), RandomSeedFlag(1)])
  224. self.assertEqual(tests_in_iteration1, tests_with_seed1)
  225. # Make sure running the tests with random seed 2 gets the same
  226. # order as in iteration 2 above. Success means that Google Test
  227. # correctly restores the test order before re-shuffling at the
  228. # beginning of iteration 2.
  229. [tests_with_seed2] = GetTestsForAllIterations(
  230. {}, [ShuffleFlag(), RandomSeedFlag(2)])
  231. self.assertEqual(tests_in_iteration2, tests_with_seed2)
  232. # Make sure running the tests with random seed 3 gets the same
  233. # order as in iteration 3 above. Success means that Google Test
  234. # correctly restores the test order before re-shuffling at the
  235. # beginning of iteration 3.
  236. [tests_with_seed3] = GetTestsForAllIterations(
  237. {}, [ShuffleFlag(), RandomSeedFlag(3)])
  238. self.assertEqual(tests_in_iteration3, tests_with_seed3)
  239. def testShuffleGeneratesNewOrderInEachIteration(self):
  240. [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
  241. GetTestsForAllIterations(
  242. {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))
  243. self.assert_(tests_in_iteration1 != tests_in_iteration2,
  244. tests_in_iteration1)
  245. self.assert_(tests_in_iteration1 != tests_in_iteration3,
  246. tests_in_iteration1)
  247. self.assert_(tests_in_iteration2 != tests_in_iteration3,
  248. tests_in_iteration2)
  249. def testShuffleShardedTestsPreservesPartition(self):
  250. # If we run M tests on N shards, the same M tests should be run in
  251. # total, regardless of the random seeds used by the shards.
  252. [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
  253. SHARD_INDEX_ENV_VAR: '0'},
  254. [ShuffleFlag(), RandomSeedFlag(1)])
  255. [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
  256. SHARD_INDEX_ENV_VAR: '1'},
  257. [ShuffleFlag(), RandomSeedFlag(20)])
  258. [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
  259. SHARD_INDEX_ENV_VAR: '2'},
  260. [ShuffleFlag(), RandomSeedFlag(25)])
  261. sorted_sharded_tests = tests1 + tests2 + tests3
  262. sorted_sharded_tests.sort()
  263. sorted_active_tests = []
  264. sorted_active_tests.extend(ACTIVE_TESTS)
  265. sorted_active_tests.sort()
  266. self.assertEqual(sorted_active_tests, sorted_sharded_tests)
  267. if __name__ == '__main__':
  268. gtest_test_utils.Main()