/Unittests/googletest/test/gtest_shuffle_test.py

  1#!/usr/bin/env python
  2#
  3# Copyright 2009 Google Inc. All Rights Reserved.
  4#
  5# Redistribution and use in source and binary forms, with or without
  6# modification, are permitted provided that the following conditions are
  7# met:
  8#
  9#     * Redistributions of source code must retain the above copyright
 10# notice, this list of conditions and the following disclaimer.
 11#     * Redistributions in binary form must reproduce the above
 12# copyright notice, this list of conditions and the following disclaimer
 13# in the documentation and/or other materials provided with the
 14# distribution.
 15#     * Neither the name of Google Inc. nor the names of its
 16# contributors may be used to endorse or promote products derived from
 17# this software without specific prior written permission.
 18#
 19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 30
 31"""Verifies that test shuffling works."""
 32
 33__author__ = 'wan@google.com (Zhanyong Wan)'
 34
 35import os
 36import gtest_test_utils
 37
 38# Command to run the gtest_shuffle_test_ program.
 39COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_shuffle_test_')
 40
 41# The environment variables for test sharding.
 42TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS'
 43SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX'
 44
 45TEST_FILTER = 'A*.A:A*.B:C*'
 46
 47ALL_TESTS = []
 48ACTIVE_TESTS = []
 49FILTERED_TESTS = []
 50SHARDED_TESTS = []
 51
 52SHUFFLED_ALL_TESTS = []
 53SHUFFLED_ACTIVE_TESTS = []
 54SHUFFLED_FILTERED_TESTS = []
 55SHUFFLED_SHARDED_TESTS = []
 56
 57
 58def AlsoRunDisabledTestsFlag():
 59  return '--gtest_also_run_disabled_tests'
 60
 61
 62def FilterFlag(test_filter):
 63  return '--gtest_filter=%s' % (test_filter,)
 64
 65
 66def RepeatFlag(n):
 67  return '--gtest_repeat=%s' % (n,)
 68
 69
 70def ShuffleFlag():
 71  return '--gtest_shuffle'
 72
 73
 74def RandomSeedFlag(n):
 75  return '--gtest_random_seed=%s' % (n,)
 76
 77
 78def RunAndReturnOutput(extra_env, args):
 79  """Runs the test program and returns its output."""
 80
 81  try:
 82    original_env = os.environ.copy()
 83    os.environ.update(extra_env)
 84    return gtest_test_utils.Subprocess([COMMAND] + args).output
 85  finally:
 86    for key in extra_env.iterkeys():
 87      if key in original_env:
 88        os.environ[key] = original_env[key]
 89      else:
 90        del os.environ[key]
 91
 92
 93def GetTestsForAllIterations(extra_env, args):
 94  """Runs the test program and returns a list of test lists.
 95
 96  Args:
 97    extra_env: a map from environment variables to their values
 98    args: command line flags to pass to gtest_shuffle_test_
 99
100  Returns:
101    A list where the i-th element is the list of tests run in the i-th
102    test iteration.
103  """
104
105  test_iterations = []
106  for line in RunAndReturnOutput(extra_env, args).split('\n'):
107    if line.startswith('----'):
108      tests = []
109      test_iterations.append(tests)
110    elif line.strip():
111      tests.append(line.strip())  # 'TestCaseName.TestName'
112
113  return test_iterations
114
115
116def GetTestCases(tests):
117  """Returns a list of test cases in the given full test names.
118
119  Args:
120    tests: a list of full test names
121
122  Returns:
123    A list of test cases from 'tests', in their original order.
124    Consecutive duplicates are removed.
125  """
126
127  test_cases = []
128  for test in tests:
129    test_case = test.split('.')[0]
130    if not test_case in test_cases:
131      test_cases.append(test_case)
132
133  return test_cases
134
135
136def CalculateTestLists():
137  """Calculates the list of tests run under different flags."""
138
139  if not ALL_TESTS:
140    ALL_TESTS.extend(
141        GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0])
142
143  if not ACTIVE_TESTS:
144    ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0])
145
146  if not FILTERED_TESTS:
147    FILTERED_TESTS.extend(
148        GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0])
149
150  if not SHARDED_TESTS:
151    SHARDED_TESTS.extend(
152        GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
153                                  SHARD_INDEX_ENV_VAR: '1'},
154                                 [])[0])
155
156  if not SHUFFLED_ALL_TESTS:
157    SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations(
158        {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0])
159
160  if not SHUFFLED_ACTIVE_TESTS:
161    SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations(
162        {}, [ShuffleFlag(), RandomSeedFlag(1)])[0])
163
164  if not SHUFFLED_FILTERED_TESTS:
165    SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations(
166        {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0])
167
168  if not SHUFFLED_SHARDED_TESTS:
169    SHUFFLED_SHARDED_TESTS.extend(
170        GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
171                                  SHARD_INDEX_ENV_VAR: '1'},
172                                 [ShuffleFlag(), RandomSeedFlag(1)])[0])
173
174
175class GTestShuffleUnitTest(gtest_test_utils.TestCase):
176  """Tests test shuffling."""
177
178  def setUp(self):
179    CalculateTestLists()
180
181  def testShufflePreservesNumberOfTests(self):
182    self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS))
183    self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS))
184    self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS))
185    self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS))
186
187  def testShuffleChangesTestOrder(self):
188    self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS)
189    self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS)
190    self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS,
191                 SHUFFLED_FILTERED_TESTS)
192    self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS,
193                 SHUFFLED_SHARDED_TESTS)
194
195  def testShuffleChangesTestCaseOrder(self):
196    self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS),
197                 GetTestCases(SHUFFLED_ALL_TESTS))
198    self.assert_(
199        GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS),
200        GetTestCases(SHUFFLED_ACTIVE_TESTS))
201    self.assert_(
202        GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS),
203        GetTestCases(SHUFFLED_FILTERED_TESTS))
204    self.assert_(
205        GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS),
206        GetTestCases(SHUFFLED_SHARDED_TESTS))
207
208  def testShuffleDoesNotRepeatTest(self):
209    for test in SHUFFLED_ALL_TESTS:
210      self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test),
211                       '%s appears more than once' % (test,))
212    for test in SHUFFLED_ACTIVE_TESTS:
213      self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test),
214                       '%s appears more than once' % (test,))
215    for test in SHUFFLED_FILTERED_TESTS:
216      self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test),
217                       '%s appears more than once' % (test,))
218    for test in SHUFFLED_SHARDED_TESTS:
219      self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test),
220                       '%s appears more than once' % (test,))
221
222  def testShuffleDoesNotCreateNewTest(self):
223    for test in SHUFFLED_ALL_TESTS:
224      self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,))
225    for test in SHUFFLED_ACTIVE_TESTS:
226      self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,))
227    for test in SHUFFLED_FILTERED_TESTS:
228      self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,))
229    for test in SHUFFLED_SHARDED_TESTS:
230      self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,))
231
232  def testShuffleIncludesAllTests(self):
233    for test in ALL_TESTS:
234      self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,))
235    for test in ACTIVE_TESTS:
236      self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,))
237    for test in FILTERED_TESTS:
238      self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,))
239    for test in SHARDED_TESTS:
240      self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,))
241
242  def testShuffleLeavesDeathTestsAtFront(self):
243    non_death_test_found = False
244    for test in SHUFFLED_ACTIVE_TESTS:
245      if 'DeathTest.' in test:
246        self.assert_(not non_death_test_found,
247                     '%s appears after a non-death test' % (test,))
248      else:
249        non_death_test_found = True
250
251  def _VerifyTestCasesDoNotInterleave(self, tests):
252    test_cases = []
253    for test in tests:
254      [test_case, _] = test.split('.')
255      if test_cases and test_cases[-1] != test_case:
256        test_cases.append(test_case)
257        self.assertEqual(1, test_cases.count(test_case),
258                         'Test case %s is not grouped together in %s' %
259                         (test_case, tests))
260
261  def testShuffleDoesNotInterleaveTestCases(self):
262    self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS)
263    self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS)
264    self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS)
265    self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS)
266
267  def testShuffleRestoresOrderAfterEachIteration(self):
268    # Get the test lists in all 3 iterations, using random seed 1, 2,
269    # and 3 respectively.  Google Test picks a different seed in each
270    # iteration, and this test depends on the current implementation
271    # picking successive numbers.  This dependency is not ideal, but
272    # makes the test much easier to write.
273    [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
274        GetTestsForAllIterations(
275            {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))
276
277    # Make sure running the tests with random seed 1 gets the same
278    # order as in iteration 1 above.
279    [tests_with_seed1] = GetTestsForAllIterations(
280        {}, [ShuffleFlag(), RandomSeedFlag(1)])
281    self.assertEqual(tests_in_iteration1, tests_with_seed1)
282
283    # Make sure running the tests with random seed 2 gets the same
284    # order as in iteration 2 above.  Success means that Google Test
285    # correctly restores the test order before re-shuffling at the
286    # beginning of iteration 2.
287    [tests_with_seed2] = GetTestsForAllIterations(
288        {}, [ShuffleFlag(), RandomSeedFlag(2)])
289    self.assertEqual(tests_in_iteration2, tests_with_seed2)
290
291    # Make sure running the tests with random seed 3 gets the same
292    # order as in iteration 3 above.  Success means that Google Test
293    # correctly restores the test order before re-shuffling at the
294    # beginning of iteration 3.
295    [tests_with_seed3] = GetTestsForAllIterations(
296        {}, [ShuffleFlag(), RandomSeedFlag(3)])
297    self.assertEqual(tests_in_iteration3, tests_with_seed3)
298
299  def testShuffleGeneratesNewOrderInEachIteration(self):
300    [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
301        GetTestsForAllIterations(
302            {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))
303
304    self.assert_(tests_in_iteration1 != tests_in_iteration2,
305                 tests_in_iteration1)
306    self.assert_(tests_in_iteration1 != tests_in_iteration3,
307                 tests_in_iteration1)
308    self.assert_(tests_in_iteration2 != tests_in_iteration3,
309                 tests_in_iteration2)
310
311  def testShuffleShardedTestsPreservesPartition(self):
312    # If we run M tests on N shards, the same M tests should be run in
313    # total, regardless of the random seeds used by the shards.
314    [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
315                                         SHARD_INDEX_ENV_VAR: '0'},
316                                        [ShuffleFlag(), RandomSeedFlag(1)])
317    [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
318                                         SHARD_INDEX_ENV_VAR: '1'},
319                                        [ShuffleFlag(), RandomSeedFlag(20)])
320    [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
321                                         SHARD_INDEX_ENV_VAR: '2'},
322                                        [ShuffleFlag(), RandomSeedFlag(25)])
323    sorted_sharded_tests = tests1 + tests2 + tests3
324    sorted_sharded_tests.sort()
325    sorted_active_tests = []
326    sorted_active_tests.extend(ACTIVE_TESTS)
327    sorted_active_tests.sort()
328    self.assertEqual(sorted_active_tests, sorted_sharded_tests)
329
330if __name__ == '__main__':
331  gtest_test_utils.Main()