PageRenderTime 52ms CodeModel.GetById 12ms app.highlight 34ms RepoModel.GetById 1ms app.codeStats 0ms

/Unittests/googletest/test/gtest_filter_unittest.py

http://unladen-swallow.googlecode.com/
Python | 594 lines | 519 code | 23 blank | 52 comment | 2 complexity | d2ae60d101361cd71f3e4b2d0459391a MD5 | raw file
  1#!/usr/bin/env python
  2#
  3# Copyright 2005 Google Inc. All Rights Reserved.
  4#
  5# Redistribution and use in source and binary forms, with or without
  6# modification, are permitted provided that the following conditions are
  7# met:
  8#
  9#     * Redistributions of source code must retain the above copyright
 10# notice, this list of conditions and the following disclaimer.
 11#     * Redistributions in binary form must reproduce the above
 12# copyright notice, this list of conditions and the following disclaimer
 13# in the documentation and/or other materials provided with the
 14# distribution.
 15#     * Neither the name of Google Inc. nor the names of its
 16# contributors may be used to endorse or promote products derived from
 17# this software without specific prior written permission.
 18#
 19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 30
 31"""Unit test for Google Test test filters.
 32
 33A user can specify which test(s) in a Google Test program to run via either
 34the GTEST_FILTER environment variable or the --gtest_filter flag.
 35This script tests such functionality by invoking
 36gtest_filter_unittest_ (a program written with Google Test) with different
 37environments and command line flags.
 38
 39Note that test sharding may also influence which tests are filtered. Therefore,
 40we test that here also.
 41"""
 42
 43__author__ = 'wan@google.com (Zhanyong Wan)'
 44
 45import os
 46import re
 47import sets
 48import gtest_test_utils
 49
 50# Constants.
 51
 52IS_WINDOWS = os.name == 'nt'
 53
 54# The environment variable for specifying the test filters.
 55FILTER_ENV_VAR = 'GTEST_FILTER'
 56
 57# The environment variables for test sharding.
 58TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS'
 59SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX'
 60SHARD_STATUS_FILE_ENV_VAR = 'GTEST_SHARD_STATUS_FILE'
 61
 62# The command line flag for specifying the test filters.
 63FILTER_FLAG = 'gtest_filter'
 64
 65# The command line flag for including disabled tests.
 66ALSO_RUN_DISABED_TESTS_FLAG = 'gtest_also_run_disabled_tests'
 67
 68# Command to run the gtest_filter_unittest_ program.
 69COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_filter_unittest_')
 70
 71# Regex for determining whether parameterized tests are enabled in the binary.
 72PARAM_TEST_REGEX = re.compile(r'/ParamTest')
 73
 74# Regex for parsing test case names from Google Test's output.
 75TEST_CASE_REGEX = re.compile(r'^\[\-+\] \d+ tests? from (\w+(/\w+)?)')
 76
 77# Regex for parsing test names from Google Test's output.
 78TEST_REGEX = re.compile(r'^\[\s*RUN\s*\].*\.(\w+(/\w+)?)')
 79
 80# Full names of all tests in gtest_filter_unittests_.
 81PARAM_TESTS = [
 82    'SeqP/ParamTest.TestX/0',
 83    'SeqP/ParamTest.TestX/1',
 84    'SeqP/ParamTest.TestY/0',
 85    'SeqP/ParamTest.TestY/1',
 86    'SeqQ/ParamTest.TestX/0',
 87    'SeqQ/ParamTest.TestX/1',
 88    'SeqQ/ParamTest.TestY/0',
 89    'SeqQ/ParamTest.TestY/1',
 90    ]
 91
 92DISABLED_TESTS = [
 93    'BarTest.DISABLED_TestFour',
 94    'BarTest.DISABLED_TestFive',
 95    'BazTest.DISABLED_TestC',
 96    'DISABLED_FoobarTest.Test1',
 97    'DISABLED_FoobarTest.DISABLED_Test2',
 98    'DISABLED_FoobarbazTest.TestA',
 99    ]
100
101# All the non-disabled tests.
102ACTIVE_TESTS = [
103    'FooTest.Abc',
104    'FooTest.Xyz',
105
106    'BarTest.TestOne',
107    'BarTest.TestTwo',
108    'BarTest.TestThree',
109
110    'BazTest.TestOne',
111    'BazTest.TestA',
112    'BazTest.TestB',
113
114    'HasDeathTest.Test1',
115    'HasDeathTest.Test2',
116    ] + PARAM_TESTS
117
118param_tests_present = None
119
120# Utilities.
121
122
123def SetEnvVar(env_var, value):
124  """Sets the env variable to 'value'; unsets it when 'value' is None."""
125
126  if value is not None:
127    os.environ[env_var] = value
128  elif env_var in os.environ:
129    del os.environ[env_var]
130
131
132def RunAndReturnOutput(args = None):
133  """Runs the test program and returns its output."""
134
135  return gtest_test_utils.Subprocess([COMMAND] + (args or [])).output
136
137
138def RunAndExtractTestList(args = None):
139  """Runs the test program and returns its exit code and a list of tests run."""
140
141  p = gtest_test_utils.Subprocess([COMMAND] + (args or []))
142  tests_run = []
143  test_case = ''
144  test = ''
145  for line in p.output.split('\n'):
146    match = TEST_CASE_REGEX.match(line)
147    if match is not None:
148      test_case = match.group(1)
149    else:
150      match = TEST_REGEX.match(line)
151      if match is not None:
152        test = match.group(1)
153        tests_run.append(test_case + '.' + test)
154  return (tests_run, p.exit_code)
155
156
157def InvokeWithModifiedEnv(extra_env, function, *args, **kwargs):
158  """Runs the given function and arguments in a modified environment."""
159  try:
160    original_env = os.environ.copy()
161    os.environ.update(extra_env)
162    return function(*args, **kwargs)
163  finally:
164    for key in extra_env.iterkeys():
165      if key in original_env:
166        os.environ[key] = original_env[key]
167      else:
168        del os.environ[key]
169
170
171def RunWithSharding(total_shards, shard_index, command):
172  """Runs a test program shard and returns exit code and a list of tests run."""
173
174  extra_env = {SHARD_INDEX_ENV_VAR: str(shard_index),
175               TOTAL_SHARDS_ENV_VAR: str(total_shards)}
176  return InvokeWithModifiedEnv(extra_env, RunAndExtractTestList, command)
177
178# The unit test.
179
180
181class GTestFilterUnitTest(gtest_test_utils.TestCase):
182  """Tests GTEST_FILTER env variable or --gtest_filter flag to filter tests."""
183
184  # Utilities.
185
186  def AssertSetEqual(self, lhs, rhs):
187    """Asserts that two sets are equal."""
188
189    for elem in lhs:
190      self.assert_(elem in rhs, '%s in %s' % (elem, rhs))
191
192    for elem in rhs:
193      self.assert_(elem in lhs, '%s in %s' % (elem, lhs))
194
195  def AssertPartitionIsValid(self, set_var, list_of_sets):
196    """Asserts that list_of_sets is a valid partition of set_var."""
197
198    full_partition = []
199    for slice_var in list_of_sets:
200      full_partition.extend(slice_var)
201    self.assertEqual(len(set_var), len(full_partition))
202    self.assertEqual(sets.Set(set_var), sets.Set(full_partition))
203
204  def AdjustForParameterizedTests(self, tests_to_run):
205    """Adjust tests_to_run in case value parameterized tests are disabled."""
206
207    global param_tests_present
208    if not param_tests_present:
209      return list(sets.Set(tests_to_run) - sets.Set(PARAM_TESTS))
210    else:
211      return tests_to_run
212
213  def RunAndVerify(self, gtest_filter, tests_to_run):
214    """Checks that the binary runs correct set of tests for the given filter."""
215
216    tests_to_run = self.AdjustForParameterizedTests(tests_to_run)
217
218    # First, tests using GTEST_FILTER.
219
220    # Windows removes empty variables from the environment when passing it
221    # to a new process. This means it is impossible to pass an empty filter
222    # into a process using the GTEST_FILTER environment variable. However,
223    # we can still test the case when the variable is not supplied (i.e.,
224    # gtest_filter is None).
225    # pylint: disable-msg=C6403
226    if not IS_WINDOWS or gtest_filter != '':
227      SetEnvVar(FILTER_ENV_VAR, gtest_filter)
228      tests_run = RunAndExtractTestList()[0]
229      SetEnvVar(FILTER_ENV_VAR, None)
230      self.AssertSetEqual(tests_run, tests_to_run)
231    # pylint: enable-msg=C6403
232
233    # Next, tests using --gtest_filter.
234
235    if gtest_filter is None:
236      args = []
237    else:
238      args = ['--%s=%s' % (FILTER_FLAG, gtest_filter)]
239
240    tests_run = RunAndExtractTestList(args)[0]
241    self.AssertSetEqual(tests_run, tests_to_run)
242
243  def RunAndVerifyWithSharding(self, gtest_filter, total_shards, tests_to_run,
244                               args=None, check_exit_0=False):
245    """Checks that binary runs correct tests for the given filter and shard.
246
247    Runs all shards of gtest_filter_unittest_ with the given filter, and
248    verifies that the right set of tests were run. The union of tests run
249    on each shard should be identical to tests_to_run, without duplicates.
250
251    Args:
252      gtest_filter: A filter to apply to the tests.
253      total_shards: A total number of shards to split test run into.
254      tests_to_run: A set of tests expected to run.
255      args   :      Arguments to pass to the to the test binary.
256      check_exit_0: When set to a true value, make sure that all shards
257                    return 0.
258    """
259
260    tests_to_run = self.AdjustForParameterizedTests(tests_to_run)
261
262    # Windows removes empty variables from the environment when passing it
263    # to a new process. This means it is impossible to pass an empty filter
264    # into a process using the GTEST_FILTER environment variable. However,
265    # we can still test the case when the variable is not supplied (i.e.,
266    # gtest_filter is None).
267    # pylint: disable-msg=C6403
268    if not IS_WINDOWS or gtest_filter != '':
269      SetEnvVar(FILTER_ENV_VAR, gtest_filter)
270      partition = []
271      for i in range(0, total_shards):
272        (tests_run, exit_code) = RunWithSharding(total_shards, i, args)
273        if check_exit_0:
274          self.assertEqual(0, exit_code)
275        partition.append(tests_run)
276
277      self.AssertPartitionIsValid(tests_to_run, partition)
278      SetEnvVar(FILTER_ENV_VAR, None)
279    # pylint: enable-msg=C6403
280
281  def RunAndVerifyAllowingDisabled(self, gtest_filter, tests_to_run):
282    """Checks that the binary runs correct set of tests for the given filter.
283
284    Runs gtest_filter_unittest_ with the given filter, and enables
285    disabled tests. Verifies that the right set of tests were run.
286
287    Args:
288      gtest_filter: A filter to apply to the tests.
289      tests_to_run: A set of tests expected to run.
290    """
291
292    tests_to_run = self.AdjustForParameterizedTests(tests_to_run)
293
294    # Construct the command line.
295    args = ['--%s' % ALSO_RUN_DISABED_TESTS_FLAG]
296    if gtest_filter is not None:
297      args.append('--%s=%s' % (FILTER_FLAG, gtest_filter))
298
299    tests_run = RunAndExtractTestList(args)[0]
300    self.AssertSetEqual(tests_run, tests_to_run)
301
302  def setUp(self):
303    """Sets up test case.
304
305    Determines whether value-parameterized tests are enabled in the binary and
306    sets the flags accordingly.
307    """
308
309    global param_tests_present
310    if param_tests_present is None:
311      param_tests_present = PARAM_TEST_REGEX.search(
312          RunAndReturnOutput()) is not None
313
314  def testDefaultBehavior(self):
315    """Tests the behavior of not specifying the filter."""
316
317    self.RunAndVerify(None, ACTIVE_TESTS)
318
319  def testDefaultBehaviorWithShards(self):
320    """Tests the behavior without the filter, with sharding enabled."""
321
322    self.RunAndVerifyWithSharding(None, 1, ACTIVE_TESTS)
323    self.RunAndVerifyWithSharding(None, 2, ACTIVE_TESTS)
324    self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) - 1, ACTIVE_TESTS)
325    self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS), ACTIVE_TESTS)
326    self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) + 1, ACTIVE_TESTS)
327
328  def testEmptyFilter(self):
329    """Tests an empty filter."""
330
331    self.RunAndVerify('', [])
332    self.RunAndVerifyWithSharding('', 1, [])
333    self.RunAndVerifyWithSharding('', 2, [])
334
335  def testBadFilter(self):
336    """Tests a filter that matches nothing."""
337
338    self.RunAndVerify('BadFilter', [])
339    self.RunAndVerifyAllowingDisabled('BadFilter', [])
340
341  def testFullName(self):
342    """Tests filtering by full name."""
343
344    self.RunAndVerify('FooTest.Xyz', ['FooTest.Xyz'])
345    self.RunAndVerifyAllowingDisabled('FooTest.Xyz', ['FooTest.Xyz'])
346    self.RunAndVerifyWithSharding('FooTest.Xyz', 5, ['FooTest.Xyz'])
347
348  def testUniversalFilters(self):
349    """Tests filters that match everything."""
350
351    self.RunAndVerify('*', ACTIVE_TESTS)
352    self.RunAndVerify('*.*', ACTIVE_TESTS)
353    self.RunAndVerifyWithSharding('*.*', len(ACTIVE_TESTS) - 3, ACTIVE_TESTS)
354    self.RunAndVerifyAllowingDisabled('*', ACTIVE_TESTS + DISABLED_TESTS)
355    self.RunAndVerifyAllowingDisabled('*.*', ACTIVE_TESTS + DISABLED_TESTS)
356
357  def testFilterByTestCase(self):
358    """Tests filtering by test case name."""
359
360    self.RunAndVerify('FooTest.*', ['FooTest.Abc', 'FooTest.Xyz'])
361
362    BAZ_TESTS = ['BazTest.TestOne', 'BazTest.TestA', 'BazTest.TestB']
363    self.RunAndVerify('BazTest.*', BAZ_TESTS)
364    self.RunAndVerifyAllowingDisabled('BazTest.*',
365                                      BAZ_TESTS + ['BazTest.DISABLED_TestC'])
366
367  def testFilterByTest(self):
368    """Tests filtering by test name."""
369
370    self.RunAndVerify('*.TestOne', ['BarTest.TestOne', 'BazTest.TestOne'])
371
372  def testFilterDisabledTests(self):
373    """Select only the disabled tests to run."""
374
375    self.RunAndVerify('DISABLED_FoobarTest.Test1', [])
376    self.RunAndVerifyAllowingDisabled('DISABLED_FoobarTest.Test1',
377                                      ['DISABLED_FoobarTest.Test1'])
378
379    self.RunAndVerify('*DISABLED_*', [])
380    self.RunAndVerifyAllowingDisabled('*DISABLED_*', DISABLED_TESTS)
381
382    self.RunAndVerify('*.DISABLED_*', [])
383    self.RunAndVerifyAllowingDisabled('*.DISABLED_*', [
384        'BarTest.DISABLED_TestFour',
385        'BarTest.DISABLED_TestFive',
386        'BazTest.DISABLED_TestC',
387        'DISABLED_FoobarTest.DISABLED_Test2',
388        ])
389
390    self.RunAndVerify('DISABLED_*', [])
391    self.RunAndVerifyAllowingDisabled('DISABLED_*', [
392        'DISABLED_FoobarTest.Test1',
393        'DISABLED_FoobarTest.DISABLED_Test2',
394        'DISABLED_FoobarbazTest.TestA',
395        ])
396
397  def testWildcardInTestCaseName(self):
398    """Tests using wildcard in the test case name."""
399
400    self.RunAndVerify('*a*.*', [
401        'BarTest.TestOne',
402        'BarTest.TestTwo',
403        'BarTest.TestThree',
404
405        'BazTest.TestOne',
406        'BazTest.TestA',
407        'BazTest.TestB',
408
409        'HasDeathTest.Test1',
410        'HasDeathTest.Test2', ] + PARAM_TESTS)
411
412  def testWildcardInTestName(self):
413    """Tests using wildcard in the test name."""
414
415    self.RunAndVerify('*.*A*', ['FooTest.Abc', 'BazTest.TestA'])
416
417  def testFilterWithoutDot(self):
418    """Tests a filter that has no '.' in it."""
419
420    self.RunAndVerify('*z*', [
421        'FooTest.Xyz',
422
423        'BazTest.TestOne',
424        'BazTest.TestA',
425        'BazTest.TestB',
426        ])
427
428  def testTwoPatterns(self):
429    """Tests filters that consist of two patterns."""
430
431    self.RunAndVerify('Foo*.*:*A*', [
432        'FooTest.Abc',
433        'FooTest.Xyz',
434
435        'BazTest.TestA',
436        ])
437
438    # An empty pattern + a non-empty one
439    self.RunAndVerify(':*A*', ['FooTest.Abc', 'BazTest.TestA'])
440
441  def testThreePatterns(self):
442    """Tests filters that consist of three patterns."""
443
444    self.RunAndVerify('*oo*:*A*:*One', [
445        'FooTest.Abc',
446        'FooTest.Xyz',
447
448        'BarTest.TestOne',
449
450        'BazTest.TestOne',
451        'BazTest.TestA',
452        ])
453
454    # The 2nd pattern is empty.
455    self.RunAndVerify('*oo*::*One', [
456        'FooTest.Abc',
457        'FooTest.Xyz',
458
459        'BarTest.TestOne',
460
461        'BazTest.TestOne',
462        ])
463
464    # The last 2 patterns are empty.
465    self.RunAndVerify('*oo*::', [
466        'FooTest.Abc',
467        'FooTest.Xyz',
468        ])
469
470  def testNegativeFilters(self):
471    self.RunAndVerify('*-HasDeathTest.Test1', [
472        'FooTest.Abc',
473        'FooTest.Xyz',
474
475        'BarTest.TestOne',
476        'BarTest.TestTwo',
477        'BarTest.TestThree',
478
479        'BazTest.TestOne',
480        'BazTest.TestA',
481        'BazTest.TestB',
482
483        'HasDeathTest.Test2',
484        ] + PARAM_TESTS)
485
486    self.RunAndVerify('*-FooTest.Abc:HasDeathTest.*', [
487        'FooTest.Xyz',
488
489        'BarTest.TestOne',
490        'BarTest.TestTwo',
491        'BarTest.TestThree',
492
493        'BazTest.TestOne',
494        'BazTest.TestA',
495        'BazTest.TestB',
496        ] + PARAM_TESTS)
497
498    self.RunAndVerify('BarTest.*-BarTest.TestOne', [
499        'BarTest.TestTwo',
500        'BarTest.TestThree',
501        ])
502
503    # Tests without leading '*'.
504    self.RunAndVerify('-FooTest.Abc:FooTest.Xyz:HasDeathTest.*', [
505        'BarTest.TestOne',
506        'BarTest.TestTwo',
507        'BarTest.TestThree',
508
509        'BazTest.TestOne',
510        'BazTest.TestA',
511        'BazTest.TestB',
512        ] + PARAM_TESTS)
513
514    # Value parameterized tests.
515    self.RunAndVerify('*/*', PARAM_TESTS)
516
517    # Value parameterized tests filtering by the sequence name.
518    self.RunAndVerify('SeqP/*', [
519        'SeqP/ParamTest.TestX/0',
520        'SeqP/ParamTest.TestX/1',
521        'SeqP/ParamTest.TestY/0',
522        'SeqP/ParamTest.TestY/1',
523        ])
524
525    # Value parameterized tests filtering by the test name.
526    self.RunAndVerify('*/0', [
527        'SeqP/ParamTest.TestX/0',
528        'SeqP/ParamTest.TestY/0',
529        'SeqQ/ParamTest.TestX/0',
530        'SeqQ/ParamTest.TestY/0',
531        ])
532
533  def testFlagOverridesEnvVar(self):
534    """Tests that the filter flag overrides the filtering env. variable."""
535
536    SetEnvVar(FILTER_ENV_VAR, 'Foo*')
537    args = ['--%s=%s' % (FILTER_FLAG, '*One')]
538    tests_run = RunAndExtractTestList(args)[0]
539    SetEnvVar(FILTER_ENV_VAR, None)
540
541    self.AssertSetEqual(tests_run, ['BarTest.TestOne', 'BazTest.TestOne'])
542
543  def testShardStatusFileIsCreated(self):
544    """Tests that the shard file is created if specified in the environment."""
545
546    shard_status_file = os.path.join(gtest_test_utils.GetTempDir(),
547                                     'shard_status_file')
548    self.assert_(not os.path.exists(shard_status_file))
549
550    extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file}
551    try:
552      InvokeWithModifiedEnv(extra_env, RunAndReturnOutput)
553    finally:
554      self.assert_(os.path.exists(shard_status_file))
555      os.remove(shard_status_file)
556
557  def testShardStatusFileIsCreatedWithListTests(self):
558    """Tests that the shard file is created with --gtest_list_tests."""
559
560    shard_status_file = os.path.join(gtest_test_utils.GetTempDir(),
561                                     'shard_status_file2')
562    self.assert_(not os.path.exists(shard_status_file))
563
564    extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file}
565    try:
566      InvokeWithModifiedEnv(extra_env,
567                            RunAndReturnOutput,
568                            ['--gtest_list_tests'])
569    finally:
570      self.assert_(os.path.exists(shard_status_file))
571      os.remove(shard_status_file)
572
573  def testShardingWorksWithDeathTests(self):
574    """Tests integration with death tests and sharding."""
575    gtest_filter = 'HasDeathTest.*:SeqP/*'
576    expected_tests = [
577        'HasDeathTest.Test1',
578        'HasDeathTest.Test2',
579
580        'SeqP/ParamTest.TestX/0',
581        'SeqP/ParamTest.TestX/1',
582        'SeqP/ParamTest.TestY/0',
583        'SeqP/ParamTest.TestY/1',
584        ]
585
586    for flag in ['--gtest_death_test_style=threadsafe',
587                 '--gtest_death_test_style=fast']:
588      self.RunAndVerifyWithSharding(gtest_filter, 3, expected_tests,
589                                    check_exit_0=True, args=[flag])
590      self.RunAndVerifyWithSharding(gtest_filter, 5, expected_tests,
591                                    check_exit_0=True, args=[flag])
592
593if __name__ == '__main__':
594  gtest_test_utils.Main()