/behave/runner.py

https://github.com/atteroTheGreatest/behave · Python · 684 lines · 491 code · 115 blank · 78 comment · 119 complexity · d661779f5113f438f5201e8dc8e9f7c7 MD5 · raw file

  1. # -*- coding: utf-8 -*-
  2. from __future__ import with_statement
  3. import contextlib
  4. import os.path
  5. import StringIO
  6. import sys
  7. import traceback
  8. import warnings
  9. import weakref
  10. from behave import matchers
  11. from behave.step_registry import setup_step_decorators
  12. from behave.formatter import formatters
  13. from behave.configuration import ConfigError
  14. from behave.log_capture import LoggingCapture
  15. from behave.runner_util import \
  16. collect_feature_locations, parse_features
  17. class ContextMaskWarning(UserWarning):
  18. '''Raised if a context variable is being overwritten in some situations.
  19. If the variable was originally set by user code then this will be raised if
  20. *behave* overwites the value.
  21. If the variable was originally set by *behave* then this will be raised if
  22. user code overwites the value.
  23. '''
  24. pass
  25. class Context(object):
  26. '''Hold contextual information during the running of tests.
  27. This object is a place to store information related to the tests you're
  28. running. You may add arbitrary attributes to it of whatever value you need.
  29. During the running of your tests the object will have additional layers of
  30. namespace added and removed automatically. There is a "root" namespace and
  31. additional namespaces for features and scenarios.
  32. Certain names are used by *behave*; be wary of using them yourself as
  33. *behave* may overwrite the value you set. These names are:
  34. .. attribute:: feature
  35. This is set when we start testing a new feature and holds a
  36. :class:`~behave.model.Feature`. It will not be present outside of a
  37. feature (i.e. within the scope of the environment before_all and
  38. after_all).
  39. .. attribute:: scenario
  40. This is set when we start testing a new scenario (including the
  41. individual scenarios of a scenario outline) and holds a
  42. :class:`~behave.model.Scenario`. It will not be present outside of the
  43. scope of a scenario.
  44. .. attribute:: tags
  45. The current set of active tags (as a Python set containing instances of
  46. :class:`~behave.model.Tag` which are basically just glorified strings)
  47. combined from the feature and scenario. This attribute will not be
  48. present outside of a feature scope.
  49. .. attribute:: aborted
  50. This is set to true in the root namespace when the user aborts a test run
  51. (:exc:`KeyboardInterrupt` exception). Initially: False.
  52. .. attribute:: failed
  53. This is set to true in the root namespace as soon as a step fails.
  54. Initially: False.
  55. .. attribute:: table
  56. This is set at the step level and holds any :class:`~behave.model.Table`
  57. associated with the step.
  58. .. attribute:: text
  59. This is set at the step level and holds any multiline text associated
  60. with the step.
  61. .. attribute:: config
  62. The configuration of *behave* as determined by configuration files and
  63. command-line options. The attributes of this object are the same as the
  64. `configuration file settion names`_.
  65. .. attribute:: active_outline
  66. This is set for each scenario in a scenario outline and references the
  67. :class:`~behave.model.Row` that is active for the current scenario. It is
  68. present mostly for debugging, but may be useful otherwise.
  69. .. attribute:: log_capture
  70. If logging capture is enabled then this attribute contains the captured
  71. logging as an instance of :class:`~behave.log_capture.LoggingCapture`.
  72. It is not present if logging is not being captured.
  73. .. attribute:: stdout_capture
  74. If stdout capture is enabled then this attribute contains the captured
  75. output as a StringIO instance. It is not present if stdout is not being
  76. captured.
  77. .. attribute:: stderr_capture
  78. If stderr capture is enabled then this attribute contains the captured
  79. output as a StringIO instance. It is not present if stderr is not being
  80. captured.
  81. If an attempt made by user code to overwrite one of these variables, or
  82. indeed by *behave* to overwite a user-set variable, then a
  83. :class:`behave.runner.ContextMaskWarning` warning will be raised.
  84. You may use the "in" operator to test whether a certain value has been set
  85. on the context, for example:
  86. 'feature' in context
  87. checks whether there is a "feature" value in the context.
  88. Values may be deleted from the context using "del" but only at the level
  89. they are set. You can't delete a value set by a feature at a scenario level
  90. but you can delete a value set for a scenario in that scenario.
  91. .. _`configuration file settion names`: behave.html#configuration-files
  92. '''
  93. BEHAVE = 'behave'
  94. USER = 'user'
  95. def __init__(self, runner):
  96. self._runner = weakref.proxy(runner)
  97. self._config = runner.config
  98. d = self._root = {
  99. 'aborted': False,
  100. 'failed': False,
  101. 'config': self._config,
  102. 'active_outline': None,
  103. }
  104. self._stack = [d]
  105. self._record = {}
  106. self._origin = {}
  107. self._mode = self.BEHAVE
  108. self.feature = None
  109. def _push(self):
  110. self._stack.insert(0, {})
  111. def _pop(self):
  112. self._stack.pop(0)
  113. @contextlib.contextmanager
  114. def user_mode(self):
  115. try:
  116. self._mode = self.USER
  117. yield
  118. finally:
  119. # -- NOTE: Otherwise skipped if AssertionError/Exception is raised.
  120. self._mode = self.BEHAVE
  121. def _set_root_attribute(self, attr, value):
  122. for frame in self.__dict__['_stack']:
  123. if frame is self.__dict__['_root']:
  124. continue
  125. if attr in frame:
  126. record = self.__dict__['_record'][attr]
  127. params = {
  128. 'attr': attr,
  129. 'filename': record[0],
  130. 'line': record[1],
  131. 'function': record[3],
  132. }
  133. self._emit_warning(attr, params)
  134. self.__dict__['_root'][attr] = value
  135. if attr not in self._origin:
  136. self._origin[attr] = self._mode
  137. def _emit_warning(self, attr, params):
  138. msg = ''
  139. if self._mode is self.BEHAVE and self._origin[attr] is not self.BEHAVE:
  140. msg = "behave runner is masking context attribute '%(attr)s' " \
  141. "orignally set in %(function)s (%(filename)s:%(line)s)"
  142. elif self._mode is self.USER:
  143. if self._origin[attr] is not self.USER:
  144. msg = "user code is masking context attribute '%(attr)s' " \
  145. "orignally set by behave"
  146. elif self._config.verbose:
  147. msg = "user code is masking context attribute " \
  148. "'%(attr)s'; see the tutorial for what this means"
  149. if msg:
  150. msg = msg % params
  151. warnings.warn(msg, ContextMaskWarning, stacklevel=3)
  152. def _dump(self):
  153. for level, frame in enumerate(self._stack):
  154. print 'Level %d' % level
  155. print repr(frame)
  156. def __getattr__(self, attr):
  157. if attr[0] == '_':
  158. return self.__dict__[attr]
  159. for frame in self._stack:
  160. if attr in frame:
  161. return frame[attr]
  162. msg = "'{0}' object has no attribute '{1}'"
  163. msg = msg.format(self.__class__.__name__, attr)
  164. raise AttributeError(msg)
  165. def __setattr__(self, attr, value):
  166. if attr[0] == '_':
  167. self.__dict__[attr] = value
  168. return
  169. for frame in self._stack[1:]:
  170. if attr in frame:
  171. record = self._record[attr]
  172. params = {
  173. 'attr': attr,
  174. 'filename': record[0],
  175. 'line': record[1],
  176. 'function': record[3],
  177. }
  178. self._emit_warning(attr, params)
  179. stack_frame = traceback.extract_stack(limit=2)[0]
  180. self._record[attr] = stack_frame
  181. frame = self._stack[0]
  182. frame[attr] = value
  183. if attr not in self._origin:
  184. self._origin[attr] = self._mode
  185. def __delattr__(self, attr):
  186. frame = self._stack[0]
  187. if attr in frame:
  188. del frame[attr]
  189. del self._record[attr]
  190. else:
  191. msg = "'{0}' object has no attribute '{1}' at the current level"
  192. msg = msg.format(self.__class__.__name__, attr)
  193. raise AttributeError(msg)
  194. def __contains__(self, attr):
  195. if attr[0] == '_':
  196. return attr in self.__dict__
  197. for frame in self._stack:
  198. if attr in frame:
  199. return True
  200. return False
  201. def execute_steps(self, steps_text):
  202. '''The steps identified in the "steps" text string will be parsed and
  203. executed in turn just as though they were defined in a feature file.
  204. If the execute_steps call fails (either through error or failure
  205. assertion) then the step invoking it will fail.
  206. ValueError will be raised if this is invoked outside a feature context.
  207. Returns boolean False if the steps are not parseable, True otherwise.
  208. '''
  209. assert isinstance(steps_text, unicode), "Steps must be unicode."
  210. if not self.feature:
  211. raise ValueError('execute_steps() called outside of feature')
  212. # -- PREPARE: Save original context data for current step.
  213. # Needed if step definition that called this method uses .table/.text
  214. original_table = getattr(self, "table", None)
  215. original_text = getattr(self, "text", None)
  216. self.feature.parser.variant = 'steps'
  217. steps = self.feature.parser.parse_steps(steps_text)
  218. for step in steps:
  219. passed = step.run(self._runner, quiet=True, capture=False)
  220. if not passed:
  221. # -- ISSUE #96: Provide more substep info to diagnose problem.
  222. step_line = u"%s %s" % (step.keyword, step.name)
  223. message = "%s SUB-STEP: %s" % (step.status.upper(), step_line)
  224. if step.error_message:
  225. message += "\nSubstep info: %s" % step.error_message
  226. assert False, message
  227. # -- FINALLY: Restore original context data for current step.
  228. self.table = original_table
  229. self.text = original_text
  230. return True
  231. def exec_file(filename, globals={}, locals=None):
  232. if locals is None:
  233. locals = globals
  234. locals['__file__'] = filename
  235. if sys.version_info[0] == 3:
  236. with open(filename) as f:
  237. # -- FIX issue #80: exec(f.read(), globals, locals)
  238. filename2 = os.path.relpath(filename, os.getcwd())
  239. code = compile(f.read(), filename2, 'exec')
  240. exec(code, globals, locals)
  241. else:
  242. execfile(filename, globals, locals)
  243. def path_getrootdir(path):
  244. """
  245. Extract rootdir from path in a platform independent way.
  246. POSIX-PATH EXAMPLE:
  247. rootdir = path_getrootdir("/foo/bar/one.feature")
  248. assert rootdir == "/"
  249. WINDOWS-PATH EXAMPLE:
  250. rootdir = path_getrootdir("D:\\foo\\bar\\one.feature")
  251. assert rootdir == r"D:\"
  252. """
  253. drive, _ = os.path.splitdrive(path)
  254. if drive:
  255. # -- WINDOWS:
  256. return drive + os.path.sep
  257. # -- POSIX:
  258. return os.path.sep
  259. class PathManager(object):
  260. """
  261. Context manager to add paths to sys.path (python search path) within a scope
  262. """
  263. def __init__(self, paths=None):
  264. self.initial_paths = paths or []
  265. self.paths = None
  266. def __enter__(self):
  267. self.paths = list(self.initial_paths)
  268. sys.path = self.paths + sys.path
  269. def __exit__(self, *crap):
  270. for path in self.paths:
  271. sys.path.remove(path)
  272. self.paths = None
  273. def add(self, path):
  274. if self.paths is None:
  275. # -- CALLED OUTSIDE OF CONTEXT:
  276. self.initial_paths.append(path)
  277. else:
  278. sys.path.insert(0, path)
  279. self.paths.append(path)
  280. class ModelRunner(object):
  281. """
  282. Test runner for a behave model (features).
  283. Provides the core functionality of a test runner and
  284. the functional API needed by model elements.
  285. .. attribute:: aborted
  286. This is set to true when the user aborts a test run
  287. (:exc:`KeyboardInterrupt` exception). Initially: False.
  288. Stored as derived attribute in :attr:`Context.aborted`.
  289. """
  290. def __init__(self, config, features=None):
  291. self.config = config
  292. self.features = features or []
  293. self.hooks = {}
  294. self.formatters = []
  295. self.undefined_steps = []
  296. self.context = None
  297. self.feature = None
  298. self.stdout_capture = None
  299. self.stderr_capture = None
  300. self.log_capture = None
  301. self.old_stdout = None
  302. self.old_stderr = None
  303. # @property
  304. def _get_aborted(self):
  305. value = False
  306. if self.context:
  307. value = self.context.aborted
  308. return value
  309. # @aborted.setter
  310. def _set_aborted(self, value):
  311. assert self.context
  312. self.context._set_root_attribute('aborted', bool(value))
  313. aborted = property(_get_aborted, _set_aborted,
  314. doc="Indicates that test run is aborted by the user.")
  315. def run_hook(self, name, context, *args):
  316. if not self.config.dry_run and (name in self.hooks):
  317. # try:
  318. with context.user_mode():
  319. self.hooks[name](context, *args)
  320. # except KeyboardInterrupt:
  321. # self.aborted = True
  322. # if name not in ("before_all", "after_all"):
  323. # raise
  324. def setup_capture(self):
  325. if not self.context:
  326. self.context = Context(self)
  327. if self.config.stdout_capture:
  328. self.stdout_capture = StringIO.StringIO()
  329. self.context.stdout_capture = self.stdout_capture
  330. if self.config.stderr_capture:
  331. self.stderr_capture = StringIO.StringIO()
  332. self.context.stderr_capture = self.stderr_capture
  333. if self.config.log_capture:
  334. self.log_capture = LoggingCapture(self.config)
  335. self.log_capture.inveigle()
  336. self.context.log_capture = self.log_capture
  337. def start_capture(self):
  338. if self.config.stdout_capture:
  339. # -- REPLACE ONLY: In non-capturing mode.
  340. if not self.old_stdout:
  341. self.old_stdout = sys.stdout
  342. sys.stdout = self.stdout_capture
  343. assert sys.stdout is self.stdout_capture
  344. if self.config.stderr_capture:
  345. # -- REPLACE ONLY: In non-capturing mode.
  346. if not self.old_stderr:
  347. self.old_stderr = sys.stderr
  348. sys.stderr = self.stderr_capture
  349. assert sys.stderr is self.stderr_capture
  350. def stop_capture(self):
  351. if self.config.stdout_capture:
  352. # -- RESTORE ONLY: In capturing mode.
  353. if self.old_stdout:
  354. sys.stdout = self.old_stdout
  355. self.old_stdout = None
  356. assert sys.stdout is not self.stdout_capture
  357. if self.config.stderr_capture:
  358. # -- RESTORE ONLY: In capturing mode.
  359. if self.old_stderr:
  360. sys.stderr = self.old_stderr
  361. self.old_stderr = None
  362. assert sys.stderr is not self.stderr_capture
  363. def teardown_capture(self):
  364. if self.config.log_capture:
  365. self.log_capture.abandon()
  366. def run_model(self, features=None):
  367. if not self.context:
  368. self.context = Context(self)
  369. if features is None:
  370. features = self.features
  371. # -- ENSURE: context.execute_steps() works in weird cases (hooks, ...)
  372. context = self.context
  373. self.setup_capture()
  374. self.run_hook('before_all', context)
  375. run_feature = not self.aborted
  376. failed_count = 0
  377. undefined_steps_initial_size = len(self.undefined_steps)
  378. for feature in features:
  379. if run_feature:
  380. try:
  381. self.feature = feature
  382. for formatter in self.formatters:
  383. formatter.uri(feature.filename)
  384. failed = feature.run(self)
  385. if failed:
  386. failed_count += 1
  387. if self.config.stop or self.aborted:
  388. # -- FAIL-EARLY: After first failure.
  389. run_feature = False
  390. except KeyboardInterrupt:
  391. self.aborted = True
  392. failed_count += 1
  393. run_feature = False
  394. # -- ALWAYS: Report run/not-run feature to reporters.
  395. # REQUIRED-FOR: Summary to keep track of untested features.
  396. for reporter in self.config.reporters:
  397. reporter.feature(feature)
  398. # -- AFTER-ALL:
  399. if self.aborted:
  400. print "\nABORTED: By user."
  401. for formatter in self.formatters:
  402. formatter.close()
  403. self.run_hook('after_all', self.context)
  404. for reporter in self.config.reporters:
  405. reporter.end()
  406. # if self.aborted:
  407. # print "\nABORTED: By user."
  408. failed = ((failed_count > 0) or self.aborted or
  409. (len(self.undefined_steps) > undefined_steps_initial_size))
  410. return failed
  411. def run(self):
  412. """
  413. Implements the run method by running the model.
  414. """
  415. self.context = Context(self)
  416. return self.run_model()
  417. class Runner(ModelRunner):
  418. """
  419. Standard test runner for behave:
  420. * setup paths
  421. * loads environment hooks
  422. * loads step definitions
  423. * select feature files, parses them and creates model (elements)
  424. """
  425. def __init__(self, config):
  426. super(Runner, self).__init__(config)
  427. self.path_manager = PathManager()
  428. self.base_dir = None
  429. def setup_paths(self):
  430. if self.config.paths:
  431. if self.config.verbose:
  432. print 'Supplied path:', \
  433. ', '.join('"%s"' % path for path in self.config.paths)
  434. first_path = self.config.paths[0]
  435. if hasattr(first_path, "filename"):
  436. # -- BETTER: isinstance(first_path, FileLocation):
  437. first_path = first_path.filename
  438. base_dir = first_path
  439. if base_dir.startswith('@'):
  440. # -- USE: behave @features.txt
  441. base_dir = base_dir[1:]
  442. file_locations = self.feature_locations()
  443. if file_locations:
  444. base_dir = os.path.dirname(file_locations[0].filename)
  445. base_dir = os.path.abspath(base_dir)
  446. # supplied path might be to a feature file
  447. if os.path.isfile(base_dir):
  448. if self.config.verbose:
  449. print 'Primary path is to a file so using its directory'
  450. base_dir = os.path.dirname(base_dir)
  451. else:
  452. if self.config.verbose:
  453. print 'Using default path "./features"'
  454. base_dir = os.path.abspath('features')
  455. # Get the root. This is not guaranteed to be '/' because Windows.
  456. root_dir = path_getrootdir(base_dir)
  457. new_base_dir = base_dir
  458. while True:
  459. if self.config.verbose:
  460. print 'Trying base directory:', new_base_dir
  461. if os.path.isdir(os.path.join(new_base_dir, 'steps')):
  462. break
  463. if os.path.isfile(os.path.join(new_base_dir, 'environment.py')):
  464. break
  465. if new_base_dir == root_dir:
  466. break
  467. new_base_dir = os.path.dirname(new_base_dir)
  468. if new_base_dir == root_dir:
  469. if self.config.verbose:
  470. if not self.config.paths:
  471. print 'ERROR: Could not find "steps" directory. Please '\
  472. 'specify where to find your features.'
  473. else:
  474. print 'ERROR: Could not find "steps" directory in your '\
  475. 'specified path "%s"' % base_dir
  476. raise ConfigError('No steps directory in "%s"' % base_dir)
  477. base_dir = new_base_dir
  478. self.config.base_dir = base_dir
  479. for dirpath, dirnames, filenames in os.walk(base_dir):
  480. if [fn for fn in filenames if fn.endswith('.feature')]:
  481. break
  482. else:
  483. if self.config.verbose:
  484. if not self.config.paths:
  485. print 'ERROR: Could not find any "<name>.feature" files. '\
  486. 'Please specify where to find your features.'
  487. else:
  488. print 'ERROR: Could not find any "<name>.feature" files '\
  489. 'in your specified path "%s"' % base_dir
  490. raise ConfigError('No feature files in "%s"' % base_dir)
  491. self.base_dir = base_dir
  492. self.path_manager.add(base_dir)
  493. if not self.config.paths:
  494. self.config.paths = [base_dir]
  495. if base_dir != os.getcwd():
  496. self.path_manager.add(os.getcwd())
  497. def before_all_default_hook(self, context):
  498. """
  499. Default implementation for :func:`before_all()` hook.
  500. Setup the logging subsystem based on the configuration data.
  501. """
  502. context.config.setup_logging()
  503. def load_hooks(self, filename='environment.py'):
  504. hooks_path = os.path.join(self.base_dir, filename)
  505. if os.path.exists(hooks_path):
  506. exec_file(hooks_path, self.hooks)
  507. if 'before_all' not in self.hooks:
  508. self.hooks['before_all'] = self.before_all_default_hook
  509. def load_step_definitions(self, extra_step_paths=[]):
  510. step_globals = {
  511. 'use_step_matcher': matchers.use_step_matcher,
  512. 'step_matcher': matchers.step_matcher, # -- DEPRECATING
  513. }
  514. setup_step_decorators(step_globals)
  515. # -- Allow steps to import other stuff from the steps dir
  516. # NOTE: Default matcher can be overridden in "environment.py" hook.
  517. steps_dir = os.path.join(self.base_dir, 'steps')
  518. paths = [steps_dir] + list(extra_step_paths)
  519. with PathManager(paths):
  520. default_matcher = matchers.current_matcher
  521. for path in paths:
  522. for name in sorted(os.listdir(path)):
  523. if name.endswith('.py'):
  524. # -- LOAD STEP DEFINITION:
  525. # Reset to default matcher after each step-definition.
  526. # A step-definition may change the matcher 0..N times.
  527. # ENSURE: Each step definition has clean globals.
  528. step_module_globals = step_globals.copy()
  529. exec_file(os.path.join(path, name), step_module_globals)
  530. matchers.current_matcher = default_matcher
  531. def feature_locations(self):
  532. return collect_feature_locations(self.config.paths)
  533. def run(self):
  534. with self.path_manager:
  535. self.setup_paths()
  536. return self.run_with_paths()
  537. def run_with_paths(self):
  538. self.context = Context(self)
  539. self.load_hooks()
  540. self.load_step_definitions()
  541. # -- ENSURE: context.execute_steps() works in weird cases (hooks, ...)
  542. # self.setup_capture()
  543. # self.run_hook('before_all', self.context)
  544. # -- STEP: Parse all feature files (by using their file location).
  545. feature_locations = [ filename for filename in self.feature_locations()
  546. if not self.config.exclude(filename) ]
  547. features = parse_features(feature_locations, language=self.config.lang)
  548. self.features.extend(features)
  549. # -- STEP: Run all features.
  550. stream_openers = self.config.outputs
  551. self.formatters = formatters.get_formatter(self.config, stream_openers)
  552. return self.run_model()