PageRenderTime 334ms CodeModel.GetById 319ms RepoModel.GetById 1ms app.codeStats 0ms

/circuits/core/workers.py

https://bitbucket.org/prologic/circuits/
Python | 88 lines | 82 code | 1 blank | 5 comment | 1 complexity | c397bf4978d621c6d151c9314458e7a1 MD5 | raw file
  1. # Module: workers
  2. # Date: 6th February 2011
  3. # Author: James Mills, prologic at shortcircuit dot net dot au
  4. """Workers
  5. Worker components used to perform "work" in independent threads or
  6. processes. Worker(s) are typically used by a Pool (circuits.core.pools)
  7. to create a pool of workers. Worker(s) are not registered with a Manager
  8. or another Component - instead they are managed by the Pool. If a Worker
  9. is used independently it should not be registered as it causes its
  10. main event handler ``_on_task`` to execute in the other thread blocking it.
  11. """
  12. from threading import current_thread
  13. from weakref import WeakKeyDictionary
  14. from multiprocessing import cpu_count
  15. from multiprocessing.pool import ThreadPool
  16. from multiprocessing import Pool as ProcessPool
  17. from .events import Event
  18. from .handlers import handler
  19. from .components import BaseComponent
  20. DEFAULT_WORKERS = 10
  21. class task(Event):
  22. """task Event
  23. This Event is used to initiate a new task to be performed by a Worker
  24. or a Pool of Worker(s).
  25. :param f: The function to be executed.
  26. :type f: function
  27. :param args: Arguments to pass to the function
  28. :type args: tuple
  29. :param kwargs: Keyword Arguments to pass to the function
  30. :type kwargs: dict
  31. """
  32. success = True
  33. failure = True
  34. def __init__(self, f, *args, **kwargs):
  35. "x.__init__(...) initializes x; see x.__class__.__doc__ for signature"
  36. super(task, self).__init__(f, *args, **kwargs)
  37. class Worker(BaseComponent):
  38. """A thread/process Worker Component
  39. This Component creates a Worker (either a thread or process) which
  40. when given a ``Task``, will execute the given function in the task
  41. in the background in its thread/process.
  42. :param process: True to start this Worker as a process (Thread otherwise)
  43. :type process: bool
  44. """
  45. channel = "worker"
  46. def init(self, process=False, workers=None, channel=channel):
  47. if not hasattr(current_thread(), "_children"):
  48. current_thread()._children = WeakKeyDictionary()
  49. self.workers = workers or (cpu_count() if process else DEFAULT_WORKERS)
  50. Pool = ProcessPool if process else ThreadPool
  51. self.pool = Pool(self.workers)
  52. @handler("stopped", "unregistered", channel="*")
  53. def _on_stopped(self, event, *args):
  54. if event.name == "unregistered" and args[0] is not self:
  55. return
  56. self.pool.close()
  57. self.pool.join()
  58. @handler("task")
  59. def _on_task(self, f, *args, **kwargs):
  60. result = self.pool.apply_async(f, args, kwargs)
  61. while not result.ready():
  62. yield
  63. yield result.get()