/lib/core/async_pool.py

https://github.com/orleven/Tentacle · Python · 166 lines · 124 code · 34 blank · 8 comment · 28 complexity · b90c7652a3233bfdd3922215d029ca03 MD5 · raw file

  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. # @author: 'orleven'
  4. import asyncio
  5. from lib.core.data import logger
  6. class WorkItem:
  7. def __init__(self, _func, *args, **kwargs):
  8. self._func = _func
  9. self._args = args
  10. self._kwargs = kwargs
  11. self.future = asyncio.Future()
  12. class AsyncWorker:
  13. def __init__(self, pool):
  14. self.pool = pool
  15. self.is_running = False
  16. def start(self):
  17. self._fut = asyncio.ensure_future(self.run())
  18. async def stop(self, timeout=None):
  19. await asyncio.wait_for(self._fut, timeout)
  20. async def run(self):
  21. while True:
  22. item = await self.pool.work_queue.get()
  23. if item == None:
  24. break
  25. try:
  26. self.is_running = True
  27. result = await item._func(*item._args, **item._kwargs)
  28. item.future.set_result(result)
  29. except Exception as ex:
  30. item.future.set_exception(ex)
  31. finally:
  32. self.is_running = False
  33. class AsyncPool:
  34. def __init__(self, num_workers, backlog=0):
  35. self.num_workers = num_workers
  36. self.workers = []
  37. self.work_queue = asyncio.Queue(backlog)
  38. for _ in range(num_workers):
  39. worker = AsyncWorker(self)
  40. worker.start()
  41. self.workers.append(worker)
  42. self._closed = False
  43. self._finish_left = num_workers
  44. async def submit(self, _func, *args, **kwargs) -> asyncio.Future:
  45. if self._closed:
  46. raise RuntimeError('submit after shutdown')
  47. item = WorkItem(_func, *args, **kwargs)
  48. await self.work_queue.put(item)
  49. return item.future
  50. async def shutdown(self, timeout=None, cancel_queued=False):
  51. self._closed = True
  52. if cancel_queued:
  53. # cancel all existing tasks
  54. while not self.work_queue.empty():
  55. item = await self.work_queue.get()
  56. if item:
  57. item.future.set_exception(asyncio.CancelledError())
  58. # explicit yield to wake up other putters
  59. await asyncio.sleep(0)
  60. # put finishing item
  61. # note that putting more than num_workers times may block shutdown forever
  62. while self._finish_left > 0:
  63. self._finish_left -= 1
  64. await self.work_queue.put(None)
  65. # wait workers to complete
  66. await asyncio.gather(
  67. *(worker.stop(timeout) for worker in self.workers), return_exceptions=True
  68. )
  69. @property
  70. def is_finished(self):
  71. return (not any(w.is_running for w in self.workers)) and self.work_queue.empty()
  72. class PoolCollector:
  73. def __init__(self, pool: AsyncPool):
  74. self._pool = pool
  75. self._queue = asyncio.Queue()
  76. @classmethod
  77. def create(cls, num_workers, backlog=0):
  78. pool = AsyncPool(num_workers, backlog=backlog)
  79. return cls(pool)
  80. @property
  81. def remain_task_count(self):
  82. return self._pool.work_queue.qsize()
  83. @property
  84. def scanning_task_count(self):
  85. return len([w for w in self._pool.workers if w.is_running])
  86. async def submit(self, _func, *args, **kwargs):
  87. future = await self._pool.submit(_func, *args, **kwargs)
  88. future.add_done_callback(self._queue.put_nowait)
  89. return future
  90. async def submit_all(self, items):
  91. for item in items:
  92. await self.submit(item._func, *item._args, **item._kwargs)
  93. await self.shutdown()
  94. async def shutdown(self):
  95. await self._pool.shutdown()
  96. await self._queue.put(None)
  97. @property
  98. def is_finished(self):
  99. return self._pool.is_finished
  100. async def __aenter__(self):
  101. return self
  102. async def __aexit__(self, exc_type, exc_value, traceback):
  103. await self.close()
  104. async def close(self):
  105. await self._pool.shutdown(timeout=0, cancel_queued=True)
  106. while not self._queue.empty():
  107. future = await self._queue.get()
  108. ignore_cancelled(future)
  109. async def iter(self):
  110. while True:
  111. item = await self._queue.get()
  112. if item is not None:
  113. yield item
  114. else:
  115. break
  116. def ignore_cancelled(future):
  117. if not future:
  118. return
  119. try:
  120. future.result()
  121. except Exception as ex:
  122. if not isinstance(ex, asyncio.CancelledError):
  123. logger.warning('future exception', exc_info=True)
  124. async def finish_detect_daemon(manager: PoolCollector, time_interval=5):
  125. last_status = True
  126. while True:
  127. await asyncio.sleep(time_interval)
  128. this_status = bool(manager.is_finished)
  129. if last_status and this_status:
  130. await manager.shutdown()
  131. break
  132. last_status = this_status