/core/liftbridge/queue.py

https://github.com/nocproject/noc · Python · 123 lines · 84 code · 11 blank · 28 comment · 27 complexity · a0f318ac2e8c9f762cea0a9037b448f4 MD5 · raw file

  1. # ----------------------------------------------------------------------
  2. # LiftBridge Publisher Queue
  3. # ----------------------------------------------------------------------
  4. # Copyright (C) 2007-2020 The NOC Project
  5. # See LICENSE for details
  6. # ----------------------------------------------------------------------
  7. # Python modules
  8. from collections import deque
  9. from threading import Lock
  10. import asyncio
  11. from typing import Optional, Dict, Any
  12. # NOC modules
  13. from .api_pb2 import PublishRequest
  14. class LiftBridgeQueue(object):
  15. def __init__(self, loop: Optional[asyncio.BaseEventLoop] = None):
  16. self.queue: deque = deque()
  17. self.lock = Lock()
  18. self.waiter: Optional[asyncio.Event] = None
  19. self.drain_waiter: Optional[asyncio.Event] = None
  20. self.loop: Optional[asyncio.BaseEventLoop] = loop
  21. self.req_puts: int = 0
  22. self.req_gets: int = 0
  23. self.to_shutdown: bool = False
  24. def _notify_waiter(self, waiter: asyncio.Event) -> None:
  25. if self.loop:
  26. self.loop.call_soon_threadsafe(waiter.set)
  27. else:
  28. waiter.set()
  29. def put(self, req: PublishRequest, fifo: bool = True) -> None:
  30. """
  31. Put request into queue
  32. :param req:
  33. :param fifo:
  34. :return:
  35. """
  36. with self.lock:
  37. if fifo:
  38. self.queue.append(req)
  39. else:
  40. self.queue.appendleft(req)
  41. self.req_puts += 1
  42. # Notify waiters
  43. if not self.waiter:
  44. return
  45. self._notify_waiter(self.waiter)
  46. async def get(self, timeout: Optional[float] = None) -> Optional[PublishRequest]:
  47. """
  48. Get request from queue. Wait forever, if timeout is None,
  49. of return None if timeout is expired.
  50. :param timeout:
  51. :return:
  52. """
  53. with self.lock:
  54. # Direct path, in case the queue is not empty
  55. if len(self.queue):
  56. self.req_gets += 1
  57. return self.queue.popleft()
  58. self.waiter = asyncio.Event()
  59. # Wait until waiter is set
  60. if timeout:
  61. try:
  62. await asyncio.wait_for(self.waiter.wait(), timeout)
  63. except asyncio.TimeoutError:
  64. return None
  65. else:
  66. await self.waiter.wait()
  67. # Pop message and reset waiter
  68. with self.lock:
  69. if self.drain_waiter and not len(self.queue):
  70. self._notify_waiter(self.drain_waiter)
  71. self.waiter = None
  72. self.req_gets += 1
  73. try:
  74. return self.queue.popleft()
  75. except IndexError:
  76. return None # Triggered by shutdown
  77. def apply_metrics(self, data: Dict[str, Any]) -> None:
  78. data.update(
  79. {
  80. "liftbridge_publish_puts": self.req_puts,
  81. "liftbridge_publish_gets": self.req_gets,
  82. "liftbridge_publish_queue": len(self.queue),
  83. }
  84. )
  85. def shutdown(self) -> bool:
  86. with self.lock:
  87. if self.waiter:
  88. self._notify_waiter(self.waiter)
  89. self.to_shutdown = True
  90. async def drain(self, timeout: Optional[float] = None) -> bool:
  91. """
  92. Wait until queue is empty
  93. :return:
  94. """
  95. with self.lock:
  96. if self.waiter:
  97. self._notify_waiter(self.waiter)
  98. if not len(self.queue):
  99. return True
  100. self.drain_waiter = asyncio.Event()
  101. if timeout:
  102. try:
  103. await asyncio.wait_for(self.drain_waiter.wait(), timeout)
  104. except asyncio.TimeoutError:
  105. pass
  106. else:
  107. await self.drain_waiter.wait()
  108. self.drain_waiter = None
  109. return len(self.queue) == 0
  110. def qsize(self) -> int:
  111. with self.lock:
  112. return len(self.queue)