/circuits/core/pollers.py

https://bitbucket.org/prologic/circuits/ · Python · 549 lines · 396 code · 101 blank · 52 comment · 109 complexity · df182f3b2833e930ab35d295248592ae MD5 · raw file

  1. # Module: pollers
  2. # Date: 15th September 2008
  3. # Author: James Mills <prologic@shortcircuit.net.au>
  4. """Poller Components for asynchronous file and socket I/O.
  5. This module contains Poller components that enable polling of file or socket
  6. descriptors for read/write events. Pollers:
  7. - Select
  8. - Poll
  9. - EPoll
  10. """
  11. import os
  12. import select
  13. import platform
  14. from errno import EBADF, EINTR
  15. from select import error as SelectError
  16. from socket import error as SocketError, create_connection, \
  17. socket as create_socket, AF_INET, SOCK_STREAM, socket
  18. from threading import Thread
  19. from circuits.core.handlers import handler
  20. from .events import Event
  21. from .components import BaseComponent
  22. class _read(Event):
  23. """_read Event"""
  24. class _write(Event):
  25. """_write Event"""
  26. class _error(Event):
  27. """_error Event"""
  28. class _disconnect(Event):
  29. """_disconnect Event"""
  30. class BasePoller(BaseComponent):
  31. channel = None
  32. def __init__(self, channel=channel):
  33. super(BasePoller, self).__init__(channel=channel)
  34. self._read = []
  35. self._write = []
  36. self._targets = {}
  37. self._ctrl_recv, self._ctrl_send = self._create_control_con()
  38. def _create_control_con(self):
  39. if platform.system() == "Linux":
  40. return os.pipe()
  41. server = create_socket(AF_INET, SOCK_STREAM)
  42. server.bind(("localhost", 0))
  43. server.listen(1)
  44. res_list = []
  45. def accept():
  46. sock, _ = server.accept()
  47. sock.setblocking(False)
  48. res_list.append(sock)
  49. at = Thread(target=accept)
  50. at.start()
  51. clnt_sock = create_connection(server.getsockname())
  52. at.join()
  53. return (res_list[0], clnt_sock)
  54. @handler("generate_events", priority=-9)
  55. def _on_generate_events(self, event):
  56. """
  57. Pollers have slightly higher priority than the default handler
  58. from Manager to ensure that they are invoked before the
  59. default handler. They act as event filters to avoid the additional
  60. invocation of the default handler which would be unnecessary
  61. overhead.
  62. """
  63. event.stop()
  64. self._generate_events(event)
  65. def resume(self):
  66. if isinstance(self._ctrl_send, socket):
  67. self._ctrl_send.send(b"\0")
  68. else:
  69. os.write(self._ctrl_send, b"\0")
  70. def _read_ctrl(self):
  71. try:
  72. if isinstance(self._ctrl_recv, socket):
  73. return self._ctrl_recv.recv(1)
  74. else:
  75. return os.read(self._ctrl_recv, 1)
  76. except:
  77. return b"\0"
  78. def addReader(self, source, fd):
  79. channel = getattr(source, "channel", "*")
  80. self._read.append(fd)
  81. self._targets[fd] = channel
  82. def addWriter(self, source, fd):
  83. channel = getattr(source, "channel", "*")
  84. self._write.append(fd)
  85. self._targets[fd] = channel
  86. def removeReader(self, fd):
  87. if fd in self._read:
  88. self._read.remove(fd)
  89. if not (fd in self._read or fd in self._write) and fd in self._targets:
  90. del self._targets[fd]
  91. def removeWriter(self, fd):
  92. if fd in self._write:
  93. self._write.remove(fd)
  94. if not (fd in self._read or fd in self._write) and fd in self._targets:
  95. del self._targets[fd]
  96. def isReading(self, fd):
  97. return fd in self._read
  98. def isWriting(self, fd):
  99. return fd in self._write
  100. def discard(self, fd):
  101. if fd in self._read:
  102. self._read.remove(fd)
  103. if fd in self._write:
  104. self._write.remove(fd)
  105. if fd in self._targets:
  106. del self._targets[fd]
  107. def getTarget(self, fd):
  108. return self._targets.get(fd, self.parent)
  109. class Select(BasePoller):
  110. """Select(...) -> new Select Poller Component
  111. Creates a new Select Poller Component that uses the select poller
  112. implementation. This poller is not recommended but is available for legacy
  113. reasons as most systems implement select-based polling for backwards
  114. compatibility.
  115. """
  116. channel = "select"
  117. def __init__(self, channel=channel):
  118. super(Select, self).__init__(channel=channel)
  119. self._read.append(self._ctrl_recv)
  120. def _preenDescriptors(self):
  121. for socks in (self._read[:], self._write[:]):
  122. for sock in socks:
  123. try:
  124. select.select([sock], [sock], [sock], 0)
  125. except Exception:
  126. self.discard(sock)
  127. def _generate_events(self, event):
  128. try:
  129. if not any([self._read, self._write]):
  130. return
  131. timeout = event.time_left
  132. if timeout < 0:
  133. r, w, _ = select.select(self._read, self._write, [])
  134. else:
  135. r, w, _ = select.select(self._read, self._write, [], timeout)
  136. except ValueError as e:
  137. # Possibly a file descriptor has gone negative?
  138. return self._preenDescriptors()
  139. except TypeError as e:
  140. # Something *totally* invalid (object w/o fileno, non-integral
  141. # result) was passed
  142. return self._preenDescriptors()
  143. except (SelectError, SocketError, IOError) as e:
  144. # select(2) encountered an error
  145. if e.args[0] in (0, 2):
  146. # windows does this if it got an empty list
  147. if (not self._read) and (not self._write):
  148. return
  149. else:
  150. raise
  151. elif e.args[0] == EINTR:
  152. return
  153. elif e.args[0] == EBADF:
  154. return self._preenDescriptors()
  155. else:
  156. # OK, I really don't know what's going on. Blow up.
  157. raise
  158. for sock in w:
  159. if self.isWriting(sock):
  160. self.fire(_write(sock), self.getTarget(sock))
  161. for sock in r:
  162. if sock == self._ctrl_recv:
  163. self._read_ctrl()
  164. continue
  165. if self.isReading(sock):
  166. self.fire(_read(sock), self.getTarget(sock))
  167. class Poll(BasePoller):
  168. """Poll(...) -> new Poll Poller Component
  169. Creates a new Poll Poller Component that uses the poll poller
  170. implementation.
  171. """
  172. channel = "poll"
  173. def __init__(self, channel=channel):
  174. super(Poll, self).__init__(channel=channel)
  175. self._map = {}
  176. self._poller = select.poll()
  177. self._disconnected_flag = (
  178. select.POLLHUP
  179. | select.POLLERR
  180. | select.POLLNVAL
  181. )
  182. self._read.append(self._ctrl_recv)
  183. self._updateRegistration(self._ctrl_recv)
  184. def _updateRegistration(self, fd):
  185. fileno = fd.fileno() if not isinstance(fd, int) else fd
  186. try:
  187. self._poller.unregister(fileno)
  188. except (KeyError, ValueError):
  189. pass
  190. mask = 0
  191. if fd in self._read:
  192. mask = mask | select.POLLIN
  193. if fd in self._write:
  194. mask = mask | select.POLLOUT
  195. if mask:
  196. self._poller.register(fd, mask)
  197. self._map[fileno] = fd
  198. else:
  199. super(Poll, self).discard(fd)
  200. try:
  201. del self._map[fileno]
  202. except KeyError:
  203. pass
  204. def addReader(self, source, fd):
  205. super(Poll, self).addReader(source, fd)
  206. self._updateRegistration(fd)
  207. def addWriter(self, source, fd):
  208. super(Poll, self).addWriter(source, fd)
  209. self._updateRegistration(fd)
  210. def removeReader(self, fd):
  211. super(Poll, self).removeReader(fd)
  212. self._updateRegistration(fd)
  213. def removeWriter(self, fd):
  214. super(Poll, self).removeWriter(fd)
  215. self._updateRegistration(fd)
  216. def discard(self, fd):
  217. super(Poll, self).discard(fd)
  218. self._updateRegistration(fd)
  219. def _generate_events(self, event):
  220. try:
  221. timeout = event.time_left
  222. if timeout < 0:
  223. l = self._poller.poll()
  224. else:
  225. l = self._poller.poll(1000 * timeout)
  226. except SelectError as e:
  227. if e.args[0] == EINTR:
  228. return
  229. else:
  230. raise
  231. for fileno, event in l:
  232. self._process(fileno, event)
  233. def _process(self, fileno, event):
  234. if fileno not in self._map:
  235. return
  236. fd = self._map[fileno]
  237. if fd == self._ctrl_recv:
  238. self._read_ctrl()
  239. return
  240. if event & self._disconnected_flag and not (event & select.POLLIN):
  241. self.fire(_disconnect(fd), self.getTarget(fd))
  242. self._poller.unregister(fileno)
  243. super(Poll, self).discard(fd)
  244. del self._map[fileno]
  245. else:
  246. try:
  247. if event & select.POLLIN:
  248. self.fire(_read(fd), self.getTarget(fd))
  249. if event & select.POLLOUT:
  250. self.fire(_write(fd), self.getTarget(fd))
  251. except Exception as e:
  252. self.fire(_error(fd, e), self.getTarget(fd))
  253. self.fire(_disconnect(fd), self.getTarget(fd))
  254. self._poller.unregister(fileno)
  255. super(Poll, self).discard(fd)
  256. del self._map[fileno]
  257. class EPoll(BasePoller):
  258. """EPoll(...) -> new EPoll Poller Component
  259. Creates a new EPoll Poller Component that uses the epoll poller
  260. implementation.
  261. """
  262. channel = "epoll"
  263. def __init__(self, channel=channel):
  264. super(EPoll, self).__init__(channel=channel)
  265. self._map = {}
  266. self._poller = select.epoll()
  267. self._disconnected_flag = (select.EPOLLHUP | select.EPOLLERR)
  268. self._read.append(self._ctrl_recv)
  269. self._updateRegistration(self._ctrl_recv)
  270. def _updateRegistration(self, fd):
  271. try:
  272. fileno = fd.fileno() if not isinstance(fd, int) else fd
  273. self._poller.unregister(fileno)
  274. except (SocketError, IOError, ValueError) as e:
  275. if e.args[0] == EBADF:
  276. keys = [k for k, v in list(self._map.items()) if v == fd]
  277. for key in keys:
  278. del self._map[key]
  279. mask = 0
  280. if fd in self._read:
  281. mask = mask | select.EPOLLIN
  282. if fd in self._write:
  283. mask = mask | select.EPOLLOUT
  284. if mask:
  285. self._poller.register(fd, mask)
  286. self._map[fileno] = fd
  287. else:
  288. super(EPoll, self).discard(fd)
  289. def addReader(self, source, fd):
  290. super(EPoll, self).addReader(source, fd)
  291. self._updateRegistration(fd)
  292. def addWriter(self, source, fd):
  293. super(EPoll, self).addWriter(source, fd)
  294. self._updateRegistration(fd)
  295. def removeReader(self, fd):
  296. super(EPoll, self).removeReader(fd)
  297. self._updateRegistration(fd)
  298. def removeWriter(self, fd):
  299. super(EPoll, self).removeWriter(fd)
  300. self._updateRegistration(fd)
  301. def discard(self, fd):
  302. super(EPoll, self).discard(fd)
  303. self._updateRegistration(fd)
  304. def _generate_events(self, event):
  305. try:
  306. timeout = event.time_left
  307. if timeout < 0:
  308. l = self._poller.poll()
  309. else:
  310. l = self._poller.poll(timeout)
  311. except IOError as e:
  312. if e.args[0] == EINTR:
  313. return
  314. except SelectError as e:
  315. if e.args[0] == EINTR:
  316. return
  317. else:
  318. raise
  319. for fileno, event in l:
  320. self._process(fileno, event)
  321. def _process(self, fileno, event):
  322. if fileno not in self._map:
  323. return
  324. fd = self._map[fileno]
  325. if fd == self._ctrl_recv:
  326. self._read_ctrl()
  327. return
  328. if event & self._disconnected_flag and not (event & select.POLLIN):
  329. self.fire(_disconnect(fd), self.getTarget(fd))
  330. self._poller.unregister(fileno)
  331. super(EPoll, self).discard(fd)
  332. del self._map[fileno]
  333. else:
  334. try:
  335. if event & select.EPOLLIN:
  336. self.fire(_read(fd), self.getTarget(fd))
  337. if event & select.EPOLLOUT:
  338. self.fire(_write(fd), self.getTarget(fd))
  339. except Exception as e:
  340. self.fire(_error(fd, e), self.getTarget(fd))
  341. self.fire(_disconnect(fd), self.getTarget(fd))
  342. self._poller.unregister(fileno)
  343. super(EPoll, self).discard(fd)
  344. del self._map[fileno]
  345. class KQueue(BasePoller):
  346. """KQueue(...) -> new KQueue Poller Component
  347. Creates a new KQueue Poller Component that uses the kqueue poller
  348. implementation.
  349. """
  350. channel = "kqueue"
  351. def __init__(self, channel=channel):
  352. super(KQueue, self).__init__(channel=channel)
  353. self._map = {}
  354. self._poller = select.kqueue()
  355. self._read.append(self._ctrl_recv)
  356. self._map[self._ctrl_recv.fileno()] = self._ctrl_recv
  357. self._poller.control(
  358. [
  359. select.kevent(
  360. self._ctrl_recv, select.KQ_FILTER_READ, select.KQ_EV_ADD
  361. )
  362. ], 0
  363. )
  364. def addReader(self, source, sock):
  365. super(KQueue, self).addReader(source, sock)
  366. self._map[sock.fileno()] = sock
  367. self._poller.control(
  368. [select.kevent(sock, select.KQ_FILTER_READ, select.KQ_EV_ADD)], 0
  369. )
  370. def addWriter(self, source, sock):
  371. super(KQueue, self).addWriter(source, sock)
  372. self._map[sock.fileno()] = sock
  373. self._poller.control(
  374. [select.kevent(sock, select.KQ_FILTER_WRITE, select.KQ_EV_ADD)], 0
  375. )
  376. def removeReader(self, sock):
  377. super(KQueue, self).removeReader(sock)
  378. self._poller.control(
  379. [
  380. select.kevent(sock, select.KQ_FILTER_READ, select.KQ_EV_DELETE)
  381. ],
  382. 0
  383. )
  384. def removeWriter(self, sock):
  385. super(KQueue, self).removeWriter(sock)
  386. self._poller.control(
  387. [select.kevent(sock, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE)],
  388. 0
  389. )
  390. def discard(self, sock):
  391. super(KQueue, self).discard(sock)
  392. del self._map[sock.fileno()]
  393. self._poller.control(
  394. [
  395. select.kevent(
  396. sock,
  397. select.KQ_FILTER_WRITE | select.KQ_FILTER_READ,
  398. select.KQ_EV_DELETE
  399. )
  400. ],
  401. 0
  402. )
  403. def _generate_events(self, event):
  404. try:
  405. timeout = event.time_left
  406. if timeout < 0:
  407. l = self._poller.control(None, 1000)
  408. else:
  409. l = self._poller.control(None, 1000, timeout)
  410. except SelectError as e:
  411. if e[0] == EINTR:
  412. return
  413. else:
  414. raise
  415. for event in l:
  416. self._process(event)
  417. def _process(self, event):
  418. if event.ident not in self._map:
  419. # shouldn't happen ?
  420. # we unregister the socket since we don't care about it anymore
  421. self._poller.control(
  422. [
  423. select.kevent(
  424. event.ident, event.filter, select.KQ_EV_DELETE
  425. )
  426. ],
  427. 0
  428. )
  429. return
  430. sock = self._map[event.ident]
  431. if sock == self._ctrl_recv:
  432. self._read_ctrl()
  433. return
  434. if event.flags & select.KQ_EV_ERROR:
  435. self.fire(_error(sock, "error"), self.getTarget(sock))
  436. elif event.flags & select.KQ_EV_EOF:
  437. self.fire(_disconnect(sock), self.getTarget(sock))
  438. elif event.filter == select.KQ_FILTER_WRITE:
  439. self.fire(_write(sock), self.getTarget(sock))
  440. elif event.filter == select.KQ_FILTER_READ:
  441. self.fire(_read(sock), self.getTarget(sock))
  442. Poller = Select
  443. __all__ = ("BasePoller", "Poller", "Select", "Poll", "EPoll", "KQueue")