PageRenderTime 40ms CodeModel.GetById 1ms app.highlight 35ms RepoModel.GetById 1ms app.codeStats 0ms

/circuits/core/pollers.py

https://bitbucket.org/prologic/circuits/
Python | 549 lines | 396 code | 101 blank | 52 comment | 105 complexity | df182f3b2833e930ab35d295248592ae MD5 | raw file
  1# Module:   pollers
  2# Date:     15th September 2008
  3# Author:   James Mills <prologic@shortcircuit.net.au>
  4
  5"""Poller Components for asynchronous file and socket I/O.
  6
  7This module contains Poller components that enable polling of file or socket
  8descriptors for read/write events. Pollers:
  9- Select
 10- Poll
 11- EPoll
 12"""
 13
 14import os
 15import select
 16import platform
 17from errno import EBADF, EINTR
 18from select import error as SelectError
 19from socket import error as SocketError, create_connection, \
 20    socket as create_socket, AF_INET, SOCK_STREAM, socket
 21from threading import Thread
 22from circuits.core.handlers import handler
 23
 24from .events import Event
 25from .components import BaseComponent
 26
 27
 28class _read(Event):
 29    """_read Event"""
 30
 31
 32class _write(Event):
 33    """_write Event"""
 34
 35
 36class _error(Event):
 37    """_error Event"""
 38
 39
 40class _disconnect(Event):
 41    """_disconnect Event"""
 42
 43
 44class BasePoller(BaseComponent):
 45
 46    channel = None
 47
 48    def __init__(self, channel=channel):
 49        super(BasePoller, self).__init__(channel=channel)
 50
 51        self._read = []
 52        self._write = []
 53        self._targets = {}
 54
 55        self._ctrl_recv, self._ctrl_send = self._create_control_con()
 56
 57    def _create_control_con(self):
 58        if platform.system() == "Linux":
 59            return os.pipe()
 60        server = create_socket(AF_INET, SOCK_STREAM)
 61        server.bind(("localhost", 0))
 62        server.listen(1)
 63        res_list = []
 64
 65        def accept():
 66            sock, _ = server.accept()
 67            sock.setblocking(False)
 68            res_list.append(sock)
 69        at = Thread(target=accept)
 70        at.start()
 71        clnt_sock = create_connection(server.getsockname())
 72        at.join()
 73        return (res_list[0], clnt_sock)
 74
 75    @handler("generate_events", priority=-9)
 76    def _on_generate_events(self, event):
 77        """
 78        Pollers have slightly higher priority than the default handler
 79        from Manager to ensure that they are invoked before the
 80        default handler. They act as event filters to avoid the additional
 81        invocation of the default handler which would be unnecessary
 82        overhead.
 83        """
 84
 85        event.stop()
 86        self._generate_events(event)
 87
 88    def resume(self):
 89        if isinstance(self._ctrl_send, socket):
 90            self._ctrl_send.send(b"\0")
 91        else:
 92            os.write(self._ctrl_send, b"\0")
 93
 94    def _read_ctrl(self):
 95        try:
 96            if isinstance(self._ctrl_recv, socket):
 97                return self._ctrl_recv.recv(1)
 98            else:
 99                return os.read(self._ctrl_recv, 1)
100        except:
101            return b"\0"
102
103    def addReader(self, source, fd):
104        channel = getattr(source, "channel", "*")
105        self._read.append(fd)
106        self._targets[fd] = channel
107
108    def addWriter(self, source, fd):
109        channel = getattr(source, "channel", "*")
110        self._write.append(fd)
111        self._targets[fd] = channel
112
113    def removeReader(self, fd):
114        if fd in self._read:
115            self._read.remove(fd)
116        if not (fd in self._read or fd in self._write) and fd in self._targets:
117            del self._targets[fd]
118
119    def removeWriter(self, fd):
120        if fd in self._write:
121            self._write.remove(fd)
122        if not (fd in self._read or fd in self._write) and fd in self._targets:
123            del self._targets[fd]
124
125    def isReading(self, fd):
126        return fd in self._read
127
128    def isWriting(self, fd):
129        return fd in self._write
130
131    def discard(self, fd):
132        if fd in self._read:
133            self._read.remove(fd)
134        if fd in self._write:
135            self._write.remove(fd)
136        if fd in self._targets:
137            del self._targets[fd]
138
139    def getTarget(self, fd):
140        return self._targets.get(fd, self.parent)
141
142
143class Select(BasePoller):
144    """Select(...) -> new Select Poller Component
145
146    Creates a new Select Poller Component that uses the select poller
147    implementation. This poller is not recommended but is available for legacy
148    reasons as most systems implement select-based polling for backwards
149    compatibility.
150    """
151
152    channel = "select"
153
154    def __init__(self, channel=channel):
155        super(Select, self).__init__(channel=channel)
156
157        self._read.append(self._ctrl_recv)
158
159    def _preenDescriptors(self):
160        for socks in (self._read[:], self._write[:]):
161            for sock in socks:
162                try:
163                    select.select([sock], [sock], [sock], 0)
164                except Exception:
165                    self.discard(sock)
166
167    def _generate_events(self, event):
168        try:
169            if not any([self._read, self._write]):
170                return
171            timeout = event.time_left
172            if timeout < 0:
173                r, w, _ = select.select(self._read, self._write, [])
174            else:
175                r, w, _ = select.select(self._read, self._write, [], timeout)
176        except ValueError as e:
177            # Possibly a file descriptor has gone negative?
178            return self._preenDescriptors()
179        except TypeError as e:
180            # Something *totally* invalid (object w/o fileno, non-integral
181            # result) was passed
182            return self._preenDescriptors()
183        except (SelectError, SocketError, IOError) as e:
184            # select(2) encountered an error
185            if e.args[0] in (0, 2):
186                # windows does this if it got an empty list
187                if (not self._read) and (not self._write):
188                    return
189                else:
190                    raise
191            elif e.args[0] == EINTR:
192                return
193            elif e.args[0] == EBADF:
194                return self._preenDescriptors()
195            else:
196                # OK, I really don't know what's going on.  Blow up.
197                raise
198
199        for sock in w:
200            if self.isWriting(sock):
201                self.fire(_write(sock), self.getTarget(sock))
202
203        for sock in r:
204            if sock == self._ctrl_recv:
205                self._read_ctrl()
206                continue
207            if self.isReading(sock):
208                self.fire(_read(sock), self.getTarget(sock))
209
210
211class Poll(BasePoller):
212    """Poll(...) -> new Poll Poller Component
213
214    Creates a new Poll Poller Component that uses the poll poller
215    implementation.
216    """
217
218    channel = "poll"
219
220    def __init__(self, channel=channel):
221        super(Poll, self).__init__(channel=channel)
222
223        self._map = {}
224        self._poller = select.poll()
225
226        self._disconnected_flag = (
227            select.POLLHUP
228            | select.POLLERR
229            | select.POLLNVAL
230        )
231
232        self._read.append(self._ctrl_recv)
233        self._updateRegistration(self._ctrl_recv)
234
235    def _updateRegistration(self, fd):
236        fileno = fd.fileno() if not isinstance(fd, int) else fd
237
238        try:
239            self._poller.unregister(fileno)
240        except (KeyError, ValueError):
241            pass
242
243        mask = 0
244
245        if fd in self._read:
246            mask = mask | select.POLLIN
247        if fd in self._write:
248            mask = mask | select.POLLOUT
249
250        if mask:
251            self._poller.register(fd, mask)
252            self._map[fileno] = fd
253        else:
254            super(Poll, self).discard(fd)
255            try:
256                del self._map[fileno]
257            except KeyError:
258                pass
259
260    def addReader(self, source, fd):
261        super(Poll, self).addReader(source, fd)
262        self._updateRegistration(fd)
263
264    def addWriter(self, source, fd):
265        super(Poll, self).addWriter(source, fd)
266        self._updateRegistration(fd)
267
268    def removeReader(self, fd):
269        super(Poll, self).removeReader(fd)
270        self._updateRegistration(fd)
271
272    def removeWriter(self, fd):
273        super(Poll, self).removeWriter(fd)
274        self._updateRegistration(fd)
275
276    def discard(self, fd):
277        super(Poll, self).discard(fd)
278        self._updateRegistration(fd)
279
280    def _generate_events(self, event):
281        try:
282            timeout = event.time_left
283            if timeout < 0:
284                l = self._poller.poll()
285            else:
286                l = self._poller.poll(1000 * timeout)
287        except SelectError as e:
288            if e.args[0] == EINTR:
289                return
290            else:
291                raise
292
293        for fileno, event in l:
294            self._process(fileno, event)
295
296    def _process(self, fileno, event):
297        if fileno not in self._map:
298            return
299
300        fd = self._map[fileno]
301        if fd == self._ctrl_recv:
302            self._read_ctrl()
303            return
304
305        if event & self._disconnected_flag and not (event & select.POLLIN):
306            self.fire(_disconnect(fd), self.getTarget(fd))
307            self._poller.unregister(fileno)
308            super(Poll, self).discard(fd)
309            del self._map[fileno]
310        else:
311            try:
312                if event & select.POLLIN:
313                    self.fire(_read(fd), self.getTarget(fd))
314                if event & select.POLLOUT:
315                    self.fire(_write(fd), self.getTarget(fd))
316            except Exception as e:
317                self.fire(_error(fd, e), self.getTarget(fd))
318                self.fire(_disconnect(fd), self.getTarget(fd))
319                self._poller.unregister(fileno)
320                super(Poll, self).discard(fd)
321                del self._map[fileno]
322
323
324class EPoll(BasePoller):
325    """EPoll(...) -> new EPoll Poller Component
326
327    Creates a new EPoll Poller Component that uses the epoll poller
328    implementation.
329    """
330
331    channel = "epoll"
332
333    def __init__(self, channel=channel):
334        super(EPoll, self).__init__(channel=channel)
335
336        self._map = {}
337        self._poller = select.epoll()
338
339        self._disconnected_flag = (select.EPOLLHUP | select.EPOLLERR)
340
341        self._read.append(self._ctrl_recv)
342        self._updateRegistration(self._ctrl_recv)
343
344    def _updateRegistration(self, fd):
345        try:
346            fileno = fd.fileno() if not isinstance(fd, int) else fd
347            self._poller.unregister(fileno)
348        except (SocketError, IOError, ValueError) as e:
349            if e.args[0] == EBADF:
350                keys = [k for k, v in list(self._map.items()) if v == fd]
351                for key in keys:
352                    del self._map[key]
353
354        mask = 0
355
356        if fd in self._read:
357            mask = mask | select.EPOLLIN
358        if fd in self._write:
359            mask = mask | select.EPOLLOUT
360
361        if mask:
362            self._poller.register(fd, mask)
363            self._map[fileno] = fd
364        else:
365            super(EPoll, self).discard(fd)
366
367    def addReader(self, source, fd):
368        super(EPoll, self).addReader(source, fd)
369        self._updateRegistration(fd)
370
371    def addWriter(self, source, fd):
372        super(EPoll, self).addWriter(source, fd)
373        self._updateRegistration(fd)
374
375    def removeReader(self, fd):
376        super(EPoll, self).removeReader(fd)
377        self._updateRegistration(fd)
378
379    def removeWriter(self, fd):
380        super(EPoll, self).removeWriter(fd)
381        self._updateRegistration(fd)
382
383    def discard(self, fd):
384        super(EPoll, self).discard(fd)
385        self._updateRegistration(fd)
386
387    def _generate_events(self, event):
388        try:
389            timeout = event.time_left
390            if timeout < 0:
391                l = self._poller.poll()
392            else:
393                l = self._poller.poll(timeout)
394        except IOError as e:
395            if e.args[0] == EINTR:
396                return
397        except SelectError as e:
398            if e.args[0] == EINTR:
399                return
400            else:
401                raise
402
403        for fileno, event in l:
404            self._process(fileno, event)
405
406    def _process(self, fileno, event):
407        if fileno not in self._map:
408            return
409
410        fd = self._map[fileno]
411        if fd == self._ctrl_recv:
412            self._read_ctrl()
413            return
414
415        if event & self._disconnected_flag and not (event & select.POLLIN):
416            self.fire(_disconnect(fd), self.getTarget(fd))
417            self._poller.unregister(fileno)
418            super(EPoll, self).discard(fd)
419            del self._map[fileno]
420        else:
421            try:
422                if event & select.EPOLLIN:
423                    self.fire(_read(fd), self.getTarget(fd))
424                if event & select.EPOLLOUT:
425                    self.fire(_write(fd), self.getTarget(fd))
426            except Exception as e:
427                self.fire(_error(fd, e), self.getTarget(fd))
428                self.fire(_disconnect(fd), self.getTarget(fd))
429                self._poller.unregister(fileno)
430                super(EPoll, self).discard(fd)
431                del self._map[fileno]
432
433
434class KQueue(BasePoller):
435    """KQueue(...) -> new KQueue Poller Component
436
437    Creates a new KQueue Poller Component that uses the kqueue poller
438    implementation.
439    """
440
441    channel = "kqueue"
442
443    def __init__(self, channel=channel):
444        super(KQueue, self).__init__(channel=channel)
445        self._map = {}
446        self._poller = select.kqueue()
447
448        self._read.append(self._ctrl_recv)
449        self._map[self._ctrl_recv.fileno()] = self._ctrl_recv
450        self._poller.control(
451            [
452                select.kevent(
453                    self._ctrl_recv, select.KQ_FILTER_READ, select.KQ_EV_ADD
454                )
455            ], 0
456        )
457
458    def addReader(self, source, sock):
459        super(KQueue, self).addReader(source, sock)
460        self._map[sock.fileno()] = sock
461        self._poller.control(
462            [select.kevent(sock, select.KQ_FILTER_READ, select.KQ_EV_ADD)], 0
463        )
464
465    def addWriter(self, source, sock):
466        super(KQueue, self).addWriter(source, sock)
467        self._map[sock.fileno()] = sock
468        self._poller.control(
469            [select.kevent(sock, select.KQ_FILTER_WRITE, select.KQ_EV_ADD)], 0
470        )
471
472    def removeReader(self, sock):
473        super(KQueue, self).removeReader(sock)
474        self._poller.control(
475            [
476                select.kevent(sock, select.KQ_FILTER_READ, select.KQ_EV_DELETE)
477            ],
478            0
479        )
480
481    def removeWriter(self, sock):
482        super(KQueue, self).removeWriter(sock)
483        self._poller.control(
484            [select.kevent(sock, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE)],
485            0
486        )
487
488    def discard(self, sock):
489        super(KQueue, self).discard(sock)
490        del self._map[sock.fileno()]
491        self._poller.control(
492            [
493                select.kevent(
494                    sock,
495                    select.KQ_FILTER_WRITE | select.KQ_FILTER_READ,
496                    select.KQ_EV_DELETE
497                )
498            ],
499            0
500        )
501
502    def _generate_events(self, event):
503        try:
504            timeout = event.time_left
505            if timeout < 0:
506                l = self._poller.control(None, 1000)
507            else:
508                l = self._poller.control(None, 1000, timeout)
509        except SelectError as e:
510            if e[0] == EINTR:
511                return
512            else:
513                raise
514
515        for event in l:
516            self._process(event)
517
518    def _process(self, event):
519        if event.ident not in self._map:
520            # shouldn't happen ?
521            # we unregister the socket since we don't care about it anymore
522            self._poller.control(
523                [
524                    select.kevent(
525                        event.ident, event.filter, select.KQ_EV_DELETE
526                    )
527                ],
528                0
529            )
530
531            return
532
533        sock = self._map[event.ident]
534        if sock == self._ctrl_recv:
535            self._read_ctrl()
536            return
537
538        if event.flags & select.KQ_EV_ERROR:
539            self.fire(_error(sock, "error"), self.getTarget(sock))
540        elif event.flags & select.KQ_EV_EOF:
541            self.fire(_disconnect(sock), self.getTarget(sock))
542        elif event.filter == select.KQ_FILTER_WRITE:
543            self.fire(_write(sock), self.getTarget(sock))
544        elif event.filter == select.KQ_FILTER_READ:
545            self.fire(_read(sock), self.getTarget(sock))
546
547Poller = Select
548
549__all__ = ("BasePoller", "Poller", "Select", "Poll", "EPoll", "KQueue")