/tv/lib/net.py

  1# Miro - an RSS based video player application
  2# Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011
  3# Participatory Culture Foundation
  4#
  5# This program is free software; you can redistribute it and/or modify
  6# it under the terms of the GNU General Public License as published by
  7# the Free Software Foundation; either version 2 of the License, or
  8# (at your option) any later version.
  9#
 10# This program is distributed in the hope that it will be useful,
 11# but WITHOUT ANY WARRANTY; without even the implied warranty of
 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13# GNU General Public License for more details.
 14#
 15# You should have received a copy of the GNU General Public License
 16# along with this program; if not, write to the Free Software
 17# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 18#
 19# In addition, as a special exception, the copyright holders give
 20# permission to link the code of portions of this program with the OpenSSL
 21# library.
 22#
 23# You must obey the GNU General Public License in all respects for all of
 24# the code used other than OpenSSL. If you modify file(s) with this
 25# exception, you may extend this exception to your version of the file(s),
 26# but you are not obligated to do so. If you do not wish to do so, delete
 27# this exception statement from your version. If you delete this exception
 28# statement from all source files in the program, then also delete it here.
 29
 30"""net.py -- Low-level Networking Code
 31
 32The main class here is ConnectionHandler, which is a state-based socket
 33handling class.  It's used for communication with the downloader daemon via a
 34socket.
 35
 36This class also defines the base Exception classes used by httpclient.
 37
 38"""
 39
 40import errno
 41import logging
 42import socket
 43
 44from miro import app
 45from miro import eventloop
 46from miro import util
 47from miro import prefs
 48from miro import signals
 49from miro import trapcall
 50from miro.clock import clock
 51from miro.gtcache import gettext as _
 52
 53SOCKET_READ_TIMEOUT = 60
 54SOCKET_INITIAL_READ_TIMEOUT = 30
 55SOCKET_CONNECT_TIMEOUT = 15
 56
 57# socket.ssl is deprecated as of Python 2.6, so we use socket_ssl for
 58# pre Python 2.6 and ssl.wrap_socket for Python 2.6 and later.
 59try:
 60    import ssl
 61    ssl.wrap_socket
 62    def convert_to_ssl(sock):
 63        return ssl.wrap_socket(sock)
 64except (ImportError, AttributeError):
 65    def convert_to_ssl(sock):
 66        return socket.ssl(sock)
 67
 68class NetworkError(StandardError):
 69    """Base class for all errors that will be passed to errbacks from get_url
 70    and friends.  NetworkErrors can be display in 2 ways:
 71
 72    getFriendlyDescription() -- short, newbie friendly description
 73    getLongDescription() -- detailed description
 74    """
 75    def __init__(self, shortDescription, longDescription=None):
 76        if longDescription is None:
 77            longDescription = shortDescription
 78        self.friendlyDescription = _("Error: %(msg)s",
 79                                     {"msg": shortDescription})
 80        self.longDescription = longDescription
 81
 82    def getFriendlyDescription(self):
 83        return self.friendlyDescription
 84
 85    def getLongDescription(self):
 86        return self.longDescription
 87
 88    def __str__(self):
 89        return "%s: %s -- %s" % (self.__class__,
 90                util.stringify(self.getFriendlyDescription()),
 91                util.stringify(self.getLongDescription()))
 92
 93class ConnectionError(NetworkError):
 94    def __init__(self, errorMessage):
 95        self.friendlyDescription = _("Can't connect")
 96        self.longDescription = _("Connection Error: %(msg)s",
 97                                 {"msg": util.unicodify(errorMessage)})
 98
 99class SSLConnectionError(ConnectionError):
100    def __init__(self):
101        self.friendlyDescription = _("Can't connect")
102        self.longDescription = _("SSL connection error")
103
104class ConnectionTimeout(NetworkError):
105    def __init__(self, host):
106        NetworkError.__init__(self, _('Timeout'),
107                _('Connection to %(host)s timed out', {"host": host}))
108
109def trap_call(object, function, *args, **kwargs):
110    """Convenience function do a trapcall.trap_call, where when is
111    'While talking to the network'
112    """
113    return trapcall.time_trap_call("Calling %s on %s" % (function, object),
114                                   function, *args, **kwargs)
115
116class NetworkBuffer(object):
117    """Responsible for storing incomming network data and doing some basic
118    parsing of it.  I think this is about as fast as we can do things in pure
119    python, someday we may want to make it C...
120    """
121    def __init__(self):
122        self.chunks = []
123        self.length = 0
124
125    def addData(self, data):
126        self.chunks.append(data)
127        self.length += len(data)
128
129    def _mergeChunks(self):
130        self.chunks = [''.join(self.chunks)]
131
132    def has_data(self):
133        return self.length > 0
134
135    def discard_data(self):
136        self.chunks = []
137        self.length = 0
138
139    def read(self, size=None):
140        """Read at most size bytes from the data that has been added to the
141        buffer.  """
142
143        self._mergeChunks()
144        if size is not None:
145            rv = self.chunks[0][:size]
146            self.chunks[0] = self.chunks[0][len(rv):]
147        else:
148            rv = self.chunks[0]
149            self.chunks = []
150        self.length -= len(rv)
151        return rv
152
153    def readline(self):
154        """Like a file readline, with several difference:  
155        * If there isn't a full line ready to be read we return None.  
156        * Doesn't include the trailing line separator.
157        * Both "\r\n" and "\n" act as a line ender
158        """
159
160        self._mergeChunks()
161        split = self.chunks[0].split("\n", 1)
162        if len(split) == 2:
163            self.chunks[0] = split[1]
164            self.length = len(self.chunks[0])
165            if split[0].endswith("\r"):
166                return split[0][:-1]
167            else:
168                return split[0]
169        else:
170            return None
171
172    def unread(self, data):
173        """Put back read data.  This make is like the data was never read at
174        all.
175        """
176        self.chunks.insert(0, data)
177        self.length += len(data)
178
179    def getValue(self):
180        self._mergeChunks()
181        return self.chunks[0]
182
183class _Packet(object):
184    """A packet of data for the AsyncSocket class
185    """
186    def __init__(self, data, callback=None):
187        self.data = data
188        self.callback = callback
189
190class AsyncSocket(object):
191    """Socket class that uses the eventloop module.
192    """
193
194    MEMORY_ERROR_LIMIT = 5
195
196    def __init__(self, closeCallback=None):
197        """Create an AsyncSocket.  If closeCallback is given, it will be
198        called if we detect that the socket has been closed durring a
199        read/write operation.  The arguments will be the AsyncSocket object
200        and either socket.SHUT_RD or socket.SHUT_WR.
201        """
202        self.toSend = []
203        self.to_send_length = 0
204        self.readSize = 4096
205        self.socket = None
206        self.readCallback = None
207        self.closeCallback = closeCallback
208        self.readTimeout = None
209        self.timedOut = False
210        self.connectionErrback = None
211        self.disable_read_timeout = False
212        self.readSomeData = False
213        self.name = ""
214        self.lastClock = None
215        self.memoryErrors = 0
216
217    def __str__(self):
218        if self.name:
219            return "%s: %s" % (type(self).__name__, self.name)
220        else:
221            return "Unknown %s" % (type(self).__name__,)
222
223    # The complication in the timeout code is because creating and
224    # cancelling a timeout costs some memory (timeout is in memory
225    # until it goes off, even if cancelled.)
226    def startReadTimeout(self):
227        if self.disable_read_timeout:
228            return
229        self.lastClock = clock()
230        if self.readTimeout is not None:
231            return
232        self.readTimeout = eventloop.add_timeout(SOCKET_INITIAL_READ_TIMEOUT,
233                                                 self.onReadTimeout,
234                                                 "AsyncSocket.onReadTimeout")
235
236    def stopReadTimeout(self):
237        if self.readTimeout is not None:
238            self.readTimeout.cancel()
239            self.readTimeout = None
240
241    def _pick_address(self, addresses):
242        """Pick the best entry to use from a list of addresses
243        
244        :param addresses: list of address tuples returned by getaddrinfo()
245        :returns: one of the tuples, or None if no address could be found
246        """
247        if not app.config.get(prefs.DISABLE_IPV6) and util.use_ipv6():
248            # prefer ipv6 if possible
249            for entry in addresses:
250                if entry[0] == socket.AF_INET6:
251                    return entry
252        # fall back on ipv4
253        for entry in addresses:
254            if entry[0] == socket.AF_INET:
255                return entry
256        return None
257
258    def open_connection(self, host, port, callback, errback,
259                        disable_read_timeout=None):
260        """Open a connection.  On success, callback will be called with this
261        object.
262        """
263        if disable_read_timeout is not None:
264            self.disable_read_timeout = disable_read_timeout
265        self.name = "Outgoing %s:%s" % (host, port)
266
267        self.connectionErrback = errback
268        def handleGetAddrInfoException(e):
269            if self.connectionErrback is None:
270                # called connectionErrback while we were waiting for
271                # getaddrinfo
272                return
273            trap_call(self, errback,
274                      ConnectionError(e[1] + " (host: %s)" % host))
275        def createSocketHandle(family):
276            try:
277                self.socket = socket.socket(family, socket.SOCK_STREAM)
278            except socket.error, e:
279                trap_call(self, errback, ConnectionError(e[1]))
280                return
281
282            self.socket.setblocking(0)
283            return self.socket
284
285        def onAddressLookup(addresses):
286            if self.connectionErrback is None:
287                # called connectionErrback while we were waiting for
288                # getaddrinfo
289                return
290            entry = self._pick_address(addresses)
291            if entry is None:
292                # FIXME - wtf kind of user message is this?  it's too
293                # technical and there's no way a user would know what
294                # to do about it.
295                msg = _("Couldn't find address family to use")
296                trap_call(self, errback, ConnectionError(msg))
297                return
298            try:
299                self.socket = socket.socket(entry[0], socket.SOCK_STREAM)
300            except socket.error, e:
301                trap_call(self, errback, ConnectionError(e[1]))
302                return
303            self.socket.setblocking(0)
304            try:
305                rv = self.socket.connect_ex(entry[4])
306            except socket.gaierror:
307                trap_call(self, errback, ConnectionError('gaierror'))
308                return
309            if rv in (0, errno.EINPROGRESS, errno.EWOULDBLOCK):
310                eventloop.add_write_callback(self.socket, onWriteReady)
311                self.socketConnectTimeout = eventloop.add_timeout(
312                         SOCKET_CONNECT_TIMEOUT, onWriteTimeout,
313                        "socket connect timeout")
314            else:
315                fullmsg = "Connection failed"
316                trap_call(self, errback, ConnectionError(fullmsg))
317        def onWriteReady():
318            eventloop.remove_write_callback(self.socket)
319            self.socketConnectTimeout.cancel()
320            rv = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
321            if rv == 0:
322                trap_call(self, callback, self)
323            else:
324                msg = errno.errorcode.get(rv, _('Unknown Error code'))
325                trap_call(self, errback, ConnectionError(msg))
326            self.connectionErrback = None
327        def onWriteTimeout():
328            eventloop.remove_write_callback(self.socket)
329            trap_call(self, errback, ConnectionTimeout(host))
330            self.connectionErrback = None
331        eventloop.call_in_thread(onAddressLookup, handleGetAddrInfoException,
332                                 socket.getaddrinfo,
333                                 "getAddrInfo - %s:%s" % (host, port),
334                                 host, port)
335
336    def accept_connection(self, family, host, port, callback, errback):
337        def finishAccept():
338            eventloop.remove_read_callback(self.socket)
339            (self.socket, addr) = self.socket.accept()
340            trap_call(self, callback, self)
341            self.connectionErrback = None
342
343        self.name = "Incoming %s:%s" % (host, port)
344        self.connectionErrback = errback
345
346        try:
347            self.socket = socket.socket(family, socket.SOCK_STREAM)
348            self.socket.bind((host, port))
349        except socket.error, e:
350            trap_call(self, errback, ConnectionError(e[1]))
351            return
352
353        if self.socket.family == socket.AF_INET:
354            (self.addr, self.port) = self.socket.getsockname()
355        elif self.socket.family == socket.AF_INET6:
356            (self.addr, self.port, self.flowinfo,
357             self.scopeid) = self.socket.getsockname()
358        else:
359            raise ValueError("Unknown socket family: %s", self.socket.family)
360        self.socket.listen(63)
361        eventloop.add_read_callback(self.socket, finishAccept)
362
363    def close_connection(self):
364        if self.isOpen():
365            eventloop.stop_handling_socket(self.socket)
366            self.stopReadTimeout()
367            self.socket.close()
368            self.socket = None
369        if self.connectionErrback is not None:
370            error = NetworkError(_("Connection closed"))
371            trap_call(self, self.connectionErrback, error)
372            self.connectionErrback = None
373
374    def isOpen(self):
375        return self.socket is not None
376
377    def send_data(self, data, callback=None):
378        """Send data out to the socket when it becomes ready.
379        
380        NOTE: currently we have no way of detecting when the data gets sent
381        out, or if errors happen.
382        """
383
384        if not self.isOpen():
385            raise ValueError("Socket not connected")
386        self.toSend.append(_Packet(data, callback))
387        self.to_send_length += len(data)
388        eventloop.add_write_callback(self.socket, self.onWriteReady)
389
390    def startReading(self, readCallback):
391        """Start reading from the socket.  When data becomes available it will
392        be passed to readCallback.  If there is already a read callback, it
393        will be replaced.
394        """
395
396        if not self.isOpen():
397            raise ValueError("Socket not connected")
398        self.readCallback = readCallback
399        eventloop.add_read_callback(self.socket, self.onReadReady)
400        self.startReadTimeout()
401
402    def stopReading(self):
403        """Stop reading from the socket."""
404
405        if not self.isOpen():
406            raise ValueError("Socket not connected")
407        self.readCallback = None
408        eventloop.remove_read_callback(self.socket)
409        self.stopReadTimeout()
410
411    def onReadTimeout(self):
412        if self.readSomeData:
413            timeout = SOCKET_READ_TIMEOUT
414        else:
415            timeout = SOCKET_INITIAL_READ_TIMEOUT
416
417        if clock() < self.lastClock + timeout:
418            self.readTimeout = eventloop.add_timeout(
419                self.lastClock + timeout - clock(), self.onReadTimeout,
420                "AsyncSocket.onReadTimeout")
421        else:
422            self.readTimeout = None
423            self.timedOut = True
424            self.handleEarlyClose('read')
425
426    def handleSocketError(self, code, msg, operation):
427        if code in (errno.EWOULDBLOCK, errno.EINTR):
428            return
429
430        if operation == "write":
431            expectedErrors = (errno.EPIPE, errno.ECONNRESET)
432        else:
433            expectedErrors = (errno.ECONNREFUSED, errno.ECONNRESET)
434        if code not in expectedErrors:
435            logging.warning("WARNING, got unexpected error during %s",
436                            operation)
437            logging.warning("%s: %s", errno.errorcode.get(code), msg)
438        self.handleEarlyClose(operation)
439
440    def onWriteReady(self):
441        try:
442            if len(self.toSend) > 0:
443                sent = self.socket.send(self.toSend[0].data)
444            else:
445                sent = 0
446        except socket.error, (code, msg):
447            self.handleSocketError(code, msg, "write")
448        else:
449            self.handleSentData(sent)
450
451    def handleSentData(self, sent):
452        if len(self.toSend) > 0:
453            self.toSend[0].data = self.toSend[0].data[sent:]
454            if len(self.toSend[0].data) == 0:
455                if self.toSend[0].callback:
456                    self.toSend[0].callback()
457                self.toSend = self.toSend[1:]
458        self.to_send_length -= sent
459        if len(self.toSend) == 0:
460            eventloop.remove_write_callback(self.socket)
461
462    def onReadReady(self):
463        try:
464            data = self.socket.recv(self.readSize)
465        except socket.error, (code, msg):
466            self.handleSocketError(code, msg, "read")
467        except MemoryError:
468            # This happens because of a windows bug in the socket code (see
469            # #4373).  Let's hope that things clear themselves up next time we
470            # read.
471            self.memoryErrors += 1
472            if self.memoryErrors > self.MEMORY_ERROR_LIMIT:
473                logging.error("ERROR: Too many MemoryErrors on %s", self)
474                self.handleEarlyClose('read')
475            else:
476                logging.warning(
477                    "WARNING: Memory error while reading from %s", self)
478        else:
479            self.memoryErrors = 0
480            self.handleReadData(data)
481
482    def handleReadData(self, data):
483        self.startReadTimeout()
484        if data == '':
485            if self.closeCallback:
486                trap_call(self, self.closeCallback, self, socket.SHUT_RD)
487        else:
488            self.readSomeData = True
489            trap_call(self, self.readCallback, data)
490
491    def handleEarlyClose(self, operation):
492        self.close_connection()
493        if self.closeCallback:
494            if operation == 'read':
495                typ = socket.SHUT_RD
496            else:
497                typ = socket.SHUT_WR
498            trap_call(self, self.closeCallback, self, typ)
499
500class AsyncSSLStream(AsyncSocket):
501    def __init__(self, closeCallback=None):
502        super(AsyncSSLStream, self).__init__(closeCallback)
503        self.interruptedOperation = None
504
505    def open_connection(self, host, port, callback, errback,
506                        disable_read_timeout=None):
507        def onSocketOpen(self):
508            self.socket.setblocking(1)
509            eventloop.call_in_thread(onSSLOpen, handleSSLError, convert_to_ssl,
510                                   "AsyncSSL onSocketOpen()",
511                                   self.socket)
512        def onSSLOpen(ssl):
513            if self.socket is None:
514                # the connection was closed while we were calling
515                # convert_to_ssl
516                return
517            self.socket.setblocking(0)
518            self.ssl = ssl
519            # finally we can call the actuall callback
520            callback(self)
521        def handleSSLError(error):
522            logging.error("handleSSLError: %r", error)
523            errback(SSLConnectionError())
524        super(AsyncSSLStream, self).open_connection(host, port, onSocketOpen,
525                errback, disable_read_timeout)
526
527    def resumeNormalCallbacks(self):
528        if self.readCallback is not None:
529            eventloop.add_read_callback(self.socket, self.onReadReady)
530        if len(self.toSend) != 0:
531            eventloop.add_write_callback(self.socket, self.onWriteReady)
532
533    def handleSocketError(self, code, msg, operation):
534        if code in (socket.SSL_ERROR_WANT_READ, socket.SSL_ERROR_WANT_WRITE):
535            if self.interruptedOperation is None:
536                self.interruptedOperation = operation
537            elif self.interruptedOperation != operation:
538                signals.system.failed("When talking to the network", 
539                details="socket error for the wrong SSL operation")
540                self.close_connection()
541                return
542            eventloop.stop_handling_socket(self.socket)
543            if code == socket.SSL_ERROR_WANT_READ:
544                eventloop.add_read_callback(self.socket, self.onReadReady)
545            else:
546                eventloop.add_write_callback(self.socket, self.onWriteReady)
547        elif code in (socket.SSL_ERROR_ZERO_RETURN, socket.SSL_ERROR_SSL,
548                socket.SSL_ERROR_SYSCALL, socket.SSL_ERROR_EOF):
549            self.handleEarlyClose(operation)
550        else:
551            super(AsyncSSLStream, self).handleSocketError(code, msg,
552                    operation)
553
554    def onWriteReady(self):
555        if self.interruptedOperation == 'read':
556            return self.onReadReady()
557        try:
558            if len(self.toSend) > 0:
559                sent = self.ssl.write(self.toSend[0].data)
560            else:
561                sent = 0
562        except socket.error, (code, msg):
563            self.handleSocketError(code, msg, "write")
564        else:
565            if self.interruptedOperation == 'write':
566                self.resumeNormalCallbacks()
567                self.interruptedOperation = None
568            self.handleSentData(sent)
569
570    def onReadReady(self):
571        if self.interruptedOperation == 'write':
572            return self.onWriteReady()
573        try:
574            data = self.ssl.read(self.readSize)
575        except socket.error, (code, msg):
576            self.handleSocketError(code, msg, "read")
577        else:
578            if self.interruptedOperation == 'read':
579                self.resumeNormalCallbacks()
580                self.interruptedOperation = None
581            self.handleReadData(data)
582
583class ConnectionHandler(object):
584    """Base class to handle asynchronous network streams.  It implements a
585    simple state machine to deal with incomming data.
586
587    Sending data: Use the send_data() method.
588
589    Reading Data: Add entries to the state dictionary, which maps strings to
590    methods.  The state methods will be called when there is data available,
591    which can be read from the buffer variable.  The states dictionary can
592    contain a None value, to signal that the handler isn't interested in
593    reading at that point.  Use change_state() to switch states.
594
595    Subclasses should override the handle_close() method to handle the
596    socket closing.
597    """
598    stream_factory = AsyncSocket
599
600    def __init__(self):
601        self.buffer = NetworkBuffer()
602        self.states = {'initializing': None, 'closed': None}
603        self.stream = self.stream_factory(closeCallback=self.closeCallback)
604        self.change_state('initializing')
605        self.name = ""
606
607    def __str__(self):
608        return "%s -- %s" % (self.__class__, self.state)
609
610    def open_connection(self, host, port, callback, errback,
611                        disable_read_timeout=None):
612        self.name = "Outgoing %s:%s" % (host, port)
613        self.host = host
614        self.port = port
615        def callbackIntercept(asyncSocket):
616            if callback:
617                trap_call(self, callback, self)
618        self.stream.open_connection(host, port, callbackIntercept, errback,
619                                    disable_read_timeout)
620
621    def close_connection(self):
622        if self.stream.isOpen():
623            self.stream.close_connection()
624        self.change_state('closed')
625        self.buffer.discard_data()
626
627    def send_data(self, data, callback=None):
628        self.stream.send_data(data, callback)
629
630    def change_state(self, newState):
631        self.readHandler = self.states[newState]
632        self.state = newState
633        self.updateReadCallback()
634
635    def updateReadCallback(self):
636        if self.readHandler is not None:
637            self.stream.startReading(self.handleData)
638        elif self.stream.isOpen():
639            try:
640                self.stream.stopReading()
641            except KeyError:
642                pass
643
644    def handleData(self, data):
645        self.buffer.addData(data)
646        lastState = self.state
647        self.readHandler()
648        # If we switch states, continue processing the buffer.  There may be
649        # extra data that the last read handler didn't read in
650        while self.readHandler is not None and lastState != self.state:
651            lastState = self.state
652            self.readHandler()
653
654    def closeCallback(self, stream, typ):
655        self.handle_close(typ)
656
657    def handle_close(self, typ):
658        """Handle our stream becoming closed.  Type is either socket.SHUT_RD,
659        or socket.SHUT_WR.
660        """
661        raise NotImplementedError()