/tv/lib/net.py
Python | 661 lines | 583 code | 7 blank | 71 comment | 7 complexity | 323e67f38159e24e7d2c80ec7d6f67ce MD5 | raw file
1# Miro - an RSS based video player application 2# Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 3# Participatory Culture Foundation 4# 5# This program is free software; you can redistribute it and/or modify 6# it under the terms of the GNU General Public License as published by 7# the Free Software Foundation; either version 2 of the License, or 8# (at your option) any later version. 9# 10# This program is distributed in the hope that it will be useful, 11# but WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13# GNU General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with this program; if not, write to the Free Software 17# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 18# 19# In addition, as a special exception, the copyright holders give 20# permission to link the code of portions of this program with the OpenSSL 21# library. 22# 23# You must obey the GNU General Public License in all respects for all of 24# the code used other than OpenSSL. If you modify file(s) with this 25# exception, you may extend this exception to your version of the file(s), 26# but you are not obligated to do so. If you do not wish to do so, delete 27# this exception statement from your version. If you delete this exception 28# statement from all source files in the program, then also delete it here. 29 30"""net.py -- Low-level Networking Code 31 32The main class here is ConnectionHandler, which is a state-based socket 33handling class. It's used for communication with the downloader daemon via a 34socket. 35 36This class also defines the base Exception classes used by httpclient. 37 38""" 39 40import errno 41import logging 42import socket 43 44from miro import app 45from miro import eventloop 46from miro import util 47from miro import prefs 48from miro import signals 49from miro import trapcall 50from miro.clock import clock 51from miro.gtcache import gettext as _ 52 53SOCKET_READ_TIMEOUT = 60 54SOCKET_INITIAL_READ_TIMEOUT = 30 55SOCKET_CONNECT_TIMEOUT = 15 56 57# socket.ssl is deprecated as of Python 2.6, so we use socket_ssl for 58# pre Python 2.6 and ssl.wrap_socket for Python 2.6 and later. 59try: 60 import ssl 61 ssl.wrap_socket 62 def convert_to_ssl(sock): 63 return ssl.wrap_socket(sock) 64except (ImportError, AttributeError): 65 def convert_to_ssl(sock): 66 return socket.ssl(sock) 67 68class NetworkError(StandardError): 69 """Base class for all errors that will be passed to errbacks from get_url 70 and friends. NetworkErrors can be display in 2 ways: 71 72 getFriendlyDescription() -- short, newbie friendly description 73 getLongDescription() -- detailed description 74 """ 75 def __init__(self, shortDescription, longDescription=None): 76 if longDescription is None: 77 longDescription = shortDescription 78 self.friendlyDescription = _("Error: %(msg)s", 79 {"msg": shortDescription}) 80 self.longDescription = longDescription 81 82 def getFriendlyDescription(self): 83 return self.friendlyDescription 84 85 def getLongDescription(self): 86 return self.longDescription 87 88 def __str__(self): 89 return "%s: %s -- %s" % (self.__class__, 90 util.stringify(self.getFriendlyDescription()), 91 util.stringify(self.getLongDescription())) 92 93class ConnectionError(NetworkError): 94 def __init__(self, errorMessage): 95 self.friendlyDescription = _("Can't connect") 96 self.longDescription = _("Connection Error: %(msg)s", 97 {"msg": util.unicodify(errorMessage)}) 98 99class SSLConnectionError(ConnectionError): 100 def __init__(self): 101 self.friendlyDescription = _("Can't connect") 102 self.longDescription = _("SSL connection error") 103 104class ConnectionTimeout(NetworkError): 105 def __init__(self, host): 106 NetworkError.__init__(self, _('Timeout'), 107 _('Connection to %(host)s timed out', {"host": host})) 108 109def trap_call(object, function, *args, **kwargs): 110 """Convenience function do a trapcall.trap_call, where when is 111 'While talking to the network' 112 """ 113 return trapcall.time_trap_call("Calling %s on %s" % (function, object), 114 function, *args, **kwargs) 115 116class NetworkBuffer(object): 117 """Responsible for storing incomming network data and doing some basic 118 parsing of it. I think this is about as fast as we can do things in pure 119 python, someday we may want to make it C... 120 """ 121 def __init__(self): 122 self.chunks = [] 123 self.length = 0 124 125 def addData(self, data): 126 self.chunks.append(data) 127 self.length += len(data) 128 129 def _mergeChunks(self): 130 self.chunks = [''.join(self.chunks)] 131 132 def has_data(self): 133 return self.length > 0 134 135 def discard_data(self): 136 self.chunks = [] 137 self.length = 0 138 139 def read(self, size=None): 140 """Read at most size bytes from the data that has been added to the 141 buffer. """ 142 143 self._mergeChunks() 144 if size is not None: 145 rv = self.chunks[0][:size] 146 self.chunks[0] = self.chunks[0][len(rv):] 147 else: 148 rv = self.chunks[0] 149 self.chunks = [] 150 self.length -= len(rv) 151 return rv 152 153 def readline(self): 154 """Like a file readline, with several difference: 155 * If there isn't a full line ready to be read we return None. 156 * Doesn't include the trailing line separator. 157 * Both "\r\n" and "\n" act as a line ender 158 """ 159 160 self._mergeChunks() 161 split = self.chunks[0].split("\n", 1) 162 if len(split) == 2: 163 self.chunks[0] = split[1] 164 self.length = len(self.chunks[0]) 165 if split[0].endswith("\r"): 166 return split[0][:-1] 167 else: 168 return split[0] 169 else: 170 return None 171 172 def unread(self, data): 173 """Put back read data. This make is like the data was never read at 174 all. 175 """ 176 self.chunks.insert(0, data) 177 self.length += len(data) 178 179 def getValue(self): 180 self._mergeChunks() 181 return self.chunks[0] 182 183class _Packet(object): 184 """A packet of data for the AsyncSocket class 185 """ 186 def __init__(self, data, callback=None): 187 self.data = data 188 self.callback = callback 189 190class AsyncSocket(object): 191 """Socket class that uses the eventloop module. 192 """ 193 194 MEMORY_ERROR_LIMIT = 5 195 196 def __init__(self, closeCallback=None): 197 """Create an AsyncSocket. If closeCallback is given, it will be 198 called if we detect that the socket has been closed durring a 199 read/write operation. The arguments will be the AsyncSocket object 200 and either socket.SHUT_RD or socket.SHUT_WR. 201 """ 202 self.toSend = [] 203 self.to_send_length = 0 204 self.readSize = 4096 205 self.socket = None 206 self.readCallback = None 207 self.closeCallback = closeCallback 208 self.readTimeout = None 209 self.timedOut = False 210 self.connectionErrback = None 211 self.disable_read_timeout = False 212 self.readSomeData = False 213 self.name = "" 214 self.lastClock = None 215 self.memoryErrors = 0 216 217 def __str__(self): 218 if self.name: 219 return "%s: %s" % (type(self).__name__, self.name) 220 else: 221 return "Unknown %s" % (type(self).__name__,) 222 223 # The complication in the timeout code is because creating and 224 # cancelling a timeout costs some memory (timeout is in memory 225 # until it goes off, even if cancelled.) 226 def startReadTimeout(self): 227 if self.disable_read_timeout: 228 return 229 self.lastClock = clock() 230 if self.readTimeout is not None: 231 return 232 self.readTimeout = eventloop.add_timeout(SOCKET_INITIAL_READ_TIMEOUT, 233 self.onReadTimeout, 234 "AsyncSocket.onReadTimeout") 235 236 def stopReadTimeout(self): 237 if self.readTimeout is not None: 238 self.readTimeout.cancel() 239 self.readTimeout = None 240 241 def _pick_address(self, addresses): 242 """Pick the best entry to use from a list of addresses 243 244 :param addresses: list of address tuples returned by getaddrinfo() 245 :returns: one of the tuples, or None if no address could be found 246 """ 247 if not app.config.get(prefs.DISABLE_IPV6) and util.use_ipv6(): 248 # prefer ipv6 if possible 249 for entry in addresses: 250 if entry[0] == socket.AF_INET6: 251 return entry 252 # fall back on ipv4 253 for entry in addresses: 254 if entry[0] == socket.AF_INET: 255 return entry 256 return None 257 258 def open_connection(self, host, port, callback, errback, 259 disable_read_timeout=None): 260 """Open a connection. On success, callback will be called with this 261 object. 262 """ 263 if disable_read_timeout is not None: 264 self.disable_read_timeout = disable_read_timeout 265 self.name = "Outgoing %s:%s" % (host, port) 266 267 self.connectionErrback = errback 268 def handleGetAddrInfoException(e): 269 if self.connectionErrback is None: 270 # called connectionErrback while we were waiting for 271 # getaddrinfo 272 return 273 trap_call(self, errback, 274 ConnectionError(e[1] + " (host: %s)" % host)) 275 def createSocketHandle(family): 276 try: 277 self.socket = socket.socket(family, socket.SOCK_STREAM) 278 except socket.error, e: 279 trap_call(self, errback, ConnectionError(e[1])) 280 return 281 282 self.socket.setblocking(0) 283 return self.socket 284 285 def onAddressLookup(addresses): 286 if self.connectionErrback is None: 287 # called connectionErrback while we were waiting for 288 # getaddrinfo 289 return 290 entry = self._pick_address(addresses) 291 if entry is None: 292 # FIXME - wtf kind of user message is this? it's too 293 # technical and there's no way a user would know what 294 # to do about it. 295 msg = _("Couldn't find address family to use") 296 trap_call(self, errback, ConnectionError(msg)) 297 return 298 try: 299 self.socket = socket.socket(entry[0], socket.SOCK_STREAM) 300 except socket.error, e: 301 trap_call(self, errback, ConnectionError(e[1])) 302 return 303 self.socket.setblocking(0) 304 try: 305 rv = self.socket.connect_ex(entry[4]) 306 except socket.gaierror: 307 trap_call(self, errback, ConnectionError('gaierror')) 308 return 309 if rv in (0, errno.EINPROGRESS, errno.EWOULDBLOCK): 310 eventloop.add_write_callback(self.socket, onWriteReady) 311 self.socketConnectTimeout = eventloop.add_timeout( 312 SOCKET_CONNECT_TIMEOUT, onWriteTimeout, 313 "socket connect timeout") 314 else: 315 fullmsg = "Connection failed" 316 trap_call(self, errback, ConnectionError(fullmsg)) 317 def onWriteReady(): 318 eventloop.remove_write_callback(self.socket) 319 self.socketConnectTimeout.cancel() 320 rv = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) 321 if rv == 0: 322 trap_call(self, callback, self) 323 else: 324 msg = errno.errorcode.get(rv, _('Unknown Error code')) 325 trap_call(self, errback, ConnectionError(msg)) 326 self.connectionErrback = None 327 def onWriteTimeout(): 328 eventloop.remove_write_callback(self.socket) 329 trap_call(self, errback, ConnectionTimeout(host)) 330 self.connectionErrback = None 331 eventloop.call_in_thread(onAddressLookup, handleGetAddrInfoException, 332 socket.getaddrinfo, 333 "getAddrInfo - %s:%s" % (host, port), 334 host, port) 335 336 def accept_connection(self, family, host, port, callback, errback): 337 def finishAccept(): 338 eventloop.remove_read_callback(self.socket) 339 (self.socket, addr) = self.socket.accept() 340 trap_call(self, callback, self) 341 self.connectionErrback = None 342 343 self.name = "Incoming %s:%s" % (host, port) 344 self.connectionErrback = errback 345 346 try: 347 self.socket = socket.socket(family, socket.SOCK_STREAM) 348 self.socket.bind((host, port)) 349 except socket.error, e: 350 trap_call(self, errback, ConnectionError(e[1])) 351 return 352 353 if self.socket.family == socket.AF_INET: 354 (self.addr, self.port) = self.socket.getsockname() 355 elif self.socket.family == socket.AF_INET6: 356 (self.addr, self.port, self.flowinfo, 357 self.scopeid) = self.socket.getsockname() 358 else: 359 raise ValueError("Unknown socket family: %s", self.socket.family) 360 self.socket.listen(63) 361 eventloop.add_read_callback(self.socket, finishAccept) 362 363 def close_connection(self): 364 if self.isOpen(): 365 eventloop.stop_handling_socket(self.socket) 366 self.stopReadTimeout() 367 self.socket.close() 368 self.socket = None 369 if self.connectionErrback is not None: 370 error = NetworkError(_("Connection closed")) 371 trap_call(self, self.connectionErrback, error) 372 self.connectionErrback = None 373 374 def isOpen(self): 375 return self.socket is not None 376 377 def send_data(self, data, callback=None): 378 """Send data out to the socket when it becomes ready. 379 380 NOTE: currently we have no way of detecting when the data gets sent 381 out, or if errors happen. 382 """ 383 384 if not self.isOpen(): 385 raise ValueError("Socket not connected") 386 self.toSend.append(_Packet(data, callback)) 387 self.to_send_length += len(data) 388 eventloop.add_write_callback(self.socket, self.onWriteReady) 389 390 def startReading(self, readCallback): 391 """Start reading from the socket. When data becomes available it will 392 be passed to readCallback. If there is already a read callback, it 393 will be replaced. 394 """ 395 396 if not self.isOpen(): 397 raise ValueError("Socket not connected") 398 self.readCallback = readCallback 399 eventloop.add_read_callback(self.socket, self.onReadReady) 400 self.startReadTimeout() 401 402 def stopReading(self): 403 """Stop reading from the socket.""" 404 405 if not self.isOpen(): 406 raise ValueError("Socket not connected") 407 self.readCallback = None 408 eventloop.remove_read_callback(self.socket) 409 self.stopReadTimeout() 410 411 def onReadTimeout(self): 412 if self.readSomeData: 413 timeout = SOCKET_READ_TIMEOUT 414 else: 415 timeout = SOCKET_INITIAL_READ_TIMEOUT 416 417 if clock() < self.lastClock + timeout: 418 self.readTimeout = eventloop.add_timeout( 419 self.lastClock + timeout - clock(), self.onReadTimeout, 420 "AsyncSocket.onReadTimeout") 421 else: 422 self.readTimeout = None 423 self.timedOut = True 424 self.handleEarlyClose('read') 425 426 def handleSocketError(self, code, msg, operation): 427 if code in (errno.EWOULDBLOCK, errno.EINTR): 428 return 429 430 if operation == "write": 431 expectedErrors = (errno.EPIPE, errno.ECONNRESET) 432 else: 433 expectedErrors = (errno.ECONNREFUSED, errno.ECONNRESET) 434 if code not in expectedErrors: 435 logging.warning("WARNING, got unexpected error during %s", 436 operation) 437 logging.warning("%s: %s", errno.errorcode.get(code), msg) 438 self.handleEarlyClose(operation) 439 440 def onWriteReady(self): 441 try: 442 if len(self.toSend) > 0: 443 sent = self.socket.send(self.toSend[0].data) 444 else: 445 sent = 0 446 except socket.error, (code, msg): 447 self.handleSocketError(code, msg, "write") 448 else: 449 self.handleSentData(sent) 450 451 def handleSentData(self, sent): 452 if len(self.toSend) > 0: 453 self.toSend[0].data = self.toSend[0].data[sent:] 454 if len(self.toSend[0].data) == 0: 455 if self.toSend[0].callback: 456 self.toSend[0].callback() 457 self.toSend = self.toSend[1:] 458 self.to_send_length -= sent 459 if len(self.toSend) == 0: 460 eventloop.remove_write_callback(self.socket) 461 462 def onReadReady(self): 463 try: 464 data = self.socket.recv(self.readSize) 465 except socket.error, (code, msg): 466 self.handleSocketError(code, msg, "read") 467 except MemoryError: 468 # This happens because of a windows bug in the socket code (see 469 # #4373). Let's hope that things clear themselves up next time we 470 # read. 471 self.memoryErrors += 1 472 if self.memoryErrors > self.MEMORY_ERROR_LIMIT: 473 logging.error("ERROR: Too many MemoryErrors on %s", self) 474 self.handleEarlyClose('read') 475 else: 476 logging.warning( 477 "WARNING: Memory error while reading from %s", self) 478 else: 479 self.memoryErrors = 0 480 self.handleReadData(data) 481 482 def handleReadData(self, data): 483 self.startReadTimeout() 484 if data == '': 485 if self.closeCallback: 486 trap_call(self, self.closeCallback, self, socket.SHUT_RD) 487 else: 488 self.readSomeData = True 489 trap_call(self, self.readCallback, data) 490 491 def handleEarlyClose(self, operation): 492 self.close_connection() 493 if self.closeCallback: 494 if operation == 'read': 495 typ = socket.SHUT_RD 496 else: 497 typ = socket.SHUT_WR 498 trap_call(self, self.closeCallback, self, typ) 499 500class AsyncSSLStream(AsyncSocket): 501 def __init__(self, closeCallback=None): 502 super(AsyncSSLStream, self).__init__(closeCallback) 503 self.interruptedOperation = None 504 505 def open_connection(self, host, port, callback, errback, 506 disable_read_timeout=None): 507 def onSocketOpen(self): 508 self.socket.setblocking(1) 509 eventloop.call_in_thread(onSSLOpen, handleSSLError, convert_to_ssl, 510 "AsyncSSL onSocketOpen()", 511 self.socket) 512 def onSSLOpen(ssl): 513 if self.socket is None: 514 # the connection was closed while we were calling 515 # convert_to_ssl 516 return 517 self.socket.setblocking(0) 518 self.ssl = ssl 519 # finally we can call the actuall callback 520 callback(self) 521 def handleSSLError(error): 522 logging.error("handleSSLError: %r", error) 523 errback(SSLConnectionError()) 524 super(AsyncSSLStream, self).open_connection(host, port, onSocketOpen, 525 errback, disable_read_timeout) 526 527 def resumeNormalCallbacks(self): 528 if self.readCallback is not None: 529 eventloop.add_read_callback(self.socket, self.onReadReady) 530 if len(self.toSend) != 0: 531 eventloop.add_write_callback(self.socket, self.onWriteReady) 532 533 def handleSocketError(self, code, msg, operation): 534 if code in (socket.SSL_ERROR_WANT_READ, socket.SSL_ERROR_WANT_WRITE): 535 if self.interruptedOperation is None: 536 self.interruptedOperation = operation 537 elif self.interruptedOperation != operation: 538 signals.system.failed("When talking to the network", 539 details="socket error for the wrong SSL operation") 540 self.close_connection() 541 return 542 eventloop.stop_handling_socket(self.socket) 543 if code == socket.SSL_ERROR_WANT_READ: 544 eventloop.add_read_callback(self.socket, self.onReadReady) 545 else: 546 eventloop.add_write_callback(self.socket, self.onWriteReady) 547 elif code in (socket.SSL_ERROR_ZERO_RETURN, socket.SSL_ERROR_SSL, 548 socket.SSL_ERROR_SYSCALL, socket.SSL_ERROR_EOF): 549 self.handleEarlyClose(operation) 550 else: 551 super(AsyncSSLStream, self).handleSocketError(code, msg, 552 operation) 553 554 def onWriteReady(self): 555 if self.interruptedOperation == 'read': 556 return self.onReadReady() 557 try: 558 if len(self.toSend) > 0: 559 sent = self.ssl.write(self.toSend[0].data) 560 else: 561 sent = 0 562 except socket.error, (code, msg): 563 self.handleSocketError(code, msg, "write") 564 else: 565 if self.interruptedOperation == 'write': 566 self.resumeNormalCallbacks() 567 self.interruptedOperation = None 568 self.handleSentData(sent) 569 570 def onReadReady(self): 571 if self.interruptedOperation == 'write': 572 return self.onWriteReady() 573 try: 574 data = self.ssl.read(self.readSize) 575 except socket.error, (code, msg): 576 self.handleSocketError(code, msg, "read") 577 else: 578 if self.interruptedOperation == 'read': 579 self.resumeNormalCallbacks() 580 self.interruptedOperation = None 581 self.handleReadData(data) 582 583class ConnectionHandler(object): 584 """Base class to handle asynchronous network streams. It implements a 585 simple state machine to deal with incomming data. 586 587 Sending data: Use the send_data() method. 588 589 Reading Data: Add entries to the state dictionary, which maps strings to 590 methods. The state methods will be called when there is data available, 591 which can be read from the buffer variable. The states dictionary can 592 contain a None value, to signal that the handler isn't interested in 593 reading at that point. Use change_state() to switch states. 594 595 Subclasses should override the handle_close() method to handle the 596 socket closing. 597 """ 598 stream_factory = AsyncSocket 599 600 def __init__(self): 601 self.buffer = NetworkBuffer() 602 self.states = {'initializing': None, 'closed': None} 603 self.stream = self.stream_factory(closeCallback=self.closeCallback) 604 self.change_state('initializing') 605 self.name = "" 606 607 def __str__(self): 608 return "%s -- %s" % (self.__class__, self.state) 609 610 def open_connection(self, host, port, callback, errback, 611 disable_read_timeout=None): 612 self.name = "Outgoing %s:%s" % (host, port) 613 self.host = host 614 self.port = port 615 def callbackIntercept(asyncSocket): 616 if callback: 617 trap_call(self, callback, self) 618 self.stream.open_connection(host, port, callbackIntercept, errback, 619 disable_read_timeout) 620 621 def close_connection(self): 622 if self.stream.isOpen(): 623 self.stream.close_connection() 624 self.change_state('closed') 625 self.buffer.discard_data() 626 627 def send_data(self, data, callback=None): 628 self.stream.send_data(data, callback) 629 630 def change_state(self, newState): 631 self.readHandler = self.states[newState] 632 self.state = newState 633 self.updateReadCallback() 634 635 def updateReadCallback(self): 636 if self.readHandler is not None: 637 self.stream.startReading(self.handleData) 638 elif self.stream.isOpen(): 639 try: 640 self.stream.stopReading() 641 except KeyError: 642 pass 643 644 def handleData(self, data): 645 self.buffer.addData(data) 646 lastState = self.state 647 self.readHandler() 648 # If we switch states, continue processing the buffer. There may be 649 # extra data that the last read handler didn't read in 650 while self.readHandler is not None and lastState != self.state: 651 lastState = self.state 652 self.readHandler() 653 654 def closeCallback(self, stream, typ): 655 self.handle_close(typ) 656 657 def handle_close(self, typ): 658 """Handle our stream becoming closed. Type is either socket.SHUT_RD, 659 or socket.SHUT_WR. 660 """ 661 raise NotImplementedError()