/scrapy/utils/response.py

https://github.com/yoyo2k/scrapy · Python · 84 lines · 68 code · 7 blank · 9 comment · 11 complexity · 154b40e63da15d57e09db43dccc58da7 MD5 · raw file 

    

  1. """
    2. 

  2. This module provides some useful functions for working with
    3. 

  3. scrapy.http.Response objects
    4. 

  4. """
    5. 

    6. 

  6. import os
    7. 

  7. import weakref
    8. 

  8. import webbrowser
    9. 

  9. import tempfile
    10. 

  10. 

  11. from twisted.web import http
    12. 

  12. from twisted.web.http import RESPONSES
    13. 

  13. from w3lib import html
    14. 

  14. 

  15. from scrapy.xlib.BeautifulSoup import BeautifulSoup
    16. 

  16. from scrapy.http import Response, HtmlResponse
    17. 

  17. 

  18. def body_or_str(obj, unicode=True):
    19. 

  19.     assert isinstance(obj, (Response, basestring)), \
    20. 

  20.         "obj must be Response or basestring, not %s" % type(obj).__name__
    21. 

  21.     if isinstance(obj, Response):
    22. 

  22.         return obj.body_as_unicode() if unicode else obj.body
    23. 

  23.     elif isinstance(obj, str):
    24. 

  24.         return obj.decode('utf-8') if unicode else obj
    25. 

  25.     else:
    26. 

  26.         return obj if unicode else obj.encode('utf-8')
    27. 

  27. 

  28. _baseurl_cache = weakref.WeakKeyDictionary()
    29. 

  29. def get_base_url(response):
    30. 

  30.     """Return the base url of the given response, joined with the response url"""
    31. 

  31.     if response not in _baseurl_cache:
    32. 

  32.         text = response.body_as_unicode()[0:4096]
    33. 

  33.         _baseurl_cache[response] = html.get_base_url(text, response.url, \
    34. 

  34.             response.encoding)
    35. 

  35.     return _baseurl_cache[response]
    36. 

  36. 

  37. _metaref_cache = weakref.WeakKeyDictionary()
    38. 

  38. def get_meta_refresh(response):
    39. 

  39.     """Parse the http-equiv refrsh parameter from the given response"""
    40. 

  40.     if response not in _metaref_cache:
    41. 

  41.         text = response.body_as_unicode()[0:4096]
    42. 

  42.         _metaref_cache[response] = html.get_meta_refresh(text, response.url, \
    43. 

  43.             response.encoding)
    44. 

  44.     return _metaref_cache[response]
    45. 

  45. 

  46. def response_status_message(status):
    47. 

  47.     """Return status code plus status text descriptive message
    48. 

    49. 

  49.     >>> response_status_message(200)
    50. 

  50.     '200 OK'
    51. 

  51. 

  52.     >>> response_status_message(404)
    53. 

  53.     '404 Not Found'
    54. 

  54.     """
    55. 

  55.     return '%s %s' % (status, http.responses.get(int(status)))
    56. 

  56. 

  57. def response_httprepr(response):
    58. 

  58.     """Return raw HTTP representation (as string) of the given response. This
    59. 

  59.     is provided only for reference, since it's not the exact stream of bytes
    60. 

  60.     that was received (that's not exposed by Twisted).
    61. 

  61.     """
    62. 

    63. 

  63.     s = "HTTP/1.1 %d %s\r\n" % (response.status, RESPONSES.get(response.status, ''))
    64. 

  64.     if response.headers:
    65. 

  65.         s += response.headers.to_string() + "\r\n"
    66. 

  66.     s += "\r\n"
    67. 

  67.     s += response.body
    68. 

  68.     return s
    69. 

  69. 

  70. def open_in_browser(response, _openfunc=webbrowser.open):
    71. 

  71.     """Open the given response in a local web browser, populating the <base>
    72. 

  72.     tag for external links to work
    73. 

  73.     """
    74. 

  74.     # XXX: this implementation is a bit dirty and could be improved
    75. 

  75.     if not isinstance(response, HtmlResponse):
    76. 

  76.         raise TypeError("Unsupported response type: %s" % \
    77. 

  77.             response.__class__.__name__)
    78. 

  78.     body = response.body
    79. 

  79.     if '<base' not in body:
    80. 

  80.         body = body.replace('<head>', '<head><base href="%s">' % response.url)
    81. 

  81.     fd, fname = tempfile.mkstemp('.html')
    82. 

  82.     os.write(fd, body)
    83. 

  83.     os.close(fd)
    84. 

  84.     return _openfunc("file://%s" % fname)
    85.