/tests/test_response.py

https://gitlab.com/czm1129/pyspider
Python | 90 lines | 70 code | 14 blank | 6 comment | 2 complexity | 01cd94c0be04154159332b82f8f07006 MD5 | raw file
  1. #!/usr/bin/env python
  2. # -*- encoding: utf-8 -*-
  3. # vim: set et sw=4 ts=4 sts=4 ff=unix fenc=utf8:
  4. # Author: Binux<roy@binux.me>
  5. # http://binux.me
  6. # Created on 2015-01-18 11:10:27
  7. import os
  8. import copy
  9. import time
  10. import httpbin
  11. import unittest2 as unittest
  12. import logging
  13. import logging.config
  14. logging.config.fileConfig("pyspider/logging.conf")
  15. from pyspider.libs import utils
  16. from pyspider.libs.response import rebuild_response
  17. from pyspider.fetcher.tornado_fetcher import Fetcher
  18. class TestResponse(unittest.TestCase):
  19. sample_task_http = {
  20. 'taskid': 'taskid',
  21. 'project': 'project',
  22. 'url': '',
  23. }
  24. @classmethod
  25. def setUpClass(self):
  26. self.fetcher = Fetcher(None, None, async=False)
  27. self.httpbin_thread = utils.run_in_subprocess(httpbin.app.run, port=14887)
  28. self.httpbin = 'http://127.0.0.1:14887'
  29. time.sleep(0.5)
  30. @classmethod
  31. def tearDownClass(self):
  32. self.httpbin_thread.terminate()
  33. def get(self, url, **kwargs):
  34. if not url.startswith('http://'):
  35. url = self.httpbin + url
  36. request = copy.deepcopy(self.sample_task_http)
  37. request['url'] = url
  38. request.update(kwargs)
  39. task, result = self.fetcher.fetch(request)
  40. response = rebuild_response(result)
  41. return response
  42. def test_10_html(self):
  43. response = self.get('/html')
  44. self.assertEqual(response.status_code, 200)
  45. self.assertIsNotNone(response.doc('h1'))
  46. def test_20_xml(self):
  47. response = self.get('/xml')
  48. self.assertEqual(response.status_code, 200)
  49. self.assertIsNotNone(response.doc('item'))
  50. def test_30_gzip(self):
  51. response = self.get('/gzip')
  52. self.assertEqual(response.status_code, 200)
  53. self.assertIn('gzipped', response.text)
  54. def test_40_deflate(self):
  55. response = self.get('/deflate')
  56. self.assertEqual(response.status_code, 200)
  57. self.assertIn('deflated', response.text)
  58. def test_50_ok(self):
  59. response = self.get('/status/200')
  60. self.assertTrue(response.ok)
  61. self.assertTrue(response)
  62. response = self.get('/status/302')
  63. self.assertTrue(response.ok)
  64. self.assertTrue(response)
  65. with self.assertRaises(Exception):
  66. self.raise_for_status(allow_redirects=False)
  67. def test_60_not_ok(self):
  68. response = self.get('/status/400')
  69. self.assertFalse(response.ok)
  70. self.assertFalse(response)
  71. response = self.get('/status/500')
  72. self.assertFalse(response.ok)
  73. self.assertFalse(response)
  74. response = self.get('/status/600')
  75. self.assertFalse(response.ok)
  76. self.assertFalse(response)