PageRenderTime 52ms CodeModel.GetById 24ms RepoModel.GetById 0ms app.codeStats 0ms

/historical/hive_test.py

https://bitbucket.org/lindenlab/apiary/
Python | 287 lines | 257 code | 6 blank | 24 comment | 0 complexity | 95c578d508d0e244018c414471e52228 MD5 | raw file
  1. #
  2. # $LicenseInfo:firstyear=2010&license=mit$
  3. #
  4. # Copyright (c) 2010, Linden Research, Inc.
  5. #
  6. # Permission is hereby granted, free of charge, to any person obtaining a copy
  7. # of this software and associated documentation files (the "Software"), to deal
  8. # in the Software without restriction, including without limitation the rights
  9. # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10. # copies of the Software, and to permit persons to whom the Software is
  11. # furnished to do so, subject to the following conditions:
  12. #
  13. # The above copyright notice and this permission notice shall be included in
  14. # all copies or substantial portions of the Software.
  15. #
  16. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  19. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22. # THE SOFTWARE.
  23. # $/LicenseInfo$
  24. #
  25. import sys
  26. import time
  27. import thread
  28. import unittest
  29. import hive
  30. def create_worker(cls):
  31. thread.start_new_thread(hive.run_worker, (cls,))
  32. class SimpleWorker(hive.Worker):
  33. def __init__(self, options, arguments):
  34. hive.Worker.__init__(self, options, arguments)
  35. self._result = ''
  36. def start(self):
  37. self._result = '('
  38. def event(self, data):
  39. self._result += data
  40. def end(self):
  41. self._result += ')'
  42. return self._result
  43. def main(self):
  44. #print "worker up..."
  45. hive.Worker.main(self)
  46. #print "worker down..."
  47. class SimpleCentral(hive.Central):
  48. def __init__(self, options, arguments):
  49. hive.Central.__init__(self, options, arguments)
  50. self.results = []
  51. self._events = []
  52. self._startfn = self.start
  53. self._eventfn = self.event
  54. self._endfn = self.end
  55. def addstart(self, seq):
  56. self._events.append([self._startfn, seq])
  57. def addevent(self, seq, data):
  58. self._events.append([self._eventfn, seq, data])
  59. def addend(self, seq):
  60. self._events.append([self._endfn, seq])
  61. def addstringseq(self, seq, data):
  62. self.addstart(seq)
  63. for c in data:
  64. self.addevent(seq, c)
  65. self.addend(seq)
  66. def addsleep(self, t=0.25):
  67. self._events.append([time.sleep, t])
  68. def next(self):
  69. if not self._events:
  70. return False
  71. e = self._events.pop(0)
  72. e[0](*e[1:])
  73. return True
  74. def result(self, seq, data):
  75. self.results.append(data)
  76. class HiveTestCase(unittest.TestCase):
  77. def setUp(self):
  78. self.options = hive.Hive(None, None).default_options()
  79. self._transport = None
  80. self._open = False
  81. self._threads = []
  82. hive.clean(self.options)
  83. def tearDown(self):
  84. self.close()
  85. self.join_workers()
  86. hive.clean(self.options)
  87. def connect(self):
  88. if self._transport is None:
  89. self._transport = hive.Transport(self.options)
  90. self.send = self._transport.send
  91. self.recv = self._transport.recv
  92. self._transport.connect()
  93. self._open = True
  94. def close(self):
  95. if self._open:
  96. self._transport.close()
  97. self._open = False
  98. def start_workers(self, cls=SimpleWorker, n=1):
  99. threads = hive.start_workers(cls, n, self.options, [])
  100. f = sys._getframe(1)
  101. name = "Worker-%s:%d-" % (f.f_code.co_name, f.f_lineno)
  102. for i in xrange(0,len(threads)):
  103. threads[i].setName(name + str(i+1))
  104. self._threads += threads
  105. def join_workers(self):
  106. for t in self._threads:
  107. t.join(5.0)
  108. if t.isAlive():
  109. raise Exception("thread %s never died" % t.getName())
  110. self._threads = []
  111. class TestWorker(HiveTestCase):
  112. def testAMQP(self):
  113. self.connect()
  114. qname = self._transport.queue()
  115. self.send(qname, 'Hello')
  116. d = self.recv(qname)
  117. self.assertEqual(d, 'Hello')
  118. self.close()
  119. def testNoWork(self):
  120. self.start_workers(SimpleWorker, 1)
  121. time.sleep(0.1)
  122. self.connect()
  123. self._transport.queue('worker-job')
  124. self.close()
  125. self.join_workers()
  126. class TestBasics(HiveTestCase):
  127. def testEmptyCentral(self):
  128. c = SimpleCentral(self.options, [])
  129. c.main()
  130. def testOneSeq(self):
  131. c = SimpleCentral(self.options, [])
  132. c.addstringseq('aaa', 'abc')
  133. self.start_workers(SimpleWorker, 1)
  134. c.main()
  135. r = ','.join(c.results)
  136. self.assert_('(abc)' in r)
  137. self.join_workers()
  138. def testTwoSeq(self):
  139. c = SimpleCentral(self.options, [])
  140. c.addstringseq('aaa', 'abc')
  141. c.addstringseq('xxx', 'xyz')
  142. self.start_workers(SimpleWorker, 1)
  143. c.main()
  144. r = ','.join(c.results)
  145. self.assert_('(abc)' in r)
  146. self.assert_('(xyz)' in r)
  147. self.join_workers()
  148. def testTwoSeq2(self):
  149. c = SimpleCentral(self.options, [])
  150. c.addstringseq('aaa', 'abc')
  151. c.addstringseq('xxx', 'xyz')
  152. self.start_workers(SimpleWorker, 2)
  153. c.main()
  154. r = ','.join(c.results)
  155. self.assert_('(abc)' in r)
  156. self.assert_('(xyz)' in r)
  157. self.join_workers()
  158. def testTwoSeq2Interleaved(self):
  159. c = SimpleCentral(self.options, [])
  160. c.addstart('aaa')
  161. c.addstart('xxx')
  162. c.addsleep()
  163. c.addevent('aaa', 'a')
  164. c.addevent('xxx', 'x')
  165. c.addsleep()
  166. c.addevent('xxx', 'y')
  167. c.addevent('aaa', 'b')
  168. c.addsleep()
  169. c.addevent('aaa', 'c')
  170. c.addevent('xxx', 'z')
  171. c.addsleep()
  172. c.addend('aaa')
  173. c.addend('xxx')
  174. self.start_workers(SimpleWorker, 2)
  175. c.main()
  176. r = ','.join(c.results)
  177. self.assert_('(abc)' in r)
  178. self.assert_('(xyz)' in r)
  179. self.join_workers()
  180. class TestTimeouts(HiveTestCase):
  181. def setUp(self):
  182. HiveTestCase.setUp(self)
  183. self.options.timeout = 0.5
  184. def testWorkerNoJobsTimeout(self):
  185. # Worker w/no jobs stops after timeout
  186. w = SimpleWorker(self.options, [])
  187. t = time.time()
  188. self.assertRaises(hive.TimeoutError, w.main)
  189. t = time.time() - t
  190. self.assert_(t < 1.0, "%f < 1.0" % t)
  191. def testWorkerNoEndTimeout(self):
  192. # Worker w/job, but no job end, stops after timeout
  193. self.connect()
  194. qname = self._transport.queue()
  195. self.send(qname, 'Hello')
  196. self.send(qname, 'Hold on...')
  197. self._transport.queue('worker-job')
  198. self.send('worker-job', qname)
  199. w = SimpleWorker(self.options, [])
  200. t = time.time()
  201. self.assertRaises(hive.TimeoutError, w.main)
  202. t = time.time() - t
  203. self.assert_(t < 1.0, "%f < 1.0" % t)
  204. def testCentralNoWorkersTimeout(self):
  205. # Central w/no workers stops after timeout
  206. c = SimpleCentral(self.options, [])
  207. c.addstringseq('aaa', 'abc') # needs to have some work, or just exits!
  208. t = time.time()
  209. self.assertRaises(hive.TimeoutError, c.main)
  210. t = time.time() - t
  211. self.assert_(t < 1.0, "%f < 1.0" % t)
  212. def xtestCentralNoWorkersCantStartTimeout(self):
  213. # Central w/no workers stops after timeout, when stalled on starting
  214. c = SimpleCentral(self.options, [])
  215. c.addstringseq('aaa', 'abc') # needs to three units of work...
  216. c.addstringseq('bbb', 'def')
  217. c.addstringseq('ccc', 'ghi') # ...so this one stalls on starting
  218. t = time.time()
  219. self.assertRaises(hive.TimeoutError, c.main)
  220. t = time.time() - t
  221. self.assert_(t < 1.0, "%f < 1.0" % t)
  222. if __name__ == '__main__':
  223. unittest.main()