PageRenderTime 36ms CodeModel.GetById 31ms RepoModel.GetById 0ms app.codeStats 0ms

/testing/trigger_scripts/perf_device_trigger_unittest.py

https://github.com/chromium/chromium
Python | 353 lines | 313 code | 19 blank | 21 comment | 13 complexity | a80a3dffaa6c5828b935aaad8ee42588 MD5 | raw file
Possible License(s): MPL-2.0-no-copyleft-exception, Apache-2.0, BSD-3-Clause
  1. #!/usr/bin/env vpython3
  2. # Copyright 2018 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Tests for perf_device_trigger_unittest.py."""
  6. import unittest
  7. import perf_device_trigger
  8. class Args(object): # pylint: disable=useless-object-inheritance
  9. def __init__(self):
  10. self.shards = 1
  11. self.shard_index = None
  12. self.dump_json = ''
  13. self.multiple_trigger_configs = None
  14. self.multiple_dimension_script_verbose = False
  15. self.use_dynamic_shards = False
  16. class FakeTriggerer(perf_device_trigger.PerfDeviceTriggerer):
  17. def __init__(self, args, swarming_args, files, list_bots_result,
  18. list_tasks_results):
  19. self._bot_statuses = []
  20. self._swarming_runs = []
  21. self._files = files
  22. self._temp_file_id = 0
  23. self._triggered_with_swarming_go = 0
  24. self._list_bots_result = list_bots_result
  25. self._list_tasks_results = list_tasks_results
  26. # pylint: disable=super-with-arguments
  27. super(FakeTriggerer, self).__init__(args, swarming_args)
  28. # pylint: enable=super-with-arguments
  29. def set_files(self, files):
  30. self._files = files
  31. def make_temp_file(self, prefix=None, suffix=None):
  32. result = prefix + str(self._temp_file_id) + suffix
  33. self._temp_file_id += 1
  34. return result
  35. def delete_temp_file(self, temp_file):
  36. pass
  37. def read_json_from_temp_file(self, temp_file):
  38. return self._files[temp_file]
  39. def read_encoded_json_from_temp_file(self, temp_file):
  40. return self._files[temp_file]
  41. def write_json_to_file(self, merged_json, output_file):
  42. self._files[output_file] = merged_json
  43. def list_bots(self,
  44. dimensions,
  45. server='chromium-swarm.appspot.com'):
  46. return self._list_bots_result
  47. def list_tasks(self, tags, limit=None,
  48. server='chromium-swarm.appspot.com'):
  49. res, self._list_tasks_results = self._list_tasks_results[
  50. 0], self._list_tasks_results[1:]
  51. return res
  52. def run_swarming(self, args):
  53. self._swarming_runs.append(args)
  54. def run_swarming_go(self,
  55. args,
  56. _json_path,
  57. _shard_index,
  58. _shard,
  59. _merged_json=None):
  60. self._triggered_with_swarming_go += 1
  61. self.run_swarming(args)
  62. class UnitTest(unittest.TestCase):
  63. def setup_and_trigger(self,
  64. previous_task_assignment_map,
  65. alive_bots,
  66. dead_bots,
  67. use_dynamic_shards=False):
  68. args = Args()
  69. args.shards = len(previous_task_assignment_map)
  70. args.dump_json = 'output.json'
  71. args.multiple_dimension_script_verbose = True
  72. if use_dynamic_shards:
  73. args.use_dynamic_shards = True
  74. swarming_args = [
  75. 'trigger',
  76. '--swarming',
  77. 'http://foo_server',
  78. '--dimension',
  79. 'pool',
  80. 'chrome-perf-fyi',
  81. '--dimension',
  82. 'os',
  83. 'windows',
  84. '--',
  85. 'benchmark1',
  86. ]
  87. triggerer = FakeTriggerer(
  88. args, swarming_args, self.get_files(args.shards),
  89. self.generate_list_of_eligible_bots_query_response(
  90. alive_bots, dead_bots), [
  91. self.generate_last_task_to_shard_query_response(
  92. i, previous_task_assignment_map.get(i))
  93. for i in range(args.shards)
  94. ])
  95. triggerer.trigger_tasks(args, swarming_args)
  96. return triggerer
  97. def get_files(self, num_shards):
  98. files = {}
  99. file_index = 0
  100. file_index = file_index + 1
  101. # Perf device trigger will call swarming n times:
  102. # 1. Once for all eligible bots
  103. # 2. once per shard to determine last bot run
  104. # Shard builders is a list of build ids that represents
  105. # the last build that ran the shard that corresponds to that
  106. # index. If that shard hasn't been run before the entry
  107. # should be an empty string.
  108. for i in range(num_shards):
  109. task = {
  110. 'tasks': [{
  111. 'request': {
  112. 'task_id': 'f%d' % i,
  113. },
  114. }],
  115. }
  116. files['base_trigger_dimensions%d.json' % file_index] = task
  117. file_index = file_index + 1
  118. return files
  119. def generate_last_task_to_shard_query_response(self, shard, bot_id):
  120. if len(bot_id):
  121. # Test both cases where bot_id is present and you have to parse
  122. # out of the tags.
  123. if shard % 2:
  124. return [{'bot_id': bot_id}]
  125. return [{'tags': ['id:%s' % bot_id]}]
  126. return []
  127. def generate_list_of_eligible_bots_query_response(self, alive_bots,
  128. dead_bots):
  129. if len(alive_bots) == 0 and len(dead_bots) == 0:
  130. return {}
  131. bots = []
  132. for bot_id in alive_bots:
  133. bots.append({
  134. 'bot_id': ('%s' % bot_id),
  135. 'is_dead': False,
  136. 'quarantined': False
  137. })
  138. is_dead = True
  139. for bot_id in dead_bots:
  140. is_quarantined = (not is_dead)
  141. bots.append({
  142. 'bot_id': ('%s' % bot_id),
  143. 'is_dead': is_dead,
  144. 'quarantined': is_quarantined
  145. })
  146. is_dead = (not is_dead)
  147. return bots
  148. def list_contains_sublist(self, main_list, sub_list):
  149. return any(sub_list == main_list[offset:offset + len(sub_list)]
  150. for offset in range(len(main_list) - (len(sub_list) - 1)))
  151. def get_triggered_shard_to_bot(self, triggerer):
  152. triggered_map = {}
  153. for run in triggerer._swarming_runs:
  154. if not 'trigger' in run:
  155. continue
  156. bot_id = run[(run.index('id') + 1)]
  157. g = 'GTEST_SHARD_INDEX='
  158. shard = [int(r[len(g):]) for r in run if r.startswith(g)][0]
  159. triggered_map[shard] = bot_id
  160. return triggered_map
  161. def test_all_healthy_shards(self):
  162. triggerer = self.setup_and_trigger(
  163. previous_task_assignment_map={
  164. 0: 'build3',
  165. 1: 'build4',
  166. 2: 'build5'
  167. },
  168. alive_bots=['build3', 'build4', 'build5'],
  169. dead_bots=['build1', 'build2'])
  170. expected_task_assignment = self.get_triggered_shard_to_bot(triggerer)
  171. self.assertEquals(len(set(expected_task_assignment.values())), 3)
  172. # All three bots were healthy so we should expect the task assignment to
  173. # stay the same
  174. self.assertEquals(expected_task_assignment.get(0), 'build3')
  175. self.assertEquals(expected_task_assignment.get(1), 'build4')
  176. self.assertEquals(expected_task_assignment.get(2), 'build5')
  177. def test_no_bot_returned(self):
  178. with self.assertRaises(ValueError) as context:
  179. self.setup_and_trigger(previous_task_assignment_map={0: 'build1'},
  180. alive_bots=[],
  181. dead_bots=[])
  182. err_msg = 'Not enough available machines exist in swarming pool'
  183. self.assertTrue(err_msg in str(context.exception))
  184. def test_previously_healthy_now_dead(self):
  185. # Test that it swaps out build1 and build2 that are dead
  186. # for two healthy bots
  187. triggerer = self.setup_and_trigger(
  188. previous_task_assignment_map={
  189. 0: 'build1',
  190. 1: 'build2',
  191. 2: 'build3'
  192. },
  193. alive_bots=['build3', 'build4', 'build5'],
  194. dead_bots=['build1', 'build2'])
  195. expected_task_assignment = self.get_triggered_shard_to_bot(triggerer)
  196. self.assertEquals(len(set(expected_task_assignment.values())), 3)
  197. # The first two should be assigned to one of the unassigned healthy bots
  198. new_healthy_bots = ['build4', 'build5']
  199. self.assertIn(expected_task_assignment.get(0), new_healthy_bots)
  200. self.assertIn(expected_task_assignment.get(1), new_healthy_bots)
  201. self.assertEquals(expected_task_assignment.get(2), 'build3')
  202. def test_not_enough_healthy_bots(self):
  203. triggerer = self.setup_and_trigger(
  204. previous_task_assignment_map={
  205. 0: 'build1',
  206. 1: 'build2',
  207. 2: 'build3',
  208. 3: 'build4',
  209. 4: 'build5'
  210. },
  211. alive_bots=['build3', 'build4', 'build5'],
  212. dead_bots=['build1', 'build2'])
  213. expected_task_assignment = self.get_triggered_shard_to_bot(triggerer)
  214. self.assertEquals(len(set(expected_task_assignment.values())), 5)
  215. # We have 5 shards and 5 bots that ran them, but two
  216. # are now dead and there aren't any other healthy bots
  217. # to swap out to. Make sure they still assign to the
  218. # same shards.
  219. self.assertEquals(expected_task_assignment.get(0), 'build1')
  220. self.assertEquals(expected_task_assignment.get(1), 'build2')
  221. self.assertEquals(expected_task_assignment.get(2), 'build3')
  222. self.assertEquals(expected_task_assignment.get(3), 'build4')
  223. self.assertEquals(expected_task_assignment.get(4), 'build5')
  224. def test_not_enough_healthy_bots_shard_not_seen(self):
  225. triggerer = self.setup_and_trigger(
  226. previous_task_assignment_map={
  227. 0: 'build1',
  228. 1: '',
  229. 2: 'build3',
  230. 3: 'build4',
  231. 4: 'build5'
  232. },
  233. alive_bots=['build3', 'build4', 'build5'],
  234. dead_bots=['build1', 'build2'])
  235. expected_task_assignment = self.get_triggered_shard_to_bot(triggerer)
  236. self.assertEquals(len(set(expected_task_assignment.values())), 5)
  237. # Not enough healthy bots so make sure shard 0 is still assigned to its
  238. # same dead bot.
  239. self.assertEquals(expected_task_assignment.get(0), 'build1')
  240. # Shard 1 had not been triggered yet, but there weren't enough
  241. # healthy bots. Make sure it got assigned to the other dead bot.
  242. self.assertEquals(expected_task_assignment.get(1), 'build2')
  243. # The rest of the assignments should stay the same.
  244. self.assertEquals(expected_task_assignment.get(2), 'build3')
  245. self.assertEquals(expected_task_assignment.get(3), 'build4')
  246. self.assertEquals(expected_task_assignment.get(4), 'build5')
  247. def test_shards_not_triggered_yet(self):
  248. # First time this configuration has been seen. Choose three
  249. # healthy shards to trigger jobs on
  250. triggerer = self.setup_and_trigger(
  251. previous_task_assignment_map={
  252. 0: '',
  253. 1: '',
  254. 2: ''
  255. },
  256. alive_bots=['build3', 'build4', 'build5'],
  257. dead_bots=['build1', 'build2'])
  258. expected_task_assignment = self.get_triggered_shard_to_bot(triggerer)
  259. self.assertEquals(len(set(expected_task_assignment.values())), 3)
  260. new_healthy_bots = ['build3', 'build4', 'build5']
  261. self.assertIn(expected_task_assignment.get(0), new_healthy_bots)
  262. self.assertIn(expected_task_assignment.get(1), new_healthy_bots)
  263. self.assertIn(expected_task_assignment.get(2), new_healthy_bots)
  264. def test_previously_duplicate_task_assignments(self):
  265. triggerer = self.setup_and_trigger(
  266. previous_task_assignment_map={
  267. 0: 'build3',
  268. 1: 'build3',
  269. 2: 'build5',
  270. 3: 'build6'
  271. },
  272. alive_bots=['build3', 'build4', 'build5', 'build7'],
  273. dead_bots=['build1', 'build6'])
  274. expected_task_assignment = self.get_triggered_shard_to_bot(triggerer)
  275. # Test that the new assignment will add a new bot to avoid
  276. # assign 'build3' to both shard 0 & shard 1 as before.
  277. # It also replaces the dead 'build6' bot.
  278. self.assertEquals(set(expected_task_assignment.values()),
  279. {'build3', 'build4', 'build5', 'build7'})
  280. def test_dynamic_sharding(self):
  281. triggerer = self.setup_and_trigger(
  282. # The previous map should not matter.
  283. previous_task_assignment_map={
  284. 0: 'build301',
  285. 1: 'build1--',
  286. 2: 'build-blah'
  287. },
  288. alive_bots=['build1', 'build2', 'build3', 'build4', 'build5'],
  289. dead_bots=[],
  290. use_dynamic_shards=True)
  291. expected_task_assignment = self.get_triggered_shard_to_bot(triggerer)
  292. self.assertEquals(set(expected_task_assignment.values()),
  293. {'build1', 'build2', 'build3', 'build4', 'build5'})
  294. def test_dynamic_sharding_with_dead_bots(self):
  295. triggerer = self.setup_and_trigger(
  296. # The previous map should not matter.
  297. previous_task_assignment_map={
  298. 0: 'build301',
  299. 1: 'build1--',
  300. 2: 'build-blah'
  301. },
  302. alive_bots=['build2', 'build5', 'build3'],
  303. dead_bots=['build1', 'build4'],
  304. use_dynamic_shards=True)
  305. expected_task_assignment = self.get_triggered_shard_to_bot(triggerer)
  306. self.assertEquals(set(expected_task_assignment.values()),
  307. {'build2', 'build3', 'build5'})
  308. if __name__ == '__main__':
  309. unittest.main()