/js/src/metrics/gc/gc-test.py

http://github.com/zpao/v8monkey · Python · 178 lines · 146 code · 30 blank · 2 comment · 40 complexity · f5be5048a49175d8a69a83ddeb4fcfdc MD5 · raw file

  1. # Works with python2.6
  2. import datetime, os, re, sys, traceback
  3. import math, string, copy
  4. import subprocess
  5. from subprocess import *
  6. from operator import itemgetter
  7. class Test:
  8. def __init__(self, path, name):
  9. self.path = path
  10. self.name = name
  11. @classmethod
  12. def from_file(cls, path, name, options):
  13. return cls(path, name)
  14. def find_tests(dir, substring = None):
  15. ans = []
  16. for dirpath, dirnames, filenames in os.walk(dir):
  17. if dirpath == '.':
  18. continue
  19. for filename in filenames:
  20. if not filename.endswith('.js'):
  21. continue
  22. test = os.path.join(dirpath, filename)
  23. if substring is None or substring in os.path.relpath(test, dir):
  24. ans.append([test, filename])
  25. return ans
  26. def get_test_cmd(path):
  27. return [ JS, '-f', path ]
  28. def avg(seq):
  29. return sum(seq) / len(seq)
  30. def stddev(seq, mean):
  31. diffs = ((float(item) - mean) ** 2 for item in seq)
  32. return math.sqrt(sum(diffs) / len(seq))
  33. def run_test(test):
  34. env = os.environ.copy()
  35. env['MOZ_GCTIMER'] = 'stderr'
  36. cmd = get_test_cmd(test.path)
  37. total = []
  38. mark = []
  39. sweep = []
  40. close_fds = sys.platform != 'win32'
  41. p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=close_fds, env=env)
  42. out, err = p.communicate()
  43. out, err = out.decode(), err.decode()
  44. float_array = [float(_) for _ in err.split()]
  45. if len(float_array) == 0:
  46. print('Error: No data from application. Configured with --enable-gctimer?')
  47. sys.exit(1)
  48. for i, currItem in enumerate(float_array):
  49. if (i % 3 == 0):
  50. total.append(currItem)
  51. else:
  52. if (i % 3 == 1):
  53. mark.append(currItem)
  54. else:
  55. sweep.append(currItem)
  56. return max(total), avg(total), max(mark), avg(mark), max(sweep), avg(sweep)
  57. def run_tests(tests, test_dir):
  58. bench_map = {}
  59. try:
  60. for i, test in enumerate(tests):
  61. filename_str = '"%s"' % test.name
  62. TMax, TAvg, MMax, MAvg, SMax, SAvg = run_test(test)
  63. bench_map[test.name] = [TMax, TAvg, MMax, MAvg, SMax, SAvg]
  64. fmt = '%20s: {"TMax": %4.1f, "TAvg": %4.1f, "MMax": %4.1f, "MAvg": %4.1f, "SMax": %4.1f, "SAvg": %4.1f}'
  65. if (i != len(tests) - 1):
  66. fmt += ','
  67. print(fmt %(filename_str ,TMax, TAvg, MMax, MAvg, SMax, MAvg))
  68. except KeyboardInterrupt:
  69. print('fail')
  70. return dict((filename, dict(TMax=TMax, TAvg=TAvg, MMax=MMax, MAvg=MAvg, SMax=SMax, SAvg=SAvg))
  71. for filename, (TMax, TAvg, MMax, MAvg, SMax, SAvg) in bench_map.iteritems())
  72. def compare(current, baseline):
  73. percent_speedups = []
  74. for key, current_result in current.iteritems():
  75. try:
  76. baseline_result = baseline[key]
  77. except KeyError:
  78. print key, 'missing from baseline'
  79. continue
  80. val_getter = itemgetter('TMax', 'TAvg', 'MMax', 'MAvg', 'SMax', 'SAvg')
  81. BTMax, BTAvg, BMMax, BMAvg, BSMax, BSAvg = val_getter(baseline_result)
  82. CTMax, CTAvg, CMMax, CMAvg, CSMax, CSAvg = val_getter(current_result)
  83. fmt = '%30s: %s'
  84. if CTAvg <= BTAvg:
  85. speedup = (CTAvg / BTAvg - 1) * 100
  86. result = 'faster: %6.2f < baseline %6.2f (%+6.2f%%)' % \
  87. (CTAvg, BTAvg, speedup)
  88. percent_speedups.append(speedup)
  89. else:
  90. slowdown = (CTAvg / BTAvg - 1) * 100
  91. result = 'SLOWER: %6.2f > baseline %6.2f (%+6.2f%%) ' % \
  92. (CTAvg, BTAvg, slowdown)
  93. percent_speedups.append(slowdown)
  94. print '%30s: %s' % (key, result)
  95. if percent_speedups:
  96. print 'Average speedup: %.2f%%' % avg(percent_speedups)
  97. def try_import_json():
  98. try:
  99. import json
  100. return json
  101. except ImportError:
  102. try:
  103. import simplejson as json
  104. return json
  105. except ImportError:
  106. pass
  107. if __name__ == '__main__':
  108. script_path = os.path.abspath(__file__)
  109. script_dir = os.path.dirname(script_path)
  110. test_dir = os.path.join(script_dir, 'tests')
  111. from optparse import OptionParser
  112. op = OptionParser(usage='%prog [options] JS_SHELL [TESTS]')
  113. op.add_option('-b', '--baseline', metavar='JSON_PATH',
  114. dest='baseline_path', help='json file with baseline values to '
  115. 'compare against')
  116. (OPTIONS, args) = op.parse_args()
  117. if len(args) < 1:
  118. op.error('missing JS_SHELL argument')
  119. # We need to make sure we are using backslashes on Windows.
  120. JS, test_args = os.path.normpath(args[0]), args[1:]
  121. test_list = []
  122. bench_map = {}
  123. test_list = find_tests(test_dir)
  124. if not test_list:
  125. print >> sys.stderr, "No tests found matching command line arguments."
  126. sys.exit(0)
  127. test_list = [ Test.from_file(tst, name, OPTIONS) for tst, name in test_list ]
  128. if OPTIONS.baseline_path:
  129. json = try_import_json()
  130. if not json:
  131. print('You need a json lib for baseline comparison')
  132. sys.exit(1)
  133. try:
  134. print("{")
  135. bench_map = run_tests(test_list, test_dir)
  136. print("}")
  137. except OSError:
  138. if not os.path.exists(JS):
  139. print >> sys.stderr, "JS shell argument: file does not exist: '%s'"%JS
  140. sys.exit(1)
  141. else:
  142. raise
  143. if OPTIONS.baseline_path:
  144. baseline_map = []
  145. fh = open(OPTIONS.baseline_path, 'r')
  146. baseline_map = json.load(fh)
  147. fh.close()
  148. compare(current=bench_map, baseline=baseline_map)