/js/src/metrics/gc/gc-test.py

  1# Works with python2.6
  2
  3import datetime, os, re, sys, traceback
  4import math, string, copy
  5import subprocess
  6from subprocess import *
  7from operator import itemgetter
  8
  9class Test:
 10    def __init__(self, path, name):
 11        self.path = path
 12        self.name = name
 13
 14    @classmethod
 15    def from_file(cls, path, name, options):
 16        return cls(path, name)
 17
 18def find_tests(dir, substring = None):
 19    ans = []
 20    for dirpath, dirnames, filenames in os.walk(dir):
 21        if dirpath == '.':
 22            continue
 23        for filename in filenames:
 24            if not filename.endswith('.js'):
 25                continue
 26            test = os.path.join(dirpath, filename)
 27            if substring is None or substring in os.path.relpath(test, dir):
 28                ans.append([test, filename])
 29    return ans
 30
 31def get_test_cmd(path):
 32    return [ JS, '-f', path ]
 33
 34def avg(seq):
 35    return sum(seq) / len(seq)
 36
 37def stddev(seq, mean):
 38    diffs = ((float(item) - mean) ** 2 for item in seq)
 39    return math.sqrt(sum(diffs) / len(seq))
 40
 41def run_test(test):
 42    env = os.environ.copy()
 43    env['MOZ_GCTIMER'] = 'stderr'
 44    cmd = get_test_cmd(test.path)
 45    total = []
 46    mark = []
 47    sweep = []
 48    close_fds = sys.platform != 'win32'
 49    p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=close_fds, env=env)
 50    out, err = p.communicate()
 51    out, err = out.decode(), err.decode()
 52
 53    float_array = [float(_) for _ in err.split()]
 54    
 55    if len(float_array) == 0:
 56        print('Error: No data from application. Configured with --enable-gctimer?')
 57        sys.exit(1)
 58
 59    for i, currItem in enumerate(float_array):
 60        if (i % 3 == 0):
 61            total.append(currItem)
 62        else:
 63            if (i % 3 == 1):
 64                mark.append(currItem)
 65            else:
 66                sweep.append(currItem)
 67
 68    return max(total), avg(total), max(mark), avg(mark), max(sweep), avg(sweep)
 69
 70def run_tests(tests, test_dir):
 71    bench_map = {}
 72
 73    try:
 74        for i, test in enumerate(tests):
 75            filename_str = '"%s"' % test.name
 76            TMax, TAvg, MMax, MAvg, SMax, SAvg = run_test(test)
 77            bench_map[test.name] = [TMax, TAvg, MMax, MAvg, SMax, SAvg]
 78            fmt = '%20s: {"TMax": %4.1f, "TAvg": %4.1f, "MMax": %4.1f, "MAvg": %4.1f, "SMax": %4.1f, "SAvg": %4.1f}'
 79            if (i != len(tests) - 1):
 80                fmt += ','
 81            print(fmt %(filename_str ,TMax, TAvg, MMax, MAvg, SMax, MAvg))
 82    except KeyboardInterrupt:
 83        print('fail')
 84
 85    return dict((filename, dict(TMax=TMax, TAvg=TAvg, MMax=MMax, MAvg=MAvg, SMax=SMax, SAvg=SAvg))
 86            for filename, (TMax, TAvg, MMax, MAvg, SMax, SAvg) in bench_map.iteritems())
 87
 88def compare(current, baseline):
 89    percent_speedups = []
 90    for key, current_result in current.iteritems():
 91        try:
 92            baseline_result = baseline[key]
 93        except KeyError:
 94            print key, 'missing from baseline'
 95            continue
 96
 97        val_getter = itemgetter('TMax', 'TAvg', 'MMax', 'MAvg', 'SMax', 'SAvg')
 98        BTMax, BTAvg, BMMax, BMAvg, BSMax, BSAvg = val_getter(baseline_result)
 99        CTMax, CTAvg, CMMax, CMAvg, CSMax, CSAvg = val_getter(current_result)
100
101        fmt = '%30s: %s'
102        if CTAvg <= BTAvg:
103            speedup = (CTAvg / BTAvg - 1) * 100
104            result = 'faster: %6.2f < baseline %6.2f (%+6.2f%%)' % \
105                    (CTAvg, BTAvg, speedup)
106            percent_speedups.append(speedup)
107        else:
108            slowdown = (CTAvg / BTAvg - 1) * 100
109            result = 'SLOWER: %6.2f > baseline %6.2f (%+6.2f%%) ' % \
110                    (CTAvg, BTAvg, slowdown)
111            percent_speedups.append(slowdown)
112        print '%30s: %s' % (key, result)
113    if percent_speedups:
114        print 'Average speedup: %.2f%%' % avg(percent_speedups)
115
116def try_import_json():
117    try:
118        import json
119        return json
120    except ImportError:
121        try:
122            import simplejson as json
123            return json
124        except ImportError:
125            pass
126
127if __name__ == '__main__':
128    script_path = os.path.abspath(__file__)
129    script_dir = os.path.dirname(script_path)
130    test_dir = os.path.join(script_dir, 'tests')
131
132    from optparse import OptionParser
133    op = OptionParser(usage='%prog [options] JS_SHELL [TESTS]')
134
135    op.add_option('-b', '--baseline', metavar='JSON_PATH',
136            dest='baseline_path', help='json file with baseline values to '
137            'compare against')
138
139    (OPTIONS, args) = op.parse_args()
140    if len(args) < 1:
141        op.error('missing JS_SHELL argument')
142    # We need to make sure we are using backslashes on Windows.
143    JS, test_args = os.path.normpath(args[0]), args[1:]
144
145    test_list = []
146    bench_map = {}
147
148    test_list = find_tests(test_dir)
149
150    if not test_list:
151        print >> sys.stderr, "No tests found matching command line arguments."
152        sys.exit(0)
153
154    test_list = [ Test.from_file(tst, name, OPTIONS) for tst, name in test_list ]
155
156    if OPTIONS.baseline_path:
157        json = try_import_json()
158        if not json:
159            print('You need a json lib for baseline comparison')
160            sys.exit(1)
161    try:
162        print("{")
163        bench_map = run_tests(test_list, test_dir)
164        print("}")
165
166    except OSError:
167        if not os.path.exists(JS):
168            print >> sys.stderr, "JS shell argument: file does not exist: '%s'"%JS
169            sys.exit(1)
170        else:
171            raise
172
173    if OPTIONS.baseline_path:
174        baseline_map = []
175        fh = open(OPTIONS.baseline_path, 'r')
176        baseline_map = json.load(fh)
177        fh.close()
178        compare(current=bench_map, baseline=baseline_map)