/js/src/jit-test/jit_test.py

  1#!/usr/bin/env python
  2
  3# jit_test.py -- Python harness for JavaScript trace tests.
  4
  5import datetime, os, re, sys, tempfile, traceback, time, shlex
  6import subprocess
  7from subprocess import *
  8from threading import Thread
  9
 10DEBUGGER_INFO = {
 11  "gdb": {
 12    "interactive": True,
 13    "args": "-q --args"
 14  },
 15
 16  "valgrind": {
 17    "interactive": False,
 18    "args": "--leak-check=full"
 19  }
 20}
 21
 22# Backported from Python 3.1 posixpath.py
 23def _relpath(path, start=None):
 24    """Return a relative version of a path"""
 25
 26    if not path:
 27        raise ValueError("no path specified")
 28
 29    if start is None:
 30        start = os.curdir
 31
 32    start_list = os.path.abspath(start).split(os.sep)
 33    path_list = os.path.abspath(path).split(os.sep)
 34
 35    # Work out how much of the filepath is shared by start and path.
 36    i = len(os.path.commonprefix([start_list, path_list]))
 37
 38    rel_list = [os.pardir] * (len(start_list)-i) + path_list[i:]
 39    if not rel_list:
 40        return os.curdir
 41    return os.path.join(*rel_list)
 42
 43os.path.relpath = _relpath
 44
 45class Test:
 46    def __init__(self, path):
 47        self.path = path       # path to test file
 48        
 49        self.jitflags = []     # jit flags to enable
 50        self.slow = False      # True means the test is slow-running
 51        self.allow_oom = False # True means that OOM is not considered a failure
 52        self.valgrind = False  # True means run under valgrind
 53        self.expect_error = '' # Errors to expect and consider passing
 54        self.expect_status = 0 # Exit status to expect from shell
 55
 56    def copy(self):
 57        t = Test(self.path)
 58        t.jitflags = self.jitflags[:]
 59        t.slow = self.slow
 60        t.allow_oom = self.allow_oom
 61        t.valgrind = self.valgrind
 62        t.expect_error = self.expect_error
 63        t.expect_status = self.expect_status
 64        return t
 65
 66    COOKIE = '|jit-test|'
 67
 68    @classmethod
 69    def from_file(cls, path, options):
 70        test = cls(path)
 71
 72        line = open(path).readline()
 73        i = line.find(cls.COOKIE)
 74        if i != -1:
 75            meta = line[i + len(cls.COOKIE):].strip('\n')
 76            parts = meta.split(';')
 77            for part in parts:
 78                part = part.strip()
 79                if not part:
 80                    continue
 81                name, _, value = part.partition(':')
 82                if value:
 83                    value = value.strip()
 84                    if name == 'error':
 85                        test.expect_error = value
 86                    elif name == 'exitstatus':
 87                        try:
 88                            test.expect_status = int(value, 0);
 89                        except ValueError:
 90                            print("warning: couldn't parse exit status %s"%value)
 91                    else:
 92                        print('warning: unrecognized |jit-test| attribute %s'%part)
 93                else:
 94                    if name == 'slow':
 95                        test.slow = True
 96                    elif name == 'allow-oom':
 97                        test.allow_oom = True
 98                    elif name == 'valgrind':
 99                        test.valgrind = options.valgrind
100                    elif name == 'mjitalways':
101                        test.jitflags.append('-a')
102                    elif name == 'debug':
103                        test.jitflags.append('-d')
104                    elif name == 'mjit':
105                        test.jitflags.append('-m')
106                    else:
107                        print('warning: unrecognized |jit-test| attribute %s'%part)
108
109        if options.valgrind_all:
110            test.valgrind = True
111
112        return test
113
114def find_tests(dir, substring = None):
115    ans = []
116    for dirpath, dirnames, filenames in os.walk(dir):
117        dirnames.sort()
118        filenames.sort()
119        if dirpath == '.':
120            continue
121        for filename in filenames:
122            if not filename.endswith('.js'):
123                continue
124            if filename in ('shell.js', 'browser.js', 'jsref.js'):
125                continue
126            test = os.path.join(dirpath, filename)
127            if substring is None or substring in os.path.relpath(test, dir):
128                ans.append(test)
129    return ans
130
131def get_test_cmd(path, jitflags, lib_dir, shell_args):
132    libdir_var = lib_dir
133    if not libdir_var.endswith('/'):
134        libdir_var += '/'
135    expr = "const platform=%r; const libdir=%r;"%(sys.platform, libdir_var)
136    # We may have specified '-a' or '-d' twice: once via --jitflags, once
137    # via the "|jit-test|" line.  Remove dups because they are toggles.
138    return ([ JS ] + list(set(jitflags)) + shell_args +
139            [ '-e', expr, '-f', os.path.join(lib_dir, 'prolog.js'), '-f', path ])
140
141def set_limits():
142    # resource module not supported on all platforms
143    try:
144        import resource
145        GB = 2**30
146        resource.setrlimit(resource.RLIMIT_AS, (1*GB, 1*GB))
147    except:
148        return
149
150def tmppath(token):
151    fd, path = tempfile.mkstemp(prefix=token)
152    os.close(fd)
153    return path
154
155def read_and_unlink(path):
156    f = open(path)
157    d = f.read()
158    f.close()
159    os.unlink(path)
160    return d
161
162def th_run_cmd(cmdline, options, l):
163    # close_fds and preexec_fn are not supported on Windows and will
164    # cause a ValueError.
165    if sys.platform != 'win32':
166        options["close_fds"] = True
167        options["preexec_fn"] = set_limits
168    p = Popen(cmdline, stdin=PIPE, stdout=PIPE, stderr=PIPE, **options)
169
170    l[0] = p
171    out, err = p.communicate()
172    l[1] = (out, err, p.returncode)
173
174def run_timeout_cmd(cmdline, options, timeout=60.0):
175    l = [ None, None ]
176    timed_out = False
177    th = Thread(target=th_run_cmd, args=(cmdline, options, l))
178    th.start()
179    th.join(timeout)
180    while th.isAlive():
181        if l[0] is not None:
182            try:
183                # In Python 3, we could just do l[0].kill().
184                import signal
185                if sys.platform != 'win32':
186                    os.kill(l[0].pid, signal.SIGKILL)
187                time.sleep(.1)
188                timed_out = True
189            except OSError:
190                # Expecting a "No such process" error
191                pass
192    th.join()
193    (out, err, code) = l[1]
194    return (out, err, code, timed_out)
195
196def run_cmd(cmdline, env, timeout):
197    return run_timeout_cmd(cmdline, { 'env': env }, timeout)
198
199def run_cmd_avoid_stdio(cmdline, env, timeout):
200    stdoutPath, stderrPath = tmppath('jsstdout'), tmppath('jsstderr')
201    env['JS_STDOUT'] = stdoutPath
202    env['JS_STDERR'] = stderrPath       
203    _, __, code = run_timeout_cmd(cmdline, { 'env': env }, timeout)
204    return read_and_unlink(stdoutPath), read_and_unlink(stderrPath), code
205
206def run_test(test, lib_dir, shell_args):
207    cmd = get_test_cmd(test.path, test.jitflags, lib_dir, shell_args)
208
209    if (test.valgrind and
210        any([os.path.exists(os.path.join(d, 'valgrind'))
211             for d in os.environ['PATH'].split(os.pathsep)])):
212        valgrind_prefix = [ 'valgrind',
213                            '-q',
214                            '--smc-check=all',
215                            '--error-exitcode=1',
216                            '--leak-check=full']
217        if os.uname()[0] == 'Darwin':
218            valgrind_prefix += ['--dsymutil=yes']
219        cmd = valgrind_prefix + cmd
220
221    if OPTIONS.show_cmd:
222        print(subprocess.list2cmdline(cmd))
223
224    if OPTIONS.avoid_stdio:
225        run = run_cmd_avoid_stdio
226    else:
227        run = run_cmd
228    out, err, code, timed_out = run(cmd, os.environ, OPTIONS.timeout)
229
230    if OPTIONS.show_output:
231        sys.stdout.write(out)
232        sys.stdout.write(err)
233        sys.stdout.write('Exit code: %s\n' % code)
234    if test.valgrind:
235        sys.stdout.write(err)
236    return (check_output(out, err, code, test),
237            out, err, code, timed_out)
238
239def check_output(out, err, rc, test):
240    if test.expect_error:
241        return test.expect_error in err
242
243    for line in out.split('\n'):
244        if line.startswith('Trace stats check failed'):
245            return False
246
247    for line in err.split('\n'):
248        if 'Assertion failed:' in line:
249            return False
250
251    if rc != test.expect_status:
252        # Allow a non-zero exit code if we want to allow OOM, but only if we
253        # actually got OOM.
254        return test.allow_oom and ': out of memory' in err
255
256    return True
257
258def print_tinderbox(label, test, message=None):
259    jitflags = " ".join(test.jitflags)
260    result = "%s | jit_test.py %-15s| %s" % (label, jitflags, test.path)
261    if message:
262        result += ": " + message
263    print result
264
265def run_tests(tests, test_dir, lib_dir, shell_args):
266    pb = None
267    if not OPTIONS.hide_progress and not OPTIONS.show_cmd:
268        try:
269            from progressbar import ProgressBar
270            pb = ProgressBar('', len(tests), 24)
271        except ImportError:
272            pass
273
274    failures = []
275    timeouts = 0
276    complete = False
277    doing = 'before starting'
278    try:
279        for i, test in enumerate(tests):
280            doing = 'on %s'%test.path
281            ok, out, err, code, timed_out = run_test(test, lib_dir, shell_args)
282            doing = 'after %s'%test.path
283
284            if not ok:
285                failures.append([ test, out, err, code, timed_out ])
286            if timed_out:
287                timeouts += 1
288
289            if OPTIONS.tinderbox:
290                if ok:
291                    print_tinderbox("TEST-PASS", test);
292                else:
293                    lines = [ _ for _ in out.split('\n') + err.split('\n')
294                              if _ != '' ]
295                    if len(lines) >= 1:
296                        msg = lines[-1]
297                    else:
298                        msg = ''
299                    print_tinderbox("TEST-UNEXPECTED-FAIL", test, msg);
300
301            n = i + 1
302            if pb:
303                pb.label = '[%4d|%4d|%4d|%4d]'%(n - len(failures), len(failures), timeouts, n)
304                pb.update(n)
305        complete = True
306    except KeyboardInterrupt:
307        print_tinderbox("TEST-UNEXPECTED-FAIL", test);
308
309    if pb:
310        pb.finish()
311
312    if failures:
313        if OPTIONS.write_failures:
314            try:
315                out = open(OPTIONS.write_failures, 'w')
316                # Don't write duplicate entries when we are doing multiple failures per job.
317                written = set()
318                for test, fout, ferr, fcode, _ in failures:
319                    if test.path not in written:
320                        out.write(os.path.relpath(test.path, test_dir) + '\n')
321                        if OPTIONS.write_failure_output:
322                            out.write(fout)
323                            out.write(ferr)
324                            out.write('Exit code: ' + str(fcode) + "\n")
325                        written.add(test.path)
326                out.close()
327            except IOError:
328                sys.stderr.write("Exception thrown trying to write failure file '%s'\n"%
329                                 OPTIONS.write_failures)
330                traceback.print_exc()
331                sys.stderr.write('---\n')
332
333        def show_test(test):
334            if OPTIONS.show_failed:
335                print('    ' + subprocess.list2cmdline(get_test_cmd(test.path, test.jitflags, lib_dir, shell_args)))
336            else:
337                print('    ' + ' '.join(test.jitflags + [ test.path ]))
338
339        print('FAILURES:')
340        for test, _, __, ___, timed_out in failures:
341            if not timed_out:
342                show_test(test)
343
344        print('TIMEOUTS:')
345        for test, _, __, ___, timed_out in failures:
346            if timed_out:
347                show_test(test)
348
349        return False
350    else:
351        print('PASSED ALL' + ('' if complete else ' (partial run -- interrupted by user %s)'%doing))
352        return True
353
354def parse_jitflags():
355    jitflags = [ [ '-' + flag for flag in flags ] 
356                 for flags in OPTIONS.jitflags.split(',') ]
357    for flags in jitflags:
358        for flag in flags:
359            if flag not in ('-m', '-a', '-p', '-d', '-n'):
360                print('Invalid jit flag: "%s"'%flag)
361                sys.exit(1)
362    return jitflags
363
364def platform_might_be_android():
365    try:
366        # The python package for SL4A provides an |android| module.
367        # If that module is present, we're likely in SL4A-python on
368        # device.  False positives and negatives are possible,
369        # however.
370        import android
371        return True
372    except ImportError:
373        return False
374
375def stdio_might_be_broken():
376    return platform_might_be_android()
377
378JS = None
379OPTIONS = None
380def main(argv):
381    global JS, OPTIONS
382
383    script_path = os.path.abspath(__file__)
384    script_dir = os.path.dirname(script_path)
385    test_dir = os.path.join(script_dir, 'tests')
386    lib_dir = os.path.join(script_dir, 'lib')
387
388    # The [TESTS] optional arguments are paths of test files relative
389    # to the jit-test/tests directory.
390
391    from optparse import OptionParser
392    op = OptionParser(usage='%prog [options] JS_SHELL [TESTS]')
393    op.add_option('-s', '--show-cmd', dest='show_cmd', action='store_true',
394                  help='show js shell command run')
395    op.add_option('-f', '--show-failed-cmd', dest='show_failed', 
396                  action='store_true', help='show command lines of failed tests')
397    op.add_option('-o', '--show-output', dest='show_output', action='store_true',
398                  help='show output from js shell')
399    op.add_option('-x', '--exclude', dest='exclude', action='append',
400                  help='exclude given test dir or path')
401    op.add_option('--no-slow', dest='run_slow', action='store_false',
402                  help='do not run tests marked as slow')
403    op.add_option('-t', '--timeout', dest='timeout',  type=float, default=150.0,
404                  help='set test timeout in seconds')
405    op.add_option('--no-progress', dest='hide_progress', action='store_true',
406                  help='hide progress bar')
407    op.add_option('--tinderbox', dest='tinderbox', action='store_true',
408                  help='Tinderbox-parseable output format')
409    op.add_option('--args', dest='shell_args', default='',
410                  help='extra args to pass to the JS shell')
411    op.add_option('-w', '--write-failures', dest='write_failures', metavar='FILE',
412                  help='Write a list of failed tests to [FILE]')
413    op.add_option('-r', '--read-tests', dest='read_tests', metavar='FILE',
414                  help='Run test files listed in [FILE]')
415    op.add_option('-R', '--retest', dest='retest', metavar='FILE',
416                  help='Retest using test list file [FILE]')
417    op.add_option('-g', '--debug', dest='debug', action='store_true',
418                  help='Run test in gdb')
419    op.add_option('--valgrind', dest='valgrind', action='store_true',
420                  help='Enable the |valgrind| flag, if valgrind is in $PATH.')
421    op.add_option('--valgrind-all', dest='valgrind_all', action='store_true',
422                  help='Run all tests with valgrind, if valgrind is in $PATH.')
423    op.add_option('--jitflags', dest='jitflags', default='m,mn',
424                  help='Example: --jitflags=m,mn to run each test with -m, -m -n [default=%default]')
425    op.add_option('--avoid-stdio', dest='avoid_stdio', action='store_true',
426                  help='Use js-shell file indirection instead of piping stdio.')
427    op.add_option('--write-failure-output', dest='write_failure_output', action='store_true',
428                  help='With --write-failures=FILE, additionally write the output of failed tests to [FILE]')
429    (OPTIONS, args) = op.parse_args(argv)
430    if len(args) < 1:
431        op.error('missing JS_SHELL argument')
432    # We need to make sure we are using backslashes on Windows.
433    JS, test_args = os.path.normpath(args[0]), args[1:]
434    JS = os.path.realpath(JS) # Burst through the symlinks!
435
436    if stdio_might_be_broken():
437        # Prefer erring on the side of caution and not using stdio if
438        # it might be broken on this platform.  The file-redirect
439        # fallback should work on any platform, so at worst by
440        # guessing wrong we might have slowed down the tests a bit.
441        #
442        # XXX technically we could check for broken stdio, but it
443        # really seems like overkill.
444        OPTIONS.avoid_stdio = True
445
446    if OPTIONS.retest:
447        OPTIONS.read_tests = OPTIONS.retest
448        OPTIONS.write_failures = OPTIONS.retest
449
450    test_list = []
451    read_all = True
452
453    if test_args:
454        read_all = False
455        for arg in test_args:
456            test_list += find_tests(test_dir, arg)
457
458    if OPTIONS.read_tests:
459        read_all = False
460        try:
461            f = open(OPTIONS.read_tests)
462            for line in f:
463                test_list.append(os.path.join(test_dir, line.strip('\n')))
464            f.close()
465        except IOError:
466            if OPTIONS.retest:
467                read_all = True
468            else:
469                sys.stderr.write("Exception thrown trying to read test file '%s'\n"%
470                                 OPTIONS.read_tests)
471                traceback.print_exc()
472                sys.stderr.write('---\n')
473
474    if read_all:
475        test_list = find_tests(test_dir)
476
477    if OPTIONS.exclude:
478        exclude_list = []
479        for exclude in OPTIONS.exclude:
480            exclude_list += find_tests(test_dir, exclude)
481        test_list = [ test for test in test_list if test not in set(exclude_list) ]
482
483    if not test_list:
484        print >> sys.stderr, "No tests found matching command line arguments."
485        sys.exit(0)
486
487    test_list = [ Test.from_file(_, OPTIONS) for _ in test_list ]
488
489    if not OPTIONS.run_slow:
490        test_list = [ _ for _ in test_list if not _.slow ]
491
492    # The full test list is ready. Now create copies for each JIT configuration.
493    job_list = []
494    jitflags_list = parse_jitflags()
495    for test in test_list:
496        for jitflags in jitflags_list:
497            new_test = test.copy()
498            new_test.jitflags.extend(jitflags)
499            job_list.append(new_test)
500    
501
502    shell_args = shlex.split(OPTIONS.shell_args)
503
504    if OPTIONS.debug:
505        if len(job_list) > 1:
506            print('Multiple tests match command line arguments, debugger can only run one')
507            for tc in job_list:
508                print('    %s'%tc.path)
509            sys.exit(1)
510
511        tc = job_list[0]
512        cmd = [ 'gdb', '--args' ] + get_test_cmd(tc.path, tc.jitflags, lib_dir, shell_args)
513        call(cmd)
514        sys.exit()
515
516    try:
517        ok = run_tests(job_list, test_dir, lib_dir, shell_args)
518        if not ok:
519            sys.exit(2)
520    except OSError:
521        if not os.path.exists(JS):
522            print >> sys.stderr, "JS shell argument: file does not exist: '%s'"%JS
523            sys.exit(1)
524        else:
525            raise
526
527if __name__ == '__main__':
528    main(sys.argv[1:])