/build/manifestparser.py

   1#!/usr/bin/env python
   2
   3# ***** BEGIN LICENSE BLOCK *****
   4# Version: MPL 1.1/GPL 2.0/LGPL 2.1
   5# 
   6# The contents of this file are subject to the Mozilla Public License Version
   7# 1.1 (the "License"); you may not use this file except in compliance with
   8# the License. You may obtain a copy of the License at
   9# http://www.mozilla.org/MPL/
  10# 
  11# Software distributed under the License is distributed on an "AS IS" basis,
  12# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  13# for the specific language governing rights and limitations under the
  14# License.
  15# 
  16# The Original Code is mozilla.org code.
  17# 
  18# The Initial Developer of the Original Code is
  19# Mozilla.org.
  20# Portions created by the Initial Developer are Copyright (C) 2010
  21# the Initial Developer. All Rights Reserved.
  22# 
  23# Contributor(s):
  24#     Jeff Hammel <jhammel@mozilla.com>     (Original author)
  25# 
  26# Alternatively, the contents of this file may be used under the terms of
  27# either of the GNU General Public License Version 2 or later (the "GPL"),
  28# or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29# in which case the provisions of the GPL or the LGPL are applicable instead
  30# of those above. If you wish to allow use of your version of this file only
  31# under the terms of either the GPL or the LGPL, and not to allow others to
  32# use your version of this file under the terms of the MPL, indicate your
  33# decision by deleting the provisions above and replace them with the notice
  34# and other provisions required by the GPL or the LGPL. If you do not delete
  35# the provisions above, a recipient may use your version of this file under
  36# the terms of any one of the MPL, the GPL or the LGPL.
  37# 
  38# ***** END LICENSE BLOCK *****
  39
  40"""
  41Mozilla universal manifest parser
  42"""
  43
  44# this file lives at
  45# http://hg.mozilla.org/automation/ManifestDestiny/raw-file/tip/manifestparser.py
  46
  47__all__ = ['read_ini', # .ini reader
  48           'ManifestParser', 'TestManifest', 'convert', # manifest handling
  49           'parse', 'ParseError', 'ExpressionParser'] # conditional expression parser
  50
  51import os
  52import re
  53import shutil
  54import sys
  55from fnmatch import fnmatch
  56from optparse import OptionParser
  57
  58version = '0.5.3' # package version
  59try:
  60    from setuptools import setup
  61except:
  62    setup = None
  63
  64# we need relpath, but it is introduced in python 2.6
  65# http://docs.python.org/library/os.path.html
  66try:
  67    relpath = os.path.relpath
  68except AttributeError:
  69    def relpath(path, start):
  70        """
  71        Return a relative version of a path
  72        from /usr/lib/python2.6/posixpath.py
  73        """
  74
  75        if not path:
  76            raise ValueError("no path specified")
  77
  78        start_list = os.path.abspath(start).split(os.path.sep)
  79        path_list = os.path.abspath(path).split(os.path.sep)
  80
  81        # Work out how much of the filepath is shared by start and path.
  82        i = len(os.path.commonprefix([start_list, path_list]))
  83
  84        rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:]
  85        if not rel_list:
  86            return start
  87        return os.path.join(*rel_list)
  88
  89# expr.py
  90# from:
  91# http://k0s.org/mozilla/hg/expressionparser
  92# http://hg.mozilla.org/users/tmielczarek_mozilla.com/expressionparser
  93
  94# Implements a top-down parser/evaluator for simple boolean expressions.
  95# ideas taken from http://effbot.org/zone/simple-top-down-parsing.htm
  96#
  97# Rough grammar:
  98# expr := literal
  99#       | '(' expr ')'
 100#       | expr '&&' expr
 101#       | expr '||' expr
 102#       | expr '==' expr
 103#       | expr '!=' expr
 104# literal := BOOL
 105#          | INT
 106#          | STRING
 107#          | IDENT
 108# BOOL   := true|false
 109# INT    := [0-9]+
 110# STRING := "[^"]*"
 111# IDENT  := [A-Za-z_]\w*
 112
 113# Identifiers take their values from a mapping dictionary passed as the second
 114# argument.
 115
 116# Glossary (see above URL for details):
 117# - nud: null denotation
 118# - led: left detonation
 119# - lbp: left binding power
 120# - rbp: right binding power
 121
 122class ident_token(object):
 123    def __init__(self, value):
 124        self.value = value
 125    def nud(self, parser):
 126        # identifiers take their value from the value mappings passed
 127        # to the parser
 128        return parser.value(self.value)
 129
 130class literal_token(object):
 131    def __init__(self, value):
 132        self.value = value
 133    def nud(self, parser):
 134        return self.value
 135
 136class eq_op_token(object):
 137    "=="
 138    def led(self, parser, left):
 139        return left == parser.expression(self.lbp)
 140    
 141class neq_op_token(object):
 142    "!="
 143    def led(self, parser, left):
 144        return left != parser.expression(self.lbp)
 145
 146class not_op_token(object):
 147    "!"
 148    def nud(self, parser):
 149        return not parser.expression()
 150
 151class and_op_token(object):
 152    "&&"
 153    def led(self, parser, left):
 154        right = parser.expression(self.lbp)
 155        return left and right
 156    
 157class or_op_token(object):
 158    "||"
 159    def led(self, parser, left):
 160        right = parser.expression(self.lbp)
 161        return left or right
 162
 163class lparen_token(object):
 164    "("
 165    def nud(self, parser):
 166        expr = parser.expression()
 167        parser.advance(rparen_token)
 168        return expr
 169
 170class rparen_token(object):
 171    ")"
 172
 173class end_token(object):
 174    """always ends parsing"""
 175
 176### derived literal tokens
 177
 178class bool_token(literal_token):
 179    def __init__(self, value):
 180        value = {'true':True, 'false':False}[value]
 181        literal_token.__init__(self, value)
 182
 183class int_token(literal_token):
 184    def __init__(self, value):
 185        literal_token.__init__(self, int(value))
 186
 187class string_token(literal_token):
 188    def __init__(self, value):
 189        literal_token.__init__(self, value[1:-1])
 190
 191precedence = [(end_token, rparen_token),
 192              (or_op_token,),
 193              (and_op_token,),
 194              (eq_op_token, neq_op_token),
 195              (lparen_token,),
 196              ]
 197for index, rank in enumerate(precedence):
 198    for token in rank:
 199        token.lbp = index # lbp = lowest left binding power
 200
 201class ParseError(Exception):
 202    """errror parsing conditional expression"""
 203
 204class ExpressionParser(object):
 205    def __init__(self, text, valuemapping, strict=False):
 206        """
 207        Initialize the parser with input |text|, and |valuemapping| as
 208        a dict mapping identifier names to values.
 209        """
 210        self.text = text
 211        self.valuemapping = valuemapping
 212        self.strict = strict
 213
 214    def _tokenize(self):
 215        """
 216        Lex the input text into tokens and yield them in sequence.
 217        """
 218        # scanner callbacks
 219        def bool_(scanner, t): return bool_token(t)
 220        def identifier(scanner, t): return ident_token(t)
 221        def integer(scanner, t): return int_token(t)
 222        def eq(scanner, t): return eq_op_token()
 223        def neq(scanner, t): return neq_op_token()
 224        def or_(scanner, t): return or_op_token()
 225        def and_(scanner, t): return and_op_token()
 226        def lparen(scanner, t): return lparen_token()
 227        def rparen(scanner, t): return rparen_token()
 228        def string_(scanner, t): return string_token(t)
 229        def not_(scanner, t): return not_op_token()
 230
 231        scanner = re.Scanner([
 232            (r"true|false", bool_),
 233            (r"[a-zA-Z_]\w*", identifier),
 234            (r"[0-9]+", integer),
 235            (r'("[^"]*")|(\'[^\']*\')', string_),
 236            (r"==", eq),
 237            (r"!=", neq),
 238            (r"\|\|", or_),
 239            (r"!", not_),
 240            (r"&&", and_),
 241            (r"\(", lparen),
 242            (r"\)", rparen),
 243            (r"\s+", None), # skip whitespace
 244            ])
 245        tokens, remainder = scanner.scan(self.text)
 246        for t in tokens:
 247            yield t
 248        yield end_token()
 249
 250    def value(self, ident):
 251        """
 252        Look up the value of |ident| in the value mapping passed in the
 253        constructor.
 254        """
 255        if self.strict:
 256            return self.valuemapping[ident]
 257        else:
 258            return self.valuemapping.get(ident, None)
 259
 260    def advance(self, expected):
 261        """
 262        Assert that the next token is an instance of |expected|, and advance
 263        to the next token.
 264        """
 265        if not isinstance(self.token, expected):
 266            raise Exception, "Unexpected token!"
 267        self.token = self.iter.next()
 268        
 269    def expression(self, rbp=0):
 270        """
 271        Parse and return the value of an expression until a token with
 272        right binding power greater than rbp is encountered.
 273        """
 274        t = self.token
 275        self.token = self.iter.next()
 276        left = t.nud(self)
 277        while rbp < self.token.lbp:
 278            t = self.token
 279            self.token = self.iter.next()
 280            left = t.led(self, left)
 281        return left
 282
 283    def parse(self):
 284        """
 285        Parse and return the value of the expression in the text
 286        passed to the constructor. Raises a ParseError if the expression
 287        could not be parsed.
 288        """
 289        try:
 290            self.iter = self._tokenize()
 291            self.token = self.iter.next()
 292            return self.expression()
 293        except:
 294            raise ParseError("could not parse: %s; variables: %s" % (self.text, self.valuemapping))
 295
 296    __call__ = parse
 297
 298def parse(text, **values):
 299    """
 300    Parse and evaluate a boolean expression in |text|. Use |values| to look
 301    up the value of identifiers referenced in the expression. Returns the final
 302    value of the expression. A ParseError will be raised if parsing fails.
 303    """
 304    return ExpressionParser(text, values).parse()
 305
 306def normalize_path(path):
 307    """normalize a relative path"""
 308    if sys.platform.startswith('win'):
 309        return path.replace('/', os.path.sep)
 310    return path
 311
 312def denormalize_path(path):
 313    """denormalize a relative path"""
 314    if sys.platform.startswith('win'):
 315        return path.replace(os.path.sep, '/')
 316    return path
 317    
 318
 319def read_ini(fp, variables=None, default='DEFAULT',
 320             comments=';#', separators=('=', ':'),
 321             strict=True):
 322    """
 323    read an .ini file and return a list of [(section, values)]
 324    - fp : file pointer or path to read
 325    - variables : default set of variables
 326    - default : name of the section for the default section
 327    - comments : characters that if they start a line denote a comment
 328    - separators : strings that denote key, value separation in order
 329    - strict : whether to be strict about parsing
 330    """
 331
 332    if variables is None:
 333        variables = {}
 334
 335    if isinstance(fp, basestring):
 336        fp = file(fp)
 337
 338    sections = []
 339    key = value = None
 340    section_names = set([])
 341
 342    # read the lines
 343    for line in fp.readlines():
 344
 345        stripped = line.strip()
 346
 347        # ignore blank lines
 348        if not stripped:
 349            # reset key and value to avoid continuation lines
 350            key = value = None
 351            continue
 352
 353        # ignore comment lines
 354        if stripped[0] in comments:
 355            continue
 356
 357        # check for a new section
 358        if len(stripped) > 2 and stripped[0] == '[' and stripped[-1] == ']':
 359            section = stripped[1:-1].strip()
 360            key = value = None
 361
 362            # deal with DEFAULT section
 363            if section.lower() == default.lower():
 364                if strict:
 365                    assert default not in section_names
 366                section_names.add(default)
 367                current_section = variables
 368                continue
 369
 370            if strict:
 371                # make sure this section doesn't already exist
 372                assert section not in section_names
 373
 374            section_names.add(section)
 375            current_section = {}
 376            sections.append((section, current_section))
 377            continue
 378
 379        # if there aren't any sections yet, something bad happen
 380        if not section_names:
 381            raise Exception('No sections found')
 382
 383        # (key, value) pair
 384        for separator in separators:
 385            if separator in stripped:
 386                key, value = stripped.split(separator, 1)
 387                key = key.strip()
 388                value = value.strip()
 389
 390                if strict:
 391                    # make sure this key isn't already in the section or empty
 392                    assert key
 393                    if current_section is not variables:
 394                        assert key not in current_section
 395
 396                current_section[key] = value
 397                break
 398        else:
 399            # continuation line ?
 400            if line[0].isspace() and key:
 401                value = '%s%s%s' % (value, os.linesep, stripped)
 402                current_section[key] = value
 403            else:
 404                # something bad happen!
 405                raise Exception("Not sure what you're trying to do")
 406
 407    # interpret the variables
 408    def interpret_variables(global_dict, local_dict):
 409        variables = global_dict.copy()
 410        variables.update(local_dict)
 411        return variables
 412
 413    sections = [(i, interpret_variables(variables, j)) for i, j in sections]
 414    return sections
 415
 416
 417### objects for parsing manifests
 418
 419class ManifestParser(object):
 420    """read .ini manifests"""
 421
 422    ### methods for reading manifests
 423
 424    def __init__(self, manifests=(), defaults=None, strict=True):
 425        self._defaults = defaults or {}
 426        self.tests = []
 427        self.strict = strict
 428        self.rootdir = None
 429        self.relativeRoot = None
 430        if manifests:
 431            self.read(*manifests)
 432
 433    def getRelativeRoot(self, root):
 434        return root
 435
 436    def read(self, *filenames, **defaults):
 437
 438        # ensure all files exist
 439        missing = [ filename for filename in filenames
 440                    if not os.path.exists(filename) ]
 441        if missing:
 442            raise IOError('Missing files: %s' % ', '.join(missing))
 443
 444        # process each file
 445        for filename in filenames:
 446
 447            # set the per file defaults
 448            defaults = defaults.copy() or self._defaults.copy()
 449            here = os.path.dirname(os.path.abspath(filename))
 450            defaults['here'] = here
 451
 452            if self.rootdir is None:
 453                # set the root directory
 454                # == the directory of the first manifest given
 455                self.rootdir = here
 456
 457            # read the configuration
 458            sections = read_ini(fp=filename, variables=defaults, strict=self.strict)
 459
 460            # get the tests
 461            for section, data in sections:
 462
 463                # a file to include
 464                # TODO: keep track of included file structure:
 465                # self.manifests = {'manifest.ini': 'relative/path.ini'}
 466                if section.startswith('include:'):
 467                    include_file = section.split('include:', 1)[-1]
 468                    include_file = normalize_path(include_file)
 469                    if not os.path.isabs(include_file):
 470                        include_file = os.path.join(self.getRelativeRoot(here), include_file)
 471                    if not os.path.exists(include_file):
 472                        if self.strict:
 473                            raise IOError("File '%s' does not exist" % include_file)
 474                        else:
 475                            continue
 476                    include_defaults = data.copy()
 477                    self.read(include_file, **include_defaults)
 478                    continue
 479
 480                # otherwise an item
 481                test = data
 482                test['name'] = section
 483                test['manifest'] = os.path.abspath(filename)
 484
 485                # determine the path
 486                path = test.get('path', section)
 487                if '://' not in path: # don't futz with URLs
 488                    path = normalize_path(path)
 489                    if not os.path.isabs(path):
 490                        path = os.path.join(here, path)
 491                test['path'] = path
 492
 493                # append the item
 494                self.tests.append(test)
 495
 496    ### methods for querying manifests
 497
 498    def query(self, *checks, **kw):
 499        """
 500        general query function for tests
 501        - checks : callable conditions to test if the test fulfills the query
 502        """
 503        tests = kw.get('tests', None)
 504        if tests is None:
 505            tests = self.tests
 506        retval = []
 507        for test in tests:
 508            for check in checks:
 509                if not check(test):
 510                    break
 511            else:
 512                retval.append(test)
 513        return retval
 514
 515    def get(self, _key=None, inverse=False, tags=None, tests=None, **kwargs):
 516        # TODO: pass a dict instead of kwargs since you might hav
 517        # e.g. 'inverse' as a key in the dict
 518
 519        # TODO: tags should just be part of kwargs with None values
 520        # (None == any is kinda weird, but probably still better)
 521
 522        # fix up tags
 523        if tags:
 524            tags = set(tags)
 525        else:
 526            tags = set()
 527
 528        # make some check functions
 529        if inverse:
 530            has_tags = lambda test: not tags.intersection(test.keys())
 531            def dict_query(test):
 532                for key, value in kwargs.items():
 533                    if test.get(key) == value:
 534                        return False
 535                return True
 536        else:
 537            has_tags = lambda test: tags.issubset(test.keys())
 538            def dict_query(test):
 539                for key, value in kwargs.items():
 540                    if test.get(key) != value:
 541                        return False
 542                return True
 543
 544        # query the tests
 545        tests = self.query(has_tags, dict_query, tests=tests)
 546
 547        # if a key is given, return only a list of that key
 548        # useful for keys like 'name' or 'path'
 549        if _key:
 550            return [test[_key] for test in tests]
 551
 552        # return the tests
 553        return tests
 554
 555    def missing(self, tests=None):
 556        """return list of tests that do not exist on the filesystem"""
 557        if tests is None:
 558            tests = self.tests
 559        return [test for test in tests
 560                if not os.path.exists(test['path'])]
 561
 562    def manifests(self, tests=None):
 563        """
 564        return manifests in order in which they appear in the tests
 565        """
 566        if tests is None:
 567            tests = self.tests
 568        manifests = []
 569        for test in tests:
 570            manifest = test.get('manifest')
 571            if not manifest:
 572                continue
 573            if manifest not in manifests:
 574                manifests.append(manifest)
 575        return manifests
 576
 577    ### methods for outputting from manifests
 578
 579    def write(self, fp=sys.stdout, rootdir=None,
 580              global_tags=None, global_kwargs=None,
 581              local_tags=None, local_kwargs=None):
 582        """
 583        write a manifest given a query
 584        global and local options will be munged to do the query
 585        globals will be written to the top of the file
 586        locals (if given) will be written per test
 587        """
 588
 589        # root directory
 590        if rootdir is None:
 591            rootdir = self.rootdir
 592
 593        # sanitize input
 594        global_tags = global_tags or set()
 595        local_tags = local_tags or set()
 596        global_kwargs = global_kwargs or {}
 597        local_kwargs = local_kwargs or {}
 598        
 599        # create the query
 600        tags = set([])
 601        tags.update(global_tags)
 602        tags.update(local_tags)
 603        kwargs = {}
 604        kwargs.update(global_kwargs)
 605        kwargs.update(local_kwargs)
 606
 607        # get matching tests
 608        tests = self.get(tags=tags, **kwargs)
 609
 610        # print the .ini manifest
 611        if global_tags or global_kwargs:
 612            print >> fp, '[DEFAULT]'
 613            for tag in global_tags:
 614                print >> fp, '%s =' % tag
 615            for key, value in global_kwargs.items():
 616                print >> fp, '%s = %s' % (key, value)
 617            print >> fp
 618
 619        for test in tests:
 620            test = test.copy() # don't overwrite
 621
 622            path = test['name']
 623            if not os.path.isabs(path):
 624                path = denormalize_path(relpath(test['path'], self.rootdir))
 625            print >> fp, '[%s]' % path
 626          
 627            # reserved keywords:
 628            reserved = ['path', 'name', 'here', 'manifest']
 629            for key in sorted(test.keys()):
 630                if key in reserved:
 631                    continue
 632                if key in global_kwargs:
 633                    continue
 634                if key in global_tags and not test[key]:
 635                    continue
 636                print >> fp, '%s = %s' % (key, test[key])
 637            print >> fp
 638
 639    def copy(self, directory, rootdir=None, *tags, **kwargs):
 640        """
 641        copy the manifests and associated tests
 642        - directory : directory to copy to
 643        - rootdir : root directory to copy to (if not given from manifests)
 644        - tags : keywords the tests must have
 645        - kwargs : key, values the tests must match
 646        """
 647        # XXX note that copy does *not* filter the tests out of the
 648        # resulting manifest; it just stupidly copies them over.
 649        # ideally, it would reread the manifests and filter out the
 650        # tests that don't match *tags and **kwargs
 651        
 652        # destination
 653        if not os.path.exists(directory):
 654            os.path.makedirs(directory)
 655        else:
 656            # sanity check
 657            assert os.path.isdir(directory)
 658
 659        # tests to copy
 660        tests = self.get(tags=tags, **kwargs)
 661        if not tests:
 662            return # nothing to do!
 663
 664        # root directory
 665        if rootdir is None:
 666            rootdir = self.rootdir
 667
 668        # copy the manifests + tests
 669        manifests = [relpath(manifest, rootdir) for manifest in self.manifests()]
 670        for manifest in manifests:
 671            destination = os.path.join(directory, manifest)
 672            dirname = os.path.dirname(destination)
 673            if not os.path.exists(dirname):
 674                os.makedirs(dirname)
 675            else:
 676                # sanity check
 677                assert os.path.isdir(dirname)
 678            shutil.copy(os.path.join(rootdir, manifest), destination)
 679        for test in tests:
 680            if os.path.isabs(test['name']):
 681                continue
 682            source = test['path']
 683            if not os.path.exists(source):
 684                print >> sys.stderr, "Missing test: '%s' does not exist!" % source
 685                continue
 686                # TODO: should err on strict
 687            destination = os.path.join(directory, relpath(test['path'], rootdir))
 688            shutil.copy(source, destination)
 689            # TODO: ensure that all of the tests are below the from_dir
 690
 691    def update(self, from_dir, rootdir=None, *tags, **kwargs):
 692        """
 693        update the tests as listed in a manifest from a directory
 694        - from_dir : directory where the tests live
 695        - rootdir : root directory to copy to (if not given from manifests)
 696        - tags : keys the tests must have
 697        - kwargs : key, values the tests must match
 698        """
 699    
 700        # get the tests
 701        tests = self.get(tags=tags, **kwargs)
 702
 703        # get the root directory
 704        if not rootdir:
 705            rootdir = self.rootdir
 706
 707        # copy them!
 708        for test in tests:
 709            if not os.path.isabs(test['name']):
 710                _relpath = relpath(test['path'], rootdir)
 711                source = os.path.join(from_dir, _relpath)
 712                if not os.path.exists(source):
 713                    # TODO err on strict
 714                    print >> sys.stderr, "Missing test: '%s'; skipping" % test['name']
 715                    continue
 716                destination = os.path.join(rootdir, _relpath)
 717                shutil.copy(source, destination)
 718
 719
 720class TestManifest(ManifestParser):
 721    """
 722    apply logic to manifests;  this is your integration layer :)
 723    specific harnesses may subclass from this if they need more logic
 724    """
 725
 726    def filter(self, values, tests):
 727        """
 728        filter on a specific list tag, e.g.:
 729        run-if.os = win linux
 730        skip-if.os = mac
 731        """
 732
 733        # tags:
 734        run_tag = 'run-if'
 735        skip_tag = 'skip-if'
 736        fail_tag = 'fail-if'
 737
 738        # loop over test
 739        for test in tests:
 740            reason = None # reason to disable
 741            
 742            # tagged-values to run
 743            if run_tag in test:
 744                condition = test[run_tag]
 745                if not parse(condition, **values):
 746                    reason = '%s: %s' % (run_tag, condition)
 747
 748            # tagged-values to skip
 749            if skip_tag in test:
 750                condition = test[skip_tag]
 751                if parse(condition, **values):
 752                    reason = '%s: %s' % (skip_tag, condition)
 753
 754            # mark test as disabled if there's a reason
 755            if reason:
 756                test.setdefault('disabled', reason)        
 757
 758            # mark test as a fail if so indicated
 759            if fail_tag in test:
 760                condition = test[fail_tag]
 761                if parse(condition, **values):
 762                    test['expected'] = 'fail'
 763
 764    def active_tests(self, exists=True, disabled=True, **values):
 765        """
 766        - exists : return only existing tests
 767        - disabled : whether to return disabled tests
 768        - tags : keys and values to filter on (e.g. `os = linux mac`)
 769        """
 770
 771        tests = [i.copy() for i in self.tests] # shallow copy
 772
 773        # mark all tests as passing unless indicated otherwise
 774        for test in tests:
 775            test['expected'] = test.get('expected', 'pass')
 776        
 777        # ignore tests that do not exist
 778        if exists:
 779            tests = [test for test in tests if os.path.exists(test['path'])]
 780
 781        # filter by tags
 782        self.filter(values, tests)
 783
 784        # ignore disabled tests if specified
 785        if not disabled:
 786            tests = [test for test in tests
 787                     if not 'disabled' in test]
 788
 789        # return active tests
 790        return tests
 791
 792    def test_paths(self):
 793        return [test['path'] for test in self.active_tests()]
 794
 795
 796### utility function(s); probably belongs elsewhere
 797
 798def convert(directories, pattern=None, ignore=(), write=None):
 799    """
 800    convert directories to a simple manifest
 801    """
 802
 803    retval = []
 804    include = []
 805    for directory in directories:
 806        for dirpath, dirnames, filenames in os.walk(directory):
 807
 808            # filter out directory names
 809            dirnames = [ i for i in dirnames if i not in ignore ]
 810            dirnames.sort()
 811
 812            # reference only the subdirectory
 813            _dirpath = dirpath
 814            dirpath = dirpath.split(directory, 1)[-1].strip(os.path.sep)
 815
 816            if dirpath.split(os.path.sep)[0] in ignore:
 817                continue
 818
 819            # filter by glob
 820            if pattern:
 821                filenames = [filename for filename in filenames
 822                             if fnmatch(filename, pattern)]
 823
 824            filenames.sort()
 825
 826            # write a manifest for each directory
 827            if write and (dirnames or filenames):
 828                manifest = file(os.path.join(_dirpath, write), 'w')
 829                for dirname in dirnames:
 830                    print >> manifest, '[include:%s]' % os.path.join(dirname, write)
 831                for filename in filenames:
 832                    print >> manifest, '[%s]' % filename
 833                manifest.close()
 834
 835            # add to the list
 836            retval.extend([denormalize_path(os.path.join(dirpath, filename))
 837                           for filename in filenames])
 838
 839    if write:
 840        return # the manifests have already been written!
 841  
 842    retval.sort()
 843    retval = ['[%s]' % filename for filename in retval]
 844    return '\n'.join(retval)
 845
 846### command line attributes
 847
 848class ParserError(Exception):
 849  """error for exceptions while parsing the command line"""
 850
 851def parse_args(_args):
 852    """
 853    parse and return:
 854    --keys=value (or --key value)
 855    -tags
 856    args
 857    """
 858
 859    # return values
 860    _dict = {}
 861    tags = []
 862    args = []
 863
 864    # parse the arguments
 865    key = None
 866    for arg in _args:
 867        if arg.startswith('---'):
 868            raise ParserError("arguments should start with '-' or '--' only")
 869        elif arg.startswith('--'):
 870            if key:
 871                raise ParserError("Key %s still open" % key)
 872            key = arg[2:]
 873            if '=' in key:
 874                key, value = key.split('=', 1)
 875                _dict[key] = value
 876                key = None
 877                continue
 878        elif arg.startswith('-'):
 879            if key:
 880                raise ParserError("Key %s still open" % key)
 881            tags.append(arg[1:])
 882            continue
 883        else:
 884            if key:
 885                _dict[key] = arg
 886                continue
 887            args.append(arg)
 888
 889    # return values
 890    return (_dict, tags, args)
 891
 892
 893### classes for subcommands
 894
 895class CLICommand(object):
 896    usage = '%prog [options] command'
 897    def __init__(self, parser):
 898      self._parser = parser # master parser
 899    def parser(self):
 900      return OptionParser(usage=self.usage, description=self.__doc__,
 901                          add_help_option=False)
 902
 903class Copy(CLICommand):
 904    usage = '%prog [options] copy manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
 905    def __call__(self, options, args):
 906      # parse the arguments
 907      try:
 908        kwargs, tags, args = parse_args(args)
 909      except ParserError, e:
 910        self._parser.error(e.message)
 911
 912      # make sure we have some manifests, otherwise it will
 913      # be quite boring
 914      if not len(args) == 2:
 915        HelpCLI(self._parser)(options, ['copy'])
 916        return
 917
 918      # read the manifests
 919      # TODO: should probably ensure these exist here
 920      manifests = ManifestParser()
 921      manifests.read(args[0])
 922
 923      # print the resultant query
 924      manifests.copy(args[1], None, *tags, **kwargs)
 925
 926
 927class CreateCLI(CLICommand):
 928    """
 929    create a manifest from a list of directories
 930    """
 931    usage = '%prog [options] create directory <directory> <...>'
 932
 933    def parser(self):
 934        parser = CLICommand.parser(self)
 935        parser.add_option('-p', '--pattern', dest='pattern',
 936                          help="glob pattern for files")
 937        parser.add_option('-i', '--ignore', dest='ignore',
 938                          default=[], action='append',
 939                          help='directories to ignore')
 940        parser.add_option('-w', '--in-place', dest='in_place',
 941                          help='Write .ini files in place; filename to write to')
 942        return parser
 943
 944    def __call__(self, _options, args):
 945        parser = self.parser()
 946        options, args = parser.parse_args(args)
 947
 948        # need some directories
 949        if not len(args):
 950            parser.print_usage()
 951            return
 952
 953        # add the directories to the manifest
 954        for arg in args:
 955            assert os.path.exists(arg)
 956            assert os.path.isdir(arg)
 957            manifest = convert(args, pattern=options.pattern, ignore=options.ignore,
 958                               write=options.in_place)
 959        if manifest:
 960            print manifest
 961
 962
 963class WriteCLI(CLICommand):
 964    """
 965    write a manifest based on a query
 966    """
 967    usage = '%prog [options] write manifest <manifest> -tag1 -tag2 --key1=value1 --key2=value2 ...'
 968    def __call__(self, options, args):
 969
 970        # parse the arguments
 971        try:
 972            kwargs, tags, args = parse_args(args)
 973        except ParserError, e:
 974            self._parser.error(e.message)
 975
 976        # make sure we have some manifests, otherwise it will
 977        # be quite boring
 978        if not args:
 979            HelpCLI(self._parser)(options, ['write'])
 980            return
 981
 982        # read the manifests
 983        # TODO: should probably ensure these exist here
 984        manifests = ManifestParser()
 985        manifests.read(*args)
 986
 987        # print the resultant query
 988        manifests.write(global_tags=tags, global_kwargs=kwargs)
 989      
 990
 991class HelpCLI(CLICommand):
 992    """
 993    get help on a command
 994    """
 995    usage = '%prog [options] help [command]'
 996
 997    def __call__(self, options, args):
 998        if len(args) == 1 and args[0] in commands:
 999            commands[args[0]](self._parser).parser().print_help()
1000        else:
1001            self._parser.print_help()
1002            print '\nCommands:'
1003            for command in sorted(commands):
1004                print '  %s : %s' % (command, commands[command].__doc__.strip())
1005
1006class SetupCLI(CLICommand):
1007    """
1008    setup using setuptools
1009    """
1010    # use setup.py from the repo when you want to distribute to python!
1011    # otherwise setuptools will complain that it can't find setup.py
1012    # and result in a useless package
1013    
1014    usage = '%prog [options] setup [setuptools options]'
1015    
1016    def __call__(self, options, args):
1017        sys.argv = [sys.argv[0]] + args
1018        assert setup is not None, "You must have setuptools installed to use SetupCLI"
1019        here = os.path.dirname(os.path.abspath(__file__))
1020        try:
1021            filename = os.path.join(here, 'README.txt')
1022            description = file(filename).read()
1023        except:    
1024            description = ''
1025        os.chdir(here)
1026
1027        setup(name='ManifestDestiny',
1028              version=version,
1029              description="Universal manifests for Mozilla test harnesses",
1030              long_description=description,
1031              classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
1032              keywords='mozilla manifests',
1033              author='Jeff Hammel',
1034              author_email='jhammel@mozilla.com',
1035              url='https://wiki.mozilla.org/Auto-tools/Projects/ManifestDestiny',
1036              license='MPL',
1037              zip_safe=False,
1038              py_modules=['manifestparser'],
1039              install_requires=[
1040                  # -*- Extra requirements: -*-
1041                  ],
1042              entry_points="""
1043              [console_scripts]
1044              manifestparser = manifestparser:main
1045              """,
1046              )
1047
1048
1049class UpdateCLI(CLICommand):
1050    """
1051    update the tests as listed in a manifest from a directory
1052    """
1053    usage = '%prog [options] update manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
1054
1055    def __call__(self, options, args):
1056        # parse the arguments
1057        try:
1058            kwargs, tags, args = parse_args(args)
1059        except ParserError, e:
1060            self._parser.error(e.message)
1061
1062        # make sure we have some manifests, otherwise it will
1063        # be quite boring
1064        if not len(args) == 2:
1065            HelpCLI(self._parser)(options, ['update'])
1066            return
1067
1068        # read the manifests
1069        # TODO: should probably ensure these exist here
1070        manifests = ManifestParser()
1071        manifests.read(args[0])
1072
1073        # print the resultant query
1074        manifests.update(args[1], None, *tags, **kwargs)
1075
1076
1077# command -> class mapping
1078commands = { 'create': CreateCLI,
1079             'help': HelpCLI,
1080             'update': UpdateCLI,
1081             'write': WriteCLI }
1082if setup is not None:
1083    commands['setup'] = SetupCLI
1084
1085def main(args=sys.argv[1:]):
1086    """console_script entry point"""
1087
1088    # set up an option parser
1089    usage = '%prog [options] [command] ...'
1090    description = __doc__
1091    parser = OptionParser(usage=usage, description=description)
1092    parser.add_option('-s', '--strict', dest='strict',
1093                      action='store_true', default=False,
1094                      help='adhere strictly to errors')
1095    parser.disable_interspersed_args()
1096
1097    options, args = parser.parse_args(args)
1098
1099    if not args:
1100        HelpCLI(parser)(options, args)
1101        parser.exit()
1102
1103    # get the command
1104    command = args[0]
1105    if command not in commands:
1106        parser.error("Command must be one of %s (you gave '%s')" % (', '.join(sorted(commands.keys())), command))
1107
1108    handler = commands[command](parser)
1109    handler(options, args[1:])
1110
1111if __name__ == '__main__':
1112    main()