/build/manifestparser.py

http://github.com/zpao/v8monkey · Python · 1112 lines · 623 code · 185 blank · 304 comment · 170 complexity · ae6bfbebc45aa5a84ac4c186e1cd34a9 MD5 · raw file

  1. #!/usr/bin/env python
  2. # ***** BEGIN LICENSE BLOCK *****
  3. # Version: MPL 1.1/GPL 2.0/LGPL 2.1
  4. #
  5. # The contents of this file are subject to the Mozilla Public License Version
  6. # 1.1 (the "License"); you may not use this file except in compliance with
  7. # the License. You may obtain a copy of the License at
  8. # http://www.mozilla.org/MPL/
  9. #
  10. # Software distributed under the License is distributed on an "AS IS" basis,
  11. # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  12. # for the specific language governing rights and limitations under the
  13. # License.
  14. #
  15. # The Original Code is mozilla.org code.
  16. #
  17. # The Initial Developer of the Original Code is
  18. # Mozilla.org.
  19. # Portions created by the Initial Developer are Copyright (C) 2010
  20. # the Initial Developer. All Rights Reserved.
  21. #
  22. # Contributor(s):
  23. # Jeff Hammel <jhammel@mozilla.com> (Original author)
  24. #
  25. # Alternatively, the contents of this file may be used under the terms of
  26. # either of the GNU General Public License Version 2 or later (the "GPL"),
  27. # or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  28. # in which case the provisions of the GPL or the LGPL are applicable instead
  29. # of those above. If you wish to allow use of your version of this file only
  30. # under the terms of either the GPL or the LGPL, and not to allow others to
  31. # use your version of this file under the terms of the MPL, indicate your
  32. # decision by deleting the provisions above and replace them with the notice
  33. # and other provisions required by the GPL or the LGPL. If you do not delete
  34. # the provisions above, a recipient may use your version of this file under
  35. # the terms of any one of the MPL, the GPL or the LGPL.
  36. #
  37. # ***** END LICENSE BLOCK *****
  38. """
  39. Mozilla universal manifest parser
  40. """
  41. # this file lives at
  42. # http://hg.mozilla.org/automation/ManifestDestiny/raw-file/tip/manifestparser.py
  43. __all__ = ['read_ini', # .ini reader
  44. 'ManifestParser', 'TestManifest', 'convert', # manifest handling
  45. 'parse', 'ParseError', 'ExpressionParser'] # conditional expression parser
  46. import os
  47. import re
  48. import shutil
  49. import sys
  50. from fnmatch import fnmatch
  51. from optparse import OptionParser
  52. version = '0.5.3' # package version
  53. try:
  54. from setuptools import setup
  55. except:
  56. setup = None
  57. # we need relpath, but it is introduced in python 2.6
  58. # http://docs.python.org/library/os.path.html
  59. try:
  60. relpath = os.path.relpath
  61. except AttributeError:
  62. def relpath(path, start):
  63. """
  64. Return a relative version of a path
  65. from /usr/lib/python2.6/posixpath.py
  66. """
  67. if not path:
  68. raise ValueError("no path specified")
  69. start_list = os.path.abspath(start).split(os.path.sep)
  70. path_list = os.path.abspath(path).split(os.path.sep)
  71. # Work out how much of the filepath is shared by start and path.
  72. i = len(os.path.commonprefix([start_list, path_list]))
  73. rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:]
  74. if not rel_list:
  75. return start
  76. return os.path.join(*rel_list)
  77. # expr.py
  78. # from:
  79. # http://k0s.org/mozilla/hg/expressionparser
  80. # http://hg.mozilla.org/users/tmielczarek_mozilla.com/expressionparser
  81. # Implements a top-down parser/evaluator for simple boolean expressions.
  82. # ideas taken from http://effbot.org/zone/simple-top-down-parsing.htm
  83. #
  84. # Rough grammar:
  85. # expr := literal
  86. # | '(' expr ')'
  87. # | expr '&&' expr
  88. # | expr '||' expr
  89. # | expr '==' expr
  90. # | expr '!=' expr
  91. # literal := BOOL
  92. # | INT
  93. # | STRING
  94. # | IDENT
  95. # BOOL := true|false
  96. # INT := [0-9]+
  97. # STRING := "[^"]*"
  98. # IDENT := [A-Za-z_]\w*
  99. # Identifiers take their values from a mapping dictionary passed as the second
  100. # argument.
  101. # Glossary (see above URL for details):
  102. # - nud: null denotation
  103. # - led: left detonation
  104. # - lbp: left binding power
  105. # - rbp: right binding power
  106. class ident_token(object):
  107. def __init__(self, value):
  108. self.value = value
  109. def nud(self, parser):
  110. # identifiers take their value from the value mappings passed
  111. # to the parser
  112. return parser.value(self.value)
  113. class literal_token(object):
  114. def __init__(self, value):
  115. self.value = value
  116. def nud(self, parser):
  117. return self.value
  118. class eq_op_token(object):
  119. "=="
  120. def led(self, parser, left):
  121. return left == parser.expression(self.lbp)
  122. class neq_op_token(object):
  123. "!="
  124. def led(self, parser, left):
  125. return left != parser.expression(self.lbp)
  126. class not_op_token(object):
  127. "!"
  128. def nud(self, parser):
  129. return not parser.expression()
  130. class and_op_token(object):
  131. "&&"
  132. def led(self, parser, left):
  133. right = parser.expression(self.lbp)
  134. return left and right
  135. class or_op_token(object):
  136. "||"
  137. def led(self, parser, left):
  138. right = parser.expression(self.lbp)
  139. return left or right
  140. class lparen_token(object):
  141. "("
  142. def nud(self, parser):
  143. expr = parser.expression()
  144. parser.advance(rparen_token)
  145. return expr
  146. class rparen_token(object):
  147. ")"
  148. class end_token(object):
  149. """always ends parsing"""
  150. ### derived literal tokens
  151. class bool_token(literal_token):
  152. def __init__(self, value):
  153. value = {'true':True, 'false':False}[value]
  154. literal_token.__init__(self, value)
  155. class int_token(literal_token):
  156. def __init__(self, value):
  157. literal_token.__init__(self, int(value))
  158. class string_token(literal_token):
  159. def __init__(self, value):
  160. literal_token.__init__(self, value[1:-1])
  161. precedence = [(end_token, rparen_token),
  162. (or_op_token,),
  163. (and_op_token,),
  164. (eq_op_token, neq_op_token),
  165. (lparen_token,),
  166. ]
  167. for index, rank in enumerate(precedence):
  168. for token in rank:
  169. token.lbp = index # lbp = lowest left binding power
  170. class ParseError(Exception):
  171. """errror parsing conditional expression"""
  172. class ExpressionParser(object):
  173. def __init__(self, text, valuemapping, strict=False):
  174. """
  175. Initialize the parser with input |text|, and |valuemapping| as
  176. a dict mapping identifier names to values.
  177. """
  178. self.text = text
  179. self.valuemapping = valuemapping
  180. self.strict = strict
  181. def _tokenize(self):
  182. """
  183. Lex the input text into tokens and yield them in sequence.
  184. """
  185. # scanner callbacks
  186. def bool_(scanner, t): return bool_token(t)
  187. def identifier(scanner, t): return ident_token(t)
  188. def integer(scanner, t): return int_token(t)
  189. def eq(scanner, t): return eq_op_token()
  190. def neq(scanner, t): return neq_op_token()
  191. def or_(scanner, t): return or_op_token()
  192. def and_(scanner, t): return and_op_token()
  193. def lparen(scanner, t): return lparen_token()
  194. def rparen(scanner, t): return rparen_token()
  195. def string_(scanner, t): return string_token(t)
  196. def not_(scanner, t): return not_op_token()
  197. scanner = re.Scanner([
  198. (r"true|false", bool_),
  199. (r"[a-zA-Z_]\w*", identifier),
  200. (r"[0-9]+", integer),
  201. (r'("[^"]*")|(\'[^\']*\')', string_),
  202. (r"==", eq),
  203. (r"!=", neq),
  204. (r"\|\|", or_),
  205. (r"!", not_),
  206. (r"&&", and_),
  207. (r"\(", lparen),
  208. (r"\)", rparen),
  209. (r"\s+", None), # skip whitespace
  210. ])
  211. tokens, remainder = scanner.scan(self.text)
  212. for t in tokens:
  213. yield t
  214. yield end_token()
  215. def value(self, ident):
  216. """
  217. Look up the value of |ident| in the value mapping passed in the
  218. constructor.
  219. """
  220. if self.strict:
  221. return self.valuemapping[ident]
  222. else:
  223. return self.valuemapping.get(ident, None)
  224. def advance(self, expected):
  225. """
  226. Assert that the next token is an instance of |expected|, and advance
  227. to the next token.
  228. """
  229. if not isinstance(self.token, expected):
  230. raise Exception, "Unexpected token!"
  231. self.token = self.iter.next()
  232. def expression(self, rbp=0):
  233. """
  234. Parse and return the value of an expression until a token with
  235. right binding power greater than rbp is encountered.
  236. """
  237. t = self.token
  238. self.token = self.iter.next()
  239. left = t.nud(self)
  240. while rbp < self.token.lbp:
  241. t = self.token
  242. self.token = self.iter.next()
  243. left = t.led(self, left)
  244. return left
  245. def parse(self):
  246. """
  247. Parse and return the value of the expression in the text
  248. passed to the constructor. Raises a ParseError if the expression
  249. could not be parsed.
  250. """
  251. try:
  252. self.iter = self._tokenize()
  253. self.token = self.iter.next()
  254. return self.expression()
  255. except:
  256. raise ParseError("could not parse: %s; variables: %s" % (self.text, self.valuemapping))
  257. __call__ = parse
  258. def parse(text, **values):
  259. """
  260. Parse and evaluate a boolean expression in |text|. Use |values| to look
  261. up the value of identifiers referenced in the expression. Returns the final
  262. value of the expression. A ParseError will be raised if parsing fails.
  263. """
  264. return ExpressionParser(text, values).parse()
  265. def normalize_path(path):
  266. """normalize a relative path"""
  267. if sys.platform.startswith('win'):
  268. return path.replace('/', os.path.sep)
  269. return path
  270. def denormalize_path(path):
  271. """denormalize a relative path"""
  272. if sys.platform.startswith('win'):
  273. return path.replace(os.path.sep, '/')
  274. return path
  275. def read_ini(fp, variables=None, default='DEFAULT',
  276. comments=';#', separators=('=', ':'),
  277. strict=True):
  278. """
  279. read an .ini file and return a list of [(section, values)]
  280. - fp : file pointer or path to read
  281. - variables : default set of variables
  282. - default : name of the section for the default section
  283. - comments : characters that if they start a line denote a comment
  284. - separators : strings that denote key, value separation in order
  285. - strict : whether to be strict about parsing
  286. """
  287. if variables is None:
  288. variables = {}
  289. if isinstance(fp, basestring):
  290. fp = file(fp)
  291. sections = []
  292. key = value = None
  293. section_names = set([])
  294. # read the lines
  295. for line in fp.readlines():
  296. stripped = line.strip()
  297. # ignore blank lines
  298. if not stripped:
  299. # reset key and value to avoid continuation lines
  300. key = value = None
  301. continue
  302. # ignore comment lines
  303. if stripped[0] in comments:
  304. continue
  305. # check for a new section
  306. if len(stripped) > 2 and stripped[0] == '[' and stripped[-1] == ']':
  307. section = stripped[1:-1].strip()
  308. key = value = None
  309. # deal with DEFAULT section
  310. if section.lower() == default.lower():
  311. if strict:
  312. assert default not in section_names
  313. section_names.add(default)
  314. current_section = variables
  315. continue
  316. if strict:
  317. # make sure this section doesn't already exist
  318. assert section not in section_names
  319. section_names.add(section)
  320. current_section = {}
  321. sections.append((section, current_section))
  322. continue
  323. # if there aren't any sections yet, something bad happen
  324. if not section_names:
  325. raise Exception('No sections found')
  326. # (key, value) pair
  327. for separator in separators:
  328. if separator in stripped:
  329. key, value = stripped.split(separator, 1)
  330. key = key.strip()
  331. value = value.strip()
  332. if strict:
  333. # make sure this key isn't already in the section or empty
  334. assert key
  335. if current_section is not variables:
  336. assert key not in current_section
  337. current_section[key] = value
  338. break
  339. else:
  340. # continuation line ?
  341. if line[0].isspace() and key:
  342. value = '%s%s%s' % (value, os.linesep, stripped)
  343. current_section[key] = value
  344. else:
  345. # something bad happen!
  346. raise Exception("Not sure what you're trying to do")
  347. # interpret the variables
  348. def interpret_variables(global_dict, local_dict):
  349. variables = global_dict.copy()
  350. variables.update(local_dict)
  351. return variables
  352. sections = [(i, interpret_variables(variables, j)) for i, j in sections]
  353. return sections
  354. ### objects for parsing manifests
  355. class ManifestParser(object):
  356. """read .ini manifests"""
  357. ### methods for reading manifests
  358. def __init__(self, manifests=(), defaults=None, strict=True):
  359. self._defaults = defaults or {}
  360. self.tests = []
  361. self.strict = strict
  362. self.rootdir = None
  363. self.relativeRoot = None
  364. if manifests:
  365. self.read(*manifests)
  366. def getRelativeRoot(self, root):
  367. return root
  368. def read(self, *filenames, **defaults):
  369. # ensure all files exist
  370. missing = [ filename for filename in filenames
  371. if not os.path.exists(filename) ]
  372. if missing:
  373. raise IOError('Missing files: %s' % ', '.join(missing))
  374. # process each file
  375. for filename in filenames:
  376. # set the per file defaults
  377. defaults = defaults.copy() or self._defaults.copy()
  378. here = os.path.dirname(os.path.abspath(filename))
  379. defaults['here'] = here
  380. if self.rootdir is None:
  381. # set the root directory
  382. # == the directory of the first manifest given
  383. self.rootdir = here
  384. # read the configuration
  385. sections = read_ini(fp=filename, variables=defaults, strict=self.strict)
  386. # get the tests
  387. for section, data in sections:
  388. # a file to include
  389. # TODO: keep track of included file structure:
  390. # self.manifests = {'manifest.ini': 'relative/path.ini'}
  391. if section.startswith('include:'):
  392. include_file = section.split('include:', 1)[-1]
  393. include_file = normalize_path(include_file)
  394. if not os.path.isabs(include_file):
  395. include_file = os.path.join(self.getRelativeRoot(here), include_file)
  396. if not os.path.exists(include_file):
  397. if self.strict:
  398. raise IOError("File '%s' does not exist" % include_file)
  399. else:
  400. continue
  401. include_defaults = data.copy()
  402. self.read(include_file, **include_defaults)
  403. continue
  404. # otherwise an item
  405. test = data
  406. test['name'] = section
  407. test['manifest'] = os.path.abspath(filename)
  408. # determine the path
  409. path = test.get('path', section)
  410. if '://' not in path: # don't futz with URLs
  411. path = normalize_path(path)
  412. if not os.path.isabs(path):
  413. path = os.path.join(here, path)
  414. test['path'] = path
  415. # append the item
  416. self.tests.append(test)
  417. ### methods for querying manifests
  418. def query(self, *checks, **kw):
  419. """
  420. general query function for tests
  421. - checks : callable conditions to test if the test fulfills the query
  422. """
  423. tests = kw.get('tests', None)
  424. if tests is None:
  425. tests = self.tests
  426. retval = []
  427. for test in tests:
  428. for check in checks:
  429. if not check(test):
  430. break
  431. else:
  432. retval.append(test)
  433. return retval
  434. def get(self, _key=None, inverse=False, tags=None, tests=None, **kwargs):
  435. # TODO: pass a dict instead of kwargs since you might hav
  436. # e.g. 'inverse' as a key in the dict
  437. # TODO: tags should just be part of kwargs with None values
  438. # (None == any is kinda weird, but probably still better)
  439. # fix up tags
  440. if tags:
  441. tags = set(tags)
  442. else:
  443. tags = set()
  444. # make some check functions
  445. if inverse:
  446. has_tags = lambda test: not tags.intersection(test.keys())
  447. def dict_query(test):
  448. for key, value in kwargs.items():
  449. if test.get(key) == value:
  450. return False
  451. return True
  452. else:
  453. has_tags = lambda test: tags.issubset(test.keys())
  454. def dict_query(test):
  455. for key, value in kwargs.items():
  456. if test.get(key) != value:
  457. return False
  458. return True
  459. # query the tests
  460. tests = self.query(has_tags, dict_query, tests=tests)
  461. # if a key is given, return only a list of that key
  462. # useful for keys like 'name' or 'path'
  463. if _key:
  464. return [test[_key] for test in tests]
  465. # return the tests
  466. return tests
  467. def missing(self, tests=None):
  468. """return list of tests that do not exist on the filesystem"""
  469. if tests is None:
  470. tests = self.tests
  471. return [test for test in tests
  472. if not os.path.exists(test['path'])]
  473. def manifests(self, tests=None):
  474. """
  475. return manifests in order in which they appear in the tests
  476. """
  477. if tests is None:
  478. tests = self.tests
  479. manifests = []
  480. for test in tests:
  481. manifest = test.get('manifest')
  482. if not manifest:
  483. continue
  484. if manifest not in manifests:
  485. manifests.append(manifest)
  486. return manifests
  487. ### methods for outputting from manifests
  488. def write(self, fp=sys.stdout, rootdir=None,
  489. global_tags=None, global_kwargs=None,
  490. local_tags=None, local_kwargs=None):
  491. """
  492. write a manifest given a query
  493. global and local options will be munged to do the query
  494. globals will be written to the top of the file
  495. locals (if given) will be written per test
  496. """
  497. # root directory
  498. if rootdir is None:
  499. rootdir = self.rootdir
  500. # sanitize input
  501. global_tags = global_tags or set()
  502. local_tags = local_tags or set()
  503. global_kwargs = global_kwargs or {}
  504. local_kwargs = local_kwargs or {}
  505. # create the query
  506. tags = set([])
  507. tags.update(global_tags)
  508. tags.update(local_tags)
  509. kwargs = {}
  510. kwargs.update(global_kwargs)
  511. kwargs.update(local_kwargs)
  512. # get matching tests
  513. tests = self.get(tags=tags, **kwargs)
  514. # print the .ini manifest
  515. if global_tags or global_kwargs:
  516. print >> fp, '[DEFAULT]'
  517. for tag in global_tags:
  518. print >> fp, '%s =' % tag
  519. for key, value in global_kwargs.items():
  520. print >> fp, '%s = %s' % (key, value)
  521. print >> fp
  522. for test in tests:
  523. test = test.copy() # don't overwrite
  524. path = test['name']
  525. if not os.path.isabs(path):
  526. path = denormalize_path(relpath(test['path'], self.rootdir))
  527. print >> fp, '[%s]' % path
  528. # reserved keywords:
  529. reserved = ['path', 'name', 'here', 'manifest']
  530. for key in sorted(test.keys()):
  531. if key in reserved:
  532. continue
  533. if key in global_kwargs:
  534. continue
  535. if key in global_tags and not test[key]:
  536. continue
  537. print >> fp, '%s = %s' % (key, test[key])
  538. print >> fp
  539. def copy(self, directory, rootdir=None, *tags, **kwargs):
  540. """
  541. copy the manifests and associated tests
  542. - directory : directory to copy to
  543. - rootdir : root directory to copy to (if not given from manifests)
  544. - tags : keywords the tests must have
  545. - kwargs : key, values the tests must match
  546. """
  547. # XXX note that copy does *not* filter the tests out of the
  548. # resulting manifest; it just stupidly copies them over.
  549. # ideally, it would reread the manifests and filter out the
  550. # tests that don't match *tags and **kwargs
  551. # destination
  552. if not os.path.exists(directory):
  553. os.path.makedirs(directory)
  554. else:
  555. # sanity check
  556. assert os.path.isdir(directory)
  557. # tests to copy
  558. tests = self.get(tags=tags, **kwargs)
  559. if not tests:
  560. return # nothing to do!
  561. # root directory
  562. if rootdir is None:
  563. rootdir = self.rootdir
  564. # copy the manifests + tests
  565. manifests = [relpath(manifest, rootdir) for manifest in self.manifests()]
  566. for manifest in manifests:
  567. destination = os.path.join(directory, manifest)
  568. dirname = os.path.dirname(destination)
  569. if not os.path.exists(dirname):
  570. os.makedirs(dirname)
  571. else:
  572. # sanity check
  573. assert os.path.isdir(dirname)
  574. shutil.copy(os.path.join(rootdir, manifest), destination)
  575. for test in tests:
  576. if os.path.isabs(test['name']):
  577. continue
  578. source = test['path']
  579. if not os.path.exists(source):
  580. print >> sys.stderr, "Missing test: '%s' does not exist!" % source
  581. continue
  582. # TODO: should err on strict
  583. destination = os.path.join(directory, relpath(test['path'], rootdir))
  584. shutil.copy(source, destination)
  585. # TODO: ensure that all of the tests are below the from_dir
  586. def update(self, from_dir, rootdir=None, *tags, **kwargs):
  587. """
  588. update the tests as listed in a manifest from a directory
  589. - from_dir : directory where the tests live
  590. - rootdir : root directory to copy to (if not given from manifests)
  591. - tags : keys the tests must have
  592. - kwargs : key, values the tests must match
  593. """
  594. # get the tests
  595. tests = self.get(tags=tags, **kwargs)
  596. # get the root directory
  597. if not rootdir:
  598. rootdir = self.rootdir
  599. # copy them!
  600. for test in tests:
  601. if not os.path.isabs(test['name']):
  602. _relpath = relpath(test['path'], rootdir)
  603. source = os.path.join(from_dir, _relpath)
  604. if not os.path.exists(source):
  605. # TODO err on strict
  606. print >> sys.stderr, "Missing test: '%s'; skipping" % test['name']
  607. continue
  608. destination = os.path.join(rootdir, _relpath)
  609. shutil.copy(source, destination)
  610. class TestManifest(ManifestParser):
  611. """
  612. apply logic to manifests; this is your integration layer :)
  613. specific harnesses may subclass from this if they need more logic
  614. """
  615. def filter(self, values, tests):
  616. """
  617. filter on a specific list tag, e.g.:
  618. run-if.os = win linux
  619. skip-if.os = mac
  620. """
  621. # tags:
  622. run_tag = 'run-if'
  623. skip_tag = 'skip-if'
  624. fail_tag = 'fail-if'
  625. # loop over test
  626. for test in tests:
  627. reason = None # reason to disable
  628. # tagged-values to run
  629. if run_tag in test:
  630. condition = test[run_tag]
  631. if not parse(condition, **values):
  632. reason = '%s: %s' % (run_tag, condition)
  633. # tagged-values to skip
  634. if skip_tag in test:
  635. condition = test[skip_tag]
  636. if parse(condition, **values):
  637. reason = '%s: %s' % (skip_tag, condition)
  638. # mark test as disabled if there's a reason
  639. if reason:
  640. test.setdefault('disabled', reason)
  641. # mark test as a fail if so indicated
  642. if fail_tag in test:
  643. condition = test[fail_tag]
  644. if parse(condition, **values):
  645. test['expected'] = 'fail'
  646. def active_tests(self, exists=True, disabled=True, **values):
  647. """
  648. - exists : return only existing tests
  649. - disabled : whether to return disabled tests
  650. - tags : keys and values to filter on (e.g. `os = linux mac`)
  651. """
  652. tests = [i.copy() for i in self.tests] # shallow copy
  653. # mark all tests as passing unless indicated otherwise
  654. for test in tests:
  655. test['expected'] = test.get('expected', 'pass')
  656. # ignore tests that do not exist
  657. if exists:
  658. tests = [test for test in tests if os.path.exists(test['path'])]
  659. # filter by tags
  660. self.filter(values, tests)
  661. # ignore disabled tests if specified
  662. if not disabled:
  663. tests = [test for test in tests
  664. if not 'disabled' in test]
  665. # return active tests
  666. return tests
  667. def test_paths(self):
  668. return [test['path'] for test in self.active_tests()]
  669. ### utility function(s); probably belongs elsewhere
  670. def convert(directories, pattern=None, ignore=(), write=None):
  671. """
  672. convert directories to a simple manifest
  673. """
  674. retval = []
  675. include = []
  676. for directory in directories:
  677. for dirpath, dirnames, filenames in os.walk(directory):
  678. # filter out directory names
  679. dirnames = [ i for i in dirnames if i not in ignore ]
  680. dirnames.sort()
  681. # reference only the subdirectory
  682. _dirpath = dirpath
  683. dirpath = dirpath.split(directory, 1)[-1].strip(os.path.sep)
  684. if dirpath.split(os.path.sep)[0] in ignore:
  685. continue
  686. # filter by glob
  687. if pattern:
  688. filenames = [filename for filename in filenames
  689. if fnmatch(filename, pattern)]
  690. filenames.sort()
  691. # write a manifest for each directory
  692. if write and (dirnames or filenames):
  693. manifest = file(os.path.join(_dirpath, write), 'w')
  694. for dirname in dirnames:
  695. print >> manifest, '[include:%s]' % os.path.join(dirname, write)
  696. for filename in filenames:
  697. print >> manifest, '[%s]' % filename
  698. manifest.close()
  699. # add to the list
  700. retval.extend([denormalize_path(os.path.join(dirpath, filename))
  701. for filename in filenames])
  702. if write:
  703. return # the manifests have already been written!
  704. retval.sort()
  705. retval = ['[%s]' % filename for filename in retval]
  706. return '\n'.join(retval)
  707. ### command line attributes
  708. class ParserError(Exception):
  709. """error for exceptions while parsing the command line"""
  710. def parse_args(_args):
  711. """
  712. parse and return:
  713. --keys=value (or --key value)
  714. -tags
  715. args
  716. """
  717. # return values
  718. _dict = {}
  719. tags = []
  720. args = []
  721. # parse the arguments
  722. key = None
  723. for arg in _args:
  724. if arg.startswith('---'):
  725. raise ParserError("arguments should start with '-' or '--' only")
  726. elif arg.startswith('--'):
  727. if key:
  728. raise ParserError("Key %s still open" % key)
  729. key = arg[2:]
  730. if '=' in key:
  731. key, value = key.split('=', 1)
  732. _dict[key] = value
  733. key = None
  734. continue
  735. elif arg.startswith('-'):
  736. if key:
  737. raise ParserError("Key %s still open" % key)
  738. tags.append(arg[1:])
  739. continue
  740. else:
  741. if key:
  742. _dict[key] = arg
  743. continue
  744. args.append(arg)
  745. # return values
  746. return (_dict, tags, args)
  747. ### classes for subcommands
  748. class CLICommand(object):
  749. usage = '%prog [options] command'
  750. def __init__(self, parser):
  751. self._parser = parser # master parser
  752. def parser(self):
  753. return OptionParser(usage=self.usage, description=self.__doc__,
  754. add_help_option=False)
  755. class Copy(CLICommand):
  756. usage = '%prog [options] copy manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
  757. def __call__(self, options, args):
  758. # parse the arguments
  759. try:
  760. kwargs, tags, args = parse_args(args)
  761. except ParserError, e:
  762. self._parser.error(e.message)
  763. # make sure we have some manifests, otherwise it will
  764. # be quite boring
  765. if not len(args) == 2:
  766. HelpCLI(self._parser)(options, ['copy'])
  767. return
  768. # read the manifests
  769. # TODO: should probably ensure these exist here
  770. manifests = ManifestParser()
  771. manifests.read(args[0])
  772. # print the resultant query
  773. manifests.copy(args[1], None, *tags, **kwargs)
  774. class CreateCLI(CLICommand):
  775. """
  776. create a manifest from a list of directories
  777. """
  778. usage = '%prog [options] create directory <directory> <...>'
  779. def parser(self):
  780. parser = CLICommand.parser(self)
  781. parser.add_option('-p', '--pattern', dest='pattern',
  782. help="glob pattern for files")
  783. parser.add_option('-i', '--ignore', dest='ignore',
  784. default=[], action='append',
  785. help='directories to ignore')
  786. parser.add_option('-w', '--in-place', dest='in_place',
  787. help='Write .ini files in place; filename to write to')
  788. return parser
  789. def __call__(self, _options, args):
  790. parser = self.parser()
  791. options, args = parser.parse_args(args)
  792. # need some directories
  793. if not len(args):
  794. parser.print_usage()
  795. return
  796. # add the directories to the manifest
  797. for arg in args:
  798. assert os.path.exists(arg)
  799. assert os.path.isdir(arg)
  800. manifest = convert(args, pattern=options.pattern, ignore=options.ignore,
  801. write=options.in_place)
  802. if manifest:
  803. print manifest
  804. class WriteCLI(CLICommand):
  805. """
  806. write a manifest based on a query
  807. """
  808. usage = '%prog [options] write manifest <manifest> -tag1 -tag2 --key1=value1 --key2=value2 ...'
  809. def __call__(self, options, args):
  810. # parse the arguments
  811. try:
  812. kwargs, tags, args = parse_args(args)
  813. except ParserError, e:
  814. self._parser.error(e.message)
  815. # make sure we have some manifests, otherwise it will
  816. # be quite boring
  817. if not args:
  818. HelpCLI(self._parser)(options, ['write'])
  819. return
  820. # read the manifests
  821. # TODO: should probably ensure these exist here
  822. manifests = ManifestParser()
  823. manifests.read(*args)
  824. # print the resultant query
  825. manifests.write(global_tags=tags, global_kwargs=kwargs)
  826. class HelpCLI(CLICommand):
  827. """
  828. get help on a command
  829. """
  830. usage = '%prog [options] help [command]'
  831. def __call__(self, options, args):
  832. if len(args) == 1 and args[0] in commands:
  833. commands[args[0]](self._parser).parser().print_help()
  834. else:
  835. self._parser.print_help()
  836. print '\nCommands:'
  837. for command in sorted(commands):
  838. print ' %s : %s' % (command, commands[command].__doc__.strip())
  839. class SetupCLI(CLICommand):
  840. """
  841. setup using setuptools
  842. """
  843. # use setup.py from the repo when you want to distribute to python!
  844. # otherwise setuptools will complain that it can't find setup.py
  845. # and result in a useless package
  846. usage = '%prog [options] setup [setuptools options]'
  847. def __call__(self, options, args):
  848. sys.argv = [sys.argv[0]] + args
  849. assert setup is not None, "You must have setuptools installed to use SetupCLI"
  850. here = os.path.dirname(os.path.abspath(__file__))
  851. try:
  852. filename = os.path.join(here, 'README.txt')
  853. description = file(filename).read()
  854. except:
  855. description = ''
  856. os.chdir(here)
  857. setup(name='ManifestDestiny',
  858. version=version,
  859. description="Universal manifests for Mozilla test harnesses",
  860. long_description=description,
  861. classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
  862. keywords='mozilla manifests',
  863. author='Jeff Hammel',
  864. author_email='jhammel@mozilla.com',
  865. url='https://wiki.mozilla.org/Auto-tools/Projects/ManifestDestiny',
  866. license='MPL',
  867. zip_safe=False,
  868. py_modules=['manifestparser'],
  869. install_requires=[
  870. # -*- Extra requirements: -*-
  871. ],
  872. entry_points="""
  873. [console_scripts]
  874. manifestparser = manifestparser:main
  875. """,
  876. )
  877. class UpdateCLI(CLICommand):
  878. """
  879. update the tests as listed in a manifest from a directory
  880. """
  881. usage = '%prog [options] update manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
  882. def __call__(self, options, args):
  883. # parse the arguments
  884. try:
  885. kwargs, tags, args = parse_args(args)
  886. except ParserError, e:
  887. self._parser.error(e.message)
  888. # make sure we have some manifests, otherwise it will
  889. # be quite boring
  890. if not len(args) == 2:
  891. HelpCLI(self._parser)(options, ['update'])
  892. return
  893. # read the manifests
  894. # TODO: should probably ensure these exist here
  895. manifests = ManifestParser()
  896. manifests.read(args[0])
  897. # print the resultant query
  898. manifests.update(args[1], None, *tags, **kwargs)
  899. # command -> class mapping
  900. commands = { 'create': CreateCLI,
  901. 'help': HelpCLI,
  902. 'update': UpdateCLI,
  903. 'write': WriteCLI }
  904. if setup is not None:
  905. commands['setup'] = SetupCLI
  906. def main(args=sys.argv[1:]):
  907. """console_script entry point"""
  908. # set up an option parser
  909. usage = '%prog [options] [command] ...'
  910. description = __doc__
  911. parser = OptionParser(usage=usage, description=description)
  912. parser.add_option('-s', '--strict', dest='strict',
  913. action='store_true', default=False,
  914. help='adhere strictly to errors')
  915. parser.disable_interspersed_args()
  916. options, args = parser.parse_args(args)
  917. if not args:
  918. HelpCLI(parser)(options, args)
  919. parser.exit()
  920. # get the command
  921. command = args[0]
  922. if command not in commands:
  923. parser.error("Command must be one of %s (you gave '%s')" % (', '.join(sorted(commands.keys())), command))
  924. handler = commands[command](parser)
  925. handler(options, args[1:])
  926. if __name__ == '__main__':
  927. main()