/js/src/config/Preprocessor.py
Python | 475 lines | 459 code | 0 blank | 16 comment | 0 complexity | b7434d97e1956a5c60553821c7002841 MD5 | raw file
Possible License(s): JSON, LGPL-2.1, LGPL-3.0, AGPL-1.0, MIT, MPL-2.0-no-copyleft-exception, Apache-2.0, GPL-2.0, BSD-2-Clause, MPL-2.0, BSD-3-Clause, 0BSD
- """
- This is a very primitive line based preprocessor, for times when using
- a C preprocessor isn't an option.
- """
- # This Source Code Form is subject to the terms of the Mozilla Public
- # License, v. 2.0. If a copy of the MPL was not distributed with this
- # file, You can obtain one at http://mozilla.org/MPL/2.0/.
- import sys
- import os
- import os.path
- import re
- from optparse import OptionParser
- # hack around win32 mangling our line endings
- # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65443
- if sys.platform == "win32":
- import msvcrt
- msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
- os.linesep = '\n'
- import Expression
- __all__ = ['Preprocessor', 'preprocess']
- class Preprocessor:
- """
- Class for preprocessing text files.
- """
- class Error(RuntimeError):
- def __init__(self, cpp, MSG, context):
- self.file = cpp.context['FILE']
- self.line = cpp.context['LINE']
- self.key = MSG
- RuntimeError.__init__(self, (self.file, self.line, self.key, context))
- def __init__(self):
- self.context = Expression.Context()
- for k,v in {'FILE': '',
- 'LINE': 0,
- 'DIRECTORY': os.path.abspath('.')}.iteritems():
- self.context[k] = v
- self.actionLevel = 0
- self.disableLevel = 0
- # ifStates can be
- # 0: hadTrue
- # 1: wantsTrue
- # 2: #else found
- self.ifStates = []
- self.checkLineNumbers = False
- self.writtenLines = 0
- self.filters = []
- self.cmds = {}
- for cmd, level in {'define': 0,
- 'undef': 0,
- 'if': sys.maxint,
- 'ifdef': sys.maxint,
- 'ifndef': sys.maxint,
- 'else': 1,
- 'elif': 1,
- 'elifdef': 1,
- 'elifndef': 1,
- 'endif': sys.maxint,
- 'expand': 0,
- 'literal': 0,
- 'filter': 0,
- 'unfilter': 0,
- 'include': 0,
- 'includesubst': 0,
- 'error': 0}.iteritems():
- self.cmds[cmd] = (level, getattr(self, 'do_' + cmd))
- self.out = sys.stdout
- self.setMarker('#')
- self.LE = '\n'
- self.varsubst = re.compile('@(?P<VAR>\w+)@', re.U)
-
- def warnUnused(self, file):
- if self.actionLevel == 0:
- sys.stderr.write('%s: WARNING: no preprocessor directives found\n' % file)
- elif self.actionLevel == 1:
- sys.stderr.write('%s: WARNING: no useful preprocessor directives found\n' % file)
- pass
- def setLineEndings(self, aLE):
- """
- Set the line endings to be used for output.
- """
- self.LE = {'cr': '\x0D', 'lf': '\x0A', 'crlf': '\x0D\x0A'}[aLE]
-
- def setMarker(self, aMarker):
- """
- Set the marker to be used for processing directives.
- Used for handling CSS files, with pp.setMarker('%'), for example.
- The given marker may be None, in which case no markers are processed.
- """
- self.marker = aMarker
- if aMarker:
- self.instruction = re.compile('%s(?P<cmd>[a-z]+)(?:\s(?P<args>.*))?$'%aMarker, re.U)
- self.comment = re.compile(aMarker, re.U)
- else:
- class NoMatch(object):
- def match(self, *args):
- return False
- self.instruction = self.comment = NoMatch()
-
- def clone(self):
- """
- Create a clone of the current processor, including line ending
- settings, marker, variable definitions, output stream.
- """
- rv = Preprocessor()
- rv.context.update(self.context)
- rv.setMarker(self.marker)
- rv.LE = self.LE
- rv.out = self.out
- return rv
-
- def applyFilters(self, aLine):
- for f in self.filters:
- aLine = f[1](aLine)
- return aLine
-
- def write(self, aLine):
- """
- Internal method for handling output.
- """
- if self.checkLineNumbers:
- self.writtenLines += 1
- ln = self.context['LINE']
- if self.writtenLines != ln:
- self.out.write('//@line %(line)d "%(file)s"%(le)s'%{'line': ln,
- 'file': self.context['FILE'],
- 'le': self.LE})
- self.writtenLines = ln
- filteredLine = self.applyFilters(aLine)
- if filteredLine != aLine:
- self.actionLevel = 2
- # ensure our line ending. Only need to handle \n, as we're reading
- # with universal line ending support, at least for files.
- filteredLine = re.sub('\n', self.LE, filteredLine)
- self.out.write(filteredLine)
-
- def handleCommandLine(self, args, defaultToStdin = False):
- """
- Parse a commandline into this parser.
- Uses OptionParser internally, no args mean sys.argv[1:].
- """
- p = self.getCommandLineParser()
- (options, args) = p.parse_args(args=args)
- includes = options.I
- if defaultToStdin and len(args) == 0:
- args = [sys.stdin]
- includes.extend(args)
- if includes:
- for f in includes:
- self.do_include(f, False)
- self.warnUnused(f)
- pass
- def getCommandLineParser(self, unescapeDefines = False):
- escapedValue = re.compile('".*"$')
- numberValue = re.compile('\d+$')
- def handleE(option, opt, value, parser):
- for k,v in os.environ.iteritems():
- self.context[k] = v
- def handleD(option, opt, value, parser):
- vals = value.split('=', 1)
- if len(vals) == 1:
- vals.append(1)
- elif unescapeDefines and escapedValue.match(vals[1]):
- # strip escaped string values
- vals[1] = vals[1][1:-1]
- elif numberValue.match(vals[1]):
- vals[1] = int(vals[1])
- self.context[vals[0]] = vals[1]
- def handleU(option, opt, value, parser):
- del self.context[value]
- def handleF(option, opt, value, parser):
- self.do_filter(value)
- def handleLE(option, opt, value, parser):
- self.setLineEndings(value)
- def handleMarker(option, opt, value, parser):
- self.setMarker(value)
- p = OptionParser()
- p.add_option('-I', action='append', type="string", default = [],
- metavar="FILENAME", help='Include file')
- p.add_option('-E', action='callback', callback=handleE,
- help='Import the environment into the defined variables')
- p.add_option('-D', action='callback', callback=handleD, type="string",
- metavar="VAR[=VAL]", help='Define a variable')
- p.add_option('-U', action='callback', callback=handleU, type="string",
- metavar="VAR", help='Undefine a variable')
- p.add_option('-F', action='callback', callback=handleF, type="string",
- metavar="FILTER", help='Enable the specified filter')
- p.add_option('--line-endings', action='callback', callback=handleLE,
- type="string", metavar="[cr|lr|crlf]",
- help='Use the specified line endings [Default: OS dependent]')
- p.add_option('--marker', action='callback', callback=handleMarker,
- type="string",
- help='Use the specified marker instead of #')
- return p
- def handleLine(self, aLine):
- """
- Handle a single line of input (internal).
- """
- if self.actionLevel == 0 and self.comment.match(aLine):
- self.actionLevel = 1
- m = self.instruction.match(aLine)
- if m:
- args = None
- cmd = m.group('cmd')
- try:
- args = m.group('args')
- except IndexError:
- pass
- if cmd not in self.cmds:
- raise Preprocessor.Error(self, 'INVALID_CMD', aLine)
- level, cmd = self.cmds[cmd]
- if (level >= self.disableLevel):
- cmd(args)
- if cmd != 'literal':
- self.actionLevel = 2
- elif self.disableLevel == 0 and not self.comment.match(aLine):
- self.write(aLine)
- pass
- # Instruction handlers
- # These are named do_'instruction name' and take one argument
-
- # Variables
- def do_define(self, args):
- m = re.match('(?P<name>\w+)(?:\s(?P<value>.*))?', args, re.U)
- if not m:
- raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
- val = 1
- if m.group('value'):
- val = self.applyFilters(m.group('value'))
- try:
- val = int(val)
- except:
- pass
- self.context[m.group('name')] = val
- def do_undef(self, args):
- m = re.match('(?P<name>\w+)$', args, re.U)
- if not m:
- raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
- if args in self.context:
- del self.context[args]
- # Logic
- def ensure_not_else(self):
- if len(self.ifStates) == 0 or self.ifStates[-1] == 2:
- sys.stderr.write('WARNING: bad nesting of #else\n')
- def do_if(self, args, replace=False):
- if self.disableLevel and not replace:
- self.disableLevel += 1
- return
- val = None
- try:
- e = Expression.Expression(args)
- val = e.evaluate(self.context)
- except Exception:
- # XXX do real error reporting
- raise Preprocessor.Error(self, 'SYNTAX_ERR', args)
- if type(val) == str:
- # we're looking for a number value, strings are false
- val = False
- if not val:
- self.disableLevel = 1
- if replace:
- if val:
- self.disableLevel = 0
- self.ifStates[-1] = self.disableLevel
- else:
- self.ifStates.append(self.disableLevel)
- pass
- def do_ifdef(self, args, replace=False):
- if self.disableLevel and not replace:
- self.disableLevel += 1
- return
- if re.match('\W', args, re.U):
- raise Preprocessor.Error(self, 'INVALID_VAR', args)
- if args not in self.context:
- self.disableLevel = 1
- if replace:
- if args in self.context:
- self.disableLevel = 0
- self.ifStates[-1] = self.disableLevel
- else:
- self.ifStates.append(self.disableLevel)
- pass
- def do_ifndef(self, args, replace=False):
- if self.disableLevel and not replace:
- self.disableLevel += 1
- return
- if re.match('\W', args, re.U):
- raise Preprocessor.Error(self, 'INVALID_VAR', args)
- if args in self.context:
- self.disableLevel = 1
- if replace:
- if args not in self.context:
- self.disableLevel = 0
- self.ifStates[-1] = self.disableLevel
- else:
- self.ifStates.append(self.disableLevel)
- pass
- def do_else(self, args, ifState = 2):
- self.ensure_not_else()
- hadTrue = self.ifStates[-1] == 0
- self.ifStates[-1] = ifState # in-else
- if hadTrue:
- self.disableLevel = 1
- return
- self.disableLevel = 0
- def do_elif(self, args):
- if self.disableLevel == 1:
- if self.ifStates[-1] == 1:
- self.do_if(args, replace=True)
- else:
- self.do_else(None, self.ifStates[-1])
- def do_elifdef(self, args):
- if self.disableLevel == 1:
- if self.ifStates[-1] == 1:
- self.do_ifdef(args, replace=True)
- else:
- self.do_else(None, self.ifStates[-1])
- def do_elifndef(self, args):
- if self.disableLevel == 1:
- if self.ifStates[-1] == 1:
- self.do_ifndef(args, replace=True)
- else:
- self.do_else(None, self.ifStates[-1])
- def do_endif(self, args):
- if self.disableLevel > 0:
- self.disableLevel -= 1
- if self.disableLevel == 0:
- self.ifStates.pop()
- # output processing
- def do_expand(self, args):
- lst = re.split('__(\w+)__', args, re.U)
- do_replace = False
- def vsubst(v):
- if v in self.context:
- return str(self.context[v])
- return ''
- for i in range(1, len(lst), 2):
- lst[i] = vsubst(lst[i])
- lst.append('\n') # add back the newline
- self.write(reduce(lambda x, y: x+y, lst, ''))
- def do_literal(self, args):
- self.write(args + self.LE)
- def do_filter(self, args):
- filters = [f for f in args.split(' ') if hasattr(self, 'filter_' + f)]
- if len(filters) == 0:
- return
- current = dict(self.filters)
- for f in filters:
- current[f] = getattr(self, 'filter_' + f)
- filterNames = current.keys()
- filterNames.sort()
- self.filters = [(fn, current[fn]) for fn in filterNames]
- return
- def do_unfilter(self, args):
- filters = args.split(' ')
- current = dict(self.filters)
- for f in filters:
- if f in current:
- del current[f]
- filterNames = current.keys()
- filterNames.sort()
- self.filters = [(fn, current[fn]) for fn in filterNames]
- return
- # Filters
- #
- # emptyLines
- # Strips blank lines from the output.
- def filter_emptyLines(self, aLine):
- if aLine == '\n':
- return ''
- return aLine
- # slashslash
- # Strips everything after //
- def filter_slashslash(self, aLine):
- [aLine, rest] = aLine.split('//', 1)
- if rest:
- aLine += '\n'
- return aLine
- # spaces
- # Collapses sequences of spaces into a single space
- def filter_spaces(self, aLine):
- return re.sub(' +', ' ', aLine).strip(' ')
- # substition
- # helper to be used by both substition and attemptSubstitution
- def filter_substitution(self, aLine, fatal=True):
- def repl(matchobj):
- varname = matchobj.group('VAR')
- if varname in self.context:
- return str(self.context[varname])
- if fatal:
- raise Preprocessor.Error(self, 'UNDEFINED_VAR', varname)
- return matchobj.group(0)
- return self.varsubst.sub(repl, aLine)
- def filter_attemptSubstitution(self, aLine):
- return self.filter_substitution(aLine, fatal=False)
- # File ops
- def do_include(self, args, filters=True):
- """
- Preprocess a given file.
- args can either be a file name, or a file-like object.
- Files should be opened, and will be closed after processing.
- """
- isName = type(args) == str or type(args) == unicode
- oldWrittenLines = self.writtenLines
- oldCheckLineNumbers = self.checkLineNumbers
- self.checkLineNumbers = False
- if isName:
- try:
- args = str(args)
- if filters:
- args = self.applyFilters(args)
- if not os.path.isabs(args):
- args = os.path.join(self.context['DIRECTORY'], args)
- args = open(args, 'rU')
- except Preprocessor.Error:
- raise
- except:
- raise Preprocessor.Error(self, 'FILE_NOT_FOUND', str(args))
- self.checkLineNumbers = bool(re.search('\.(js|jsm|java)(?:\.in)?$', args.name))
- oldFile = self.context['FILE']
- oldLine = self.context['LINE']
- oldDir = self.context['DIRECTORY']
- if args.isatty():
- # we're stdin, use '-' and '' for file and dir
- self.context['FILE'] = '-'
- self.context['DIRECTORY'] = ''
- else:
- abspath = os.path.abspath(args.name)
- self.context['FILE'] = abspath
- self.context['DIRECTORY'] = os.path.dirname(abspath)
- self.context['LINE'] = 0
- self.writtenLines = 0
- for l in args:
- self.context['LINE'] += 1
- self.handleLine(l)
- args.close()
- self.context['FILE'] = oldFile
- self.checkLineNumbers = oldCheckLineNumbers
- self.writtenLines = oldWrittenLines
- self.context['LINE'] = oldLine
- self.context['DIRECTORY'] = oldDir
- def do_includesubst(self, args):
- args = self.filter_substitution(args)
- self.do_include(args)
- def do_error(self, args):
- raise Preprocessor.Error(self, 'Error: ', str(args))
- def main():
- pp = Preprocessor()
- pp.handleCommandLine(None, True)
- return
- def preprocess(includes=[sys.stdin], defines={},
- output = sys.stdout,
- line_endings='\n', marker='#'):
- pp = Preprocessor()
- pp.context.update(defines)
- pp.setLineEndings(line_endings)
- pp.setMarker(marker)
- pp.out = output
- for f in includes:
- pp.do_include(f, False)
- if __name__ == "__main__":
- main()