/config/JarMaker.py
Python | 489 lines | 442 code | 14 blank | 33 comment | 2 complexity | b145b8d4a4a1864106cf86661c96a0ec MD5 | raw file
Possible License(s): MPL-2.0-no-copyleft-exception, LGPL-3.0, AGPL-1.0, MIT, LGPL-2.1, 0BSD, BSD-2-Clause, MPL-2.0, BSD-3-Clause, Apache-2.0, GPL-2.0, JSON
- # This Source Code Form is subject to the terms of the Mozilla Public
- # License, v. 2.0. If a copy of the MPL was not distributed with this
- # file, You can obtain one at http://mozilla.org/MPL/2.0/.
- '''jarmaker.py provides a python class to package up chrome content by
- processing jar.mn files.
- See the documentation for jar.mn on MDC for further details on the format.
- '''
- import sys
- import os
- import os.path
- import errno
- import re
- import logging
- from time import localtime
- from optparse import OptionParser
- from MozZipFile import ZipFile
- from cStringIO import StringIO
- from datetime import datetime
- from utils import pushback_iter, lockFile
- from Preprocessor import Preprocessor
- from buildlist import addEntriesToListFile
- if sys.platform == "win32":
- from ctypes import windll, WinError
- CreateHardLink = windll.kernel32.CreateHardLinkA
- __all__ = ['JarMaker']
- class ZipEntry:
- '''Helper class for jar output.
- This class defines a simple file-like object for a zipfile.ZipEntry
- so that we can consecutively write to it and then close it.
- This methods hooks into ZipFile.writestr on close().
- '''
- def __init__(self, name, zipfile):
- self._zipfile = zipfile
- self._name = name
- self._inner = StringIO()
- def write(self, content):
- 'Append the given content to this zip entry'
- self._inner.write(content)
- return
- def close(self):
- 'The close method writes the content back to the zip file.'
- self._zipfile.writestr(self._name, self._inner.getvalue())
- def getModTime(aPath):
- if not os.path.isfile(aPath):
- return 0
- mtime = os.stat(aPath).st_mtime
- return localtime(mtime)
- class JarMaker(object):
- '''JarMaker reads jar.mn files and process those into jar files or
- flat directories, along with chrome.manifest files.
- '''
- ignore = re.compile('\s*(\#.*)?$')
- jarline = re.compile('(?:(?P<jarfile>[\w\d.\-\_\\\/]+).jar\:)|(?:\s*(\#.*)?)\s*$')
- regline = re.compile('\%\s+(.*)$')
- entryre = '(?P<optPreprocess>\*)?(?P<optOverwrite>\+?)\s+'
- entryline = re.compile(entryre + '(?P<output>[\w\d.\-\_\\\/\+\@]+)\s*(\((?P<locale>\%?)(?P<source>[\w\d.\-\_\\\/\@]+)\))?\s*$')
- def __init__(self, outputFormat = 'flat', useJarfileManifest = True,
- useChromeManifest = False):
- self.outputFormat = outputFormat
- self.useJarfileManifest = useJarfileManifest
- self.useChromeManifest = useChromeManifest
- self.pp = Preprocessor()
- def getCommandLineParser(self):
- '''Get a optparse.OptionParser for jarmaker.
- This OptionParser has the options for jarmaker as well as
- the options for the inner PreProcessor.
- '''
- # HACK, we need to unescape the string variables we get,
- # the perl versions didn't grok strings right
- p = self.pp.getCommandLineParser(unescapeDefines = True)
- p.add_option('-f', type="choice", default="jar",
- choices=('jar', 'flat', 'symlink'),
- help="fileformat used for output", metavar="[jar, flat, symlink]")
- p.add_option('-v', action="store_true", dest="verbose",
- help="verbose output")
- p.add_option('-q', action="store_false", dest="verbose",
- help="verbose output")
- p.add_option('-e', action="store_true",
- help="create chrome.manifest instead of jarfile.manifest")
- p.add_option('--both-manifests', action="store_true",
- dest="bothManifests",
- help="create chrome.manifest and jarfile.manifest")
- p.add_option('-s', type="string", action="append", default=[],
- help="source directory")
- p.add_option('-t', type="string",
- help="top source directory")
- p.add_option('-c', '--l10n-src', type="string", action="append",
- help="localization directory")
- p.add_option('--l10n-base', type="string", action="append", default=[],
- help="base directory to be used for localization (multiple)")
- p.add_option('-j', type="string",
- help="jarfile directory")
- # backwards compat, not needed
- p.add_option('-a', action="store_false", default=True,
- help="NOT SUPPORTED, turn auto-registration of chrome off (installed-chrome.txt)")
- p.add_option('-d', type="string",
- help="UNUSED, chrome directory")
- p.add_option('-o', help="cross compile for auto-registration, ignored")
- p.add_option('-l', action="store_true",
- help="ignored (used to switch off locks)")
- p.add_option('-x', action="store_true",
- help="force Unix")
- p.add_option('-z', help="backwards compat, ignored")
- p.add_option('-p', help="backwards compat, ignored")
- return p
- def processIncludes(self, includes):
- '''Process given includes with the inner PreProcessor.
- Only use this for #defines, the includes shouldn't generate
- content.
- '''
- self.pp.out = StringIO()
- for inc in includes:
- self.pp.do_include(inc)
- includesvalue = self.pp.out.getvalue()
- if includesvalue:
- logging.info("WARNING: Includes produce non-empty output")
- self.pp.out = None
- pass
- def finalizeJar(self, jarPath, chromebasepath, register,
- doZip=True):
- '''Helper method to write out the chrome registration entries to
- jarfile.manifest or chrome.manifest, or both.
- The actual file processing is done in updateManifest.
- '''
- # rewrite the manifest, if entries given
- if not register:
- return
- chromeManifest = os.path.join(os.path.dirname(jarPath),
- '..', 'chrome.manifest')
- if self.useJarfileManifest:
- self.updateManifest(jarPath + '.manifest', chromebasepath % '',
- register)
- addEntriesToListFile(chromeManifest, ['manifest chrome/%s.manifest' % (os.path.basename(jarPath),)])
- if self.useChromeManifest:
- self.updateManifest(chromeManifest, chromebasepath % 'chrome/',
- register)
- def updateManifest(self, manifestPath, chromebasepath, register):
- '''updateManifest replaces the % in the chrome registration entries
- with the given chrome base path, and updates the given manifest file.
- '''
- lock = lockFile(manifestPath + '.lck')
- try:
- myregister = dict.fromkeys(map(lambda s: s.replace('%', chromebasepath),
- register.iterkeys()))
- manifestExists = os.path.isfile(manifestPath)
- mode = (manifestExists and 'r+b') or 'wb'
- mf = open(manifestPath, mode)
- if manifestExists:
- # import previous content into hash, ignoring empty ones and comments
- imf = re.compile('(#.*)?$')
- for l in re.split('[\r\n]+', mf.read()):
- if imf.match(l):
- continue
- myregister[l] = None
- mf.seek(0)
- for k in myregister.iterkeys():
- mf.write(k + os.linesep)
- mf.close()
- finally:
- lock = None
-
- def makeJar(self, infile=None,
- jardir='',
- sourcedirs=[], topsourcedir='', localedirs=None):
- '''makeJar is the main entry point to JarMaker.
- It takes the input file, the output directory, the source dirs and the
- top source dir as argument, and optionally the l10n dirs.
- '''
- if isinstance(infile, basestring):
- logging.info("processing " + infile)
- pp = self.pp.clone()
- pp.out = StringIO()
- pp.do_include(infile)
- lines = pushback_iter(pp.out.getvalue().splitlines())
- try:
- while True:
- l = lines.next()
- m = self.jarline.match(l)
- if not m:
- raise RuntimeError(l)
- if m.group('jarfile') is None:
- # comment
- continue
- self.processJarSection(m.group('jarfile'), lines,
- jardir, sourcedirs, topsourcedir,
- localedirs)
- except StopIteration:
- # we read the file
- pass
- return
- def makeJars(self, infiles, l10nbases,
- jardir='',
- sourcedirs=[], topsourcedir='', localedirs=None):
- '''makeJars is the second main entry point to JarMaker.
- It takes an iterable sequence of input file names, the l10nbases,
- the output directory, the source dirs and the
- top source dir as argument, and optionally the l10n dirs.
- It iterates over all inputs, guesses srcdir and l10ndir from the
- path and topsourcedir and calls into makeJar.
- The l10ndirs are created by guessing the relativesrcdir, and resolving
- that against the l10nbases. l10nbases can either be path strings, or
- callables. In the latter case, that will be called with the
- relativesrcdir as argument, and is expected to return a path string.
- This logic is disabled if the jar.mn path is not inside the topsrcdir.
- '''
- topsourcedir = os.path.normpath(os.path.abspath(topsourcedir))
- def resolveL10nBase(relpath):
- def _resolve(base):
- if isinstance(base, basestring):
- return os.path.join(base, relpath)
- if callable(base):
- return base(relpath)
- return base
- return _resolve
- for infile in infiles:
- srcdir = os.path.normpath(os.path.abspath(os.path.dirname(infile)))
- l10ndir = srcdir
- if os.path.basename(srcdir) == 'locales':
- l10ndir = os.path.dirname(l10ndir)
- l10ndirs = None
- # srcdir may not be a child of topsourcedir, in which case
- # we assume that the caller passed in suitable sourcedirs,
- # and just skip passing in localedirs
- if srcdir.startswith(topsourcedir):
- rell10ndir = l10ndir[len(topsourcedir):].lstrip(os.sep)
- l10ndirs = map(resolveL10nBase(rell10ndir), l10nbases)
- if localedirs is not None:
- l10ndirs += [os.path.normpath(os.path.abspath(s))
- for s in localedirs]
- srcdirs = [os.path.normpath(os.path.abspath(s))
- for s in sourcedirs] + [srcdir]
- self.makeJar(infile=infile,
- sourcedirs=srcdirs, topsourcedir=topsourcedir,
- localedirs=l10ndirs,
- jardir=jardir)
- def processJarSection(self, jarfile, lines,
- jardir, sourcedirs, topsourcedir, localedirs):
- '''Internal method called by makeJar to actually process a section
- of a jar.mn file.
- jarfile is the basename of the jarfile or the directory name for
- flat output, lines is a pushback_iterator of the lines of jar.mn,
- the remaining options are carried over from makeJar.
- '''
- # chromebasepath is used for chrome registration manifests
- # %s is getting replaced with chrome/ for chrome.manifest, and with
- # an empty string for jarfile.manifest
- chromebasepath = '%s' + os.path.basename(jarfile)
- if self.outputFormat == 'jar':
- chromebasepath = 'jar:' + chromebasepath + '.jar!'
- chromebasepath += '/'
- jarfile = os.path.join(jardir, jarfile)
- jf = None
- if self.outputFormat == 'jar':
- #jar
- jarfilepath = jarfile + '.jar'
- try:
- os.makedirs(os.path.dirname(jarfilepath))
- except OSError, error:
- if error.errno != errno.EEXIST:
- raise
- jf = ZipFile(jarfilepath, 'a', lock = True)
- outHelper = self.OutputHelper_jar(jf)
- else:
- outHelper = getattr(self, 'OutputHelper_' + self.outputFormat)(jarfile)
- register = {}
- # This loop exits on either
- # - the end of the jar.mn file
- # - an line in the jar.mn file that's not part of a jar section
- # - on an exception raised, close the jf in that case in a finally
- try:
- while True:
- try:
- l = lines.next()
- except StopIteration:
- # we're done with this jar.mn, and this jar section
- self.finalizeJar(jarfile, chromebasepath, register)
- if jf is not None:
- jf.close()
- # reraise the StopIteration for makeJar
- raise
- if self.ignore.match(l):
- continue
- m = self.regline.match(l)
- if m:
- rline = m.group(1)
- register[rline] = 1
- continue
- m = self.entryline.match(l)
- if not m:
- # neither an entry line nor chrome reg, this jar section is done
- self.finalizeJar(jarfile, chromebasepath, register)
- if jf is not None:
- jf.close()
- lines.pushback(l)
- return
- self._processEntryLine(m, sourcedirs, topsourcedir, localedirs,
- outHelper, jf)
- finally:
- if jf is not None:
- jf.close()
- return
- def _processEntryLine(self, m,
- sourcedirs, topsourcedir, localedirs,
- outHelper, jf):
- out = m.group('output')
- src = m.group('source') or os.path.basename(out)
- # pick the right sourcedir -- l10n, topsrc or src
- if m.group('locale'):
- src_base = localedirs
- elif src.startswith('/'):
- # path/in/jar/file_name.xul (/path/in/sourcetree/file_name.xul)
- # refers to a path relative to topsourcedir, use that as base
- # and strip the leading '/'
- src_base = [topsourcedir]
- src = src[1:]
- else:
- # use srcdirs and the objdir (current working dir) for relative paths
- src_base = sourcedirs + [os.getcwd()]
- # check if the source file exists
- realsrc = None
- for _srcdir in src_base:
- if os.path.isfile(os.path.join(_srcdir, src)):
- realsrc = os.path.join(_srcdir, src)
- break
- if realsrc is None:
- if jf is not None:
- jf.close()
- raise RuntimeError('File "%s" not found in %s' % (src, ', '.join(src_base)))
- if m.group('optPreprocess'):
- outf = outHelper.getOutput(out)
- inf = open(realsrc)
- pp = self.pp.clone()
- if src[-4:] == '.css':
- pp.setMarker('%')
- pp.out = outf
- pp.do_include(inf)
- pp.warnUnused(realsrc)
- outf.close()
- inf.close()
- return
- # copy or symlink if newer or overwrite
- if (m.group('optOverwrite')
- or (getModTime(realsrc) >
- outHelper.getDestModTime(m.group('output')))):
- if self.outputFormat == 'symlink':
- outHelper.symlink(realsrc, out)
- return
- outf = outHelper.getOutput(out)
- # open in binary mode, this can be images etc
- inf = open(realsrc, 'rb')
- outf.write(inf.read())
- outf.close()
- inf.close()
-
- class OutputHelper_jar(object):
- '''Provide getDestModTime and getOutput for a given jarfile.
- '''
- def __init__(self, jarfile):
- self.jarfile = jarfile
- def getDestModTime(self, aPath):
- try :
- info = self.jarfile.getinfo(aPath)
- return info.date_time
- except:
- return 0
- def getOutput(self, name):
- return ZipEntry(name, self.jarfile)
- class OutputHelper_flat(object):
- '''Provide getDestModTime and getOutput for a given flat
- output directory. The helper method ensureDirFor is used by
- the symlink subclass.
- '''
- def __init__(self, basepath):
- self.basepath = basepath
- def getDestModTime(self, aPath):
- return getModTime(os.path.join(self.basepath, aPath))
- def getOutput(self, name):
- out = self.ensureDirFor(name)
- # remove previous link or file
- try:
- os.remove(out)
- except OSError, e:
- if e.errno != errno.ENOENT:
- raise
- return open(out, 'wb')
- def ensureDirFor(self, name):
- out = os.path.join(self.basepath, name)
- outdir = os.path.dirname(out)
- if not os.path.isdir(outdir):
- try:
- os.makedirs(outdir)
- except OSError, error:
- if error.errno != errno.EEXIST:
- raise
- return out
- class OutputHelper_symlink(OutputHelper_flat):
- '''Subclass of OutputHelper_flat that provides a helper for
- creating a symlink including creating the parent directories.
- '''
- def symlink(self, src, dest):
- out = self.ensureDirFor(dest)
- # remove previous link or file
- try:
- os.remove(out)
- except OSError, e:
- if e.errno != errno.ENOENT:
- raise
- if sys.platform != "win32":
- os.symlink(src, out)
- else:
- # On Win32, use ctypes to create a hardlink
- rv = CreateHardLink(out, src, None)
- if rv == 0:
- raise WinError()
- def main():
- jm = JarMaker()
- p = jm.getCommandLineParser()
- (options, args) = p.parse_args()
- jm.processIncludes(options.I)
- jm.outputFormat = options.f
- if options.e:
- jm.useChromeManifest = True
- jm.useJarfileManifest = False
- if options.bothManifests:
- jm.useChromeManifest = True
- jm.useJarfileManifest = True
- noise = logging.INFO
- if options.verbose is not None:
- noise = (options.verbose and logging.DEBUG) or logging.WARN
- if sys.version_info[:2] > (2,3):
- logging.basicConfig(format = "%(message)s")
- else:
- logging.basicConfig()
- logging.getLogger().setLevel(noise)
- topsrc = options.t
- topsrc = os.path.normpath(os.path.abspath(topsrc))
- if not args:
- jm.makeJar(infile=sys.stdin,
- sourcedirs=options.s, topsourcedir=topsrc,
- localedirs=options.l10n_src,
- jardir=options.j)
- else:
- jm.makeJars(args, options.l10n_base,
- jardir=options.j,
- sourcedirs=options.s, topsourcedir=topsrc,
- localedirs=options.l10n_src)
- if __name__ == "__main__":
- main()