PageRenderTime 61ms CodeModel.GetById 14ms app.highlight 40ms RepoModel.GetById 2ms app.codeStats 0ms

/indra/lib/python/indra/util/llmanifest.py

https://bitbucket.org/lindenlab/viewer-beta/
Python | 658 lines | 636 code | 5 blank | 17 comment | 15 complexity | ef3f58b95bc88822fa12997df2fba07a MD5 | raw file
  1"""\
  2@file llmanifest.py
  3@author Ryan Williams
  4@brief Library for specifying operations on a set of files.
  5
  6$LicenseInfo:firstyear=2007&license=mit$
  7
  8Copyright (c) 2007-2009, Linden Research, Inc.
  9
 10Permission is hereby granted, free of charge, to any person obtaining a copy
 11of this software and associated documentation files (the "Software"), to deal
 12in the Software without restriction, including without limitation the rights
 13to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 14copies of the Software, and to permit persons to whom the Software is
 15furnished to do so, subject to the following conditions:
 16
 17The above copyright notice and this permission notice shall be included in
 18all copies or substantial portions of the Software.
 19
 20THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 21IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 22FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 23AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 24LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 25OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 26THE SOFTWARE.
 27$/LicenseInfo$
 28"""
 29
 30import commands
 31import errno
 32import filecmp
 33import fnmatch
 34import getopt
 35import glob
 36import os
 37import re
 38import shutil
 39import sys
 40import tarfile
 41import errno
 42import subprocess
 43
 44def path_ancestors(path):
 45    drive, path = os.path.splitdrive(os.path.normpath(path))
 46    result = []
 47    while len(path) > 0 and path != os.path.sep:
 48        result.append(drive+path)
 49        path, sub = os.path.split(path)
 50    return result
 51
 52def proper_windows_path(path, current_platform = sys.platform):
 53    """ This function takes an absolute Windows or Cygwin path and
 54    returns a path appropriately formatted for the platform it's
 55    running on (as determined by sys.platform)"""
 56    path = path.strip()
 57    drive_letter = None
 58    rel = None
 59    match = re.match("/cygdrive/([a-z])/(.*)", path)
 60    if not match:
 61        match = re.match('([a-zA-Z]):\\\(.*)', path)
 62    if not match:
 63        return None         # not an absolute path
 64    drive_letter = match.group(1)
 65    rel = match.group(2)
 66    if current_platform == "cygwin":
 67        return "/cygdrive/" + drive_letter.lower() + '/' + rel.replace('\\', '/')
 68    else:
 69        return drive_letter.upper() + ':\\' + rel.replace('/', '\\')
 70
 71def get_default_platform(dummy):
 72    return {'linux2':'linux',
 73            'linux1':'linux',
 74            'cygwin':'windows',
 75            'win32':'windows',
 76            'darwin':'darwin'
 77            }[sys.platform]
 78
 79def get_default_version(srctree):
 80    # look up llversion.h and parse out the version info
 81    paths = [os.path.join(srctree, x, 'llversionviewer.h') for x in ['llcommon', '../llcommon', '../../indra/llcommon.h']]
 82    for p in paths:
 83        if os.path.exists(p):
 84            contents = open(p, 'r').read()
 85            major = re.search("LL_VERSION_MAJOR\s=\s([0-9]+)", contents).group(1)
 86            minor = re.search("LL_VERSION_MINOR\s=\s([0-9]+)", contents).group(1)
 87            patch = re.search("LL_VERSION_PATCH\s=\s([0-9]+)", contents).group(1)
 88            build = re.search("LL_VERSION_BUILD\s=\s([0-9]+)", contents).group(1)
 89            return major, minor, patch, build
 90
 91def get_channel(srctree):
 92    # look up llversionserver.h and parse out the version info
 93    paths = [os.path.join(srctree, x, 'llversionviewer.h') for x in ['llcommon', '../llcommon', '../../indra/llcommon.h']]
 94    for p in paths:
 95        if os.path.exists(p):
 96            contents = open(p, 'r').read()
 97            channel = re.search("LL_CHANNEL\s=\s\"(.+)\";\s*$", contents, flags = re.M).group(1)
 98            return channel
 99    
100
101DEFAULT_SRCTREE = os.path.dirname(sys.argv[0])
102DEFAULT_CHANNEL = 'Second Life Release'
103
104ARGUMENTS=[
105    dict(name='actions',
106         description="""This argument specifies the actions that are to be taken when the
107        script is run.  The meaningful actions are currently:
108          copy     - copies the files specified by the manifest into the
109                     destination directory.
110          package  - bundles up the files in the destination directory into
111                     an installer for the current platform
112          unpacked - bundles up the files in the destination directory into
113                     a simple tarball
114        Example use: %(name)s --actions="copy unpacked" """,
115         default="copy package"),
116    dict(name='arch',
117         description="""This argument is appended to the platform string for
118        determining which manifest class to run.
119        Example use: %(name)s --arch=i686
120        On Linux this would try to use Linux_i686Manifest.""",
121         default=""),
122    dict(name='build', description='Build directory.', default=DEFAULT_SRCTREE),
123    dict(name='buildtype', description='Build type (i.e. Debug, Release, RelWithDebInfo).', default=None),
124    dict(name='configuration',
125         description="""The build configuration used.""",
126         default="Release"),
127    dict(name='dest', description='Destination directory.', default=DEFAULT_SRCTREE),
128    dict(name='grid',
129         description="""Which grid the client will try to connect to. Even
130        though it's not strictly a grid, 'firstlook' is also an acceptable
131        value for this parameter.""",
132         default=""),
133    dict(name='channel',
134         description="""The channel to use for updates, packaging, settings name, etc.""",
135         default=get_channel),
136    dict(name='login_channel',
137         description="""The channel to use for login handshake/updates only.""",
138         default=None),
139    dict(name='installer_name',
140         description=""" The name of the file that the installer should be
141        packaged up into. Only used on Linux at the moment.""",
142         default=None),
143    dict(name='login_url',
144         description="""The url that the login screen displays in the client.""",
145         default=None),
146    dict(name='platform',
147         description="""The current platform, to be used for looking up which
148        manifest class to run.""",
149         default=get_default_platform),
150    dict(name='source',
151         description='Source directory.',
152         default=DEFAULT_SRCTREE),
153    dict(name='artwork', description='Artwork directory.', default=DEFAULT_SRCTREE),
154    dict(name='touch',
155         description="""File to touch when action is finished. Touch file will
156        contain the name of the final package in a form suitable
157        for use by a .bat file.""",
158         default=None),
159    dict(name='version',
160         description="""This specifies the version of Second Life that is
161        being packaged up.""",
162         default=get_default_version)
163    ]
164
165def usage(srctree=""):
166    nd = {'name':sys.argv[0]}
167    print """Usage:
168    %(name)s [options] [destdir]
169    Options:
170    """ % nd
171    for arg in ARGUMENTS:
172        default = arg['default']
173        if hasattr(default, '__call__'):
174            default = "(computed value) \"" + str(default(srctree)) + '"'
175        elif default is not None:
176            default = '"' + default + '"'
177        print "\t--%s        Default: %s\n\t%s\n" % (
178            arg['name'],
179            default,
180            arg['description'] % nd)
181
182def main():
183    option_names = [arg['name'] + '=' for arg in ARGUMENTS]
184    option_names.append('help')
185    options, remainder = getopt.getopt(sys.argv[1:], "", option_names)
186
187    # convert options to a hash
188    args = {'source':  DEFAULT_SRCTREE,
189            'artwork': DEFAULT_SRCTREE,
190            'build':   DEFAULT_SRCTREE,
191            'dest':    DEFAULT_SRCTREE }
192    for opt in options:
193        args[opt[0].replace("--", "")] = opt[1]
194
195    for k in 'artwork build dest source'.split():
196        args[k] = os.path.normpath(args[k])
197
198    print "Source tree:", args['source']
199    print "Artwork tree:", args['artwork']
200    print "Build tree:", args['build']
201    print "Destination tree:", args['dest']
202
203    # early out for help
204    if 'help' in args:
205        # *TODO: it is a huge hack to pass around the srctree like this
206        usage(args['source'])
207        return
208
209    # defaults
210    for arg in ARGUMENTS:
211        if arg['name'] not in args:
212            default = arg['default']
213            if hasattr(default, '__call__'):
214                default = default(args['source'])
215            if default is not None:
216                args[arg['name']] = default
217
218    # fix up version
219    if isinstance(args.get('version'), str):
220        args['version'] = args['version'].split('.')
221        
222    # default and agni are default
223    if args['grid'] in ['default', 'agni']:
224        args['grid'] = ''
225
226    if 'actions' in args:
227        args['actions'] = args['actions'].split()
228
229    # debugging
230    for opt in args:
231        print "Option:", opt, "=", args[opt]
232
233    wm = LLManifest.for_platform(args['platform'], args.get('arch'))(args)
234    wm.do(*args['actions'])
235
236    # Write out the package file in this format, so that it can easily be called
237    # and used in a .bat file - yeah, it sucks, but this is the simplest...
238    touch = args.get('touch')
239    if touch:
240        fp = open(touch, 'w')
241        fp.write('set package_file=%s\n' % wm.package_file)
242        fp.close()
243        print 'touched', touch
244    return 0
245
246class LLManifestRegistry(type):
247    def __init__(cls, name, bases, dct):
248        super(LLManifestRegistry, cls).__init__(name, bases, dct)
249        match = re.match("(\w+)Manifest", name)
250        if match:
251           cls.manifests[match.group(1).lower()] = cls
252
253class LLManifest(object):
254    __metaclass__ = LLManifestRegistry
255    manifests = {}
256    def for_platform(self, platform, arch = None):
257        if arch:
258            platform = platform + '_' + arch
259        return self.manifests[platform.lower()]
260    for_platform = classmethod(for_platform)
261
262    def __init__(self, args):
263        super(LLManifest, self).__init__()
264        self.args = args
265        self.file_list = []
266        self.excludes = []
267        self.actions = []
268        self.src_prefix = [args['source']]
269        self.artwork_prefix = [args['artwork']]
270        self.build_prefix = [args['build']]
271        self.dst_prefix = [args['dest']]
272        self.created_paths = []
273        self.package_name = "Unknown"
274        
275    def default_grid(self):
276        return self.args.get('grid', None) == ''
277    def default_channel(self):
278        return self.args.get('channel', None) == DEFAULT_CHANNEL
279
280    def construct(self):
281        """ Meant to be overriden by LLManifest implementors with code that
282        constructs the complete destination hierarchy."""
283        pass # override this method
284
285    def exclude(self, glob):
286        """ Excludes all files that match the glob from being included
287        in the file list by path()."""
288        self.excludes.append(glob)
289
290    def prefix(self, src='', build=None, dst=None):
291        """ Pushes a prefix onto the stack.  Until end_prefix is
292        called, all relevant method calls (esp. to path()) will prefix
293        paths with the entire prefix stack.  Source and destination
294        prefixes can be different, though if only one is provided they
295        are both equal.  To specify a no-op, use an empty string, not
296        None."""
297        if dst is None:
298            dst = src
299        if build is None:
300            build = src
301        self.src_prefix.append(src)
302        self.artwork_prefix.append(src)
303        self.build_prefix.append(build)
304        self.dst_prefix.append(dst)
305        return True  # so that you can wrap it in an if to get indentation
306
307    def end_prefix(self, descr=None):
308        """Pops a prefix off the stack.  If given an argument, checks
309        the argument against the top of the stack.  If the argument
310        matches neither the source or destination prefixes at the top
311        of the stack, then misnesting must have occurred and an
312        exception is raised."""
313        # as an error-prevention mechanism, check the prefix and see if it matches the source or destination prefix.  If not, improper nesting may have occurred.
314        src = self.src_prefix.pop()
315        artwork = self.artwork_prefix.pop()
316        build = self.build_prefix.pop()
317        dst = self.dst_prefix.pop()
318        if descr and not(src == descr or build == descr or dst == descr):
319            raise ValueError, "End prefix '" + descr + "' didn't match '" +src+ "' or '" +dst + "'"
320
321    def get_src_prefix(self):
322        """ Returns the current source prefix."""
323        return os.path.join(*self.src_prefix)
324
325    def get_artwork_prefix(self):
326        """ Returns the current artwork prefix."""
327        return os.path.join(*self.artwork_prefix)
328
329    def get_build_prefix(self):
330        """ Returns the current build prefix."""
331        return os.path.join(*self.build_prefix)
332
333    def get_dst_prefix(self):
334        """ Returns the current destination prefix."""
335        return os.path.join(*self.dst_prefix)
336
337    def src_path_of(self, relpath):
338        """Returns the full path to a file or directory specified
339        relative to the source directory."""
340        return os.path.join(self.get_src_prefix(), relpath)
341
342    def build_path_of(self, relpath):
343        """Returns the full path to a file or directory specified
344        relative to the build directory."""
345        return os.path.join(self.get_build_prefix(), relpath)
346
347    def dst_path_of(self, relpath):
348        """Returns the full path to a file or directory specified
349        relative to the destination directory."""
350        return os.path.join(self.get_dst_prefix(), relpath)
351
352    def ensure_src_dir(self, reldir):
353        """Construct the path for a directory relative to the
354        source path, and ensures that it exists.  Returns the
355        full path."""
356        path = os.path.join(self.get_src_prefix(), reldir)
357        self.cmakedirs(path)
358        return path
359
360    def ensure_dst_dir(self, reldir):
361        """Construct the path for a directory relative to the
362        destination path, and ensures that it exists.  Returns the
363        full path."""
364        path = os.path.join(self.get_dst_prefix(), reldir)
365        self.cmakedirs(path)
366        return path
367
368    def run_command(self, command):
369        """ Runs an external command, and returns the output.  Raises
370        an exception if the command returns a nonzero status code.  For
371        debugging/informational purposes, prints out the command's
372        output as it is received."""
373        print "Running command:", command
374        sys.stdout.flush()
375        child = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
376                                 shell=True)
377        lines = []
378        while True:
379            lines.append(child.stdout.readline())
380            if lines[-1] == '':
381                break
382            else:
383                print lines[-1],
384        output = ''.join(lines)
385        child.stdout.close()
386        status = child.wait()
387        if status:
388            raise RuntimeError(
389                "Command %s returned non-zero status (%s) \noutput:\n%s"
390                % (command, status, output) )
391        return output
392
393    def created_path(self, path):
394        """ Declare that you've created a path in order to
395          a) verify that you really have created it
396          b) schedule it for cleanup"""
397        if not os.path.exists(path):
398            raise RuntimeError, "Should be something at path " + path
399        self.created_paths.append(path)
400
401    def put_in_file(self, contents, dst):
402        # write contents as dst
403        f = open(self.dst_path_of(dst), "wb")
404        f.write(contents)
405        f.close()
406
407    def replace_in(self, src, dst=None, searchdict={}):
408        if dst == None:
409            dst = src
410        # read src
411        f = open(self.src_path_of(src), "rbU")
412        contents = f.read()
413        f.close()
414        # apply dict replacements
415        for old, new in searchdict.iteritems():
416            contents = contents.replace(old, new)
417        self.put_in_file(contents, dst)
418        self.created_paths.append(dst)
419
420    def copy_action(self, src, dst):
421        if src and (os.path.exists(src) or os.path.islink(src)):
422            # ensure that destination path exists
423            self.cmakedirs(os.path.dirname(dst))
424            self.created_paths.append(dst)
425            if not os.path.isdir(src):
426                self.ccopy(src,dst)
427            else:
428                # src is a dir
429                self.ccopytree(src,dst)
430        else:
431            print "Doesn't exist:", src
432
433    def package_action(self, src, dst):
434        pass
435
436    def copy_finish(self):
437        pass
438
439    def package_finish(self):
440        pass
441
442    def unpacked_finish(self):
443        unpacked_file_name = "unpacked_%(plat)s_%(vers)s.tar" % {
444            'plat':self.args['platform'],
445            'vers':'_'.join(self.args['version'])}
446        print "Creating unpacked file:", unpacked_file_name
447        # could add a gz here but that doubles the time it takes to do this step
448        tf = tarfile.open(self.src_path_of(unpacked_file_name), 'w:')
449        # add the entire installation package, at the very top level
450        tf.add(self.get_dst_prefix(), "")
451        tf.close()
452
453    def cleanup_finish(self):
454        """ Delete paths that were specified to have been created by this script"""
455        for c in self.created_paths:
456            # *TODO is this gonna be useful?
457            print "Cleaning up " + c
458
459    def process_file(self, src, dst):
460        if self.includes(src, dst):
461#            print src, "=>", dst
462            for action in self.actions:
463                methodname = action + "_action"
464                method = getattr(self, methodname, None)
465                if method is not None:
466                    method(src, dst)
467            self.file_list.append([src, dst])
468            return 1
469        else:
470            sys.stdout.write(" (excluding %r, %r)" % (src, dst))
471            sys.stdout.flush()
472            return 0
473
474    def process_directory(self, src, dst):
475        if not self.includes(src, dst):
476            sys.stdout.write(" (excluding %r, %r)" % (src, dst))
477            sys.stdout.flush()
478            return 0
479        names = os.listdir(src)
480        self.cmakedirs(dst)
481        errors = []
482        count = 0
483        for name in names:
484            srcname = os.path.join(src, name)
485            dstname = os.path.join(dst, name)
486            if os.path.isdir(srcname):
487                count += self.process_directory(srcname, dstname)
488            else:
489                count += self.process_file(srcname, dstname)
490        return count
491
492    def includes(self, src, dst):
493        if src:
494            for excl in self.excludes:
495                if fnmatch.fnmatch(src, excl):
496                    return False
497        return True
498
499    def remove(self, *paths):
500        for path in paths:
501            if os.path.exists(path):
502                print "Removing path", path
503                if os.path.isdir(path):
504                    shutil.rmtree(path)
505                else:
506                    os.remove(path)
507
508    def ccopy(self, src, dst):
509        """ Copy a single file or symlink.  Uses filecmp to skip copying for existing files."""
510        if os.path.islink(src):
511            linkto = os.readlink(src)
512            if os.path.islink(dst) or os.path.exists(dst):
513                os.remove(dst)  # because symlinking over an existing link fails
514            os.symlink(linkto, dst)
515        else:
516            # Don't recopy file if it's up-to-date.
517            # If we seem to be not not overwriting files that have been
518            # updated, set the last arg to False, but it will take longer.
519            if os.path.exists(dst) and filecmp.cmp(src, dst, True):
520                return
521            # only copy if it's not excluded
522            if self.includes(src, dst):
523                try:
524                    os.unlink(dst)
525                except OSError, err:
526                    if err.errno != errno.ENOENT:
527                        raise
528
529                shutil.copy2(src, dst)
530
531    def ccopytree(self, src, dst):
532        """Direct copy of shutil.copytree with the additional
533        feature that the destination directory can exist.  It
534        is so dumb that Python doesn't come with this. Also it
535        implements the excludes functionality."""
536        if not self.includes(src, dst):
537            return
538        names = os.listdir(src)
539        self.cmakedirs(dst)
540        errors = []
541        for name in names:
542            srcname = os.path.join(src, name)
543            dstname = os.path.join(dst, name)
544            try:
545                if os.path.isdir(srcname):
546                    self.ccopytree(srcname, dstname)
547                else:
548                    self.ccopy(srcname, dstname)
549                    # XXX What about devices, sockets etc.?
550            except (IOError, os.error), why:
551                errors.append((srcname, dstname, why))
552        if errors:
553            raise RuntimeError, errors
554
555
556    def cmakedirs(self, path):
557        """Ensures that a directory exists, and doesn't throw an exception
558        if you call it on an existing directory."""
559#        print "making path: ", path
560        path = os.path.normpath(path)
561        self.created_paths.append(path)
562        if not os.path.exists(path):
563            os.makedirs(path)
564
565    def find_existing_file(self, *list):
566        for f in list:
567            if os.path.exists(f):
568                return f
569        # didn't find it, return last item in list
570        if len(list) > 0:
571            return list[-1]
572        else:
573            return None
574
575    def contents_of_tar(self, src_tar, dst_dir):
576        """ Extracts the contents of the tarfile (specified
577        relative to the source prefix) into the directory
578        specified relative to the destination directory."""
579        self.check_file_exists(src_tar)
580        tf = tarfile.open(self.src_path_of(src_tar), 'r')
581        for member in tf.getmembers():
582            tf.extract(member, self.ensure_dst_dir(dst_dir))
583            # TODO get actions working on these dudes, perhaps we should extract to a temporary directory and then process_directory on it?
584            self.file_list.append([src_tar,
585                           self.dst_path_of(os.path.join(dst_dir,member.name))])
586        tf.close()
587
588
589    def wildcard_regex(self, src_glob, dst_glob):
590        src_re = re.escape(src_glob)
591        src_re = src_re.replace('\*', '([-a-zA-Z0-9._ ]*)')
592        dst_temp = dst_glob
593        i = 1
594        while dst_temp.count("*") > 0:
595            dst_temp = dst_temp.replace('*', '\g<' + str(i) + '>', 1)
596            i = i+1
597        return re.compile(src_re), dst_temp
598
599    def check_file_exists(self, path):
600        if not os.path.exists(path) and not os.path.islink(path):
601            raise RuntimeError("Path %s doesn't exist" % (
602                os.path.normpath(os.path.join(os.getcwd(), path)),))
603
604
605    wildcard_pattern = re.compile('\*')
606    def expand_globs(self, src, dst):
607        src_list = glob.glob(src)
608        src_re, d_template = self.wildcard_regex(src.replace('\\', '/'),
609                                                 dst.replace('\\', '/'))
610        for s in src_list:
611            d = src_re.sub(d_template, s.replace('\\', '/'))
612            yield os.path.normpath(s), os.path.normpath(d)
613
614    def path(self, src, dst=None):
615        sys.stdout.write("Processing %s => %s ... " % (src, dst))
616        sys.stdout.flush()
617        if src == None:
618            raise RuntimeError("No source file, dst is " + dst)
619        if dst == None:
620            dst = src
621        dst = os.path.join(self.get_dst_prefix(), dst)
622
623        def try_path(src):
624            # expand globs
625            count = 0
626            if self.wildcard_pattern.search(src):
627                for s,d in self.expand_globs(src, dst):
628                    assert(s != d)
629                    count += self.process_file(s, d)
630            else:
631                # if we're specifying a single path (not a glob),
632                # we should error out if it doesn't exist
633                self.check_file_exists(src)
634                # if it's a directory, recurse through it
635                if os.path.isdir(src):
636                    count += self.process_directory(src, dst)
637                else:
638                    count += self.process_file(src, dst)
639            return count
640        try:
641            count = try_path(os.path.join(self.get_src_prefix(), src))
642        except RuntimeError:
643            try:
644                count = try_path(os.path.join(self.get_artwork_prefix(), src))
645            except RuntimeError:
646                count = try_path(os.path.join(self.get_build_prefix(), src))
647        print "%d files" % count
648
649    def do(self, *actions):
650        self.actions = actions
651        self.construct()
652        # perform finish actions
653        for action in self.actions:
654            methodname = action + "_finish"
655            method = getattr(self, methodname, None)
656            if method is not None:
657                method()
658        return self.file_list