PageRenderTime 40ms CodeModel.GetById 12ms app.highlight 22ms RepoModel.GetById 2ms app.codeStats 0ms

/scripts/template_verifier.py

https://bitbucket.org/lindenlab/viewer-beta/
Python | 331 lines | 303 code | 1 blank | 27 comment | 2 complexity | 08cab1ac9660586ba0a29afc5c327a0c MD5 | raw file
  1#!/usr/bin/env python
  2"""\
  3@file template_verifier.py
  4@brief Message template compatibility verifier.
  5
  6$LicenseInfo:firstyear=2007&license=viewerlgpl$
  7Second Life Viewer Source Code
  8Copyright (C) 2010, Linden Research, Inc.
  9
 10This library is free software; you can redistribute it and/or
 11modify it under the terms of the GNU Lesser General Public
 12License as published by the Free Software Foundation;
 13version 2.1 of the License only.
 14
 15This library is distributed in the hope that it will be useful,
 16but WITHOUT ANY WARRANTY; without even the implied warranty of
 17MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 18Lesser General Public License for more details.
 19
 20You should have received a copy of the GNU Lesser General Public
 21License along with this library; if not, write to the Free Software
 22Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 23
 24Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
 25$/LicenseInfo$
 26"""
 27
 28"""template_verifier is a script which will compare the
 29current repository message template with the "master" message template, accessible
 30via http://secondlife.com/app/message_template/master_message_template.msg
 31If [FILE] is specified, it will be checked against the master template.
 32If [FILE] [FILE] is specified, two local files will be checked against
 33each other.
 34"""
 35
 36import sys
 37import os.path
 38
 39# Look for indra/lib/python in all possible parent directories ...
 40# This is an improvement over the setup-path.py method used previously:
 41#  * the script may blocated anywhere inside the source tree
 42#  * it doesn't depend on the current directory
 43#  * it doesn't depend on another file being present.
 44
 45def add_indra_lib_path():
 46    root = os.path.realpath(__file__)
 47    # always insert the directory of the script in the search path
 48    dir = os.path.dirname(root)
 49    if dir not in sys.path:
 50        sys.path.insert(0, dir)
 51
 52    # Now go look for indra/lib/python in the parent dies
 53    while root != os.path.sep:
 54        root = os.path.dirname(root)
 55        dir = os.path.join(root, 'indra', 'lib', 'python')
 56        if os.path.isdir(dir):
 57            if dir not in sys.path:
 58                sys.path.insert(0, dir)
 59            break
 60    else:
 61        print >>sys.stderr, "This script is not inside a valid installation."
 62        sys.exit(1)
 63
 64add_indra_lib_path()
 65
 66import optparse
 67import os
 68import urllib
 69import hashlib
 70
 71from indra.ipc import compatibility
 72from indra.ipc import tokenstream
 73from indra.ipc import llmessage
 74
 75def getstatusall(command):
 76    """ Like commands.getstatusoutput, but returns stdout and 
 77    stderr separately(to get around "killed by signal 15" getting 
 78    included as part of the file).  Also, works on Windows."""
 79    (input, out, err) = os.popen3(command, 't')
 80    status = input.close() # send no input to the command
 81    output = out.read()
 82    error = err.read()
 83    status = out.close()
 84    status = err.close() # the status comes from the *last* pipe that is closed
 85    return status, output, error
 86
 87def getstatusoutput(command):
 88    status, output, error = getstatusall(command)
 89    return status, output
 90
 91
 92def die(msg):
 93    print >>sys.stderr, msg
 94    sys.exit(1)
 95
 96MESSAGE_TEMPLATE = 'message_template.msg'
 97
 98PRODUCTION_ACCEPTABLE = (compatibility.Same, compatibility.Newer)
 99DEVELOPMENT_ACCEPTABLE = (
100    compatibility.Same, compatibility.Newer,
101    compatibility.Older, compatibility.Mixed)
102
103MAX_MASTER_AGE = 60 * 60 * 4   # refresh master cache every 4 hours
104
105def retry(times, function, *args, **kwargs):
106    for i in range(times):
107        try:
108            return function(*args, **kwargs)
109        except Exception, e:
110            if i == times - 1:
111                raise e  # we retried all the times we could
112
113def compare(base_parsed, current_parsed, mode):
114    """Compare the current template against the base template using the given
115    'mode' strictness:
116
117    development: Allows Same, Newer, Older, and Mixed
118    production: Allows only Same or Newer
119
120    Print out information about whether the current template is compatible
121    with the base template.
122
123    Returns a tuple of (bool, Compatibility)
124    Return True if they are compatible in this mode, False if not.
125    """
126
127    compat = current_parsed.compatibleWithBase(base_parsed)
128    if mode == 'production':
129        acceptable = PRODUCTION_ACCEPTABLE
130    else:
131        acceptable = DEVELOPMENT_ACCEPTABLE
132
133    if type(compat) in acceptable:
134        return True, compat
135    return False, compat
136
137def fetch(url):
138    if url.startswith('file://'):
139        # just open the file directly because urllib is dumb about these things
140        file_name = url[len('file://'):]
141        return open(file_name).read()
142    else:
143        # *FIX: this doesn't throw an exception for a 404, and oddly enough the sl.com 404 page actually gets parsed successfully
144        return ''.join(urllib.urlopen(url).readlines())   
145
146def cache_master(master_url):
147    """Using the url for the master, updates the local cache, and returns an url to the local cache."""
148    master_cache = local_master_cache_filename()
149    master_cache_url = 'file://' + master_cache
150    # decide whether to refresh the master cache based on its age
151    import time
152    if (os.path.exists(master_cache)
153        and time.time() - os.path.getmtime(master_cache) < MAX_MASTER_AGE):
154        return master_cache_url  # our cache is fresh
155    # new master doesn't exist or isn't fresh
156    print "Refreshing master cache from %s" % master_url
157    def get_and_test_master():
158        new_master_contents = fetch(master_url)
159        llmessage.parseTemplateString(new_master_contents)
160        return new_master_contents
161    try:
162        new_master_contents = retry(3, get_and_test_master)
163    except IOError, e:
164        # the refresh failed, so we should just soldier on
165        print "WARNING: unable to download new master, probably due to network error.  Your message template compatibility may be suspect."
166        print "Cause: %s" % e
167        return master_cache_url
168    try:
169        tmpname = '%s.%d' % (master_cache, os.getpid())
170        mc = open(tmpname, 'wb')
171        mc.write(new_master_contents)
172        mc.close()
173        try:
174            os.rename(tmpname, master_cache)
175        except OSError:
176            # We can't rename atomically on top of an existing file on
177            # Windows.  Unlinking the existing file will fail if the
178            # file is being held open by a process, but there's only
179            # so much working around a lame I/O API one can take in
180            # a single day.
181            os.unlink(master_cache)
182            os.rename(tmpname, master_cache)
183    except IOError, e:
184        print "WARNING: Unable to write master message template to %s, proceeding without cache." % master_cache
185        print "Cause: %s" % e
186        return master_url
187    return master_cache_url
188
189def local_template_filename():
190    """Returns the message template's default location relative to template_verifier.py:
191    ./messages/message_template.msg."""
192    d = os.path.dirname(os.path.realpath(__file__))
193    return os.path.join(d, 'messages', MESSAGE_TEMPLATE)
194
195def getuser():
196    try:
197        # Unix-only.
198        import getpass
199        return getpass.getuser()
200    except ImportError:
201        import ctypes
202        MAX_PATH = 260                  # according to a recent WinDef.h
203        name = ctypes.create_unicode_buffer(MAX_PATH)
204        namelen = ctypes.c_int(len(name)) # len in chars, NOT bytes
205        if not ctypes.windll.advapi32.GetUserNameW(name, ctypes.byref(namelen)):
206            raise ctypes.WinError()
207        return name.value
208
209def local_master_cache_filename():
210    """Returns the location of the master template cache (which is in the system tempdir)
211    <temp_dir>/master_message_template_cache.msg"""
212    import tempfile
213    d = tempfile.gettempdir()
214    user = getuser()
215    return os.path.join(d, 'master_message_template_cache.%s.msg' % user)
216
217
218def run(sysargs):
219    parser = optparse.OptionParser(
220        usage="usage: %prog [FILE] [FILE]",
221        description=__doc__)
222    parser.add_option(
223        '-m', '--mode', type='string', dest='mode',
224        default='development',
225        help="""[development|production] The strictness mode to use
226while checking the template; see the wiki page for details about
227what is allowed and disallowed by each mode:
228http://wiki.secondlife.com/wiki/Template_verifier.py
229""")
230    parser.add_option(
231        '-u', '--master_url', type='string', dest='master_url',
232        default='http://bitbucket.org/lindenlab/master-message-template/raw/tip/message_template.msg',
233        help="""The url of the master message template.""")
234    parser.add_option(
235        '-c', '--cache_master', action='store_true', dest='cache_master',
236        default=False,  help="""Set to true to attempt use local cached copy of the master template.""")
237    parser.add_option(
238        '-f', '--force', action='store_true', dest='force_verification',
239        default=False, help="""Set to true to skip the sha_1 check and force template verification.""")
240
241    options, args = parser.parse_args(sysargs)
242
243    if options.mode == 'production':
244        options.cache_master = False
245
246    # both current and master supplied in positional params
247    if len(args) == 2:
248        master_filename, current_filename = args
249        print "master:", master_filename
250        print "current:", current_filename
251        master_url = 'file://%s' % master_filename
252        current_url = 'file://%s' % current_filename
253    # only current supplied in positional param
254    elif len(args) == 1:
255        master_url = None
256        current_filename = args[0]
257        print "master:", options.master_url 
258        print "current:", current_filename
259        current_url = 'file://%s' % current_filename
260    # nothing specified, use defaults for everything
261    elif len(args) == 0:
262        master_url  = None
263        current_url = None
264    else:
265        die("Too many arguments")
266
267    if master_url is None:
268        master_url = options.master_url
269        
270    if current_url is None:
271        current_filename = local_template_filename()
272        print "master:", options.master_url
273        print "current:", current_filename
274        current_url = 'file://%s' % current_filename
275
276    # retrieve the contents of the local template
277    current = fetch(current_url)
278    hexdigest = hashlib.sha1(current).hexdigest()
279    if not options.force_verification:
280        # Early exist if the template hasn't changed.
281        sha_url = "%s.sha1" % current_url
282        current_sha = fetch(sha_url)
283        if hexdigest == current_sha:
284            print "Message template SHA_1 has not changed."
285            sys.exit(0)
286
287    # and check for syntax
288    current_parsed = llmessage.parseTemplateString(current)
289
290    if options.cache_master:
291        # optionally return a url to a locally-cached master so we don't hit the network all the time
292        master_url = cache_master(master_url)
293
294    def parse_master_url():
295        master = fetch(master_url)
296        return llmessage.parseTemplateString(master)
297    try:
298        master_parsed = retry(3, parse_master_url)
299    except (IOError, tokenstream.ParseError), e:
300        if options.mode == 'production':
301            raise e
302        else:
303            print "WARNING: problems retrieving the master from %s."  % master_url
304            print "Syntax-checking the local template ONLY, no compatibility check is being run."
305            print "Cause: %s\n\n" % e
306            return 0
307        
308    acceptable, compat = compare(
309        master_parsed, current_parsed, options.mode)
310
311    def explain(header, compat):
312        print header
313        # indent compatibility explanation
314        print '\n\t'.join(compat.explain().split('\n'))
315
316    if acceptable:
317        explain("--- PASS ---", compat)
318        if options.force_verification == False:
319            print "Updating sha1 to %s" % hexdigest
320            sha_filename = "%s.sha1" % current_filename
321            sha_file = open(sha_filename, 'w')
322            sha_file.write(hexdigest)
323            sha_file.close()
324    else:
325        explain("*** FAIL ***", compat)
326        return 1
327
328if __name__ == '__main__':
329    sys.exit(run(sys.argv[1:]))
330
331