PageRenderTime 41ms CodeModel.GetById 16ms RepoModel.GetById 0ms app.codeStats 0ms

/scripts/template_verifier.py

https://bitbucket.org/lindenlab/viewer-beta/
Python | 331 lines | 303 code | 1 blank | 27 comment | 0 complexity | 08cab1ac9660586ba0a29afc5c327a0c MD5 | raw file
Possible License(s): LGPL-2.1
  1. #!/usr/bin/env python
  2. """\
  3. @file template_verifier.py
  4. @brief Message template compatibility verifier.
  5. $LicenseInfo:firstyear=2007&license=viewerlgpl$
  6. Second Life Viewer Source Code
  7. Copyright (C) 2010, Linden Research, Inc.
  8. This library is free software; you can redistribute it and/or
  9. modify it under the terms of the GNU Lesser General Public
  10. License as published by the Free Software Foundation;
  11. version 2.1 of the License only.
  12. This library is distributed in the hope that it will be useful,
  13. but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15. Lesser General Public License for more details.
  16. You should have received a copy of the GNU Lesser General Public
  17. License along with this library; if not, write to the Free Software
  18. Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19. Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
  20. $/LicenseInfo$
  21. """
  22. """template_verifier is a script which will compare the
  23. current repository message template with the "master" message template, accessible
  24. via http://secondlife.com/app/message_template/master_message_template.msg
  25. If [FILE] is specified, it will be checked against the master template.
  26. If [FILE] [FILE] is specified, two local files will be checked against
  27. each other.
  28. """
  29. import sys
  30. import os.path
  31. # Look for indra/lib/python in all possible parent directories ...
  32. # This is an improvement over the setup-path.py method used previously:
  33. # * the script may blocated anywhere inside the source tree
  34. # * it doesn't depend on the current directory
  35. # * it doesn't depend on another file being present.
  36. def add_indra_lib_path():
  37. root = os.path.realpath(__file__)
  38. # always insert the directory of the script in the search path
  39. dir = os.path.dirname(root)
  40. if dir not in sys.path:
  41. sys.path.insert(0, dir)
  42. # Now go look for indra/lib/python in the parent dies
  43. while root != os.path.sep:
  44. root = os.path.dirname(root)
  45. dir = os.path.join(root, 'indra', 'lib', 'python')
  46. if os.path.isdir(dir):
  47. if dir not in sys.path:
  48. sys.path.insert(0, dir)
  49. break
  50. else:
  51. print >>sys.stderr, "This script is not inside a valid installation."
  52. sys.exit(1)
  53. add_indra_lib_path()
  54. import optparse
  55. import os
  56. import urllib
  57. import hashlib
  58. from indra.ipc import compatibility
  59. from indra.ipc import tokenstream
  60. from indra.ipc import llmessage
  61. def getstatusall(command):
  62. """ Like commands.getstatusoutput, but returns stdout and
  63. stderr separately(to get around "killed by signal 15" getting
  64. included as part of the file). Also, works on Windows."""
  65. (input, out, err) = os.popen3(command, 't')
  66. status = input.close() # send no input to the command
  67. output = out.read()
  68. error = err.read()
  69. status = out.close()
  70. status = err.close() # the status comes from the *last* pipe that is closed
  71. return status, output, error
  72. def getstatusoutput(command):
  73. status, output, error = getstatusall(command)
  74. return status, output
  75. def die(msg):
  76. print >>sys.stderr, msg
  77. sys.exit(1)
  78. MESSAGE_TEMPLATE = 'message_template.msg'
  79. PRODUCTION_ACCEPTABLE = (compatibility.Same, compatibility.Newer)
  80. DEVELOPMENT_ACCEPTABLE = (
  81. compatibility.Same, compatibility.Newer,
  82. compatibility.Older, compatibility.Mixed)
  83. MAX_MASTER_AGE = 60 * 60 * 4 # refresh master cache every 4 hours
  84. def retry(times, function, *args, **kwargs):
  85. for i in range(times):
  86. try:
  87. return function(*args, **kwargs)
  88. except Exception, e:
  89. if i == times - 1:
  90. raise e # we retried all the times we could
  91. def compare(base_parsed, current_parsed, mode):
  92. """Compare the current template against the base template using the given
  93. 'mode' strictness:
  94. development: Allows Same, Newer, Older, and Mixed
  95. production: Allows only Same or Newer
  96. Print out information about whether the current template is compatible
  97. with the base template.
  98. Returns a tuple of (bool, Compatibility)
  99. Return True if they are compatible in this mode, False if not.
  100. """
  101. compat = current_parsed.compatibleWithBase(base_parsed)
  102. if mode == 'production':
  103. acceptable = PRODUCTION_ACCEPTABLE
  104. else:
  105. acceptable = DEVELOPMENT_ACCEPTABLE
  106. if type(compat) in acceptable:
  107. return True, compat
  108. return False, compat
  109. def fetch(url):
  110. if url.startswith('file://'):
  111. # just open the file directly because urllib is dumb about these things
  112. file_name = url[len('file://'):]
  113. return open(file_name).read()
  114. else:
  115. # *FIX: this doesn't throw an exception for a 404, and oddly enough the sl.com 404 page actually gets parsed successfully
  116. return ''.join(urllib.urlopen(url).readlines())
  117. def cache_master(master_url):
  118. """Using the url for the master, updates the local cache, and returns an url to the local cache."""
  119. master_cache = local_master_cache_filename()
  120. master_cache_url = 'file://' + master_cache
  121. # decide whether to refresh the master cache based on its age
  122. import time
  123. if (os.path.exists(master_cache)
  124. and time.time() - os.path.getmtime(master_cache) < MAX_MASTER_AGE):
  125. return master_cache_url # our cache is fresh
  126. # new master doesn't exist or isn't fresh
  127. print "Refreshing master cache from %s" % master_url
  128. def get_and_test_master():
  129. new_master_contents = fetch(master_url)
  130. llmessage.parseTemplateString(new_master_contents)
  131. return new_master_contents
  132. try:
  133. new_master_contents = retry(3, get_and_test_master)
  134. except IOError, e:
  135. # the refresh failed, so we should just soldier on
  136. print "WARNING: unable to download new master, probably due to network error. Your message template compatibility may be suspect."
  137. print "Cause: %s" % e
  138. return master_cache_url
  139. try:
  140. tmpname = '%s.%d' % (master_cache, os.getpid())
  141. mc = open(tmpname, 'wb')
  142. mc.write(new_master_contents)
  143. mc.close()
  144. try:
  145. os.rename(tmpname, master_cache)
  146. except OSError:
  147. # We can't rename atomically on top of an existing file on
  148. # Windows. Unlinking the existing file will fail if the
  149. # file is being held open by a process, but there's only
  150. # so much working around a lame I/O API one can take in
  151. # a single day.
  152. os.unlink(master_cache)
  153. os.rename(tmpname, master_cache)
  154. except IOError, e:
  155. print "WARNING: Unable to write master message template to %s, proceeding without cache." % master_cache
  156. print "Cause: %s" % e
  157. return master_url
  158. return master_cache_url
  159. def local_template_filename():
  160. """Returns the message template's default location relative to template_verifier.py:
  161. ./messages/message_template.msg."""
  162. d = os.path.dirname(os.path.realpath(__file__))
  163. return os.path.join(d, 'messages', MESSAGE_TEMPLATE)
  164. def getuser():
  165. try:
  166. # Unix-only.
  167. import getpass
  168. return getpass.getuser()
  169. except ImportError:
  170. import ctypes
  171. MAX_PATH = 260 # according to a recent WinDef.h
  172. name = ctypes.create_unicode_buffer(MAX_PATH)
  173. namelen = ctypes.c_int(len(name)) # len in chars, NOT bytes
  174. if not ctypes.windll.advapi32.GetUserNameW(name, ctypes.byref(namelen)):
  175. raise ctypes.WinError()
  176. return name.value
  177. def local_master_cache_filename():
  178. """Returns the location of the master template cache (which is in the system tempdir)
  179. <temp_dir>/master_message_template_cache.msg"""
  180. import tempfile
  181. d = tempfile.gettempdir()
  182. user = getuser()
  183. return os.path.join(d, 'master_message_template_cache.%s.msg' % user)
  184. def run(sysargs):
  185. parser = optparse.OptionParser(
  186. usage="usage: %prog [FILE] [FILE]",
  187. description=__doc__)
  188. parser.add_option(
  189. '-m', '--mode', type='string', dest='mode',
  190. default='development',
  191. help="""[development|production] The strictness mode to use
  192. while checking the template; see the wiki page for details about
  193. what is allowed and disallowed by each mode:
  194. http://wiki.secondlife.com/wiki/Template_verifier.py
  195. """)
  196. parser.add_option(
  197. '-u', '--master_url', type='string', dest='master_url',
  198. default='http://bitbucket.org/lindenlab/master-message-template/raw/tip/message_template.msg',
  199. help="""The url of the master message template.""")
  200. parser.add_option(
  201. '-c', '--cache_master', action='store_true', dest='cache_master',
  202. default=False, help="""Set to true to attempt use local cached copy of the master template.""")
  203. parser.add_option(
  204. '-f', '--force', action='store_true', dest='force_verification',
  205. default=False, help="""Set to true to skip the sha_1 check and force template verification.""")
  206. options, args = parser.parse_args(sysargs)
  207. if options.mode == 'production':
  208. options.cache_master = False
  209. # both current and master supplied in positional params
  210. if len(args) == 2:
  211. master_filename, current_filename = args
  212. print "master:", master_filename
  213. print "current:", current_filename
  214. master_url = 'file://%s' % master_filename
  215. current_url = 'file://%s' % current_filename
  216. # only current supplied in positional param
  217. elif len(args) == 1:
  218. master_url = None
  219. current_filename = args[0]
  220. print "master:", options.master_url
  221. print "current:", current_filename
  222. current_url = 'file://%s' % current_filename
  223. # nothing specified, use defaults for everything
  224. elif len(args) == 0:
  225. master_url = None
  226. current_url = None
  227. else:
  228. die("Too many arguments")
  229. if master_url is None:
  230. master_url = options.master_url
  231. if current_url is None:
  232. current_filename = local_template_filename()
  233. print "master:", options.master_url
  234. print "current:", current_filename
  235. current_url = 'file://%s' % current_filename
  236. # retrieve the contents of the local template
  237. current = fetch(current_url)
  238. hexdigest = hashlib.sha1(current).hexdigest()
  239. if not options.force_verification:
  240. # Early exist if the template hasn't changed.
  241. sha_url = "%s.sha1" % current_url
  242. current_sha = fetch(sha_url)
  243. if hexdigest == current_sha:
  244. print "Message template SHA_1 has not changed."
  245. sys.exit(0)
  246. # and check for syntax
  247. current_parsed = llmessage.parseTemplateString(current)
  248. if options.cache_master:
  249. # optionally return a url to a locally-cached master so we don't hit the network all the time
  250. master_url = cache_master(master_url)
  251. def parse_master_url():
  252. master = fetch(master_url)
  253. return llmessage.parseTemplateString(master)
  254. try:
  255. master_parsed = retry(3, parse_master_url)
  256. except (IOError, tokenstream.ParseError), e:
  257. if options.mode == 'production':
  258. raise e
  259. else:
  260. print "WARNING: problems retrieving the master from %s." % master_url
  261. print "Syntax-checking the local template ONLY, no compatibility check is being run."
  262. print "Cause: %s\n\n" % e
  263. return 0
  264. acceptable, compat = compare(
  265. master_parsed, current_parsed, options.mode)
  266. def explain(header, compat):
  267. print header
  268. # indent compatibility explanation
  269. print '\n\t'.join(compat.explain().split('\n'))
  270. if acceptable:
  271. explain("--- PASS ---", compat)
  272. if options.force_verification == False:
  273. print "Updating sha1 to %s" % hexdigest
  274. sha_filename = "%s.sha1" % current_filename
  275. sha_file = open(sha_filename, 'w')
  276. sha_file.write(hexdigest)
  277. sha_file.close()
  278. else:
  279. explain("*** FAIL ***", compat)
  280. return 1
  281. if __name__ == '__main__':
  282. sys.exit(run(sys.argv[1:]))