PageRenderTime 27ms CodeModel.GetById 12ms RepoModel.GetById 1ms app.codeStats 0ms

/silverlining/mgr-scripts/master-runner.py

https://bitbucket.org/ianb/silverlining/
Python | 192 lines | 156 code | 17 blank | 19 comment | 21 complexity | 26501bea4e02bb48e53fa37bfa29a0ff MD5 | raw file
Possible License(s): GPL-2.0
  1. """Called by mod_wsgi, this application finds and starts all requests
  2. This defines application(), which is a WSGI application that mod_wsgi looks for.
  3. It uses $SILVER_INSTANCE_NAME to figure out who to send the request to, then
  4. configures all services, and then passes the request on to the new
  5. application. This loading process only happens once.
  6. Also for each request the environment is fixed up some to represent
  7. the request properly after having gone through Varnish.
  8. """
  9. import sys
  10. sys.path.insert(0, '/usr/local/share/silverlining/lib')
  11. # Import these to work around a mod_wsgi problem:
  12. import time, _strptime
  13. import os
  14. import re
  15. import urllib
  16. import threading
  17. from silversupport.appconfig import AppConfig
  18. #don't show DeprecationWarning in error.log
  19. #TODO, make this configurable
  20. import warnings
  21. warnings.simplefilter('ignore', DeprecationWarning)
  22. found_app = None
  23. found_app_instance_name = None
  24. error_collector = None
  25. def application(environ, start_response):
  26. try:
  27. try:
  28. get_app(environ)
  29. except:
  30. import traceback
  31. bt = traceback.format_exc()
  32. start_response('500 Server Error', [('Content-type', 'text/plain')])
  33. lines = ['There was an error loading the application:', bt, '\nEnviron:']
  34. for name, value in sorted(environ.items()):
  35. try:
  36. lines.append('%s=%r' % (name, value))
  37. except:
  38. lines.append('%s=<error>' % name)
  39. return ['\n'.join(lines)]
  40. try:
  41. return found_app(environ, start_response)
  42. except:
  43. import traceback
  44. traceback.print_exc()
  45. raise
  46. finally:
  47. if error_collector is not None:
  48. error_collector.flush_request(environ)
  49. def get_app(environ):
  50. global found_app, found_app_instance_name, error_collector
  51. delete_boring_vars(environ)
  52. instance_name = environ['SILVER_INSTANCE_NAME']
  53. os.environ['SILVER_INSTANCE_NAME'] = instance_name
  54. app_config = AppConfig.from_instance_name(instance_name)
  55. if error_collector is None:
  56. log_location = os.path.join('/var/log/silverlining/apps', app_config.app_name)
  57. if not os.path.exists(log_location):
  58. os.makedirs(log_location)
  59. error_collector = ErrorCollector(os.path.join(log_location, 'errors.log'))
  60. sys.stderr = sys.stdout = error_collector
  61. error_collector.start_request()
  62. environ['silverlining.apache_errors'] = environ['wsgi.errors']
  63. environ['wsgi.errors'] = error_collector
  64. os.environ['SILVER_CANONICAL_HOSTNAME'] = app_config.canonical_hostname()
  65. ## FIXME: give a real version here...
  66. environ['SILVER_VERSION'] = os.environ['SILVER_VERSION'] = 'silverlining/0.0'
  67. if 'SILVER_MATCH_PATH' in environ:
  68. path = urllib.unquote(environ.pop('SILVER_MATCH_PATH'))
  69. if path != '/':
  70. # Only paths besides / are interesting
  71. environ['SCRIPT_NAME'] += path
  72. assert environ['PATH_INFO'].startswith(path)
  73. environ['PATH_INFO'] = environ['PATH_INFO'][len(path):]
  74. assert not environ['PATH_INFO'] or environ['PATH_INFO'].startswith('/'), (
  75. "Bad PATH_INFO: %r (SCRIPT_NAME: %r, SILVER_MATCH_PATH: %r)" %
  76. (environ['PATH_INFO'], environ['SCRIPT_NAME'], path))
  77. # Fixup port and ipaddress
  78. environ['SERVER_PORT'] = '80'
  79. if 'HTTP_X_FORWARDED_FOR' in environ:
  80. environ['REMOTE_ADDR'] = environ.pop('HTTP_X_FORWARDED_FOR', '').split(',')[0]
  81. if 'HTTP_X_VARNISH_IP' in environ:
  82. environ['SERVER_ADDR'] = environ.pop('HTTP_X_VARNISH_IP')
  83. if 'SCRIPT_URI' in environ:
  84. environ['SCRIPT_URI'] = environ['SCRIPT_URI'].replace(':8080', '')
  85. if found_app:
  86. assert found_app_instance_name == instance_name, (
  87. "second request with unexpected instance_name (first request had instance_name=%r; "
  88. "next request had instance_name=%r)" % (found_app_instance_name, instance_name))
  89. return found_app
  90. # The application group we are running:
  91. if not re.search(r'^[A-Za-z0-9._-]+$', instance_name):
  92. raise Exception("Bad instance_name: %r" % instance_name)
  93. app_config.activate_services(os.environ)
  94. app_config.activate_path()
  95. try:
  96. found_app = app_config.get_app_from_runner()
  97. except Exception, e:
  98. raise
  99. return ErrorApp(
  100. "Could not load the runner %s: %s" % (app_config.runner, e))
  101. assert found_app is not None, (
  102. "app_config %r.get_app_from_runner() returned None")
  103. found_app_instance_name = instance_name
  104. return found_app
  105. class ErrorApp(object):
  106. """Application that simply displays the error message"""
  107. def __init__(self, message):
  108. self.message = message
  109. def __call__(self, environ, start_response):
  110. start_response('500 Server Error', [('Content-type', 'text/plain')])
  111. return [self.message]
  112. # These are variables set in Apache/wsgi_runner, that are often quite
  113. # dull but used internally; we'll delete them for cleanliness:
  114. BORING_VARS = [
  115. 'SILVER_APP_DATA', 'SILVER_PROCESS_GROUP', 'SILVER_PLATFORM',
  116. 'SILVER_PHP_ROOT', 'SILVER_REDIR_HOST', 'SILVER_FORWARDED',
  117. 'SILVER_HOST', 'SILVER_REMAINING_URI']
  118. def delete_boring_vars(environ):
  119. for name in BORING_VARS:
  120. if name in environ:
  121. del environ[name]
  122. class ErrorCollector(object):
  123. def __init__(self, filename):
  124. self.filename = filename
  125. self.buffers = threading.local()
  126. def start_request(self):
  127. self.buffers.start_time = time.time()
  128. self.buffers.buffer = []
  129. def write(self, text):
  130. if isinstance(text, unicode):
  131. text = text.encode('utf8')
  132. self.buffers.buffer.append(text)
  133. def writelines(self, lines):
  134. for line in lines:
  135. self.write(line)
  136. def flush(self):
  137. ## FIXME: should this do something?
  138. pass
  139. def close(self):
  140. ## FIXME: should this exist?
  141. pass
  142. def flush_request(self, environ):
  143. if not self.buffers.buffer:
  144. return
  145. total = time.time() - self.buffers.start_time
  146. buf = self.buffers.buffer
  147. date_formatted = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(self.buffers.start_time))
  148. req_name = (
  149. environ['REQUEST_METHOD'] + ' ' +
  150. environ.get('SCRIPT_NAME', '') +
  151. environ.get('PATH_INFO', ''))
  152. buf.insert(0, 'Errors for request %s (%s, %fsec):\n'
  153. % (req_name, date_formatted, total))
  154. if not buf[-1].endswith('\n'):
  155. buf.append('\n')
  156. buf.append('Finish errors for request %s (%s)\n' % (req_name, date_formatted))
  157. complete = ''.join(buf)
  158. fp = open(self.filename, 'a')
  159. fp.write(complete)
  160. fp.flush()
  161. fp.close()
  162. self.buffers.buffer[:] = []
  163. def __repr__(self):
  164. return '<silverlining error collector %s>' % self.filename