/silverlining/mgr-scripts/master-runner.py

  1"""Called by mod_wsgi, this application finds and starts all requests
  2
  3This defines application(), which is a WSGI application that mod_wsgi looks for.
  4
  5It uses $SILVER_INSTANCE_NAME to figure out who to send the request to, then
  6configures all services, and then passes the request on to the new
  7application.  This loading process only happens once.
  8
  9Also for each request the environment is fixed up some to represent
 10the request properly after having gone through Varnish.
 11"""
 12import sys
 13sys.path.insert(0, '/usr/local/share/silverlining/lib')
 14# Import these to work around a mod_wsgi problem:
 15import time, _strptime
 16import os
 17import re
 18import urllib
 19import threading
 20from silversupport.appconfig import AppConfig
 21
 22#don't show DeprecationWarning in error.log
 23#TODO, make this configurable
 24import warnings
 25warnings.simplefilter('ignore', DeprecationWarning)
 26
 27found_app = None
 28found_app_instance_name = None
 29error_collector = None
 30
 31
 32def application(environ, start_response):
 33    try:
 34        try:
 35            get_app(environ)
 36        except:
 37            import traceback
 38            bt = traceback.format_exc()
 39            start_response('500 Server Error', [('Content-type', 'text/plain')])
 40            lines = ['There was an error loading the application:', bt, '\nEnviron:']
 41            for name, value in sorted(environ.items()):
 42                try:
 43                    lines.append('%s=%r' % (name, value))
 44                except:
 45                    lines.append('%s=<error>' % name)
 46            return ['\n'.join(lines)]
 47        try:
 48            return found_app(environ, start_response)
 49        except:
 50            import traceback
 51            traceback.print_exc()
 52            raise
 53    finally:
 54        if error_collector is not None:
 55            error_collector.flush_request(environ)
 56
 57
 58def get_app(environ):
 59    global found_app, found_app_instance_name, error_collector
 60    delete_boring_vars(environ)
 61    instance_name = environ['SILVER_INSTANCE_NAME']
 62    os.environ['SILVER_INSTANCE_NAME'] = instance_name
 63    app_config = AppConfig.from_instance_name(instance_name)
 64    if error_collector is None:
 65        log_location = os.path.join('/var/log/silverlining/apps', app_config.app_name)
 66        if not os.path.exists(log_location):
 67            os.makedirs(log_location)
 68        error_collector = ErrorCollector(os.path.join(log_location, 'errors.log'))
 69        sys.stderr = sys.stdout = error_collector
 70    error_collector.start_request()
 71    environ['silverlining.apache_errors'] = environ['wsgi.errors']
 72    environ['wsgi.errors'] = error_collector
 73    os.environ['SILVER_CANONICAL_HOSTNAME'] = app_config.canonical_hostname()
 74    ## FIXME: give a real version here...
 75    environ['SILVER_VERSION'] = os.environ['SILVER_VERSION'] = 'silverlining/0.0'
 76    if 'SILVER_MATCH_PATH' in environ:
 77        path = urllib.unquote(environ.pop('SILVER_MATCH_PATH'))
 78        if path != '/':
 79            # Only paths besides / are interesting
 80            environ['SCRIPT_NAME'] += path
 81            assert environ['PATH_INFO'].startswith(path)
 82            environ['PATH_INFO'] = environ['PATH_INFO'][len(path):]
 83            assert not environ['PATH_INFO'] or environ['PATH_INFO'].startswith('/'), (
 84                "Bad PATH_INFO: %r (SCRIPT_NAME: %r, SILVER_MATCH_PATH: %r)" %
 85                (environ['PATH_INFO'], environ['SCRIPT_NAME'], path))
 86
 87    # Fixup port and ipaddress
 88    environ['SERVER_PORT'] = '80'
 89    if 'HTTP_X_FORWARDED_FOR' in environ:
 90        environ['REMOTE_ADDR'] = environ.pop('HTTP_X_FORWARDED_FOR', '').split(',')[0]
 91    if 'HTTP_X_VARNISH_IP' in environ:
 92        environ['SERVER_ADDR'] = environ.pop('HTTP_X_VARNISH_IP')
 93    if 'SCRIPT_URI' in environ:
 94        environ['SCRIPT_URI'] = environ['SCRIPT_URI'].replace(':8080', '')
 95
 96    if found_app:
 97        assert found_app_instance_name == instance_name, (
 98            "second request with unexpected instance_name (first request had instance_name=%r; "
 99            "next request had instance_name=%r)" % (found_app_instance_name, instance_name))
100        return found_app
101    # The application group we are running:
102    if not re.search(r'^[A-Za-z0-9._-]+$', instance_name):
103        raise Exception("Bad instance_name: %r" % instance_name)
104
105    app_config.activate_services(os.environ)
106    app_config.activate_path()
107
108    try:
109        found_app = app_config.get_app_from_runner()
110    except Exception, e:
111        raise
112        return ErrorApp(
113            "Could not load the runner %s: %s" % (app_config.runner, e))
114    assert found_app is not None, (
115        "app_config %r.get_app_from_runner() returned None")
116    found_app_instance_name = instance_name
117    return found_app
118
119
120class ErrorApp(object):
121    """Application that simply displays the error message"""
122    def __init__(self, message):
123        self.message = message
124
125    def __call__(self, environ, start_response):
126        start_response('500 Server Error', [('Content-type', 'text/plain')])
127        return [self.message]
128
129# These are variables set in Apache/wsgi_runner, that are often quite
130# dull but used internally; we'll delete them for cleanliness:
131BORING_VARS = [
132    'SILVER_APP_DATA', 'SILVER_PROCESS_GROUP', 'SILVER_PLATFORM',
133    'SILVER_PHP_ROOT', 'SILVER_REDIR_HOST', 'SILVER_FORWARDED',
134    'SILVER_HOST', 'SILVER_REMAINING_URI']
135
136
137def delete_boring_vars(environ):
138    for name in BORING_VARS:
139        if name in environ:
140            del environ[name]
141
142
143class ErrorCollector(object):
144    def __init__(self, filename):
145        self.filename = filename
146        self.buffers = threading.local()
147
148    def start_request(self):
149        self.buffers.start_time = time.time()
150        self.buffers.buffer = []
151
152    def write(self, text):
153        if isinstance(text, unicode):
154            text = text.encode('utf8')
155        self.buffers.buffer.append(text)
156
157    def writelines(self, lines):
158        for line in lines:
159            self.write(line)
160
161    def flush(self):
162        ## FIXME: should this do something?
163        pass
164
165    def close(self):
166        ## FIXME: should this exist?
167        pass
168
169    def flush_request(self, environ):
170        if not self.buffers.buffer:
171            return
172        total = time.time() - self.buffers.start_time
173        buf = self.buffers.buffer
174        date_formatted = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(self.buffers.start_time))
175        req_name = (
176            environ['REQUEST_METHOD'] + ' ' +
177            environ.get('SCRIPT_NAME', '') +
178            environ.get('PATH_INFO', ''))
179        buf.insert(0, 'Errors for request %s (%s, %fsec):\n'
180                   % (req_name, date_formatted, total))
181        if not buf[-1].endswith('\n'):
182            buf.append('\n')
183        buf.append('Finish errors for request %s (%s)\n' % (req_name, date_formatted))
184        complete = ''.join(buf)
185        fp = open(self.filename, 'a')
186        fp.write(complete)
187        fp.flush()
188        fp.close()
189        self.buffers.buffer[:] = []
190
191    def __repr__(self):
192        return '<silverlining error collector %s>' % self.filename