/silverlining/mgr-scripts/master-runner.py
Python | 192 lines | 156 code | 17 blank | 19 comment | 26 complexity | 26501bea4e02bb48e53fa37bfa29a0ff MD5 | raw file
1"""Called by mod_wsgi, this application finds and starts all requests 2 3This defines application(), which is a WSGI application that mod_wsgi looks for. 4 5It uses $SILVER_INSTANCE_NAME to figure out who to send the request to, then 6configures all services, and then passes the request on to the new 7application. This loading process only happens once. 8 9Also for each request the environment is fixed up some to represent 10the request properly after having gone through Varnish. 11""" 12import sys 13sys.path.insert(0, '/usr/local/share/silverlining/lib') 14# Import these to work around a mod_wsgi problem: 15import time, _strptime 16import os 17import re 18import urllib 19import threading 20from silversupport.appconfig import AppConfig 21 22#don't show DeprecationWarning in error.log 23#TODO, make this configurable 24import warnings 25warnings.simplefilter('ignore', DeprecationWarning) 26 27found_app = None 28found_app_instance_name = None 29error_collector = None 30 31 32def application(environ, start_response): 33 try: 34 try: 35 get_app(environ) 36 except: 37 import traceback 38 bt = traceback.format_exc() 39 start_response('500 Server Error', [('Content-type', 'text/plain')]) 40 lines = ['There was an error loading the application:', bt, '\nEnviron:'] 41 for name, value in sorted(environ.items()): 42 try: 43 lines.append('%s=%r' % (name, value)) 44 except: 45 lines.append('%s=<error>' % name) 46 return ['\n'.join(lines)] 47 try: 48 return found_app(environ, start_response) 49 except: 50 import traceback 51 traceback.print_exc() 52 raise 53 finally: 54 if error_collector is not None: 55 error_collector.flush_request(environ) 56 57 58def get_app(environ): 59 global found_app, found_app_instance_name, error_collector 60 delete_boring_vars(environ) 61 instance_name = environ['SILVER_INSTANCE_NAME'] 62 os.environ['SILVER_INSTANCE_NAME'] = instance_name 63 app_config = AppConfig.from_instance_name(instance_name) 64 if error_collector is None: 65 log_location = os.path.join('/var/log/silverlining/apps', app_config.app_name) 66 if not os.path.exists(log_location): 67 os.makedirs(log_location) 68 error_collector = ErrorCollector(os.path.join(log_location, 'errors.log')) 69 sys.stderr = sys.stdout = error_collector 70 error_collector.start_request() 71 environ['silverlining.apache_errors'] = environ['wsgi.errors'] 72 environ['wsgi.errors'] = error_collector 73 os.environ['SILVER_CANONICAL_HOSTNAME'] = app_config.canonical_hostname() 74 ## FIXME: give a real version here... 75 environ['SILVER_VERSION'] = os.environ['SILVER_VERSION'] = 'silverlining/0.0' 76 if 'SILVER_MATCH_PATH' in environ: 77 path = urllib.unquote(environ.pop('SILVER_MATCH_PATH')) 78 if path != '/': 79 # Only paths besides / are interesting 80 environ['SCRIPT_NAME'] += path 81 assert environ['PATH_INFO'].startswith(path) 82 environ['PATH_INFO'] = environ['PATH_INFO'][len(path):] 83 assert not environ['PATH_INFO'] or environ['PATH_INFO'].startswith('/'), ( 84 "Bad PATH_INFO: %r (SCRIPT_NAME: %r, SILVER_MATCH_PATH: %r)" % 85 (environ['PATH_INFO'], environ['SCRIPT_NAME'], path)) 86 87 # Fixup port and ipaddress 88 environ['SERVER_PORT'] = '80' 89 if 'HTTP_X_FORWARDED_FOR' in environ: 90 environ['REMOTE_ADDR'] = environ.pop('HTTP_X_FORWARDED_FOR', '').split(',')[0] 91 if 'HTTP_X_VARNISH_IP' in environ: 92 environ['SERVER_ADDR'] = environ.pop('HTTP_X_VARNISH_IP') 93 if 'SCRIPT_URI' in environ: 94 environ['SCRIPT_URI'] = environ['SCRIPT_URI'].replace(':8080', '') 95 96 if found_app: 97 assert found_app_instance_name == instance_name, ( 98 "second request with unexpected instance_name (first request had instance_name=%r; " 99 "next request had instance_name=%r)" % (found_app_instance_name, instance_name)) 100 return found_app 101 # The application group we are running: 102 if not re.search(r'^[A-Za-z0-9._-]+$', instance_name): 103 raise Exception("Bad instance_name: %r" % instance_name) 104 105 app_config.activate_services(os.environ) 106 app_config.activate_path() 107 108 try: 109 found_app = app_config.get_app_from_runner() 110 except Exception, e: 111 raise 112 return ErrorApp( 113 "Could not load the runner %s: %s" % (app_config.runner, e)) 114 assert found_app is not None, ( 115 "app_config %r.get_app_from_runner() returned None") 116 found_app_instance_name = instance_name 117 return found_app 118 119 120class ErrorApp(object): 121 """Application that simply displays the error message""" 122 def __init__(self, message): 123 self.message = message 124 125 def __call__(self, environ, start_response): 126 start_response('500 Server Error', [('Content-type', 'text/plain')]) 127 return [self.message] 128 129# These are variables set in Apache/wsgi_runner, that are often quite 130# dull but used internally; we'll delete them for cleanliness: 131BORING_VARS = [ 132 'SILVER_APP_DATA', 'SILVER_PROCESS_GROUP', 'SILVER_PLATFORM', 133 'SILVER_PHP_ROOT', 'SILVER_REDIR_HOST', 'SILVER_FORWARDED', 134 'SILVER_HOST', 'SILVER_REMAINING_URI'] 135 136 137def delete_boring_vars(environ): 138 for name in BORING_VARS: 139 if name in environ: 140 del environ[name] 141 142 143class ErrorCollector(object): 144 def __init__(self, filename): 145 self.filename = filename 146 self.buffers = threading.local() 147 148 def start_request(self): 149 self.buffers.start_time = time.time() 150 self.buffers.buffer = [] 151 152 def write(self, text): 153 if isinstance(text, unicode): 154 text = text.encode('utf8') 155 self.buffers.buffer.append(text) 156 157 def writelines(self, lines): 158 for line in lines: 159 self.write(line) 160 161 def flush(self): 162 ## FIXME: should this do something? 163 pass 164 165 def close(self): 166 ## FIXME: should this exist? 167 pass 168 169 def flush_request(self, environ): 170 if not self.buffers.buffer: 171 return 172 total = time.time() - self.buffers.start_time 173 buf = self.buffers.buffer 174 date_formatted = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(self.buffers.start_time)) 175 req_name = ( 176 environ['REQUEST_METHOD'] + ' ' + 177 environ.get('SCRIPT_NAME', '') + 178 environ.get('PATH_INFO', '')) 179 buf.insert(0, 'Errors for request %s (%s, %fsec):\n' 180 % (req_name, date_formatted, total)) 181 if not buf[-1].endswith('\n'): 182 buf.append('\n') 183 buf.append('Finish errors for request %s (%s)\n' % (req_name, date_formatted)) 184 complete = ''.join(buf) 185 fp = open(self.filename, 'a') 186 fp.write(complete) 187 fp.flush() 188 fp.close() 189 self.buffers.buffer[:] = [] 190 191 def __repr__(self): 192 return '<silverlining error collector %s>' % self.filename