/silversupport/appconfig.py
Python | 454 lines | 440 code | 8 blank | 6 comment | 15 complexity | 5d3053814ae47da63d67f9cdb614094c MD5 | raw file
1"""Application configuration object""" 2 3import os 4import pwd 5import sys 6import warnings 7from site import addsitedir 8from silversupport.env import is_production 9from silversupport.shell import run 10from silversupport.util import asbool, read_config 11from silversupport.disabledapps import DisabledSite, is_disabled 12 13__all__ = ['AppConfig'] 14 15DEPLOYMENT_LOCATION = "/var/www" 16 17 18class AppConfig(object): 19 """This represents an application's configuration file, and by 20 extension represents the application itself""" 21 22 def __init__(self, config_file, app_name=None, 23 local_config=None): 24 if not os.path.exists(config_file): 25 raise OSError("No config file %s" % config_file) 26 self.config_file = config_file 27 self.config = read_config(config_file) 28 if not is_production(): 29 if self.config['production'].get('version'): 30 warnings.warn('version setting in %s is deprecated' % config_file) 31 if self.config['production'].get('default_hostname'): 32 warnings.warn('default_hostname setting in %s has been renamed to default_location' 33 % config_file) 34 if self.config['production'].get('default_host'): 35 warnings.warn('default_host setting in %s has been renamed to default_location' 36 % config_file) 37 if app_name is None: 38 if is_production(): 39 app_name = os.path.basename(os.path.dirname(config_file)).split('.')[0] 40 else: 41 app_name = self.config['production']['app_name'] 42 self.app_name = app_name 43 self.local_config = local_config 44 45 @classmethod 46 def from_instance_name(cls, instance_name): 47 """Loads an instance given its name; only valid in production""" 48 return cls(os.path.join(DEPLOYMENT_LOCATION, instance_name, 'app.ini')) 49 50 @classmethod 51 def from_location(cls, location): 52 """Loads an instance given its location (hostname[/path])""" 53 import appdata 54 return cls.from_instance_name( 55 appdata.instance_for_location(*appdata.normalize_location(location))) 56 57 @property 58 def platform(self): 59 """The platform of the application. 60 61 Current valid values are ``'python'`` and ``'php'`` 62 """ 63 return self.config['production'].get('platform', 'python') 64 65 @property 66 def runner(self): 67 """The filename of the runner for this application""" 68 filename = self.config['production']['runner'] 69 return os.path.join(os.path.dirname(self.config_file), filename) 70 71 @property 72 def update_fetch(self): 73 """A list (possibly empty) of all URLs to fetch on update""" 74 places = self.config['production'].get('update_fetch') 75 if not places: 76 return [] 77 return self._parse_lines(places) 78 79 @property 80 def default_location(self): 81 """The default location to upload this application to""" 82 return self.config['production'].get('default_location') 83 84 @property 85 def services(self): 86 """A dictionary of configured services (keys=service name, 87 value=any configuration)""" 88 services = {} 89 c = self.config['production'] 90 devel = not is_production() 91 if devel: 92 devel_config = self.load_devel_config() 93 else: 94 devel_config = None 95 for name, config_string in c.items(): 96 if name.startswith('service.'): 97 name = name[len('service.'):] 98 mod = self.load_service_module(name) 99 service = mod.Service(self, config_string, devel=devel, 100 devel_config=devel_config) 101 service.name = name 102 services[name] = service 103 return services 104 105 def load_devel_config(self): 106 from silversupport.develconfig import load_devel_config 107 return load_devel_config(self.app_name) 108 109 @property 110 def service_list(self): 111 return [s for n, s in sorted(self.services.items())] 112 113 @property 114 def php_root(self): 115 """The php_root location (or ``/dev/null`` if none given)""" 116 return os.path.join( 117 self.app_dir, 118 self.config['production'].get('php_root', '/dev/null')) 119 120 @property 121 def writable_root_location(self): 122 """The writable-root location, if it is available. 123 124 If not configured, ``/dev/null`` in production and ``''`` in 125 development 126 """ 127 if 'service.writable_root' in self.config['production']: 128 ## FIXME: do development too 129 return os.path.join('/var/lib/silverlining/writable-roots', self.app_name) 130 else: 131 if is_production(): 132 return '/dev/null' 133 else: 134 return '' 135 136 @property 137 def app_dir(self): 138 """The directory the application lives in""" 139 return os.path.dirname(self.config_file) 140 141 @property 142 def static_dir(self): 143 """The location of static files""" 144 return os.path.join(self.app_dir, 'static') 145 146 @property 147 def instance_name(self): 148 """The name of the instance (APP_NAME.TIMESTAMP)""" 149 return os.path.basename(os.path.dirname(self.config_file)) 150 151 @property 152 def packages(self): 153 """A list of packages that should be installed for this application""" 154 return self._parse_lines(self.config['production'].get('packages')) 155 156 @property 157 def package_install_script(self): 158 """A list of scripts to call to install package stuff like apt 159 repositories""" 160 return self._parse_lines( 161 self.config['production'].get('package_install_script'), 162 relative_to=self.app_dir) 163 164 @property 165 def after_install_script(self): 166 """A list of scripts to call after installing packages""" 167 return self._parse_lines( 168 self.config['production'].get('after_install_script'), 169 relative_to=self.app_dir) 170 171 @property 172 def config_required(self): 173 return asbool(self.config['production'].get('config.required')) 174 175 @property 176 def config_template(self): 177 tmpl = self.config['production'].get('config.template') 178 if not tmpl: 179 return None 180 return os.path.join(self.app_dir, tmpl) 181 182 @property 183 def config_checker(self): 184 obj_name = self.config['production'].get('config.checker') 185 if not obj_name: 186 return None 187 if ':' not in obj_name: 188 raise ValueError('Bad value for config.checker (%r): should be module:obj' % obj_name) 189 mod_name, attrs = obj_name.split(':', 1) 190 __import__(mod_name) 191 mod = sys.modules[mod_name] 192 obj = mod 193 for attr in attrs.split('.'): 194 obj = getattr(obj, attr) 195 return obj 196 197 def check_config(self, config_dir): 198 checker = self.config_checker 199 if not checker: 200 return 201 checker(config_dir) 202 203 @property 204 def config_default(self): 205 dir = self.config['production'].get('config.default') 206 if not dir: 207 return None 208 return os.path.join(self.app_dir, dir) 209 210 def _parse_lines(self, lines, relative_to=None): 211 """Parse a configuration value into a series of lines, 212 ignoring empty and comment lines""" 213 if not lines: 214 return [] 215 lines = [ 216 line.strip() for line in lines.splitlines() 217 if line.strip() and not line.strip().startswith('#')] 218 if relative_to: 219 lines = [os.path.normpath(os.path.join(relative_to, line)) 220 for line in lines] 221 return lines 222 223 def activate_services(self, environ=None): 224 """Activates all the services for this application/configuration. 225 226 Note, this doesn't create databases, this only typically sets 227 environmental variables indicating runtime configuration.""" 228 if environ is None: 229 environ = os.environ 230 for service in self.service_list: 231 environ.update(service.env_setup()) 232 if is_production(): 233 environ['SILVER_VERSION'] = 'silverlining/0.0' 234 if is_production() and pwd.getpwuid(os.getuid())[0] == 'www-data': 235 tmp = environ['TEMP'] = os.path.join('/var/lib/silverlining/tmp/', self.app_name) 236 if not os.path.exists(tmp): 237 os.makedirs(tmp) 238 elif not environ.get('TEMP'): 239 environ['TEMP'] = '/tmp' 240 environ['SILVER_LOGS'] = self.log_dir 241 if not is_production() and not os.path.exists(environ['SILVER_LOGS']): 242 os.makedirs(environ['SILVER_LOGS']) 243 if is_production(): 244 config_dir = os.path.join('/var/lib/silverlining/configs', self.app_name) 245 if os.path.exists(config_dir): 246 environ['SILVER_APP_CONFIG'] = config_dir 247 elif self.config_default: 248 environ['SILVER_APP_CONFIG'] = self.config_default 249 else: 250 if self.local_config: 251 environ['SILVER_APP_CONFIG'] = self.local_config 252 elif self.config_default: 253 environ['SILVER_APP_CONFIG'] = self.config_default 254 elif self.config_required: 255 raise Exception('This application requires configuration and config.devel ' 256 'is not set and no --config was given') 257 return environ 258 259 @property 260 def log_dir(self): 261 if is_production(): 262 return os.path.join('/var/log/silverlining/apps', self.app_name) 263 else: 264 return os.path.join(self.app_dir, 'silver-logs') 265 266 def install_services(self, clear=False): 267 """Installs all the services for this application. 268 269 This is run on deployment""" 270 for service in self.service_list: 271 if clear: 272 service.clear() 273 else: 274 service.install() 275 276 def load_service_module(self, service_name): 277 """Load the service module for the given service name""" 278 __import__('silversupport.service.%s' % service_name) 279 mod = sys.modules['silversupport.service.%s' % service_name] 280 return mod 281 282 def clear_services(self): 283 for service in self.service_list: 284 service.clear(self) 285 286 def backup_services(self, dest_dir): 287 for service in self.service_list: 288 service_dir = os.path.join(dest_dir, service.name) 289 if not os.path.exists(service_dir): 290 os.makedirs(service_dir) 291 service.backup(service_dir) 292 293 def restore_services(self, source_dir): 294 for service in self.service_list: 295 service_dir = os.path.join(source_dir, service.name) 296 service.restore(service_dir) 297 298 def activate_path(self): 299 """Adds any necessary entries to sys.path for this app""" 300 lib_path = os.path.join( 301 self.app_dir, 'lib', 'python%s' % sys.version[:3], 'site-packages') 302 if lib_path not in sys.path and os.path.exists(lib_path): 303 addsitedir(lib_path) 304 sitecustomize = os.path.join( 305 self.app_dir, 'lib', 'python%s' % sys.version[:3], 'sitecustomize.py') 306 if os.path.exists(sitecustomize): 307 ns = {'__file__': sitecustomize, '__name__': 'sitecustomize'} 308 execfile(sitecustomize, ns) 309 310 def get_app_from_runner(self): 311 """Returns the WSGI app that the runner indicates 312 """ 313 assert self.platform == 'python', ( 314 "get_app_from_runner() shouldn't be run on an app with the platform %s" 315 % self.platform) 316 runner = self.runner 317 if '#' in runner: 318 runner, spec = runner.split('#', 1) 319 else: 320 spec = None 321 if runner.endswith('.ini'): 322 try: 323 from paste.deploy import loadapp 324 except ImportError: 325 print >> sys.stderr, ( 326 "To use a .ini runner (%s) you must have PasteDeploy " 327 "installed in your application." % runner) 328 raise 329 from silversupport.secret import get_secret 330 runner = 'config:%s' % runner 331 global_conf = os.environ.copy() 332 global_conf['SECRET'] = get_secret() 333 app = loadapp(runner, name=spec, 334 global_conf=global_conf) 335 elif runner.endswith('.py'): 336 ## FIXME: not sure what name to give it 337 ns = {'__file__': runner, '__name__': 'main_py'} 338 execfile(runner, ns) 339 spec = spec or 'application' 340 if spec in ns: 341 app = ns[spec] 342 else: 343 raise Exception("No application %s defined in %s" 344 % (spec, runner)) 345 else: 346 raise Exception("Unknown kind of runner (%s)" % runner) 347 if is_production() and is_disabled(self.app_name): 348 disabled_appconfig = AppConfig.from_location('disabled') 349 return DisabledSite(app, disabled_appconfig.get_app_from_runner()) 350 else: 351 return app 352 353 def canonical_hostname(self): 354 """Returns the 'canonical' hostname for this application. 355 356 This only applies in production environments.""" 357 from silversupport import appdata 358 fp = open(appdata.APPDATA_MAP) 359 hostnames = [] 360 instance_name = self.instance_name 361 for line in fp: 362 if not line.strip() or line.strip().startswith('#'): 363 continue 364 hostname, path, data = line.split(None, 2) 365 line_instance_name = data.split('|')[0] 366 if line_instance_name == instance_name: 367 if hostname.startswith('.'): 368 hostname = hostname[1:] 369 hostnames.append(hostname) 370 hostnames.sort(key=lambda x: len(x)) 371 if hostnames: 372 return hostnames[0] 373 else: 374 return None 375 376 def write_php_env(self, filename=None): 377 """Writes out a PHP file that loads up all the environmental variables 378 379 This is because we don't run any Python during the actual 380 request cycle for PHP applications. 381 """ 382 assert self.platform == 'php' 383 if filename is None: 384 filename = os.path.join(self.app_dir, 'silver-env-variables.php') 385 fp = open(filename, 'w') 386 fp.write('<?\n') 387 env = {} 388 self.activate_services(env) 389 for name, value in sorted(env.iteritems()): 390 fp.write('$_SERVER[%s] = %r;\n' % (name, value)) 391 fp.write('?>') 392 fp.close() 393 394 def sync(self, host, instance_name): 395 """Synchronize this application (locally) with a remote server 396 at the given host. 397 """ 398 dest_dir = os.path.join(DEPLOYMENT_LOCATION, instance_name) 399 self._run_rsync(host, self.app_dir, dest_dir) 400 401 def sync_config(self, host, config_dir): 402 """Synchronise the given configuration (locally) with a remote server 403 at the given host (for this app/app_name)""" 404 dest_dir = os.path.join('/var/lib/silverlining/configs', self.app_name) 405 self._run_rsync(host, config_dir, dest_dir) 406 407 def _run_rsync(self, host, source, dest): 408 assert not is_production() 409 exclude_from = os.path.join(os.path.dirname(__file__), 'rsync-exclude.txt') 410 if not source.endswith('/'): 411 source += '/' 412 ## FIXME: does it matter if dest ends with /? 413 cmd = ['rsync', 414 '--recursive', 415 '--links', # Copy over symlinks as symlinks 416 '--copy-unsafe-links', # Copy symlinks that are outside of dir as real files 417 '--executability', # Copy +x modes 418 '--times', # Copy timestamp 419 '--rsh=ssh', # Use ssh 420 '--delete', # Delete files thta aren't in the source dir 421 '--compress', 422 #'--skip-compress=.zip,.egg', # Skip some already-compressed files 423 '--exclude-from=%s' % exclude_from, 424 '--progress', # I don't think this does anything given --quiet 425 '--quiet', 426 source, 427 os.path.join('%s:%s' % (host, dest)), 428 ] 429 run(cmd) 430 431 def check_service_setup(self, logger): 432 import traceback 433 for service in self.service_list: 434 try: 435 warning = service.check_setup() 436 except Exception, e: 437 logger.notify( 438 'Error with service %s:' % service.name) 439 logger.indent += 2 440 try: 441 logger.info( 442 traceback.format_exc()) 443 logger.notify('%s: %s' % 444 (e.__class__.__name__, str(e))) 445 finally: 446 logger.indent -= 2 447 else: 448 if warning: 449 logger.notify('Warning with service %s:' % service.name) 450 logger.indent += 2 451 try: 452 logger.notify(warning) 453 finally: 454 logger.indent -= 2