PageRenderTime 126ms CodeModel.GetById 17ms RepoModel.GetById 1ms app.codeStats 0ms

/silversupport/appconfig.py

https://bitbucket.org/ianb/silverlining/
Python | 454 lines | 440 code | 8 blank | 6 comment | 27 complexity | 5d3053814ae47da63d67f9cdb614094c MD5 | raw file
Possible License(s): GPL-2.0
  1. """Application configuration object"""
  2. import os
  3. import pwd
  4. import sys
  5. import warnings
  6. from site import addsitedir
  7. from silversupport.env import is_production
  8. from silversupport.shell import run
  9. from silversupport.util import asbool, read_config
  10. from silversupport.disabledapps import DisabledSite, is_disabled
  11. __all__ = ['AppConfig']
  12. DEPLOYMENT_LOCATION = "/var/www"
  13. class AppConfig(object):
  14. """This represents an application's configuration file, and by
  15. extension represents the application itself"""
  16. def __init__(self, config_file, app_name=None,
  17. local_config=None):
  18. if not os.path.exists(config_file):
  19. raise OSError("No config file %s" % config_file)
  20. self.config_file = config_file
  21. self.config = read_config(config_file)
  22. if not is_production():
  23. if self.config['production'].get('version'):
  24. warnings.warn('version setting in %s is deprecated' % config_file)
  25. if self.config['production'].get('default_hostname'):
  26. warnings.warn('default_hostname setting in %s has been renamed to default_location'
  27. % config_file)
  28. if self.config['production'].get('default_host'):
  29. warnings.warn('default_host setting in %s has been renamed to default_location'
  30. % config_file)
  31. if app_name is None:
  32. if is_production():
  33. app_name = os.path.basename(os.path.dirname(config_file)).split('.')[0]
  34. else:
  35. app_name = self.config['production']['app_name']
  36. self.app_name = app_name
  37. self.local_config = local_config
  38. @classmethod
  39. def from_instance_name(cls, instance_name):
  40. """Loads an instance given its name; only valid in production"""
  41. return cls(os.path.join(DEPLOYMENT_LOCATION, instance_name, 'app.ini'))
  42. @classmethod
  43. def from_location(cls, location):
  44. """Loads an instance given its location (hostname[/path])"""
  45. import appdata
  46. return cls.from_instance_name(
  47. appdata.instance_for_location(*appdata.normalize_location(location)))
  48. @property
  49. def platform(self):
  50. """The platform of the application.
  51. Current valid values are ``'python'`` and ``'php'``
  52. """
  53. return self.config['production'].get('platform', 'python')
  54. @property
  55. def runner(self):
  56. """The filename of the runner for this application"""
  57. filename = self.config['production']['runner']
  58. return os.path.join(os.path.dirname(self.config_file), filename)
  59. @property
  60. def update_fetch(self):
  61. """A list (possibly empty) of all URLs to fetch on update"""
  62. places = self.config['production'].get('update_fetch')
  63. if not places:
  64. return []
  65. return self._parse_lines(places)
  66. @property
  67. def default_location(self):
  68. """The default location to upload this application to"""
  69. return self.config['production'].get('default_location')
  70. @property
  71. def services(self):
  72. """A dictionary of configured services (keys=service name,
  73. value=any configuration)"""
  74. services = {}
  75. c = self.config['production']
  76. devel = not is_production()
  77. if devel:
  78. devel_config = self.load_devel_config()
  79. else:
  80. devel_config = None
  81. for name, config_string in c.items():
  82. if name.startswith('service.'):
  83. name = name[len('service.'):]
  84. mod = self.load_service_module(name)
  85. service = mod.Service(self, config_string, devel=devel,
  86. devel_config=devel_config)
  87. service.name = name
  88. services[name] = service
  89. return services
  90. def load_devel_config(self):
  91. from silversupport.develconfig import load_devel_config
  92. return load_devel_config(self.app_name)
  93. @property
  94. def service_list(self):
  95. return [s for n, s in sorted(self.services.items())]
  96. @property
  97. def php_root(self):
  98. """The php_root location (or ``/dev/null`` if none given)"""
  99. return os.path.join(
  100. self.app_dir,
  101. self.config['production'].get('php_root', '/dev/null'))
  102. @property
  103. def writable_root_location(self):
  104. """The writable-root location, if it is available.
  105. If not configured, ``/dev/null`` in production and ``''`` in
  106. development
  107. """
  108. if 'service.writable_root' in self.config['production']:
  109. ## FIXME: do development too
  110. return os.path.join('/var/lib/silverlining/writable-roots', self.app_name)
  111. else:
  112. if is_production():
  113. return '/dev/null'
  114. else:
  115. return ''
  116. @property
  117. def app_dir(self):
  118. """The directory the application lives in"""
  119. return os.path.dirname(self.config_file)
  120. @property
  121. def static_dir(self):
  122. """The location of static files"""
  123. return os.path.join(self.app_dir, 'static')
  124. @property
  125. def instance_name(self):
  126. """The name of the instance (APP_NAME.TIMESTAMP)"""
  127. return os.path.basename(os.path.dirname(self.config_file))
  128. @property
  129. def packages(self):
  130. """A list of packages that should be installed for this application"""
  131. return self._parse_lines(self.config['production'].get('packages'))
  132. @property
  133. def package_install_script(self):
  134. """A list of scripts to call to install package stuff like apt
  135. repositories"""
  136. return self._parse_lines(
  137. self.config['production'].get('package_install_script'),
  138. relative_to=self.app_dir)
  139. @property
  140. def after_install_script(self):
  141. """A list of scripts to call after installing packages"""
  142. return self._parse_lines(
  143. self.config['production'].get('after_install_script'),
  144. relative_to=self.app_dir)
  145. @property
  146. def config_required(self):
  147. return asbool(self.config['production'].get('config.required'))
  148. @property
  149. def config_template(self):
  150. tmpl = self.config['production'].get('config.template')
  151. if not tmpl:
  152. return None
  153. return os.path.join(self.app_dir, tmpl)
  154. @property
  155. def config_checker(self):
  156. obj_name = self.config['production'].get('config.checker')
  157. if not obj_name:
  158. return None
  159. if ':' not in obj_name:
  160. raise ValueError('Bad value for config.checker (%r): should be module:obj' % obj_name)
  161. mod_name, attrs = obj_name.split(':', 1)
  162. __import__(mod_name)
  163. mod = sys.modules[mod_name]
  164. obj = mod
  165. for attr in attrs.split('.'):
  166. obj = getattr(obj, attr)
  167. return obj
  168. def check_config(self, config_dir):
  169. checker = self.config_checker
  170. if not checker:
  171. return
  172. checker(config_dir)
  173. @property
  174. def config_default(self):
  175. dir = self.config['production'].get('config.default')
  176. if not dir:
  177. return None
  178. return os.path.join(self.app_dir, dir)
  179. def _parse_lines(self, lines, relative_to=None):
  180. """Parse a configuration value into a series of lines,
  181. ignoring empty and comment lines"""
  182. if not lines:
  183. return []
  184. lines = [
  185. line.strip() for line in lines.splitlines()
  186. if line.strip() and not line.strip().startswith('#')]
  187. if relative_to:
  188. lines = [os.path.normpath(os.path.join(relative_to, line))
  189. for line in lines]
  190. return lines
  191. def activate_services(self, environ=None):
  192. """Activates all the services for this application/configuration.
  193. Note, this doesn't create databases, this only typically sets
  194. environmental variables indicating runtime configuration."""
  195. if environ is None:
  196. environ = os.environ
  197. for service in self.service_list:
  198. environ.update(service.env_setup())
  199. if is_production():
  200. environ['SILVER_VERSION'] = 'silverlining/0.0'
  201. if is_production() and pwd.getpwuid(os.getuid())[0] == 'www-data':
  202. tmp = environ['TEMP'] = os.path.join('/var/lib/silverlining/tmp/', self.app_name)
  203. if not os.path.exists(tmp):
  204. os.makedirs(tmp)
  205. elif not environ.get('TEMP'):
  206. environ['TEMP'] = '/tmp'
  207. environ['SILVER_LOGS'] = self.log_dir
  208. if not is_production() and not os.path.exists(environ['SILVER_LOGS']):
  209. os.makedirs(environ['SILVER_LOGS'])
  210. if is_production():
  211. config_dir = os.path.join('/var/lib/silverlining/configs', self.app_name)
  212. if os.path.exists(config_dir):
  213. environ['SILVER_APP_CONFIG'] = config_dir
  214. elif self.config_default:
  215. environ['SILVER_APP_CONFIG'] = self.config_default
  216. else:
  217. if self.local_config:
  218. environ['SILVER_APP_CONFIG'] = self.local_config
  219. elif self.config_default:
  220. environ['SILVER_APP_CONFIG'] = self.config_default
  221. elif self.config_required:
  222. raise Exception('This application requires configuration and config.devel '
  223. 'is not set and no --config was given')
  224. return environ
  225. @property
  226. def log_dir(self):
  227. if is_production():
  228. return os.path.join('/var/log/silverlining/apps', self.app_name)
  229. else:
  230. return os.path.join(self.app_dir, 'silver-logs')
  231. def install_services(self, clear=False):
  232. """Installs all the services for this application.
  233. This is run on deployment"""
  234. for service in self.service_list:
  235. if clear:
  236. service.clear()
  237. else:
  238. service.install()
  239. def load_service_module(self, service_name):
  240. """Load the service module for the given service name"""
  241. __import__('silversupport.service.%s' % service_name)
  242. mod = sys.modules['silversupport.service.%s' % service_name]
  243. return mod
  244. def clear_services(self):
  245. for service in self.service_list:
  246. service.clear(self)
  247. def backup_services(self, dest_dir):
  248. for service in self.service_list:
  249. service_dir = os.path.join(dest_dir, service.name)
  250. if not os.path.exists(service_dir):
  251. os.makedirs(service_dir)
  252. service.backup(service_dir)
  253. def restore_services(self, source_dir):
  254. for service in self.service_list:
  255. service_dir = os.path.join(source_dir, service.name)
  256. service.restore(service_dir)
  257. def activate_path(self):
  258. """Adds any necessary entries to sys.path for this app"""
  259. lib_path = os.path.join(
  260. self.app_dir, 'lib', 'python%s' % sys.version[:3], 'site-packages')
  261. if lib_path not in sys.path and os.path.exists(lib_path):
  262. addsitedir(lib_path)
  263. sitecustomize = os.path.join(
  264. self.app_dir, 'lib', 'python%s' % sys.version[:3], 'sitecustomize.py')
  265. if os.path.exists(sitecustomize):
  266. ns = {'__file__': sitecustomize, '__name__': 'sitecustomize'}
  267. execfile(sitecustomize, ns)
  268. def get_app_from_runner(self):
  269. """Returns the WSGI app that the runner indicates
  270. """
  271. assert self.platform == 'python', (
  272. "get_app_from_runner() shouldn't be run on an app with the platform %s"
  273. % self.platform)
  274. runner = self.runner
  275. if '#' in runner:
  276. runner, spec = runner.split('#', 1)
  277. else:
  278. spec = None
  279. if runner.endswith('.ini'):
  280. try:
  281. from paste.deploy import loadapp
  282. except ImportError:
  283. print >> sys.stderr, (
  284. "To use a .ini runner (%s) you must have PasteDeploy "
  285. "installed in your application." % runner)
  286. raise
  287. from silversupport.secret import get_secret
  288. runner = 'config:%s' % runner
  289. global_conf = os.environ.copy()
  290. global_conf['SECRET'] = get_secret()
  291. app = loadapp(runner, name=spec,
  292. global_conf=global_conf)
  293. elif runner.endswith('.py'):
  294. ## FIXME: not sure what name to give it
  295. ns = {'__file__': runner, '__name__': 'main_py'}
  296. execfile(runner, ns)
  297. spec = spec or 'application'
  298. if spec in ns:
  299. app = ns[spec]
  300. else:
  301. raise Exception("No application %s defined in %s"
  302. % (spec, runner))
  303. else:
  304. raise Exception("Unknown kind of runner (%s)" % runner)
  305. if is_production() and is_disabled(self.app_name):
  306. disabled_appconfig = AppConfig.from_location('disabled')
  307. return DisabledSite(app, disabled_appconfig.get_app_from_runner())
  308. else:
  309. return app
  310. def canonical_hostname(self):
  311. """Returns the 'canonical' hostname for this application.
  312. This only applies in production environments."""
  313. from silversupport import appdata
  314. fp = open(appdata.APPDATA_MAP)
  315. hostnames = []
  316. instance_name = self.instance_name
  317. for line in fp:
  318. if not line.strip() or line.strip().startswith('#'):
  319. continue
  320. hostname, path, data = line.split(None, 2)
  321. line_instance_name = data.split('|')[0]
  322. if line_instance_name == instance_name:
  323. if hostname.startswith('.'):
  324. hostname = hostname[1:]
  325. hostnames.append(hostname)
  326. hostnames.sort(key=lambda x: len(x))
  327. if hostnames:
  328. return hostnames[0]
  329. else:
  330. return None
  331. def write_php_env(self, filename=None):
  332. """Writes out a PHP file that loads up all the environmental variables
  333. This is because we don't run any Python during the actual
  334. request cycle for PHP applications.
  335. """
  336. assert self.platform == 'php'
  337. if filename is None:
  338. filename = os.path.join(self.app_dir, 'silver-env-variables.php')
  339. fp = open(filename, 'w')
  340. fp.write('<?\n')
  341. env = {}
  342. self.activate_services(env)
  343. for name, value in sorted(env.iteritems()):
  344. fp.write('$_SERVER[%s] = %r;\n' % (name, value))
  345. fp.write('?>')
  346. fp.close()
  347. def sync(self, host, instance_name):
  348. """Synchronize this application (locally) with a remote server
  349. at the given host.
  350. """
  351. dest_dir = os.path.join(DEPLOYMENT_LOCATION, instance_name)
  352. self._run_rsync(host, self.app_dir, dest_dir)
  353. def sync_config(self, host, config_dir):
  354. """Synchronise the given configuration (locally) with a remote server
  355. at the given host (for this app/app_name)"""
  356. dest_dir = os.path.join('/var/lib/silverlining/configs', self.app_name)
  357. self._run_rsync(host, config_dir, dest_dir)
  358. def _run_rsync(self, host, source, dest):
  359. assert not is_production()
  360. exclude_from = os.path.join(os.path.dirname(__file__), 'rsync-exclude.txt')
  361. if not source.endswith('/'):
  362. source += '/'
  363. ## FIXME: does it matter if dest ends with /?
  364. cmd = ['rsync',
  365. '--recursive',
  366. '--links', # Copy over symlinks as symlinks
  367. '--copy-unsafe-links', # Copy symlinks that are outside of dir as real files
  368. '--executability', # Copy +x modes
  369. '--times', # Copy timestamp
  370. '--rsh=ssh', # Use ssh
  371. '--delete', # Delete files thta aren't in the source dir
  372. '--compress',
  373. #'--skip-compress=.zip,.egg', # Skip some already-compressed files
  374. '--exclude-from=%s' % exclude_from,
  375. '--progress', # I don't think this does anything given --quiet
  376. '--quiet',
  377. source,
  378. os.path.join('%s:%s' % (host, dest)),
  379. ]
  380. run(cmd)
  381. def check_service_setup(self, logger):
  382. import traceback
  383. for service in self.service_list:
  384. try:
  385. warning = service.check_setup()
  386. except Exception, e:
  387. logger.notify(
  388. 'Error with service %s:' % service.name)
  389. logger.indent += 2
  390. try:
  391. logger.info(
  392. traceback.format_exc())
  393. logger.notify('%s: %s' %
  394. (e.__class__.__name__, str(e)))
  395. finally:
  396. logger.indent -= 2
  397. else:
  398. if warning:
  399. logger.notify('Warning with service %s:' % service.name)
  400. logger.indent += 2
  401. try:
  402. logger.notify(warning)
  403. finally:
  404. logger.indent -= 2