/silversupport/appconfig.py

  1"""Application configuration object"""
  2
  3import os
  4import pwd
  5import sys
  6import warnings
  7from site import addsitedir
  8from silversupport.env import is_production
  9from silversupport.shell import run
 10from silversupport.util import asbool, read_config
 11from silversupport.disabledapps import DisabledSite, is_disabled
 12
 13__all__ = ['AppConfig']
 14
 15DEPLOYMENT_LOCATION = "/var/www"
 16
 17
 18class AppConfig(object):
 19    """This represents an application's configuration file, and by
 20    extension represents the application itself"""
 21
 22    def __init__(self, config_file, app_name=None,
 23                 local_config=None):
 24        if not os.path.exists(config_file):
 25            raise OSError("No config file %s" % config_file)
 26        self.config_file = config_file
 27        self.config = read_config(config_file)
 28        if not is_production():
 29            if self.config['production'].get('version'):
 30                warnings.warn('version setting in %s is deprecated' % config_file)
 31            if self.config['production'].get('default_hostname'):
 32                warnings.warn('default_hostname setting in %s has been renamed to default_location'
 33                              % config_file)
 34            if self.config['production'].get('default_host'):
 35                warnings.warn('default_host setting in %s has been renamed to default_location'
 36                              % config_file)
 37        if app_name is None:
 38            if is_production():
 39                app_name = os.path.basename(os.path.dirname(config_file)).split('.')[0]
 40            else:
 41                app_name = self.config['production']['app_name']
 42        self.app_name = app_name
 43        self.local_config = local_config
 44
 45    @classmethod
 46    def from_instance_name(cls, instance_name):
 47        """Loads an instance given its name; only valid in production"""
 48        return cls(os.path.join(DEPLOYMENT_LOCATION, instance_name, 'app.ini'))
 49
 50    @classmethod
 51    def from_location(cls, location):
 52        """Loads an instance given its location (hostname[/path])"""
 53        import appdata
 54        return cls.from_instance_name(
 55            appdata.instance_for_location(*appdata.normalize_location(location)))
 56
 57    @property
 58    def platform(self):
 59        """The platform of the application.
 60
 61        Current valid values are ``'python'`` and ``'php'``
 62        """
 63        return self.config['production'].get('platform', 'python')
 64
 65    @property
 66    def runner(self):
 67        """The filename of the runner for this application"""
 68        filename = self.config['production']['runner']
 69        return os.path.join(os.path.dirname(self.config_file), filename)
 70
 71    @property
 72    def update_fetch(self):
 73        """A list (possibly empty) of all URLs to fetch on update"""
 74        places = self.config['production'].get('update_fetch')
 75        if not places:
 76            return []
 77        return self._parse_lines(places)
 78
 79    @property
 80    def default_location(self):
 81        """The default location to upload this application to"""
 82        return self.config['production'].get('default_location')
 83
 84    @property
 85    def services(self):
 86        """A dictionary of configured services (keys=service name,
 87        value=any configuration)"""
 88        services = {}
 89        c = self.config['production']
 90        devel = not is_production()
 91        if devel:
 92            devel_config = self.load_devel_config()
 93        else:
 94            devel_config = None
 95        for name, config_string in c.items():
 96            if name.startswith('service.'):
 97                name = name[len('service.'):]
 98                mod = self.load_service_module(name)
 99                service = mod.Service(self, config_string, devel=devel,
100                                      devel_config=devel_config)
101                service.name = name
102                services[name] = service
103        return services
104
105    def load_devel_config(self):
106        from silversupport.develconfig import load_devel_config
107        return load_devel_config(self.app_name)
108
109    @property
110    def service_list(self):
111        return [s for n, s in sorted(self.services.items())]
112
113    @property
114    def php_root(self):
115        """The php_root location (or ``/dev/null`` if none given)"""
116        return os.path.join(
117            self.app_dir,
118            self.config['production'].get('php_root', '/dev/null'))
119
120    @property
121    def writable_root_location(self):
122        """The writable-root location, if it is available.
123
124        If not configured, ``/dev/null`` in production and ``''`` in
125        development
126        """
127        if 'service.writable_root' in self.config['production']:
128            ## FIXME: do development too
129            return os.path.join('/var/lib/silverlining/writable-roots', self.app_name)
130        else:
131            if is_production():
132                return '/dev/null'
133            else:
134                return ''
135
136    @property
137    def app_dir(self):
138        """The directory the application lives in"""
139        return os.path.dirname(self.config_file)
140
141    @property
142    def static_dir(self):
143        """The location of static files"""
144        return os.path.join(self.app_dir, 'static')
145
146    @property
147    def instance_name(self):
148        """The name of the instance (APP_NAME.TIMESTAMP)"""
149        return os.path.basename(os.path.dirname(self.config_file))
150
151    @property
152    def packages(self):
153        """A list of packages that should be installed for this application"""
154        return self._parse_lines(self.config['production'].get('packages'))
155
156    @property
157    def package_install_script(self):
158        """A list of scripts to call to install package stuff like apt
159        repositories"""
160        return self._parse_lines(
161            self.config['production'].get('package_install_script'),
162            relative_to=self.app_dir)
163
164    @property
165    def after_install_script(self):
166        """A list of scripts to call after installing packages"""
167        return self._parse_lines(
168            self.config['production'].get('after_install_script'),
169            relative_to=self.app_dir)
170
171    @property
172    def config_required(self):
173        return asbool(self.config['production'].get('config.required'))
174
175    @property
176    def config_template(self):
177        tmpl = self.config['production'].get('config.template')
178        if not tmpl:
179            return None
180        return os.path.join(self.app_dir, tmpl)
181
182    @property
183    def config_checker(self):
184        obj_name = self.config['production'].get('config.checker')
185        if not obj_name:
186            return None
187        if ':' not in obj_name:
188            raise ValueError('Bad value for config.checker (%r): should be module:obj' % obj_name)
189        mod_name, attrs = obj_name.split(':', 1)
190        __import__(mod_name)
191        mod = sys.modules[mod_name]
192        obj = mod
193        for attr in attrs.split('.'):
194            obj = getattr(obj, attr)
195        return obj
196
197    def check_config(self, config_dir):
198        checker = self.config_checker
199        if not checker:
200            return
201        checker(config_dir)
202
203    @property
204    def config_default(self):
205        dir = self.config['production'].get('config.default')
206        if not dir:
207            return None
208        return os.path.join(self.app_dir, dir)
209
210    def _parse_lines(self, lines, relative_to=None):
211        """Parse a configuration value into a series of lines,
212        ignoring empty and comment lines"""
213        if not lines:
214            return []
215        lines = [
216            line.strip() for line in lines.splitlines()
217            if line.strip() and not line.strip().startswith('#')]
218        if relative_to:
219            lines = [os.path.normpath(os.path.join(relative_to, line))
220                     for line in lines]
221        return lines
222
223    def activate_services(self, environ=None):
224        """Activates all the services for this application/configuration.
225
226        Note, this doesn't create databases, this only typically sets
227        environmental variables indicating runtime configuration."""
228        if environ is None:
229            environ = os.environ
230        for service in self.service_list:
231            environ.update(service.env_setup())
232        if is_production():
233            environ['SILVER_VERSION'] = 'silverlining/0.0'
234        if is_production() and pwd.getpwuid(os.getuid())[0] == 'www-data':
235            tmp = environ['TEMP'] = os.path.join('/var/lib/silverlining/tmp/', self.app_name)
236            if not os.path.exists(tmp):
237                os.makedirs(tmp)
238        elif not environ.get('TEMP'):
239            environ['TEMP'] = '/tmp'
240        environ['SILVER_LOGS'] = self.log_dir
241        if not is_production() and not os.path.exists(environ['SILVER_LOGS']):
242            os.makedirs(environ['SILVER_LOGS'])
243        if is_production():
244            config_dir = os.path.join('/var/lib/silverlining/configs', self.app_name)
245            if os.path.exists(config_dir):
246                environ['SILVER_APP_CONFIG'] = config_dir
247            elif self.config_default:
248                environ['SILVER_APP_CONFIG'] = self.config_default
249        else:
250            if self.local_config:
251                environ['SILVER_APP_CONFIG'] = self.local_config
252            elif self.config_default:
253                environ['SILVER_APP_CONFIG'] = self.config_default
254            elif self.config_required:
255                raise Exception('This application requires configuration and config.devel '
256                                'is not set and no --config was given')
257        return environ
258
259    @property
260    def log_dir(self):
261        if is_production():
262            return os.path.join('/var/log/silverlining/apps', self.app_name)
263        else:
264            return os.path.join(self.app_dir, 'silver-logs')
265
266    def install_services(self, clear=False):
267        """Installs all the services for this application.
268
269        This is run on deployment"""
270        for service in self.service_list:
271            if clear:
272                service.clear()
273            else:
274                service.install()
275
276    def load_service_module(self, service_name):
277        """Load the service module for the given service name"""
278        __import__('silversupport.service.%s' % service_name)
279        mod = sys.modules['silversupport.service.%s' % service_name]
280        return mod
281
282    def clear_services(self):
283        for service in self.service_list:
284            service.clear(self)
285
286    def backup_services(self, dest_dir):
287        for service in self.service_list:
288            service_dir = os.path.join(dest_dir, service.name)
289            if not os.path.exists(service_dir):
290                os.makedirs(service_dir)
291            service.backup(service_dir)
292
293    def restore_services(self, source_dir):
294        for service in self.service_list:
295            service_dir = os.path.join(source_dir, service.name)
296            service.restore(service_dir)
297
298    def activate_path(self):
299        """Adds any necessary entries to sys.path for this app"""
300        lib_path = os.path.join(
301            self.app_dir, 'lib', 'python%s' % sys.version[:3], 'site-packages')
302        if lib_path not in sys.path and os.path.exists(lib_path):
303            addsitedir(lib_path)
304        sitecustomize = os.path.join(
305            self.app_dir, 'lib', 'python%s' % sys.version[:3], 'sitecustomize.py')
306        if os.path.exists(sitecustomize):
307            ns = {'__file__': sitecustomize, '__name__': 'sitecustomize'}
308            execfile(sitecustomize, ns)
309
310    def get_app_from_runner(self):
311        """Returns the WSGI app that the runner indicates
312        """
313        assert self.platform == 'python', (
314            "get_app_from_runner() shouldn't be run on an app with the platform %s"
315            % self.platform)
316        runner = self.runner
317        if '#' in runner:
318            runner, spec = runner.split('#', 1)
319        else:
320            spec = None
321        if runner.endswith('.ini'):
322            try:
323                from paste.deploy import loadapp
324            except ImportError:
325                print >> sys.stderr, (
326                    "To use a .ini runner (%s) you must have PasteDeploy "
327                    "installed in your application." % runner)
328                raise
329            from silversupport.secret import get_secret
330            runner = 'config:%s' % runner
331            global_conf = os.environ.copy()
332            global_conf['SECRET'] = get_secret()
333            app = loadapp(runner, name=spec,
334                          global_conf=global_conf)
335        elif runner.endswith('.py'):
336            ## FIXME: not sure what name to give it
337            ns = {'__file__': runner, '__name__': 'main_py'}
338            execfile(runner, ns)
339            spec = spec or 'application'
340            if spec in ns:
341                app = ns[spec]
342            else:
343                raise Exception("No application %s defined in %s"
344                                % (spec, runner))
345        else:
346            raise Exception("Unknown kind of runner (%s)" % runner)
347        if is_production() and is_disabled(self.app_name):
348            disabled_appconfig = AppConfig.from_location('disabled')
349            return DisabledSite(app, disabled_appconfig.get_app_from_runner())
350        else:
351            return app
352
353    def canonical_hostname(self):
354        """Returns the 'canonical' hostname for this application.
355
356        This only applies in production environments."""
357        from silversupport import appdata
358        fp = open(appdata.APPDATA_MAP)
359        hostnames = []
360        instance_name = self.instance_name
361        for line in fp:
362            if not line.strip() or line.strip().startswith('#'):
363                continue
364            hostname, path, data = line.split(None, 2)
365            line_instance_name = data.split('|')[0]
366            if line_instance_name == instance_name:
367                if hostname.startswith('.'):
368                    hostname = hostname[1:]
369                hostnames.append(hostname)
370        hostnames.sort(key=lambda x: len(x))
371        if hostnames:
372            return hostnames[0]
373        else:
374            return None
375
376    def write_php_env(self, filename=None):
377        """Writes out a PHP file that loads up all the environmental variables
378
379        This is because we don't run any Python during the actual
380        request cycle for PHP applications.
381        """
382        assert self.platform == 'php'
383        if filename is None:
384            filename = os.path.join(self.app_dir, 'silver-env-variables.php')
385        fp = open(filename, 'w')
386        fp.write('<?\n')
387        env = {}
388        self.activate_services(env)
389        for name, value in sorted(env.iteritems()):
390            fp.write('$_SERVER[%s] = %r;\n' % (name, value))
391        fp.write('?>')
392        fp.close()
393
394    def sync(self, host, instance_name):
395        """Synchronize this application (locally) with a remote server
396        at the given host.
397        """
398        dest_dir = os.path.join(DEPLOYMENT_LOCATION, instance_name)
399        self._run_rsync(host, self.app_dir, dest_dir)
400
401    def sync_config(self, host, config_dir):
402        """Synchronise the given configuration (locally) with a remote server
403        at the given host (for this app/app_name)"""
404        dest_dir = os.path.join('/var/lib/silverlining/configs', self.app_name)
405        self._run_rsync(host, config_dir, dest_dir)
406
407    def _run_rsync(self, host, source, dest):
408        assert not is_production()
409        exclude_from = os.path.join(os.path.dirname(__file__), 'rsync-exclude.txt')
410        if not source.endswith('/'):
411            source += '/'
412        ## FIXME: does it matter if dest ends with /?
413        cmd = ['rsync',
414               '--recursive',
415               '--links',              # Copy over symlinks as symlinks
416               '--copy-unsafe-links',  # Copy symlinks that are outside of dir as real files
417               '--executability',      # Copy +x modes
418               '--times',              # Copy timestamp
419               '--rsh=ssh',            # Use ssh
420               '--delete',             # Delete files thta aren't in the source dir
421               '--compress',
422               #'--skip-compress=.zip,.egg', # Skip some already-compressed files
423               '--exclude-from=%s' % exclude_from,
424               '--progress',           # I don't think this does anything given --quiet
425               '--quiet',
426               source,
427               os.path.join('%s:%s' % (host, dest)),
428               ]
429        run(cmd)
430
431    def check_service_setup(self, logger):
432        import traceback
433        for service in self.service_list:
434            try:
435                warning = service.check_setup()
436            except Exception, e:
437                logger.notify(
438                    'Error with service %s:' % service.name)
439                logger.indent += 2
440                try:
441                    logger.info(
442                        traceback.format_exc())
443                    logger.notify('%s: %s' %
444                                  (e.__class__.__name__, str(e)))
445                finally:
446                    logger.indent -= 2
447            else:
448                if warning:
449                    logger.notify('Warning with service %s:' % service.name)
450                    logger.indent += 2
451                    try:
452                        logger.notify(warning)
453                    finally:
454                        logger.indent -= 2