PageRenderTime 86ms CodeModel.GetById 13ms app.highlight 56ms RepoModel.GetById 11ms app.codeStats 0ms

/silk/fabfile.py

https://bitbucket.org/btubbs/silk-deployment/
Python | 510 lines | 477 code | 12 blank | 21 comment | 5 complexity | 4850c2f7bcc33a38e6b4232a6d2ec6a6 MD5 | raw file
  1import sys
  2import os
  3import datetime
  4import time
  5import posixpath
  6import random
  7import re
  8import copy
  9import yaml
 10import pkg_resources
 11
 12from fabric.api import *
 13from fabric.colors import green, red, yellow
 14from fabric.contrib.files import exists, contains, upload_template
 15
 16import silk.lib
 17
 18def _join(*args):
 19    """Convenience wrapper around posixpath.join to make the rest of our
 20    functions more readable."""
 21    return posixpath.join(*args)
 22
 23SRV_ROOT = '/srv'
 24DEFAULT_ROLLBACK_CAP = 3
 25DTS_FORMAT = '%Y%m%d_%H%M%S'
 26NGINX_SITE_DIR = '/etc/nginx/sites-enabled'
 27
 28def _set_vars():
 29    """
 30    Loads deployment settings into Fabric's global 'env' dict
 31    """
 32    env.local_root = silk.lib.get_root(os.getcwd())
 33    env.config = silk.lib.get_site_config(env.local_root)
 34    env.dts = datetime.datetime.now().strftime(DTS_FORMAT)
 35
 36    if len(env.roles) == 1:
 37        env.config.update(silk.lib.get_role_config(env.roles[0]))
 38    env.site = env.config['site']
 39    env.deployment = '%s_%s' % (env.site, env.dts)
 40    env.root = _join(SRV_ROOT, env.deployment)
 41
 42    env.envdir = _join(env.root, 'env')
 43    env.rollback_cap = env.config.get('rollback_cap', DEFAULT_ROLLBACK_CAP)
 44
 45    # Use the default supervisord include location for the deployment's include file.
 46    env.supd_conf_file = '/etc/supervisor/conf.d/%s.conf' % env.deployment
 47
 48    # Set up gunicorn config
 49    env.default_bind = silk.lib.GUNICORN_BIND_PATTERN % env.deployment
 50    if 'gunicorn' in env.config:
 51        env.config['bind'] = env.config['gunicorn'].get('bind', env.default_bind)
 52    else:
 53        env.config['bind'] = env.default_bind
 54
 55_set_vars()
 56
 57# UGLY MAGIC
 58# Here we're (ab)using Fabric's built in 'role' feature to work with the way
 59# we're loading context-specific config.  Could possibly use a refactor to
 60# avoid Fabric roles altogether.
 61def _get_hosts():
 62    """Return list of hosts to push to"""
 63    return env.config['push_hosts']
 64
 65for role in silk.lib.get_role_list(env.local_root):
 66    env.roledefs[role] = _get_hosts
 67# END UGLY MAGIC
 68
 69def _tmpfile():
 70    """Generates a random filename in /tmp.  Useful for dumping stdout to a
 71    file that you want to download or read later.  Assumes the remote host has
 72    a /tmp directory."""
 73    chars = "abcdefghijklmnopqrstuvwxyz1234567890"
 74    length = 20
 75    randompart = "".join([random.choice(chars) for x in xrange(20)])
 76    return "/tmp/silk_tmp_%s" % randompart
 77
 78def _put_dir(local_dir, remote_dir, exclude=''):
 79    """
 80    Copies a local directory to a remote one, using tar and put. Silently
 81    overwrites remote directory if it exists, creates it if it does not
 82    exist.
 83    """
 84    tarball = "%s.tar.bz2" % _tmpfile()
 85
 86    tar_cmd = 'tar -C "%(local_dir)s" -cjf "%(tarball)s" %(exclude)s .' % locals()
 87    local(tar_cmd)
 88    put(tarball, tarball, use_sudo=True)
 89    local('rm -f "%(tarball)s"' % locals())
 90    sudo('rm -Rf "{0}"; mkdir -p "{0}"; tar -C "{0}" -xjf "{1}" && rm -f "{1}"'\
 91        .format(remote_dir, tarball))
 92
 93def _get_blame():
 94    """
 95    Return information about this deployment, to be written as the "blame"
 96    section in the site.yaml file.
 97    """
 98    return {'deployed_by': env.user,
 99            'deployed_from': os.uname()[1],
100            'deployed_at': datetime.datetime.now(),
101            'deployed_role': env.roles[0]}
102
103def _write_file(path, contents, use_sudo=True, chown=None):
104    file_name = _tmpfile()
105    file = open(file_name, 'w')
106    file.write(contents)
107    file.close()
108    put(file_name, path, use_sudo=use_sudo)
109    sudo('chmod +r %s' % path)
110    if chown:
111        sudo('chown %s %s' % (chown, path))
112    local('rm %s' % file_name)
113
114def _write_site_yaml():
115    """Writes the site.yaml file for the deployed site."""
116    # Make a copy of env.config
117    site_yaml_dict = copy.copy(env.config)
118    # add the blame in
119    site_yaml_dict['blame'] = _get_blame()
120    # write it to the remote host
121    file = _join(env.root, 'site.yaml')
122    _write_file(file, yaml.safe_dump(site_yaml_dict, default_flow_style=False))
123
124def _write_template(template, dest, context):
125    path = silk.lib.get_template_path(template, env.local_root)
126    upload_template(path, dest, context=context, use_sudo=True)
127
128def _ensure_dir(remote_path):
129    if not exists(remote_path, use_sudo=True):
130        sudo('mkdir -p %s' % remote_path)
131
132def _format_supervisord_env(env_dict):
133    """Takes a dictionary and returns a string in form
134    'key=val,key2=val2'"""
135    try:
136      return ','.join(['%s="%s"' % (key, env_dict[key]) for key in env_dict.keys()])
137    except AttributeError:
138      #env_dict isn't a dictionary, so they must not have included any env vars for us.
139      return ''
140
141def _green(text):
142    print green(text)
143
144def _red(text):
145    print red(text)
146
147def _yellow(text):
148    print yellow(text)
149
150def _list_dir(dirname):
151    """Given the path for a directory on the remote host, return its contents
152    as a python list."""
153    txt = sudo('ls -1 %s' % dirname)
154    return txt.split('\r\n')
155
156def _is_supervisored(site):
157    """Returns True if site 'site' has a supervisord.conf entry, else False."""
158    # Check both the default include location and the silk <=0.2.9 location.
159    old_conf_file = _join(SRV_ROOT, site, 'conf', 'supervisord.conf')
160    new_conf_file = _join('/etc/supervisor/conf.d', '%s.conf' % site)
161    return exists(new_conf_file) or exists(old_conf_file)
162
163def _socket_head_request(path):
164    """Given a path to a socket, make an HTTP HEAD request on it"""
165    # Get the local path for the unix socket http tool
166    script = pkg_resources.resource_filename('silk', 'sock_http.py')
167    # copy it over to the host
168    dest = _tmpfile()
169    put(script, dest, use_sudo=True)
170    # run it, passing in the path to the socket
171    return sudo('python %s %s HEAD / %s' % (dest, path,
172                                            env.config['listen_hosts'][0]))
173
174def _port_head_request(port):
175    """Given a port number, use curl to make an http HEAD request to it.
176    Return response status as integer."""
177    return run('curl -I http://localhost:%s' % port)
178
179def returns_good_status():
180    """Makes an HTTP request to '/' on the site and returns True if it gets
181    back a status in the 200, 300, or 400 ranges.
182
183    You can only use this as a standalone silk command if your site has a
184    hard-configured 'bind'."""
185    _yellow("Making http request to site to ensure upness.")
186    bind = env.config['bind']
187    if bind.startswith('unix:'):
188        result = _socket_head_request(bind.replace('unix:', ''))
189    else:
190        result = _port_head_request(bind.split(':')[-1])
191    first_line = result.split('\r\n')[0]
192    status = int(first_line.split()[1])
193    return 200 <= status < 500
194
195def _is_running(procname, tries=3, wait=2):
196    """Given the name of a supervisord process, tell you whether it's running
197    or not.  If status is 'starting', will wait until status has settled."""
198    # Status return from supervisorctl will look something like this::
199    # mysite_20110623_162319 RUNNING    pid 628, uptime 0:34:13
200    # mysite_20110623_231206 FATAL      Exited too quickly (process log may have details)
201
202    status_parts = sudo('supervisorctl status %s' % procname).split()
203    if status_parts[1] == 'RUNNING':
204        # For super extra credit, we're actually going to make an HTTP request
205        # to the site to verify that it's up and running.  Unfortunately we
206        # can only do that if Gunicorn is binding to a different socket or port
207        # on each deployment (Silk binds to a new socket on each deployment by
208        # default.)  If you've configured your site to bind to a port, we'll
209        # just have to wing it.
210        if env.config['bind'] == env.default_bind:
211            if returns_good_status():
212                _green("You're golden!")
213                return True
214            else:
215                _red(":(")
216                return False
217        else:
218            return True
219    elif status_parts[1] == "FATAL":
220        return False
221    elif tries > 0:
222        # It's neither running nor dead yet, so try again
223        _yellow("Waiting %s seconds for process to settle" % wait)
224        time.sleep(wait)
225
226        # decrement the tries and double the wait time for the next check.
227        return _is_running(procname, tries - 1, wait * 2)
228    else:
229        return False
230
231def _is_this_site(name):
232    """Return True if 'name' matches our site name (old style Silk deployment
233    naming' or matches our name + timestamp pattern."""
234
235    site_pattern = re.compile('%s_\d{8}_\d{6}' % env.site)
236    return (name == env.site) or (re.match(site_pattern, name) is not None)
237
238def install_server_deps():
239    """
240    Installs nginx and supervisord on remote Ubuntu host.
241    """
242    sudo('apt-get install nginx supervisor --assume-yes --quiet --no-upgrade')
243
244def push_code():
245    _green("PUSHING CODE")
246    # Push the local site to the remote root, excluding files that we don't
247    # want to leave cluttering the production server.  Exclude site.yaml
248    # because we'll be writing a new one containing the site config updated
249    # with the role config.  Omit roles because they're superfluous at that
250    # point and also may contain sensitive connection credentials.
251    exclude = "--exclude=site.yaml --exclude=roles"
252    _put_dir(env.local_root, env.root, exclude)
253
254def create_virtualenv():
255    """Create a virtualenv inside the remote root"""
256    if 'runtime' in env.config:
257        pyversion = '--python=%s' % env.config['runtime']
258    else:
259        pyversion = ''
260
261    # Put all the prereq packages into /srv/pip_cache on the remote host, if
262    # they're not already there.
263    local_dir = pkg_resources.resource_filename('silk', 'prereqs')
264    files = pkg_resources.resource_listdir('silk', 'prereqs')
265
266    for f in files:
267        remote = posixpath.join('/tmp', f)
268        local = posixpath.join(local_dir, f)
269        if not exists(remote):
270            put(local, remote, use_sudo=True)
271
272
273    tmpl_vars = vars()
274    tmpl_vars.update(env)
275
276    c = ("virtualenv --no-site-packages %(pyversion)s --extra-search-dir=/tmp "
277         "--never-download %(envdir)s") % tmpl_vars
278    sudo(c)
279
280def pip_deps():
281    """Install requirements listed in the site's requirements.txt file."""
282
283    _green("INSTALLING PYTHON DEPENDENCIES")
284    reqs_file = os.path.join(env.root, 'requirements.txt')
285    pypi = env.config.get('pypi', 'http://pypi.python.org/pypi')
286    cachedir = posixpath.join(SRV_ROOT, 'pip_cache')
287    _ensure_dir(cachedir)
288    sudo('PIP_DOWNLOAD_CACHE="%s" %s/bin/pip install -r %s -i %s ' %
289         (cachedir, env.envdir, reqs_file, pypi))
290
291def configure_supervisor():
292    """
293    Creates and upload config file for supervisord
294    """
295    _green("WRITING SUPERVISOR CONFIG")
296
297    template_vars = {
298        'cmd': silk.lib.get_gunicorn_cmd(env, bin_dir='%s/bin' % (env.envdir)),
299        'process_env': _format_supervisord_env(env.config.get('env', '')),
300        'srv_root': SRV_ROOT,
301    }
302
303    template_vars.update(env)
304    template_vars.update(env.config)
305    #make sure the logs dir is created
306    _ensure_dir(_join(env.root, 'logs'))
307
308    # Put supervisord include in default location
309    _write_template('supervisord.conf', env.supd_conf_file, template_vars)
310
311def fix_supd_config_bug():
312    """Fixes a bug from an earlier version of Silk that wrote an invalid line
313    to the master supervisord.conf"""
314    # Silk 0.2.9 and earlier included a command to configure supervisord and
315    # nginx to include files in /srv/<site>/conf, in addition to their default
316    # include directories.  While this was valid for nginx, supervisord does
317    # not allow for multiple "files" lines in its "include" section (it does
318    # allow for multiple globs on the single "files" line, though.  This command
319    # finds the offending pattern in /etc/supervisor/supervisord.conf and
320    # replaces it with the correct equivalent.
321
322    # Note that Silk 0.3.0 and later does not require any changes from the
323    # default supervisord.conf that ships with Ubuntu.  All files are included
324    # in the supervisord's conf.d directory.
325
326    file = '/etc/supervisor/supervisord.conf'
327
328    if contains(file, "files = /srv/\*/conf/supervisord.conf", use_sudo=True):
329        _green("FIXING OLD SUPERVISOR CONFIG BUG")
330        _yellow("See http://bits.btubbs.com/silk-deployment/issue/15/incorrect-supervisord-include-config-in")
331
332        bad = "\r\n".join([
333            "files = /etc/supervisor/conf.d/*.conf",
334            "files = /srv/*/conf/supervisord.conf"
335        ])
336
337        good = ("files = /etc/supervisor/conf.d/*.conf "
338                "/srv/*/conf/supervisord.conf\n")
339
340        txt = sudo('cat %s' % file)
341
342        if bad in txt:
343            txt = txt.replace(bad, good)
344            _write_file(file, txt, use_sudo=True, chown='root')
345
346def cleanup():
347    """Deletes old versions of the site that are still sitting around."""
348    _green("CLEANING UP")
349
350    folders = _list_dir(SRV_ROOT)
351    rollbacks = [x for x in folders if _is_this_site(x)]
352
353    if len(rollbacks) > env.rollback_cap:
354        # There are more rollbacks than we want to keep around.  See if we can
355        # delete some.
356        suspects = rollbacks[:-(env.rollback_cap + 1)]
357        for folder in suspects:
358            if not _is_supervisored(folder):
359                fullpath = _join(SRV_ROOT, folder)
360                sudo('rm -rf %s' % fullpath)
361
362    # Clean up old socket files in /tmp/ that have no associated site
363    # TODO: use our own list dir function and a regular expression to filter
364    # the list of /tmp sockets instead of this funky grepping.
365    with cd('/tmp'):
366        socks = run('ls -1 | grep %s | grep sock | cat -' % env.site).split('\r\n')
367    for sock in socks:
368        procname = sock.replace('.sock', '')
369        if not exists(_join(SRV_ROOT, procname)):
370            sudo('rm /tmp/%s' % sock)
371
372    # TODO: clean out the pip-* folders that can build up in /tmp
373    # TODO: figure out a way to clean out pybundle files in /srv/_silk_build
374    # that aren't needed anymore.
375
376def start_process():
377    """Tell supervisord to read the new config, then start the new process."""
378    _green('STARTING PROCESS')
379    result = sudo('supervisorctl reread')
380
381    sudo('supervisorctl add %s' % env.deployment)
382
383def _get_nginx_static_snippet(url_path, local_path):
384    return """
385    location %(url_path)s {
386        alias %(local_path)s;
387    }
388    """ % locals()
389
390def configure_nginx():
391    """Writes a new nginx config include pointing at the newly-deployed site."""
392    _green("WRITING NGINX CONFIG")
393    nginx_static = ''
394    static_dirs = env.config.get('static_dirs', None)
395
396    # Use the static_dirs values from the site config to set up static file
397    # serving in nginx.
398    if static_dirs:
399      for item in static_dirs:
400          nginx_static += _get_nginx_static_snippet(
401              item['url_path'],
402              _join(env.root, item['system_path'])
403          )
404    template_vars = {
405        'nginx_static': nginx_static,
406        'nginx_hosts': ' '.join(env.config['listen_hosts']),
407    }
408    template_vars.update(env)
409    template_vars.update(env.config)
410
411    # Create nginx include here:
412    # /etc/nginx/sites-enabled/<sitename>.conf
413    nginx_file = _join('/etc', 'nginx', 'sites-enabled', env.site)
414    sudo('rm -f %s' % nginx_file)
415    _write_template('nginx.conf', nginx_file, template_vars)
416
417def switch_nginx():
418    _green("LOADING NEW NGINX CONFIG")
419
420    # Check if there is an old-style version (within the site root) of the
421    # nginx config laying around, and rename it to something innocuous if so.
422    old_nginx = _join(SRV_ROOT, env.site, 'conf', 'nginx.conf')
423    if exists(old_nginx):
424        sudo('mv %s %s' % (old_nginx, "%s_disabled" % old_nginx))
425
426    # Tell nginx to rescan its config files
427    sudo('/etc/init.d/nginx reload')
428
429def stop_other_versions():
430    """Stop other versions of the site that are still running, and disable their
431    configs."""
432    proclist = sudo('supervisorctl status').split('\r\n')
433
434    # parse each line so we can get at just the proc names
435    proclist = [x.split() for x in proclist]
436
437    # filter proclist to include only versions of our site
438    proclist = [x for x in proclist if _is_this_site(x[0])]
439
440    live_statuses = ["RUNNING", "STARTING"]
441
442    # stop each process left in proclist that isn't the current one
443    for proc in proclist:
444        # We assume that spaces are not allowed in proc names
445        procname = proc[0]
446        procstatus = proc[1]
447        if procname != env.deployment:
448            # Stop the process
449            if procstatus in live_statuses:
450                sudo('supervisorctl stop %s' % procname)
451
452            # Remove it from live config
453            sudo('supervisorctl remove %s' % procname)
454
455            # Remove its supervisord config file
456            conf_file = '/etc/supervisor/conf.d/%s.conf' % procname
457            if exists(conf_file):
458                sudo('rm %s' % conf_file)
459
460            # Also remove old style supervisord include if it exists
461            old_conf_file = _join(SRV_ROOT, procname, 'conf/supervisord.conf')
462            if exists(old_conf_file):
463                sudo('rm %s' % old_conf_file)
464
465    sudo('supervisorctl reread')
466
467def congrats():
468    """Congratulate the user and print a link to the site."""
469    link0 = "http://%s" % env.config['listen_hosts'][0]
470    msg = ("SUCCESS!  I think.  Check that the site is running by browsing "
471           "to this url:\n\n%s" % link0)
472    _green(msg)
473
474def push():
475    """
476    The main function.  This function will put your site on the remote host and get it
477    running.
478    """
479    # Make sure nginx and supervisord are installed
480    install_server_deps()
481
482    # Fix an embarrassing config bug from earlier versions
483    fix_supd_config_bug()
484
485    # push site code and pybundle to server, in timestamped folder
486    push_code()
487    # make virtualenv on the server and run pip install on the pybundle
488    create_virtualenv()
489    pip_deps()
490    _write_site_yaml()
491    # write supervisord config for the new site
492    configure_supervisor()
493
494    ##then the magic
495
496    ##silk starts up supervisord on the new site
497    start_process()
498    # checks that the new site is running (by using supervisorctl)
499    if _is_running(env.deployment):
500        _green("Site is running.  Proceeding with nginx switch.")
501        # if the site's running fine, then silk configures nginx to forward requests
502        # to the new site
503        configure_nginx()
504        switch_nginx()
505        stop_other_versions()
506        cleanup()
507        congrats()
508    else:
509        _red("Process failed to start cleanly.  Off to the log files!")
510        sys.exit(1)