/silk/fabfile.py
Python | 510 lines | 477 code | 12 blank | 21 comment | 5 complexity | 4850c2f7bcc33a38e6b4232a6d2ec6a6 MD5 | raw file
1import sys 2import os 3import datetime 4import time 5import posixpath 6import random 7import re 8import copy 9import yaml 10import pkg_resources 11 12from fabric.api import * 13from fabric.colors import green, red, yellow 14from fabric.contrib.files import exists, contains, upload_template 15 16import silk.lib 17 18def _join(*args): 19 """Convenience wrapper around posixpath.join to make the rest of our 20 functions more readable.""" 21 return posixpath.join(*args) 22 23SRV_ROOT = '/srv' 24DEFAULT_ROLLBACK_CAP = 3 25DTS_FORMAT = '%Y%m%d_%H%M%S' 26NGINX_SITE_DIR = '/etc/nginx/sites-enabled' 27 28def _set_vars(): 29 """ 30 Loads deployment settings into Fabric's global 'env' dict 31 """ 32 env.local_root = silk.lib.get_root(os.getcwd()) 33 env.config = silk.lib.get_site_config(env.local_root) 34 env.dts = datetime.datetime.now().strftime(DTS_FORMAT) 35 36 if len(env.roles) == 1: 37 env.config.update(silk.lib.get_role_config(env.roles[0])) 38 env.site = env.config['site'] 39 env.deployment = '%s_%s' % (env.site, env.dts) 40 env.root = _join(SRV_ROOT, env.deployment) 41 42 env.envdir = _join(env.root, 'env') 43 env.rollback_cap = env.config.get('rollback_cap', DEFAULT_ROLLBACK_CAP) 44 45 # Use the default supervisord include location for the deployment's include file. 46 env.supd_conf_file = '/etc/supervisor/conf.d/%s.conf' % env.deployment 47 48 # Set up gunicorn config 49 env.default_bind = silk.lib.GUNICORN_BIND_PATTERN % env.deployment 50 if 'gunicorn' in env.config: 51 env.config['bind'] = env.config['gunicorn'].get('bind', env.default_bind) 52 else: 53 env.config['bind'] = env.default_bind 54 55_set_vars() 56 57# UGLY MAGIC 58# Here we're (ab)using Fabric's built in 'role' feature to work with the way 59# we're loading context-specific config. Could possibly use a refactor to 60# avoid Fabric roles altogether. 61def _get_hosts(): 62 """Return list of hosts to push to""" 63 return env.config['push_hosts'] 64 65for role in silk.lib.get_role_list(env.local_root): 66 env.roledefs[role] = _get_hosts 67# END UGLY MAGIC 68 69def _tmpfile(): 70 """Generates a random filename in /tmp. Useful for dumping stdout to a 71 file that you want to download or read later. Assumes the remote host has 72 a /tmp directory.""" 73 chars = "abcdefghijklmnopqrstuvwxyz1234567890" 74 length = 20 75 randompart = "".join([random.choice(chars) for x in xrange(20)]) 76 return "/tmp/silk_tmp_%s" % randompart 77 78def _put_dir(local_dir, remote_dir, exclude=''): 79 """ 80 Copies a local directory to a remote one, using tar and put. Silently 81 overwrites remote directory if it exists, creates it if it does not 82 exist. 83 """ 84 tarball = "%s.tar.bz2" % _tmpfile() 85 86 tar_cmd = 'tar -C "%(local_dir)s" -cjf "%(tarball)s" %(exclude)s .' % locals() 87 local(tar_cmd) 88 put(tarball, tarball, use_sudo=True) 89 local('rm -f "%(tarball)s"' % locals()) 90 sudo('rm -Rf "{0}"; mkdir -p "{0}"; tar -C "{0}" -xjf "{1}" && rm -f "{1}"'\ 91 .format(remote_dir, tarball)) 92 93def _get_blame(): 94 """ 95 Return information about this deployment, to be written as the "blame" 96 section in the site.yaml file. 97 """ 98 return {'deployed_by': env.user, 99 'deployed_from': os.uname()[1], 100 'deployed_at': datetime.datetime.now(), 101 'deployed_role': env.roles[0]} 102 103def _write_file(path, contents, use_sudo=True, chown=None): 104 file_name = _tmpfile() 105 file = open(file_name, 'w') 106 file.write(contents) 107 file.close() 108 put(file_name, path, use_sudo=use_sudo) 109 sudo('chmod +r %s' % path) 110 if chown: 111 sudo('chown %s %s' % (chown, path)) 112 local('rm %s' % file_name) 113 114def _write_site_yaml(): 115 """Writes the site.yaml file for the deployed site.""" 116 # Make a copy of env.config 117 site_yaml_dict = copy.copy(env.config) 118 # add the blame in 119 site_yaml_dict['blame'] = _get_blame() 120 # write it to the remote host 121 file = _join(env.root, 'site.yaml') 122 _write_file(file, yaml.safe_dump(site_yaml_dict, default_flow_style=False)) 123 124def _write_template(template, dest, context): 125 path = silk.lib.get_template_path(template, env.local_root) 126 upload_template(path, dest, context=context, use_sudo=True) 127 128def _ensure_dir(remote_path): 129 if not exists(remote_path, use_sudo=True): 130 sudo('mkdir -p %s' % remote_path) 131 132def _format_supervisord_env(env_dict): 133 """Takes a dictionary and returns a string in form 134 'key=val,key2=val2'""" 135 try: 136 return ','.join(['%s="%s"' % (key, env_dict[key]) for key in env_dict.keys()]) 137 except AttributeError: 138 #env_dict isn't a dictionary, so they must not have included any env vars for us. 139 return '' 140 141def _green(text): 142 print green(text) 143 144def _red(text): 145 print red(text) 146 147def _yellow(text): 148 print yellow(text) 149 150def _list_dir(dirname): 151 """Given the path for a directory on the remote host, return its contents 152 as a python list.""" 153 txt = sudo('ls -1 %s' % dirname) 154 return txt.split('\r\n') 155 156def _is_supervisored(site): 157 """Returns True if site 'site' has a supervisord.conf entry, else False.""" 158 # Check both the default include location and the silk <=0.2.9 location. 159 old_conf_file = _join(SRV_ROOT, site, 'conf', 'supervisord.conf') 160 new_conf_file = _join('/etc/supervisor/conf.d', '%s.conf' % site) 161 return exists(new_conf_file) or exists(old_conf_file) 162 163def _socket_head_request(path): 164 """Given a path to a socket, make an HTTP HEAD request on it""" 165 # Get the local path for the unix socket http tool 166 script = pkg_resources.resource_filename('silk', 'sock_http.py') 167 # copy it over to the host 168 dest = _tmpfile() 169 put(script, dest, use_sudo=True) 170 # run it, passing in the path to the socket 171 return sudo('python %s %s HEAD / %s' % (dest, path, 172 env.config['listen_hosts'][0])) 173 174def _port_head_request(port): 175 """Given a port number, use curl to make an http HEAD request to it. 176 Return response status as integer.""" 177 return run('curl -I http://localhost:%s' % port) 178 179def returns_good_status(): 180 """Makes an HTTP request to '/' on the site and returns True if it gets 181 back a status in the 200, 300, or 400 ranges. 182 183 You can only use this as a standalone silk command if your site has a 184 hard-configured 'bind'.""" 185 _yellow("Making http request to site to ensure upness.") 186 bind = env.config['bind'] 187 if bind.startswith('unix:'): 188 result = _socket_head_request(bind.replace('unix:', '')) 189 else: 190 result = _port_head_request(bind.split(':')[-1]) 191 first_line = result.split('\r\n')[0] 192 status = int(first_line.split()[1]) 193 return 200 <= status < 500 194 195def _is_running(procname, tries=3, wait=2): 196 """Given the name of a supervisord process, tell you whether it's running 197 or not. If status is 'starting', will wait until status has settled.""" 198 # Status return from supervisorctl will look something like this:: 199 # mysite_20110623_162319 RUNNING pid 628, uptime 0:34:13 200 # mysite_20110623_231206 FATAL Exited too quickly (process log may have details) 201 202 status_parts = sudo('supervisorctl status %s' % procname).split() 203 if status_parts[1] == 'RUNNING': 204 # For super extra credit, we're actually going to make an HTTP request 205 # to the site to verify that it's up and running. Unfortunately we 206 # can only do that if Gunicorn is binding to a different socket or port 207 # on each deployment (Silk binds to a new socket on each deployment by 208 # default.) If you've configured your site to bind to a port, we'll 209 # just have to wing it. 210 if env.config['bind'] == env.default_bind: 211 if returns_good_status(): 212 _green("You're golden!") 213 return True 214 else: 215 _red(":(") 216 return False 217 else: 218 return True 219 elif status_parts[1] == "FATAL": 220 return False 221 elif tries > 0: 222 # It's neither running nor dead yet, so try again 223 _yellow("Waiting %s seconds for process to settle" % wait) 224 time.sleep(wait) 225 226 # decrement the tries and double the wait time for the next check. 227 return _is_running(procname, tries - 1, wait * 2) 228 else: 229 return False 230 231def _is_this_site(name): 232 """Return True if 'name' matches our site name (old style Silk deployment 233 naming' or matches our name + timestamp pattern.""" 234 235 site_pattern = re.compile('%s_\d{8}_\d{6}' % env.site) 236 return (name == env.site) or (re.match(site_pattern, name) is not None) 237 238def install_server_deps(): 239 """ 240 Installs nginx and supervisord on remote Ubuntu host. 241 """ 242 sudo('apt-get install nginx supervisor --assume-yes --quiet --no-upgrade') 243 244def push_code(): 245 _green("PUSHING CODE") 246 # Push the local site to the remote root, excluding files that we don't 247 # want to leave cluttering the production server. Exclude site.yaml 248 # because we'll be writing a new one containing the site config updated 249 # with the role config. Omit roles because they're superfluous at that 250 # point and also may contain sensitive connection credentials. 251 exclude = "--exclude=site.yaml --exclude=roles" 252 _put_dir(env.local_root, env.root, exclude) 253 254def create_virtualenv(): 255 """Create a virtualenv inside the remote root""" 256 if 'runtime' in env.config: 257 pyversion = '--python=%s' % env.config['runtime'] 258 else: 259 pyversion = '' 260 261 # Put all the prereq packages into /srv/pip_cache on the remote host, if 262 # they're not already there. 263 local_dir = pkg_resources.resource_filename('silk', 'prereqs') 264 files = pkg_resources.resource_listdir('silk', 'prereqs') 265 266 for f in files: 267 remote = posixpath.join('/tmp', f) 268 local = posixpath.join(local_dir, f) 269 if not exists(remote): 270 put(local, remote, use_sudo=True) 271 272 273 tmpl_vars = vars() 274 tmpl_vars.update(env) 275 276 c = ("virtualenv --no-site-packages %(pyversion)s --extra-search-dir=/tmp " 277 "--never-download %(envdir)s") % tmpl_vars 278 sudo(c) 279 280def pip_deps(): 281 """Install requirements listed in the site's requirements.txt file.""" 282 283 _green("INSTALLING PYTHON DEPENDENCIES") 284 reqs_file = os.path.join(env.root, 'requirements.txt') 285 pypi = env.config.get('pypi', 'http://pypi.python.org/pypi') 286 cachedir = posixpath.join(SRV_ROOT, 'pip_cache') 287 _ensure_dir(cachedir) 288 sudo('PIP_DOWNLOAD_CACHE="%s" %s/bin/pip install -r %s -i %s ' % 289 (cachedir, env.envdir, reqs_file, pypi)) 290 291def configure_supervisor(): 292 """ 293 Creates and upload config file for supervisord 294 """ 295 _green("WRITING SUPERVISOR CONFIG") 296 297 template_vars = { 298 'cmd': silk.lib.get_gunicorn_cmd(env, bin_dir='%s/bin' % (env.envdir)), 299 'process_env': _format_supervisord_env(env.config.get('env', '')), 300 'srv_root': SRV_ROOT, 301 } 302 303 template_vars.update(env) 304 template_vars.update(env.config) 305 #make sure the logs dir is created 306 _ensure_dir(_join(env.root, 'logs')) 307 308 # Put supervisord include in default location 309 _write_template('supervisord.conf', env.supd_conf_file, template_vars) 310 311def fix_supd_config_bug(): 312 """Fixes a bug from an earlier version of Silk that wrote an invalid line 313 to the master supervisord.conf""" 314 # Silk 0.2.9 and earlier included a command to configure supervisord and 315 # nginx to include files in /srv/<site>/conf, in addition to their default 316 # include directories. While this was valid for nginx, supervisord does 317 # not allow for multiple "files" lines in its "include" section (it does 318 # allow for multiple globs on the single "files" line, though. This command 319 # finds the offending pattern in /etc/supervisor/supervisord.conf and 320 # replaces it with the correct equivalent. 321 322 # Note that Silk 0.3.0 and later does not require any changes from the 323 # default supervisord.conf that ships with Ubuntu. All files are included 324 # in the supervisord's conf.d directory. 325 326 file = '/etc/supervisor/supervisord.conf' 327 328 if contains(file, "files = /srv/\*/conf/supervisord.conf", use_sudo=True): 329 _green("FIXING OLD SUPERVISOR CONFIG BUG") 330 _yellow("See http://bits.btubbs.com/silk-deployment/issue/15/incorrect-supervisord-include-config-in") 331 332 bad = "\r\n".join([ 333 "files = /etc/supervisor/conf.d/*.conf", 334 "files = /srv/*/conf/supervisord.conf" 335 ]) 336 337 good = ("files = /etc/supervisor/conf.d/*.conf " 338 "/srv/*/conf/supervisord.conf\n") 339 340 txt = sudo('cat %s' % file) 341 342 if bad in txt: 343 txt = txt.replace(bad, good) 344 _write_file(file, txt, use_sudo=True, chown='root') 345 346def cleanup(): 347 """Deletes old versions of the site that are still sitting around.""" 348 _green("CLEANING UP") 349 350 folders = _list_dir(SRV_ROOT) 351 rollbacks = [x for x in folders if _is_this_site(x)] 352 353 if len(rollbacks) > env.rollback_cap: 354 # There are more rollbacks than we want to keep around. See if we can 355 # delete some. 356 suspects = rollbacks[:-(env.rollback_cap + 1)] 357 for folder in suspects: 358 if not _is_supervisored(folder): 359 fullpath = _join(SRV_ROOT, folder) 360 sudo('rm -rf %s' % fullpath) 361 362 # Clean up old socket files in /tmp/ that have no associated site 363 # TODO: use our own list dir function and a regular expression to filter 364 # the list of /tmp sockets instead of this funky grepping. 365 with cd('/tmp'): 366 socks = run('ls -1 | grep %s | grep sock | cat -' % env.site).split('\r\n') 367 for sock in socks: 368 procname = sock.replace('.sock', '') 369 if not exists(_join(SRV_ROOT, procname)): 370 sudo('rm /tmp/%s' % sock) 371 372 # TODO: clean out the pip-* folders that can build up in /tmp 373 # TODO: figure out a way to clean out pybundle files in /srv/_silk_build 374 # that aren't needed anymore. 375 376def start_process(): 377 """Tell supervisord to read the new config, then start the new process.""" 378 _green('STARTING PROCESS') 379 result = sudo('supervisorctl reread') 380 381 sudo('supervisorctl add %s' % env.deployment) 382 383def _get_nginx_static_snippet(url_path, local_path): 384 return """ 385 location %(url_path)s { 386 alias %(local_path)s; 387 } 388 """ % locals() 389 390def configure_nginx(): 391 """Writes a new nginx config include pointing at the newly-deployed site.""" 392 _green("WRITING NGINX CONFIG") 393 nginx_static = '' 394 static_dirs = env.config.get('static_dirs', None) 395 396 # Use the static_dirs values from the site config to set up static file 397 # serving in nginx. 398 if static_dirs: 399 for item in static_dirs: 400 nginx_static += _get_nginx_static_snippet( 401 item['url_path'], 402 _join(env.root, item['system_path']) 403 ) 404 template_vars = { 405 'nginx_static': nginx_static, 406 'nginx_hosts': ' '.join(env.config['listen_hosts']), 407 } 408 template_vars.update(env) 409 template_vars.update(env.config) 410 411 # Create nginx include here: 412 # /etc/nginx/sites-enabled/<sitename>.conf 413 nginx_file = _join('/etc', 'nginx', 'sites-enabled', env.site) 414 sudo('rm -f %s' % nginx_file) 415 _write_template('nginx.conf', nginx_file, template_vars) 416 417def switch_nginx(): 418 _green("LOADING NEW NGINX CONFIG") 419 420 # Check if there is an old-style version (within the site root) of the 421 # nginx config laying around, and rename it to something innocuous if so. 422 old_nginx = _join(SRV_ROOT, env.site, 'conf', 'nginx.conf') 423 if exists(old_nginx): 424 sudo('mv %s %s' % (old_nginx, "%s_disabled" % old_nginx)) 425 426 # Tell nginx to rescan its config files 427 sudo('/etc/init.d/nginx reload') 428 429def stop_other_versions(): 430 """Stop other versions of the site that are still running, and disable their 431 configs.""" 432 proclist = sudo('supervisorctl status').split('\r\n') 433 434 # parse each line so we can get at just the proc names 435 proclist = [x.split() for x in proclist] 436 437 # filter proclist to include only versions of our site 438 proclist = [x for x in proclist if _is_this_site(x[0])] 439 440 live_statuses = ["RUNNING", "STARTING"] 441 442 # stop each process left in proclist that isn't the current one 443 for proc in proclist: 444 # We assume that spaces are not allowed in proc names 445 procname = proc[0] 446 procstatus = proc[1] 447 if procname != env.deployment: 448 # Stop the process 449 if procstatus in live_statuses: 450 sudo('supervisorctl stop %s' % procname) 451 452 # Remove it from live config 453 sudo('supervisorctl remove %s' % procname) 454 455 # Remove its supervisord config file 456 conf_file = '/etc/supervisor/conf.d/%s.conf' % procname 457 if exists(conf_file): 458 sudo('rm %s' % conf_file) 459 460 # Also remove old style supervisord include if it exists 461 old_conf_file = _join(SRV_ROOT, procname, 'conf/supervisord.conf') 462 if exists(old_conf_file): 463 sudo('rm %s' % old_conf_file) 464 465 sudo('supervisorctl reread') 466 467def congrats(): 468 """Congratulate the user and print a link to the site.""" 469 link0 = "http://%s" % env.config['listen_hosts'][0] 470 msg = ("SUCCESS! I think. Check that the site is running by browsing " 471 "to this url:\n\n%s" % link0) 472 _green(msg) 473 474def push(): 475 """ 476 The main function. This function will put your site on the remote host and get it 477 running. 478 """ 479 # Make sure nginx and supervisord are installed 480 install_server_deps() 481 482 # Fix an embarrassing config bug from earlier versions 483 fix_supd_config_bug() 484 485 # push site code and pybundle to server, in timestamped folder 486 push_code() 487 # make virtualenv on the server and run pip install on the pybundle 488 create_virtualenv() 489 pip_deps() 490 _write_site_yaml() 491 # write supervisord config for the new site 492 configure_supervisor() 493 494 ##then the magic 495 496 ##silk starts up supervisord on the new site 497 start_process() 498 # checks that the new site is running (by using supervisorctl) 499 if _is_running(env.deployment): 500 _green("Site is running. Proceeding with nginx switch.") 501 # if the site's running fine, then silk configures nginx to forward requests 502 # to the new site 503 configure_nginx() 504 switch_nginx() 505 stop_other_versions() 506 cleanup() 507 congrats() 508 else: 509 _red("Process failed to start cleanly. Off to the log files!") 510 sys.exit(1)