PageRenderTime 30ms CodeModel.GetById 10ms RepoModel.GetById 1ms app.codeStats 0ms

/app/deploy/deploy.py

https://gitlab.com/gregtyka/server
Python | 283 lines | 261 code | 21 blank | 1 comment | 20 complexity | 5c68497b84146db2c776f6034f0ae400 MD5 | raw file
  1. import sys
  2. import time
  3. import glob
  4. import subprocess
  5. import os
  6. import optparse
  7. import datetime
  8. import urllib2
  9. import webbrowser
  10. import getpass
  11. import re
  12. from contextlib import contextmanager
  13. import commands
  14. dev_appserver_path = os.path.realpath(commands.getoutput("which dev_appserver.py"))
  15. gae_path = os.path.dirname(dev_appserver_path)
  16. sys.path.append(os.path.abspath("."))
  17. sys.path.append(os.path.abspath(gae_path))
  18. sys.path.append(os.path.abspath("../offline/"))
  19. import compress
  20. import npm
  21. try:
  22. import secrets
  23. hipchat_deploy_token = secrets.hipchat_deploy_token
  24. except Exception, e:
  25. print "Exception raised while trying to import secrets. Attempting to continue..."
  26. print repr(e)
  27. hipchat_deploy_token = None
  28. try:
  29. from secrets_dev import app_engine_username, app_engine_password
  30. except Exception, e:
  31. (app_engine_username, app_engine_password) = (None, None)
  32. if hipchat_deploy_token:
  33. import hipchat.room
  34. import hipchat.config
  35. hipchat.config.manual_init(hipchat_deploy_token)
  36. @contextmanager
  37. def pushd(directory):
  38. before = os.getcwd()
  39. os.chdir(directory)
  40. yield
  41. os.chdir(before)
  42. def popen_safe(args):
  43. if isinstance(args, str):
  44. args = args.split()
  45. print 'invoking command: ' + ' '.join(args)
  46. proc = subprocess.Popen(args)
  47. ret = proc.wait()
  48. if ret != 0:
  49. raise RuntimeError("Failed running {}. Return code: {}.".format(args, ret))
  50. def popen_results(args):
  51. proc = subprocess.Popen(args, stdout=subprocess.PIPE)
  52. return proc.communicate()[0]
  53. def popen_return_code(args, input=None):
  54. proc = subprocess.Popen(args, stdin=subprocess.PIPE)
  55. proc.communicate(input)
  56. return proc.returncode
  57. def get_app_engine_credentials():
  58. if app_engine_username and app_engine_password:
  59. print "Using password for %s from secrets.py" % app_engine_username
  60. return (app_engine_username, app_engine_password)
  61. else:
  62. email = raw_input("App Engine Email: ")
  63. password = getpass.getpass("Password for %s: " % email)
  64. return (email, password)
  65. def get_app_id():
  66. f = open("app.yaml", "r")
  67. contents = f.read()
  68. f.close()
  69. app_re = re.compile("^application:\s+(.+)$", re.MULTILINE)
  70. match = app_re.search(contents)
  71. return match.groups()[0]
  72. def git_status():
  73. output = popen_results(['git', 'status', '-s'])
  74. return len(output) > 0
  75. def git_pull(exercises_branch):
  76. with pushd('khan-exercises'):
  77. popen_safe('git clean -fd')
  78. with pushd('..'):
  79. popen_safe('git submodule update --init --recursive')
  80. with pushd('khan-exercises'):
  81. popen_safe('git fetch')
  82. popen_safe('git checkout origin/{}'.format(exercises_branch))
  83. return git_version()
  84. def git_version():
  85. # grab the tip changeset hash
  86. return popen_results(['git', 'rev-parse', 'HEAD']).strip()[:6]
  87. def git_revision_msg(revision_id):
  88. return popen_results(['git', 'show', '-s', '--pretty=format:%s', revision_id]).strip()
  89. def check_secrets():
  90. content = ""
  91. try:
  92. f = open("secrets.py", "r")
  93. content = f.read()
  94. f.close()
  95. except:
  96. return False
  97. # Try to find the beginning of our production facebook app secret
  98. # to verify deploy is being sent from correct directory.
  99. regex = re.compile("^facebook_app_secret = '4362.+'$", re.MULTILINE)
  100. return regex.search(content)
  101. def check_deps():
  102. """Check if npm and friends are installed"""
  103. return npm.check_dependencies()
  104. def compile_handlebar_templates():
  105. print "Compiling handlebar templates"
  106. return 0 == popen_return_code([sys.executable,
  107. 'deploy/compile_handlebar_templates.py'])
  108. def compress_js():
  109. print "Compressing javascript"
  110. compress.compress_all_javascript()
  111. def compress_css():
  112. print "Compressing stylesheets"
  113. compress.compress_all_stylesheets()
  114. def compress_exercises():
  115. print "Compressing exercises"
  116. subprocess.check_call(["ruby", "khan-exercises/build/pack.rb"])
  117. def compile_templates():
  118. print "Compiling jinja templates"
  119. return 0 == popen_return_code([sys.executable, 'deploy/compile_templates.py'])
  120. def prime_cache(version):
  121. try:
  122. resp = urllib2.urlopen("http://%s.%s.appspot.com/api/v1/autocomplete?q=calc" % (version, get_app_id()))
  123. resp.read()
  124. resp = urllib2.urlopen("http://%s.%s.appspot.com/api/v1/topics/library/compact" % (version, get_app_id()))
  125. resp.read()
  126. print "Primed cache"
  127. except urllib2.HTTPError, e:
  128. print "Error when priming cache"
  129. print e.read()
  130. def open_browser_to_ka_version(version):
  131. webbrowser.open("http://%s.%s.appspot.com" % (version, get_app_id()))
  132. def deploy(version, email, password):
  133. print "Deploying version " + str(version)
  134. return 0 == popen_return_code(['appcfg.py', '-V', str(version), "--no_oauth2", "-e", email, "--passin", "update", "."], "%s\n" % password)
  135. def set_default_version(version, email, password):
  136. print "Setting version as default" + str(version)
  137. return 0 == popen_return_code(['appcfg.py', '-V', str(version), "--no_oauth2", "-e", email, "--passin", "set_default_version", "."], "%s\n" % password)
  138. def main():
  139. start = datetime.datetime.now()
  140. parser = optparse.OptionParser()
  141. parser.add_option('-f', '--force',
  142. action="store_true", dest="force",
  143. help="Force deploy even with local changes", default=False)
  144. parser.add_option('-v', '--version',
  145. action="store", dest="version",
  146. help="Override the deployed version identifier", default="AUTO")
  147. parser.add_option('-x', '--no-up',
  148. action="store_true", dest="noup",
  149. help="Don't hg pull/up before deploy", default="")
  150. parser.add_option('-s', '--no-secrets',
  151. action="store_true", dest="nosecrets",
  152. help="Don't check for production secrets.py file before deploying", default="")
  153. parser.add_option('-d', '--dryrun',
  154. action="store_true", dest="dryrun",
  155. help="Dry run without the final deploy-to-App-Engine step", default=False)
  156. parser.add_option('-r', '--report',
  157. action="store_true", dest="report",
  158. help="Generate a report that displays minified, gzipped file size for each package element",
  159. default=False)
  160. parser.add_option('-n', '--no-npm',
  161. action="store_false", dest="node",
  162. help="Don't check for local npm modules and don't install/update them",
  163. default=True)
  164. parser.add_option('--set-default',
  165. action="store_true", dest="set_default_version",
  166. help="Set the newly created version as the default version to serve",
  167. default=False)
  168. parser.add_option('--exercises-branch', default="master")
  169. parser.add_option('--symbolab-branch', default="master")
  170. options, args = parser.parse_args()
  171. if options.node:
  172. print "Checking for node and dependencies"
  173. if not check_deps():
  174. return
  175. if options.report:
  176. print "Generating file size report"
  177. compile_handlebar_templates()
  178. compress.file_size_report()
  179. return
  180. includes_local_changes = git_status()
  181. if not options.force and includes_local_changes:
  182. print "Local changes found in this directory, canceling deploy."
  183. return
  184. version = -1
  185. if not options.noup:
  186. version = git_pull(options.exercises_branch)
  187. if version <= 0:
  188. print "Could not find version after 'hg pull', 'hg up', 'hg tip'."
  189. return
  190. if not options.nosecrets:
  191. if not check_secrets():
  192. print "Stopping deploy. It doesn't look like you're deploying from a directory with the appropriate secrets.py."
  193. return
  194. if not compile_templates():
  195. print "Failed to compile jinja templates, bailing."
  196. return
  197. if not compile_handlebar_templates():
  198. print "Failed to compile handlebars templates, bailing."
  199. return
  200. compress_js()
  201. compress_css()
  202. compress_exercises()
  203. if options.version:
  204. version = options.version
  205. elif options.noup:
  206. print 'You must supply a version when deploying with --no-up'
  207. return
  208. if version == "AUTO":
  209. version = time.strftime("%Y-%m-%d-") + git_version()
  210. print "Deploying version " + str(version)
  211. if not options.dryrun:
  212. (email, password) = get_app_engine_credentials()
  213. success = deploy(version, email, password)
  214. if success:
  215. # open_browser_to_ka_version(version)
  216. prime_cache(version)
  217. if options.set_default_version:
  218. set_default_version(version, email, password)
  219. end = datetime.datetime.now()
  220. print "Done. Duration: %s" % (end - start)
  221. return success
  222. if __name__ == "__main__":
  223. sys.exit(0 if main() else 1)