PageRenderTime 1ms CodeModel.GetById 38ms app.highlight 11ms RepoModel.GetById 0ms app.codeStats 0ms

/app/deploy/deploy.py

https://gitlab.com/gregtyka/server
Python | 283 lines | 261 code | 21 blank | 1 comment | 11 complexity | 5c68497b84146db2c776f6034f0ae400 MD5 | raw file
  1import sys
  2import time
  3import glob
  4import subprocess
  5import os
  6import optparse
  7import datetime
  8import urllib2
  9import webbrowser
 10import getpass
 11import re
 12from contextlib import contextmanager
 13
 14import commands
 15dev_appserver_path = os.path.realpath(commands.getoutput("which dev_appserver.py"))
 16gae_path = os.path.dirname(dev_appserver_path)
 17
 18sys.path.append(os.path.abspath("."))
 19sys.path.append(os.path.abspath(gae_path))
 20sys.path.append(os.path.abspath("../offline/"))
 21import compress
 22import npm
 23
 24try:
 25    import secrets
 26    hipchat_deploy_token = secrets.hipchat_deploy_token
 27except Exception, e:
 28    print "Exception raised while trying to import secrets. Attempting to continue..."
 29    print repr(e)
 30    hipchat_deploy_token = None
 31
 32try:
 33    from secrets_dev import app_engine_username, app_engine_password
 34except Exception, e:
 35    (app_engine_username, app_engine_password) = (None, None)
 36
 37if hipchat_deploy_token:
 38    import hipchat.room
 39    import hipchat.config
 40    hipchat.config.manual_init(hipchat_deploy_token)
 41
 42@contextmanager
 43def pushd(directory):
 44    before = os.getcwd()
 45    os.chdir(directory)
 46    yield
 47    os.chdir(before)
 48
 49def popen_safe(args):
 50    if isinstance(args, str):
 51        args = args.split()
 52    print 'invoking command: ' + ' '.join(args)
 53    proc = subprocess.Popen(args)
 54    ret = proc.wait()
 55    if ret != 0:
 56        raise RuntimeError("Failed running {}. Return code: {}.".format(args, ret))
 57
 58def popen_results(args):
 59    proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 60    return proc.communicate()[0]
 61
 62def popen_return_code(args, input=None):
 63    proc = subprocess.Popen(args, stdin=subprocess.PIPE)
 64    proc.communicate(input)
 65    return proc.returncode
 66
 67def get_app_engine_credentials():
 68    if app_engine_username and app_engine_password:
 69        print "Using password for %s from secrets.py" % app_engine_username
 70        return (app_engine_username, app_engine_password)
 71    else:
 72        email = raw_input("App Engine Email: ")
 73        password = getpass.getpass("Password for %s: " % email)
 74        return (email, password)
 75
 76def get_app_id():
 77    f = open("app.yaml", "r")
 78    contents = f.read()
 79    f.close()
 80
 81    app_re = re.compile("^application:\s+(.+)$", re.MULTILINE)
 82    match = app_re.search(contents)
 83
 84    return match.groups()[0]
 85
 86def git_status():
 87    output = popen_results(['git', 'status', '-s'])
 88    return len(output) > 0
 89
 90def git_pull(exercises_branch):
 91    with pushd('khan-exercises'):
 92        popen_safe('git clean -fd')
 93
 94    with pushd('..'):
 95        popen_safe('git submodule update --init --recursive')
 96
 97    with pushd('khan-exercises'):
 98        popen_safe('git fetch')
 99        popen_safe('git checkout origin/{}'.format(exercises_branch))
100
101    return git_version()
102
103def git_version():
104    # grab the tip changeset hash
105    return popen_results(['git', 'rev-parse', 'HEAD']).strip()[:6]
106
107def git_revision_msg(revision_id):
108    return popen_results(['git', 'show', '-s', '--pretty=format:%s', revision_id]).strip()
109
110def check_secrets():
111    content = ""
112
113    try:
114        f = open("secrets.py", "r")
115        content = f.read()
116        f.close()
117    except:
118        return False
119
120    # Try to find the beginning of our production facebook app secret
121    # to verify deploy is being sent from correct directory.
122    regex = re.compile("^facebook_app_secret = '4362.+'$", re.MULTILINE)
123    return regex.search(content)
124
125def check_deps():
126    """Check if npm and friends are installed"""
127    return npm.check_dependencies()
128
129def compile_handlebar_templates():
130    print "Compiling handlebar templates"
131    return 0 == popen_return_code([sys.executable,
132                                   'deploy/compile_handlebar_templates.py'])
133
134def compress_js():
135    print "Compressing javascript"
136    compress.compress_all_javascript()
137
138def compress_css():
139    print "Compressing stylesheets"
140    compress.compress_all_stylesheets()
141
142def compress_exercises():
143    print "Compressing exercises"
144    subprocess.check_call(["ruby", "khan-exercises/build/pack.rb"])
145
146def compile_templates():
147    print "Compiling jinja templates"
148    return 0 == popen_return_code([sys.executable, 'deploy/compile_templates.py'])
149
150def prime_cache(version):
151    try:
152        resp = urllib2.urlopen("http://%s.%s.appspot.com/api/v1/autocomplete?q=calc" % (version, get_app_id()))
153        resp.read()
154        resp = urllib2.urlopen("http://%s.%s.appspot.com/api/v1/topics/library/compact" % (version, get_app_id()))
155        resp.read()
156        print "Primed cache"
157    except urllib2.HTTPError, e:
158        print "Error when priming cache"
159        print e.read()
160
161def open_browser_to_ka_version(version):
162    webbrowser.open("http://%s.%s.appspot.com" % (version, get_app_id()))
163
164def deploy(version, email, password):
165    print "Deploying version " + str(version)
166    return 0 == popen_return_code(['appcfg.py', '-V', str(version), "--no_oauth2", "-e", email, "--passin", "update", "."], "%s\n" % password)
167
168def set_default_version(version, email, password):
169    print "Setting version as default" + str(version)
170    return 0 == popen_return_code(['appcfg.py', '-V', str(version), "--no_oauth2", "-e", email, "--passin", "set_default_version", "."], "%s\n" % password)
171
172def main():
173
174    start = datetime.datetime.now()
175
176    parser = optparse.OptionParser()
177
178    parser.add_option('-f', '--force',
179        action="store_true", dest="force",
180        help="Force deploy even with local changes", default=False)
181
182    parser.add_option('-v', '--version',
183        action="store", dest="version",
184        help="Override the deployed version identifier", default="AUTO")
185
186    parser.add_option('-x', '--no-up',
187        action="store_true", dest="noup",
188        help="Don't hg pull/up before deploy", default="")
189
190    parser.add_option('-s', '--no-secrets',
191        action="store_true", dest="nosecrets",
192        help="Don't check for production secrets.py file before deploying", default="")
193
194    parser.add_option('-d', '--dryrun',
195        action="store_true", dest="dryrun",
196        help="Dry run without the final deploy-to-App-Engine step", default=False)
197
198    parser.add_option('-r', '--report',
199        action="store_true", dest="report",
200        help="Generate a report that displays minified, gzipped file size for each package element",
201            default=False)
202
203    parser.add_option('-n', '--no-npm',
204        action="store_false", dest="node",
205        help="Don't check for local npm modules and don't install/update them",
206        default=True)
207
208    parser.add_option('--set-default',
209        action="store_true", dest="set_default_version",
210        help="Set the newly created version as the default version to serve",
211        default=False)
212
213    parser.add_option('--exercises-branch', default="master")
214    parser.add_option('--symbolab-branch', default="master")
215
216    options, args = parser.parse_args()
217
218    if options.node:
219        print "Checking for node and dependencies"
220        if not check_deps():
221            return
222
223    if options.report:
224        print "Generating file size report"
225        compile_handlebar_templates()
226        compress.file_size_report()
227        return
228
229    includes_local_changes = git_status()
230    if not options.force and includes_local_changes:
231        print "Local changes found in this directory, canceling deploy."
232        return
233
234    version = -1
235
236    if not options.noup:
237        version = git_pull(options.exercises_branch)
238        if version <= 0:
239            print "Could not find version after 'hg pull', 'hg up', 'hg tip'."
240            return
241
242    if not options.nosecrets:
243        if not check_secrets():
244            print "Stopping deploy. It doesn't look like you're deploying from a directory with the appropriate secrets.py."
245            return
246
247    if not compile_templates():
248        print "Failed to compile jinja templates, bailing."
249        return
250
251    if not compile_handlebar_templates():
252        print "Failed to compile handlebars templates, bailing."
253        return
254
255    compress_js()
256    compress_css()
257    compress_exercises()
258
259    if options.version:
260        version = options.version
261    elif options.noup:
262        print 'You must supply a version when deploying with --no-up'
263        return
264
265    if version == "AUTO":
266        version = time.strftime("%Y-%m-%d-") + git_version()
267    print "Deploying version " + str(version)
268
269    if not options.dryrun:
270        (email, password) = get_app_engine_credentials()
271        success = deploy(version, email, password)
272        if success:
273            # open_browser_to_ka_version(version)
274            prime_cache(version)
275        if options.set_default_version:
276            set_default_version(version, email, password)
277
278    end = datetime.datetime.now()
279    print "Done. Duration: %s" % (end - start)
280    return success
281
282if __name__ == "__main__":
283    sys.exit(0 if main() else 1)