/Packages/Emmet/emmet/pyv8loader.py
Python | 383 lines | 344 code | 34 blank | 5 comment | 21 complexity | a2ee731bff6c17d16dbaae815f012fe2 MD5 | raw file
- # coding=utf-8
- import os
- import os.path
- import sys
- import urllib
- import urllib2
- import json
- import re
- import threading
- import subprocess
- import tempfile
- PACKAGES_URL = 'https://api.github.com/repos/emmetio/pyv8-binaries/downloads'
- class LoaderDelegate():
- """
- Abstract class used to display PyV8 binary download progress,
- and provide some settings for downloader
- """
- def __init__(self, settings={}):
- self.settings = settings
- def on_start(self, *args, **kwargs):
- "Invoked when download process is initiated"
- pass
- def on_progress(self, *args, **kwargs):
- "Invoked on download progress"
- pass
- def on_complete(self, *args, **kwargs):
- "Invoked when download process was finished successfully"
- pass
- def on_error(self, *args, **kwargs):
- "Invoked when error occured during download process"
- pass
- def setting(self, name, default=None):
- "Returns specified setting name"
- return self.settings[name] if name in self.settings else default
- class ThreadProgress():
- def __init__(self, thread, delegate):
- self.thread = thread
- self.delegate = delegate
- self._callbacks = {}
- threading.Timer(0, self.run).start()
- def run(self):
- if not self.thread.is_alive():
- if self.thread.exit_code != 0:
- return self.trigger('error', exit_code=self.thread.exit_code, thread=self.thread)
-
- return self.trigger('complete', result=self.thread.result, thread=self.thread)
- self.trigger('progress', thread=self.thread)
- threading.Timer(0.1, self.run).start()
- def on(self, event_name, callback):
- if event_name not in self._callbacks:
- self._callbacks[event_name] = []
- if callable(callback):
- self._callbacks[event_name].append(callback)
- return self
- def trigger(self, event_name, *args, **kwargs):
- if event_name in self._callbacks:
- for c in self._callbacks[event_name]:
- c(*args, **kwargs)
- if self.delegate and hasattr(self.delegate, 'on_%s' % event_name):
- getattr(self.delegate, 'on_%s' % event_name)(*args, **kwargs)
- return self
- class BinaryNotFoundError(Exception):
- pass
- class NonCleanExitError(Exception):
- def __init__(self, returncode):
- self.returncode = returncode
- def __str__(self):
- return repr(self.returncode)
- class CliDownloader():
- def __init__(self, settings):
- self.settings = settings
- def find_binary(self, name):
- for dir in os.environ['PATH'].split(os.pathsep):
- path = os.path.join(dir, name)
- if os.path.exists(path):
- return path
- raise BinaryNotFoundError('The binary %s could not be located' % name)
- def execute(self, args):
- proc = subprocess.Popen(args, stdin=subprocess.PIPE,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- output = proc.stdout.read()
- returncode = proc.wait()
- if returncode != 0:
- error = NonCleanExitError(returncode)
- error.output = output
- raise error
- return output
- class WgetDownloader(CliDownloader):
- def __init__(self, settings):
- self.settings = settings
- self.wget = self.find_binary('wget')
- def clean_tmp_file(self):
- os.remove(self.tmp_file)
- def download(self, url, error_message, timeout, tries):
- if not self.wget:
- return False
- self.tmp_file = tempfile.NamedTemporaryFile().name
- command = [self.wget, '--connect-timeout=' + str(int(timeout)), '-o',
- self.tmp_file, '-O', '-', '-U', 'Emmet PyV8 Loader']
- command.append(url)
- if self.settings.get('http_proxy'):
- os.putenv('http_proxy', self.settings.get('http_proxy'))
- if not self.settings.get('https_proxy'):
- os.putenv('https_proxy', self.settings.get('http_proxy'))
- if self.settings.get('https_proxy'):
- os.putenv('https_proxy', self.settings.get('https_proxy'))
- while tries > 0:
- tries -= 1
- try:
- result = self.execute(command)
- self.clean_tmp_file()
- return result
- except (NonCleanExitError) as (e):
- error_line = ''
- with open(self.tmp_file) as f:
- for line in list(f):
- if re.search('ERROR[: ]|failed: ', line):
- error_line = line
- break
- if e.returncode == 8:
- regex = re.compile('^.*ERROR (\d+):.*', re.S)
- if re.sub(regex, '\\1', error_line) == '503':
- # GitHub and BitBucket seem to rate limit via 503
- print ('%s: Downloading %s was rate limited' +
- ', trying again') % (__name__, url)
- continue
- error_string = 'HTTP error ' + re.sub('^.*? ERROR ', '',
- error_line)
- elif e.returncode == 4:
- error_string = re.sub('^.*?failed: ', '', error_line)
- # GitHub and BitBucket seem to time out a lot
- if error_string.find('timed out') != -1:
- print ('%s: Downloading %s timed out, ' +
- 'trying again') % (__name__, url)
- continue
- else:
- error_string = re.sub('^.*?(ERROR[: ]|failed: )', '\\1',
- error_line)
- error_string = re.sub('\\.?\s*\n\s*$', '', error_string)
- print '%s: %s %s downloading %s.' % (__name__, error_message,
- error_string, url)
- self.clean_tmp_file()
- break
- return False
- class CurlDownloader(CliDownloader):
- def __init__(self, settings):
- self.settings = settings
- self.curl = self.find_binary('curl')
- def download(self, url, error_message, timeout, tries):
- if not self.curl:
- return False
- command = [self.curl, '-f', '--user-agent', 'Emmet PyV8 Loader',
- '--connect-timeout', str(int(timeout)), '-sS']
- command.append(url)
- if self.settings.get('http_proxy'):
- os.putenv('http_proxy', self.settings.get('http_proxy'))
- if not self.settings.get('https_proxy'):
- os.putenv('HTTPS_PROXY', self.settings.get('http_proxy'))
- if self.settings.get('https_proxy'):
- os.putenv('HTTPS_PROXY', self.settings.get('https_proxy'))
- while tries > 0:
- tries -= 1
- try:
- return self.execute(command)
- except (NonCleanExitError) as (e):
- if e.returncode == 22:
- code = re.sub('^.*?(\d+)\s*$', '\\1', e.output)
- if code == '503':
- # GitHub and BitBucket seem to rate limit via 503
- print ('%s: Downloading %s was rate limited' +
- ', trying again') % (__name__, url)
- continue
- error_string = 'HTTP error ' + code
- elif e.returncode == 6:
- error_string = 'URL error host not found'
- elif e.returncode == 28:
- # GitHub and BitBucket seem to time out a lot
- print ('%s: Downloading %s timed out, trying ' +
- 'again') % (__name__, url)
- continue
- else:
- error_string = e.output.rstrip()
- print '%s: %s %s downloading %s.' % (__name__, error_message,
- error_string, url)
- break
- return False
- class UrlLib2Downloader():
- def __init__(self, settings):
- self.settings = settings
- def download(self, url, error_message, timeout, tries):
- http_proxy = self.settings.get('http_proxy')
- https_proxy = self.settings.get('https_proxy')
- if http_proxy or https_proxy:
- proxies = {}
- if http_proxy:
- proxies['http'] = http_proxy
- if not https_proxy:
- proxies['https'] = http_proxy
- if https_proxy:
- proxies['https'] = https_proxy
- proxy_handler = urllib2.ProxyHandler(proxies)
- else:
- proxy_handler = urllib2.ProxyHandler()
- handlers = [proxy_handler]
- # secure_url_match = re.match('^https://([^/]+)', url)
- # if secure_url_match != None:
- # secure_domain = secure_url_match.group(1)
- # bundle_path = self.check_certs(secure_domain, timeout)
- # if not bundle_path:
- # return False
- # handlers.append(VerifiedHTTPSHandler(ca_certs=bundle_path))
- urllib2.install_opener(urllib2.build_opener(*handlers))
- while tries > 0:
- tries -= 1
- try:
- request = urllib2.Request(url, headers={"User-Agent":
- "Emmet PyV8 Loader"})
- http_file = urllib2.urlopen(request, timeout=timeout)
- return http_file.read()
- except (urllib2.HTTPError) as (e):
- # Bitbucket and Github ratelimit using 503 a decent amount
- if str(e.code) == '503':
- print ('%s: Downloading %s was rate limited, ' +
- 'trying again') % (__name__, url)
- continue
- print '%s: %s HTTP error %s downloading %s.' % (__name__,
- error_message, str(e.code), url)
- except (urllib2.URLError) as (e):
- # Bitbucket and Github timeout a decent amount
- if str(e.reason) == 'The read operation timed out' or \
- str(e.reason) == 'timed out':
- print ('%s: Downloading %s timed out, trying ' +
- 'again') % (__name__, url)
- continue
- print '%s: %s URL error %s downloading %s.' % (__name__,
- error_message, str(e.reason), url)
- break
- return False
- class PyV8Loader(threading.Thread):
- def __init__(self, arch, download_path, config):
- self.arch = arch
- self.config = config
- self.download_path = download_path
- self.exit_code = 0
- self.result = None
- self.settings = {}
- threading.Thread.__init__(self)
- self.log('Creating thread')
- def log(self, message):
- print('PyV8 Loader: %s' % message)
- def download_url(self, url, error_message):
- # TODO add settings
- has_ssl = 'ssl' in sys.modules and hasattr(urllib2, 'HTTPSHandler')
- is_ssl = re.search('^https://', url) != None
- if (is_ssl and has_ssl) or not is_ssl:
- downloader = UrlLib2Downloader(self.settings)
- else:
- for downloader_class in [CurlDownloader, WgetDownloader]:
- try:
- downloader = downloader_class(self.settings)
- break
- except (BinaryNotFoundError):
- pass
- if not downloader:
- self.log('Unable to download PyV8 binary due to invalid downloader')
- return False
- # timeout = self.settings.get('timeout', 3)
- timeout = 3
- return downloader.download(url.replace(' ', '%20'), error_message, timeout, 3)
- def run(self):
- # get list of available packages first
- packages = self.download_url(PACKAGES_URL, 'Unable to download packages list.')
- if not packages:
- self.exit_code = 1
- return
- files = json.loads(packages)
- # find package for current architecture
- cur_item = None
- for item in files:
- if self.arch in item['description'].lower():
- cur_item = item
- break
- if not cur_item:
- self.log('Unable to find binary for %s architecture' % self.arch)
- self.exit_code = 2
- return
- if cur_item['id'] == self.config['last_id']:
- self.log('You have the most recent PyV8 binary')
- return
- # Reduce HTTP roundtrips: try to download binary from
- # http://cloud.github.com directly
- url = re.sub(r'^https?:\/\/github\.com', 'http://cloud.github.com', item['html_url'])
- self.log('Loading PyV8 binary from %s' % url)
- package = self.download_url(url, 'Unable to download package from %s' % url)
- if not package:
- url = item['html_url']
- self.log('Loading PyV8 binary from %s' % url)
- package = self.download_url(url, 'Unable to download package from %s' % url)
- if not package:
- self.exit_code = 3
- return
- # we should only save downloaded package and delegate module
- # loading/unloading to main thread since improper PyV8 unload
- # may cause editor crash
-
- try:
- os.makedirs(self.download_path)
- except Exception, e:
- pass
-
- fp = open(os.path.join(self.download_path, 'pack.zip'), 'wb')
- fp.write(package)
- fp.close()
- self.result = cur_item['id']
- # Done!
-