/Packages/Emmet/emmet/pyv8loader.py

# coding=utf-8
import os
import os.path
import sys
import urllib
import urllib2
import json
import re
import threading
import subprocess
import tempfile

PACKAGES_URL = 'https://api.github.com/repos/emmetio/pyv8-binaries/downloads'

class LoaderDelegate():
	"""
	Abstract class used to display PyV8 binary download progress,
	and provide some settings for downloader
	"""
	def __init__(self, settings={}):
		self.settings = settings

	def on_start(self, *args, **kwargs):
		"Invoked when download process is initiated"
		pass

	def on_progress(self, *args, **kwargs):
		"Invoked on download progress"
		pass

	def on_complete(self, *args, **kwargs):
		"Invoked when download process was finished successfully"
		pass

	def on_error(self, *args, **kwargs):
		"Invoked when error occured during download process"
		pass

	def setting(self, name, default=None):
		"Returns specified setting name"
		return self.settings[name] if name in self.settings else default

class ThreadProgress():
	def __init__(self, thread, delegate):
		self.thread = thread
		self.delegate = delegate
		self._callbacks = {}
		threading.Timer(0, self.run).start()

	def run(self):
		if not self.thread.is_alive():
			if self.thread.exit_code != 0:
				return self.trigger('error', exit_code=self.thread.exit_code, thread=self.thread)
				
			return self.trigger('complete', result=self.thread.result, thread=self.thread)

		self.trigger('progress', thread=self.thread)
		threading.Timer(0.1, self.run).start()

	def on(self, event_name, callback):
		if event_name not in self._callbacks:
			self._callbacks[event_name] = []

		if callable(callback):
			self._callbacks[event_name].append(callback)

		return self

	def trigger(self, event_name, *args, **kwargs):
		if event_name in self._callbacks:
			for c in self._callbacks[event_name]:
				c(*args, **kwargs)

		if self.delegate and hasattr(self.delegate, 'on_%s' % event_name):
			getattr(self.delegate, 'on_%s' % event_name)(*args, **kwargs)

		return self

class BinaryNotFoundError(Exception):
	pass


class NonCleanExitError(Exception):
	def __init__(self, returncode):
		self.returncode = returncode

	def __str__(self):
		return repr(self.returncode)


class CliDownloader():
	def __init__(self, settings):
		self.settings = settings

	def find_binary(self, name):
		for dir in os.environ['PATH'].split(os.pathsep):
			path = os.path.join(dir, name)
			if os.path.exists(path):
				return path

		raise BinaryNotFoundError('The binary %s could not be located' % name)

	def execute(self, args):
		proc = subprocess.Popen(args, stdin=subprocess.PIPE,
			stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

		output = proc.stdout.read()
		returncode = proc.wait()
		if returncode != 0:
			error = NonCleanExitError(returncode)
			error.output = output
			raise error
		return output

class WgetDownloader(CliDownloader):
	def __init__(self, settings):
		self.settings = settings
		self.wget = self.find_binary('wget')

	def clean_tmp_file(self):
		os.remove(self.tmp_file)

	def download(self, url, error_message, timeout, tries):
		if not self.wget:
			return False

		self.tmp_file = tempfile.NamedTemporaryFile().name
		command = [self.wget, '--connect-timeout=' + str(int(timeout)), '-o',
			self.tmp_file, '-O', '-', '-U', 'Emmet PyV8 Loader']

		command.append(url)

		if self.settings.get('http_proxy'):
			os.putenv('http_proxy', self.settings.get('http_proxy'))
			if not self.settings.get('https_proxy'):
				os.putenv('https_proxy', self.settings.get('http_proxy'))
		if self.settings.get('https_proxy'):
			os.putenv('https_proxy', self.settings.get('https_proxy'))

		while tries > 0:
			tries -= 1
			try:
				result = self.execute(command)
				self.clean_tmp_file()
				return result
			except (NonCleanExitError) as (e):
				error_line = ''
				with open(self.tmp_file) as f:
					for line in list(f):
						if re.search('ERROR[: ]|failed: ', line):
							error_line = line
							break

				if e.returncode == 8:
					regex = re.compile('^.*ERROR (\d+):.*', re.S)
					if re.sub(regex, '\\1', error_line) == '503':
						# GitHub and BitBucket seem to rate limit via 503
						print ('%s: Downloading %s was rate limited' +
							', trying again') % (__name__, url)
						continue
					error_string = 'HTTP error ' + re.sub('^.*? ERROR ', '',
						error_line)

				elif e.returncode == 4:
					error_string = re.sub('^.*?failed: ', '', error_line)
					# GitHub and BitBucket seem to time out a lot
					if error_string.find('timed out') != -1:
						print ('%s: Downloading %s timed out, ' +
							'trying again') % (__name__, url)
						continue

				else:
					error_string = re.sub('^.*?(ERROR[: ]|failed: )', '\\1',
						error_line)

				error_string = re.sub('\\.?\s*\n\s*$', '', error_string)
				print '%s: %s %s downloading %s.' % (__name__, error_message,
					error_string, url)
			self.clean_tmp_file()
			break
		return False


class CurlDownloader(CliDownloader):
	def __init__(self, settings):
		self.settings = settings
		self.curl = self.find_binary('curl')

	def download(self, url, error_message, timeout, tries):
		if not self.curl:
			return False
		command = [self.curl, '-f', '--user-agent', 'Emmet PyV8 Loader',
			'--connect-timeout', str(int(timeout)), '-sS']

		command.append(url)

		if self.settings.get('http_proxy'):
			os.putenv('http_proxy', self.settings.get('http_proxy'))
			if not self.settings.get('https_proxy'):
				os.putenv('HTTPS_PROXY', self.settings.get('http_proxy'))
		if self.settings.get('https_proxy'):
			os.putenv('HTTPS_PROXY', self.settings.get('https_proxy'))

		while tries > 0:
			tries -= 1
			try:
				return self.execute(command)
			except (NonCleanExitError) as (e):
				if e.returncode == 22:
					code = re.sub('^.*?(\d+)\s*$', '\\1', e.output)
					if code == '503':
						# GitHub and BitBucket seem to rate limit via 503
						print ('%s: Downloading %s was rate limited' +
							', trying again') % (__name__, url)
						continue
					error_string = 'HTTP error ' + code
				elif e.returncode == 6:
					error_string = 'URL error host not found'
				elif e.returncode == 28:
					# GitHub and BitBucket seem to time out a lot
					print ('%s: Downloading %s timed out, trying ' +
						'again') % (__name__, url)
					continue
				else:
					error_string = e.output.rstrip()

				print '%s: %s %s downloading %s.' % (__name__, error_message,
					error_string, url)
			break
		return False


class UrlLib2Downloader():
	def __init__(self, settings):
		self.settings = settings

	def download(self, url, error_message, timeout, tries):
		http_proxy = self.settings.get('http_proxy')
		https_proxy = self.settings.get('https_proxy')
		if http_proxy or https_proxy:
			proxies = {}
			if http_proxy:
				proxies['http'] = http_proxy
				if not https_proxy:
					proxies['https'] = http_proxy
			if https_proxy:
				proxies['https'] = https_proxy
			proxy_handler = urllib2.ProxyHandler(proxies)
		else:
			proxy_handler = urllib2.ProxyHandler()
		handlers = [proxy_handler]

		# secure_url_match = re.match('^https://([^/]+)', url)
		# if secure_url_match != None:
		# 	secure_domain = secure_url_match.group(1)
		# 	bundle_path = self.check_certs(secure_domain, timeout)
		# 	if not bundle_path:
		# 		return False
		# 	handlers.append(VerifiedHTTPSHandler(ca_certs=bundle_path))
		urllib2.install_opener(urllib2.build_opener(*handlers))

		while tries > 0:
			tries -= 1
			try:
				request = urllib2.Request(url, headers={"User-Agent":
					"Emmet PyV8 Loader"})
				http_file = urllib2.urlopen(request, timeout=timeout)
				return http_file.read()

			except (urllib2.HTTPError) as (e):
				# Bitbucket and Github ratelimit using 503 a decent amount
				if str(e.code) == '503':
					print ('%s: Downloading %s was rate limited, ' +
						'trying again') % (__name__, url)
					continue
				print '%s: %s HTTP error %s downloading %s.' % (__name__,
					error_message, str(e.code), url)

			except (urllib2.URLError) as (e):
				# Bitbucket and Github timeout a decent amount
				if str(e.reason) == 'The read operation timed out' or \
						str(e.reason) == 'timed out':
					print ('%s: Downloading %s timed out, trying ' +
						'again') % (__name__, url)
					continue
				print '%s: %s URL error %s downloading %s.' % (__name__,
					error_message, str(e.reason), url)
			break
		return False

class PyV8Loader(threading.Thread):
	def __init__(self, arch, download_path, config):
		self.arch = arch
		self.config = config
		self.download_path = download_path
		self.exit_code = 0
		self.result = None
		self.settings = {}

		threading.Thread.__init__(self)
		self.log('Creating thread')

	def log(self, message):
		print('PyV8 Loader: %s' % message)

	def download_url(self, url, error_message):
		# TODO add settings
		has_ssl = 'ssl' in sys.modules and hasattr(urllib2, 'HTTPSHandler')
		is_ssl = re.search('^https://', url) != None

		if (is_ssl and has_ssl) or not is_ssl:
			downloader = UrlLib2Downloader(self.settings)
		else:
			for downloader_class in [CurlDownloader, WgetDownloader]:
				try:
					downloader = downloader_class(self.settings)
					break
				except (BinaryNotFoundError):
					pass

		if not downloader:
			self.log('Unable to download PyV8 binary due to invalid downloader')
			return False

		# timeout = self.settings.get('timeout', 3)
		timeout = 3
		return downloader.download(url.replace(' ', '%20'), error_message, timeout, 3)

	def run(self):
		# get list of available packages first
		packages = self.download_url(PACKAGES_URL, 'Unable to download packages list.')

		if not packages:
			self.exit_code = 1
			return

		files = json.loads(packages)

		# find package for current architecture
		cur_item = None
		for item in files:
			if self.arch in item['description'].lower():
				cur_item = item
				break

		if not cur_item:
			self.log('Unable to find binary for %s architecture' % self.arch)
			self.exit_code = 2
			return

		if cur_item['id'] == self.config['last_id']:
			self.log('You have the most recent PyV8 binary')
			return

		# Reduce HTTP roundtrips: try to download binary from 
		# http://cloud.github.com directly
		url = re.sub(r'^https?:\/\/github\.com', 'http://cloud.github.com', item['html_url'])
		self.log('Loading PyV8 binary from %s' % url)
		package = self.download_url(url, 'Unable to download package from %s' % url)
		if not package:
			url = item['html_url']
			self.log('Loading PyV8 binary from %s' % url)
			package = self.download_url(url, 'Unable to download package from %s' % url)
			if not package:
				self.exit_code = 3
				return

		# we should only save downloaded package and delegate module
		# loading/unloading to main thread since improper PyV8 unload
		# may cause editor crash
		
		try:
			os.makedirs(self.download_path)
		except Exception, e:
			pass
		
		fp = open(os.path.join(self.download_path, 'pack.zip'), 'wb')
		fp.write(package)
		fp.close()

		self.result = cur_item['id']
		# Done!