PageRenderTime 3246ms CodeModel.GetById 33ms RepoModel.GetById 1ms app.codeStats 0ms

/Packages/Emmet/emmet/pyv8loader.py

https://gitlab.com/Blueprint-Marketing/sublime-config
Python | 383 lines | 344 code | 34 blank | 5 comment | 21 complexity | a2ee731bff6c17d16dbaae815f012fe2 MD5 | raw file
  1. # coding=utf-8
  2. import os
  3. import os.path
  4. import sys
  5. import urllib
  6. import urllib2
  7. import json
  8. import re
  9. import threading
  10. import subprocess
  11. import tempfile
  12. PACKAGES_URL = 'https://api.github.com/repos/emmetio/pyv8-binaries/downloads'
  13. class LoaderDelegate():
  14. """
  15. Abstract class used to display PyV8 binary download progress,
  16. and provide some settings for downloader
  17. """
  18. def __init__(self, settings={}):
  19. self.settings = settings
  20. def on_start(self, *args, **kwargs):
  21. "Invoked when download process is initiated"
  22. pass
  23. def on_progress(self, *args, **kwargs):
  24. "Invoked on download progress"
  25. pass
  26. def on_complete(self, *args, **kwargs):
  27. "Invoked when download process was finished successfully"
  28. pass
  29. def on_error(self, *args, **kwargs):
  30. "Invoked when error occured during download process"
  31. pass
  32. def setting(self, name, default=None):
  33. "Returns specified setting name"
  34. return self.settings[name] if name in self.settings else default
  35. class ThreadProgress():
  36. def __init__(self, thread, delegate):
  37. self.thread = thread
  38. self.delegate = delegate
  39. self._callbacks = {}
  40. threading.Timer(0, self.run).start()
  41. def run(self):
  42. if not self.thread.is_alive():
  43. if self.thread.exit_code != 0:
  44. return self.trigger('error', exit_code=self.thread.exit_code, thread=self.thread)
  45. return self.trigger('complete', result=self.thread.result, thread=self.thread)
  46. self.trigger('progress', thread=self.thread)
  47. threading.Timer(0.1, self.run).start()
  48. def on(self, event_name, callback):
  49. if event_name not in self._callbacks:
  50. self._callbacks[event_name] = []
  51. if callable(callback):
  52. self._callbacks[event_name].append(callback)
  53. return self
  54. def trigger(self, event_name, *args, **kwargs):
  55. if event_name in self._callbacks:
  56. for c in self._callbacks[event_name]:
  57. c(*args, **kwargs)
  58. if self.delegate and hasattr(self.delegate, 'on_%s' % event_name):
  59. getattr(self.delegate, 'on_%s' % event_name)(*args, **kwargs)
  60. return self
  61. class BinaryNotFoundError(Exception):
  62. pass
  63. class NonCleanExitError(Exception):
  64. def __init__(self, returncode):
  65. self.returncode = returncode
  66. def __str__(self):
  67. return repr(self.returncode)
  68. class CliDownloader():
  69. def __init__(self, settings):
  70. self.settings = settings
  71. def find_binary(self, name):
  72. for dir in os.environ['PATH'].split(os.pathsep):
  73. path = os.path.join(dir, name)
  74. if os.path.exists(path):
  75. return path
  76. raise BinaryNotFoundError('The binary %s could not be located' % name)
  77. def execute(self, args):
  78. proc = subprocess.Popen(args, stdin=subprocess.PIPE,
  79. stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  80. output = proc.stdout.read()
  81. returncode = proc.wait()
  82. if returncode != 0:
  83. error = NonCleanExitError(returncode)
  84. error.output = output
  85. raise error
  86. return output
  87. class WgetDownloader(CliDownloader):
  88. def __init__(self, settings):
  89. self.settings = settings
  90. self.wget = self.find_binary('wget')
  91. def clean_tmp_file(self):
  92. os.remove(self.tmp_file)
  93. def download(self, url, error_message, timeout, tries):
  94. if not self.wget:
  95. return False
  96. self.tmp_file = tempfile.NamedTemporaryFile().name
  97. command = [self.wget, '--connect-timeout=' + str(int(timeout)), '-o',
  98. self.tmp_file, '-O', '-', '-U', 'Emmet PyV8 Loader']
  99. command.append(url)
  100. if self.settings.get('http_proxy'):
  101. os.putenv('http_proxy', self.settings.get('http_proxy'))
  102. if not self.settings.get('https_proxy'):
  103. os.putenv('https_proxy', self.settings.get('http_proxy'))
  104. if self.settings.get('https_proxy'):
  105. os.putenv('https_proxy', self.settings.get('https_proxy'))
  106. while tries > 0:
  107. tries -= 1
  108. try:
  109. result = self.execute(command)
  110. self.clean_tmp_file()
  111. return result
  112. except (NonCleanExitError) as (e):
  113. error_line = ''
  114. with open(self.tmp_file) as f:
  115. for line in list(f):
  116. if re.search('ERROR[: ]|failed: ', line):
  117. error_line = line
  118. break
  119. if e.returncode == 8:
  120. regex = re.compile('^.*ERROR (\d+):.*', re.S)
  121. if re.sub(regex, '\\1', error_line) == '503':
  122. # GitHub and BitBucket seem to rate limit via 503
  123. print ('%s: Downloading %s was rate limited' +
  124. ', trying again') % (__name__, url)
  125. continue
  126. error_string = 'HTTP error ' + re.sub('^.*? ERROR ', '',
  127. error_line)
  128. elif e.returncode == 4:
  129. error_string = re.sub('^.*?failed: ', '', error_line)
  130. # GitHub and BitBucket seem to time out a lot
  131. if error_string.find('timed out') != -1:
  132. print ('%s: Downloading %s timed out, ' +
  133. 'trying again') % (__name__, url)
  134. continue
  135. else:
  136. error_string = re.sub('^.*?(ERROR[: ]|failed: )', '\\1',
  137. error_line)
  138. error_string = re.sub('\\.?\s*\n\s*$', '', error_string)
  139. print '%s: %s %s downloading %s.' % (__name__, error_message,
  140. error_string, url)
  141. self.clean_tmp_file()
  142. break
  143. return False
  144. class CurlDownloader(CliDownloader):
  145. def __init__(self, settings):
  146. self.settings = settings
  147. self.curl = self.find_binary('curl')
  148. def download(self, url, error_message, timeout, tries):
  149. if not self.curl:
  150. return False
  151. command = [self.curl, '-f', '--user-agent', 'Emmet PyV8 Loader',
  152. '--connect-timeout', str(int(timeout)), '-sS']
  153. command.append(url)
  154. if self.settings.get('http_proxy'):
  155. os.putenv('http_proxy', self.settings.get('http_proxy'))
  156. if not self.settings.get('https_proxy'):
  157. os.putenv('HTTPS_PROXY', self.settings.get('http_proxy'))
  158. if self.settings.get('https_proxy'):
  159. os.putenv('HTTPS_PROXY', self.settings.get('https_proxy'))
  160. while tries > 0:
  161. tries -= 1
  162. try:
  163. return self.execute(command)
  164. except (NonCleanExitError) as (e):
  165. if e.returncode == 22:
  166. code = re.sub('^.*?(\d+)\s*$', '\\1', e.output)
  167. if code == '503':
  168. # GitHub and BitBucket seem to rate limit via 503
  169. print ('%s: Downloading %s was rate limited' +
  170. ', trying again') % (__name__, url)
  171. continue
  172. error_string = 'HTTP error ' + code
  173. elif e.returncode == 6:
  174. error_string = 'URL error host not found'
  175. elif e.returncode == 28:
  176. # GitHub and BitBucket seem to time out a lot
  177. print ('%s: Downloading %s timed out, trying ' +
  178. 'again') % (__name__, url)
  179. continue
  180. else:
  181. error_string = e.output.rstrip()
  182. print '%s: %s %s downloading %s.' % (__name__, error_message,
  183. error_string, url)
  184. break
  185. return False
  186. class UrlLib2Downloader():
  187. def __init__(self, settings):
  188. self.settings = settings
  189. def download(self, url, error_message, timeout, tries):
  190. http_proxy = self.settings.get('http_proxy')
  191. https_proxy = self.settings.get('https_proxy')
  192. if http_proxy or https_proxy:
  193. proxies = {}
  194. if http_proxy:
  195. proxies['http'] = http_proxy
  196. if not https_proxy:
  197. proxies['https'] = http_proxy
  198. if https_proxy:
  199. proxies['https'] = https_proxy
  200. proxy_handler = urllib2.ProxyHandler(proxies)
  201. else:
  202. proxy_handler = urllib2.ProxyHandler()
  203. handlers = [proxy_handler]
  204. # secure_url_match = re.match('^https://([^/]+)', url)
  205. # if secure_url_match != None:
  206. # secure_domain = secure_url_match.group(1)
  207. # bundle_path = self.check_certs(secure_domain, timeout)
  208. # if not bundle_path:
  209. # return False
  210. # handlers.append(VerifiedHTTPSHandler(ca_certs=bundle_path))
  211. urllib2.install_opener(urllib2.build_opener(*handlers))
  212. while tries > 0:
  213. tries -= 1
  214. try:
  215. request = urllib2.Request(url, headers={"User-Agent":
  216. "Emmet PyV8 Loader"})
  217. http_file = urllib2.urlopen(request, timeout=timeout)
  218. return http_file.read()
  219. except (urllib2.HTTPError) as (e):
  220. # Bitbucket and Github ratelimit using 503 a decent amount
  221. if str(e.code) == '503':
  222. print ('%s: Downloading %s was rate limited, ' +
  223. 'trying again') % (__name__, url)
  224. continue
  225. print '%s: %s HTTP error %s downloading %s.' % (__name__,
  226. error_message, str(e.code), url)
  227. except (urllib2.URLError) as (e):
  228. # Bitbucket and Github timeout a decent amount
  229. if str(e.reason) == 'The read operation timed out' or \
  230. str(e.reason) == 'timed out':
  231. print ('%s: Downloading %s timed out, trying ' +
  232. 'again') % (__name__, url)
  233. continue
  234. print '%s: %s URL error %s downloading %s.' % (__name__,
  235. error_message, str(e.reason), url)
  236. break
  237. return False
  238. class PyV8Loader(threading.Thread):
  239. def __init__(self, arch, download_path, config):
  240. self.arch = arch
  241. self.config = config
  242. self.download_path = download_path
  243. self.exit_code = 0
  244. self.result = None
  245. self.settings = {}
  246. threading.Thread.__init__(self)
  247. self.log('Creating thread')
  248. def log(self, message):
  249. print('PyV8 Loader: %s' % message)
  250. def download_url(self, url, error_message):
  251. # TODO add settings
  252. has_ssl = 'ssl' in sys.modules and hasattr(urllib2, 'HTTPSHandler')
  253. is_ssl = re.search('^https://', url) != None
  254. if (is_ssl and has_ssl) or not is_ssl:
  255. downloader = UrlLib2Downloader(self.settings)
  256. else:
  257. for downloader_class in [CurlDownloader, WgetDownloader]:
  258. try:
  259. downloader = downloader_class(self.settings)
  260. break
  261. except (BinaryNotFoundError):
  262. pass
  263. if not downloader:
  264. self.log('Unable to download PyV8 binary due to invalid downloader')
  265. return False
  266. # timeout = self.settings.get('timeout', 3)
  267. timeout = 3
  268. return downloader.download(url.replace(' ', '%20'), error_message, timeout, 3)
  269. def run(self):
  270. # get list of available packages first
  271. packages = self.download_url(PACKAGES_URL, 'Unable to download packages list.')
  272. if not packages:
  273. self.exit_code = 1
  274. return
  275. files = json.loads(packages)
  276. # find package for current architecture
  277. cur_item = None
  278. for item in files:
  279. if self.arch in item['description'].lower():
  280. cur_item = item
  281. break
  282. if not cur_item:
  283. self.log('Unable to find binary for %s architecture' % self.arch)
  284. self.exit_code = 2
  285. return
  286. if cur_item['id'] == self.config['last_id']:
  287. self.log('You have the most recent PyV8 binary')
  288. return
  289. # Reduce HTTP roundtrips: try to download binary from
  290. # http://cloud.github.com directly
  291. url = re.sub(r'^https?:\/\/github\.com', 'http://cloud.github.com', item['html_url'])
  292. self.log('Loading PyV8 binary from %s' % url)
  293. package = self.download_url(url, 'Unable to download package from %s' % url)
  294. if not package:
  295. url = item['html_url']
  296. self.log('Loading PyV8 binary from %s' % url)
  297. package = self.download_url(url, 'Unable to download package from %s' % url)
  298. if not package:
  299. self.exit_code = 3
  300. return
  301. # we should only save downloaded package and delegate module
  302. # loading/unloading to main thread since improper PyV8 unload
  303. # may cause editor crash
  304. try:
  305. os.makedirs(self.download_path)
  306. except Exception, e:
  307. pass
  308. fp = open(os.path.join(self.download_path, 'pack.zip'), 'wb')
  309. fp.write(package)
  310. fp.close()
  311. self.result = cur_item['id']
  312. # Done!