PageRenderTime 58ms CodeModel.GetById 29ms RepoModel.GetById 1ms app.codeStats 0ms

/local/src.zip/old.py

https://bitbucket.org/yiwang/proxy-gfw
Python | 638 lines | 631 code | 4 blank | 3 comment | 7 complexity | f1c0a979edf97b8ca07bafe12498e95f MD5 | raw file
  1. # -*- coding: utf-8 -*-
  2. def old():
  3. import_from, global_proxy = config.import_from(config)
  4. # ================================ util.crypto =================================
  5. import hashlib, itertools
  6. class XOR:
  7. '''XOR with pure Python in case no PyCrypto'''
  8. def __init__(self, key):
  9. self.key = key
  10. def encrypt(self, data):
  11. xorsize = 1024
  12. key = itertools.cycle(map(ord, self.key))
  13. dr = xrange(0, len(data), xorsize)
  14. ss = [None] * len(dr)
  15. for i,j in enumerate(dr):
  16. dd = [ord(d)^k for d,k in itertools.izip(data[j:j+xorsize], key)]
  17. ss[i] = ''.join(map(chr, dd))
  18. return ''.join(ss)
  19. decrypt = encrypt
  20. class NUL:
  21. def encrypt(self, data):
  22. return data
  23. decrypt = encrypt
  24. class Crypto:
  25. _BlockSize = {'AES':16, 'ARC2':8, 'ARC4':1, 'Blowfish':8, 'CAST':8,
  26. 'DES':8, 'DES3':8, 'IDEA':8, 'RC5':8, 'XOR':1}
  27. _Modes = ['ECB', 'CBC', 'CFB', 'OFB', 'PGP'] #CTR needs 4 args
  28. _KeySize = {'AES':[16,24,32], 'CAST':xrange(5,17),
  29. 'DES':[8], 'DES3':[16,24], 'IDEA':[16]}
  30. def __init__(self, mode='AES-CBC-32'):
  31. mode = mode.split('-')
  32. mode += [''] * (3 - len(mode))
  33. #check cipher
  34. self.cipher = mode[0] if mode[0] else 'AES'
  35. if self.cipher not in self._BlockSize:
  36. raise ValueError('Invalid cipher: '+self.cipher)
  37. #check ciphermode
  38. if self._BlockSize[self.cipher] == 1:
  39. self.ciphermode = ''
  40. else:
  41. self.ciphermode = mode[1] if mode[1] in self._Modes else 'CBC'
  42. #check keysize
  43. try:
  44. self.keysize = int(mode[2])
  45. except ValueError:
  46. self.keysize = 32
  47. if self.keysize != 0:
  48. if self.cipher in self._KeySize:
  49. keysize = self._KeySize[self.cipher]
  50. if self.keysize not in keysize:
  51. self.keysize = keysize[-1]
  52. #avoid Memmory Error
  53. if self.cipher=='RC5' and self.keysize in (1, 57): self.keysize=32
  54. #try to import Crypto.Cipher.xxxx
  55. try:
  56. cipherlib = __import__('Crypto.Cipher.'+self.cipher, fromlist='x')
  57. self._newobj = cipherlib.new
  58. if self._BlockSize[self.cipher] != 1:
  59. self._ciphermode = getattr(cipherlib, 'MODE_'+self.ciphermode)
  60. except ImportError:
  61. if self.cipher == 'XOR': self._newobj = XOR
  62. else: raise
  63. def paddata(self, data):
  64. blocksize = self._BlockSize[self.cipher]
  65. if blocksize != 1:
  66. padlen = (blocksize - len(data) - 1) % blocksize
  67. data = '%s%s%s' % (chr(padlen), ' '*padlen, data)
  68. return data
  69. def unpaddata(self, data):
  70. if self._BlockSize[self.cipher] != 1:
  71. padlen = ord(data[0])
  72. data = data[padlen+1:]
  73. return data
  74. def getcrypto(self, key):
  75. if self.keysize==0 and key=='':
  76. return NUL()
  77. khash = hashlib.sha512(key).digest()
  78. if self.keysize != 0:
  79. key = khash[:self.keysize]
  80. blocksize = self._BlockSize[self.cipher]
  81. if blocksize == 1:
  82. return self._newobj(key)
  83. return self._newobj(key, self._ciphermode, khash[-blocksize:])
  84. def encrypt(self, data, key):
  85. crypto = self.getcrypto(key)
  86. data = self.paddata(data)
  87. return crypto.encrypt(data)
  88. def decrypt(self, data, key):
  89. crypto = self.getcrypto(key)
  90. data = crypto.decrypt(data)
  91. return self.unpaddata(data)
  92. def getmode(self):
  93. return '%s-%s-%d' % (self.cipher, self.ciphermode, self.keysize)
  94. def __str__(self):
  95. return '%s("%s")' % (self.__class__, self.getmode())
  96. def getsize(self, size):
  97. blocksize = self._BlockSize[self.cipher]
  98. return (size + blocksize - 1) // blocksize * blocksize
  99. class Crypto2(Crypto):
  100. def paddata(self, data):
  101. blocksize = self._BlockSize[self.cipher]
  102. if blocksize != 1:
  103. padlen = (blocksize - len(data) - 1) % blocksize
  104. data = '%s%s%s' % (data, ' '*padlen, chr(padlen))
  105. return data
  106. def unpaddata(self, data):
  107. if self._BlockSize[self.cipher] != 1:
  108. padlen = ord(data[-1])
  109. data = data[:-(padlen+1)]
  110. return data
  111. # =============================== plugins._base ================================
  112. HeaderDict, Proxy, URLInfo, del_bad_hosts, start_new_server, unparse_netloc = import_from(utils)
  113. import time, re, random, threading, socket, os, traceback
  114. class Handler(object):
  115. _dirty_headers = ('Connection', 'Proxy-Connection', 'Proxy-Authorization',
  116. 'Content-Length', 'Host', 'Vary', 'Via', 'X-Forwarded-For')
  117. _range_re = re.compile(r'(\d+)?-(\d+)?')
  118. _crange_re = re.compile(r'bytes\s+(\d+)-(\d+)/(\d+)')
  119. crypto = Crypto('XOR--32'); key = ''
  120. proxy = global_proxy
  121. headers = HeaderDict('Content-Type: application/octet-stream')
  122. range0 = 100000; range = 500000; max_threads = 10
  123. def __init__(self, config):
  124. dic = {'crypto': Crypto, 'key': lambda v:v, 'headers': HeaderDict,
  125. 'proxy': lambda v:global_proxy if v=='default' else Proxy(v),
  126. 'range0': lambda v:v if v>=10000 else self.__class__.range0,
  127. 'range': lambda v:v if v>=100000 else self.__class__.range,
  128. 'max_threads': lambda v:v if v>0 else self.__class__.max_threads,}
  129. self.url = URLInfo(config['url'])
  130. for k,v in dic.iteritems():
  131. if k in config:
  132. setattr(self.__class__, k, v(config[k]))
  133. setattr(self, k, getattr(self.__class__, k))
  134. def __str__(self):
  135. return ' %s %s %d %d %d' % (self.url.url, self.crypto.getmode(),
  136. self.range0, self.range, self.max_threads)
  137. def dump_data(self, data):
  138. raise NotImplementedError
  139. def load_data(self, data):
  140. raise NotImplementedError
  141. def process_request(self, req, force_range):
  142. data, headers = req.read_body(), req.headers
  143. for k in self._dirty_headers:
  144. del headers[k]
  145. if req.command == 'GET':
  146. rawrange, range = self._process_range(req.headers)
  147. if force_range:
  148. headers['Range'] = range
  149. else:
  150. rawrange, range = '', ''
  151. request = {'url':req.url, 'method':req.command,
  152. 'headers':headers, 'payload':data, 'range':range}
  153. return request, rawrange
  154. def _process_range(self, headers):
  155. range = headers.get('Range', '')
  156. m = self._range_re.search(range)
  157. if m:
  158. m = m.groups()
  159. if m[0] is None:
  160. if m[1] is None: m = None
  161. else:
  162. m = 1, int(m[1])
  163. if m[1] > self.range0: range = 'bytes=-1024'
  164. else:
  165. if m[1] is None:
  166. m = 0, int(m[0])
  167. range = 'bytes=%d-%d' % (m[1], m[1]+self.range0-1)
  168. else:
  169. m = 2, int(m[0]), int(m[1])
  170. if m[2]-m[1]+1 > self.range0:
  171. range = 'bytes=%d-%d' % (m[1], m[1]+self.range0-1)
  172. if m is None:
  173. range = 'bytes=0-%d' % (self.range0 - 1)
  174. return m, range
  175. def _fetch(self, data):
  176. data = self.crypto.encrypt(data, self.key)
  177. url = self.url
  178. opener = self.proxy.get_opener(url)
  179. try:
  180. resp = opener.open(url, data, 'POST', self.headers, 0)
  181. except Exception, e:
  182. return -1, e
  183. if resp.status != 200:
  184. opener.close()
  185. return -1, '%s: %s' % (resp.status, resp.reason)
  186. return 0, resp
  187. def fetch(self, data):
  188. raise NotImplementedError
  189. def read_data(self, type, data):
  190. if type == 1: return data
  191. resp, crypto = data
  192. data = self.crypto.unpaddata(crypto.decrypt(resp.read()))
  193. resp.close()
  194. return data
  195. def write_data(self, req, type, data):
  196. sendall = req.socket.sendall
  197. if type == 1:
  198. sendall(data)
  199. else:
  200. resp, crypto = data
  201. size = self.crypto.getsize(16384)
  202. data = crypto.decrypt(resp.read(size))
  203. sendall(self.crypto.unpaddata(data))
  204. data = resp.read(size)
  205. while data:
  206. sendall(crypto.decrypt(data))
  207. data = resp.read(size)
  208. resp.close()
  209. def _need_range_fetch(self, req, res, range):
  210. headers = res[2]
  211. m = self._crange_re.search(headers.get('Content-Range', ''))
  212. if not m: return None
  213. m = map(int, m.groups())#bytes %d-%d/%d
  214. if range is None:
  215. start=0; end=m[2]-1
  216. code = 200
  217. del headers['Content-Range']
  218. else:
  219. if range[0] == 0: #bytes=%d-
  220. start=range[1]; end=m[2]-1
  221. elif range[0] == 1: #bytes=-%d
  222. start=m[2]-range[1]; end=m[2]-1
  223. else: #bytes=%d-%d
  224. start=range[1]; end=range[2]
  225. code = 206
  226. headers['Content-Range'] = 'bytes %d-%d/%d' % (start, end, m[2])
  227. headers['Content-Length'] = str(end-start+1)
  228. req.start_response(code, headers)
  229. if start == m[0]: #Valid
  230. self.write_data(req, res[0], res[3])
  231. start = m[1] + 1
  232. return start, end
  233. def range_fetch(self, req, handler, request, start, end):
  234. t = time.time()
  235. if self._range_fetch(req, handler, request, start, end):
  236. t = time.time() - t
  237. t = (end - start + 1) / 1000.0 / t
  238. print '>>>>>>>>>> Range Fetch ended (all @ %sKB/s)' % t
  239. else:
  240. req.close_connection = 1
  241. print '>>>>>>>>>> Range Fetch failed'
  242. def _range_fetch(self, req, handler, request, start, end):
  243. request['range'] = '' # disable server auto-range-fetch
  244. i, s, thread_size, tasks = 0, start, 10, []
  245. while s <= end:
  246. e = s + (i < thread_size and self.range0 or self.range) - 1
  247. if e > end: e = end
  248. tasks.append((i, s, e))
  249. i += 1; s = e + 1
  250. task_size = len(tasks)
  251. thread_size = min(task_size, len(handler)*2, self.max_threads)
  252. print ('>>>>>>>>>> Range Fetch started: threads=%d blocks=%d '
  253. 'bytes=%d-%d' % (thread_size, task_size, start, end))
  254. if thread_size == 1:
  255. return self._single_fetch(req, handler, request, tasks)
  256. handler = list(handler); random.shuffle(handler)
  257. if thread_size > len(handler): handler *= 2
  258. results = [None] * task_size
  259. mutex = threading.Lock()
  260. threads = {}
  261. for i in xrange(thread_size):
  262. t = threading.Thread(target=handler[i]._range_thread,
  263. args=(request, tasks, results, threads, mutex))
  264. threads[t] = set()
  265. t.setDaemon(True)
  266. for t in threads: t.start()
  267. i = 0; t = False
  268. while i < task_size:
  269. if results[i] is not None:
  270. try:
  271. self.write_data(req, 1, results[i])
  272. results[i] = None
  273. i += 1
  274. continue
  275. except:
  276. mutex.acquire()
  277. del tasks[:]
  278. mutex.release()
  279. break
  280. if not threads: #All threads failed
  281. if t: break
  282. t = True; continue
  283. time.sleep(1)
  284. else:
  285. return True
  286. return False
  287. def _single_fetch(self, req, handler, request, tasks):
  288. try:
  289. for task in tasks:
  290. request['headers']['Range'] = 'bytes=%d-%d' % task[1:]
  291. data = self.dump_data(request)
  292. for i in xrange(3):
  293. self = random.choice(handler)
  294. res = self.fetch(data)
  295. if res[0] == -1:
  296. time.sleep(2)
  297. elif res[1] == 206:
  298. #print res[2]
  299. print '>>>>>>>>>> block=%d bytes=%d-%d' % task
  300. self.write_data(req, res[0], res[3])
  301. break
  302. else:
  303. raise StopIteration('Failed')
  304. except:
  305. return False
  306. return True
  307. def _range_thread(self, request, tasks, results, threads, mutex):
  308. ct = threading.current_thread()
  309. while True:
  310. mutex.acquire()
  311. try:
  312. if threads[ct].intersection(*threads.itervalues()):
  313. raise StopIteration('All threads failed')
  314. for i,task in enumerate(tasks):
  315. if task[0] not in threads[ct]:
  316. task = tasks.pop(i)
  317. break
  318. else:
  319. raise StopIteration('No task for me')
  320. request['headers']['Range'] = 'bytes=%d-%d' % task[1:]
  321. data = self.dump_data(request)
  322. except StopIteration, e:
  323. #print '>>>>>>>>>> %s: %s' % (ct.name, e)
  324. del threads[ct]
  325. break
  326. finally:
  327. mutex.release()
  328. success = False
  329. for i in xrange(2):
  330. res = self.fetch(data)
  331. if res[0] == -1:
  332. time.sleep(2)
  333. elif res[1] == 206:
  334. try: data = self.read_data(res[0], res[3])
  335. except: continue
  336. if len(data) == task[2]-task[1]+1:
  337. success = True
  338. break
  339. mutex.acquire()
  340. if success:
  341. print '>>>>>>>>>> block=%d bytes=%d-%d'%task, len(data)
  342. results[task[0]] = data
  343. else:
  344. threads[ct].add(task[0])
  345. tasks.append(task)
  346. tasks.sort(key=lambda x: x[0])
  347. mutex.release()
  348. def handle(self, handler, req, force_range):
  349. req.handler_name = handler[0].handler_name
  350. if len(handler) == 1:
  351. handlers = handler[0], handler[0]
  352. else:
  353. handlers = random.sample(handler, 2)
  354. request, range = self.process_request(req, force_range)
  355. data = self.dump_data(request)
  356. errors = []
  357. for self in handlers:
  358. res = self.fetch(data)
  359. if res[0] != -1: break
  360. e = res[1]; es = str(e); errors.append(es)
  361. if not es.startswith('Server: '): del_bad_hosts()
  362. else:
  363. return req.send_error(502, str(errors))
  364. if res[1]==206 and req.command=='GET':
  365. data = self._need_range_fetch(req, res, range)
  366. if data:
  367. start, end = data
  368. if start > end: return #end
  369. return self.range_fetch(req, handler, request, start, end)
  370. req.start_response(res[1], res[2])
  371. self.write_data(req, res[0], res[3])
  372. def _base_init(cls, config, listen=None):
  373. name = cls.handler_name
  374. print 'Initializing %s for old version.' % name
  375. server = [None] * len(config)
  376. for i,v in enumerate(config):
  377. if isinstance(v, basestring):
  378. v = {'url': v}
  379. try:
  380. server[i] = cls(v)
  381. print server[i]
  382. except:
  383. traceback.print_exc()
  384. def handler(req, force_range=False):
  385. return server[0].handle(server, req, force_range)
  386. if listen:
  387. def find_handler(req):
  388. if req.proxy_type.endswith('http'):
  389. return handler
  390. listen = data['%s_server'%name] = start_new_server(listen, find_handler)
  391. print ' %s listen on: %s' % (name, unparse_netloc(listen.server_address[:2]))
  392. return handler
  393. # ============================== plugins.gaeproxy ==============================
  394. import zlib, struct, cPickle as pickle
  395. class GAEHandler(Handler):
  396. handler_name = 'OGAE'
  397. def dump_data(self, data):
  398. return zlib.compress(pickle.dumps(data, 1))
  399. def load_data(self, data):
  400. return pickle.loads(data)
  401. def process_request(self, req, force_range):
  402. data, headers = req.read_body(), req.headers
  403. for k in self._dirty_headers:
  404. del headers[k]
  405. if req.command == 'GET':
  406. rawrange, range = self._process_range(req.headers)
  407. if force_range:
  408. headers['Range'] = range
  409. else:
  410. rawrange, range = '', ''
  411. request = {'url':req.url, 'method':req.command, 'payload':data,
  412. 'headers':headers.__getstate__(), 'range':range}
  413. return request, rawrange
  414. def fetch(self, data):
  415. data, resp = self._fetch(data)
  416. if data == -1: return data, resp
  417. crypto = self.crypto.getcrypto(self.key)
  418. headers = HeaderDict()
  419. try:
  420. raw_data = resp.read(7)
  421. zip, code, hlen = struct.unpack('>BHI', raw_data)
  422. if zip == 1:
  423. data = self.crypto.unpaddata(crypto.decrypt(resp.read()))
  424. data = zlib.decompress(data)
  425. content = data[hlen:]
  426. if code == 555:
  427. raise ValueError('Server: '+content)
  428. headers.__setstate__(self.load_data(data[:hlen]))
  429. resp.close()
  430. return 1, code, headers, content
  431. elif zip == 0:
  432. h = crypto.decrypt(resp.read(hlen))
  433. headers.__setstate__(self.load_data(self.crypto.unpaddata(h)))
  434. if code == 555:
  435. content = crypto.decrypt(resp.read())
  436. raise ValueError('Server: '+self.crypto.unpaddata(content))
  437. return 0, code, headers, (resp, crypto)
  438. else:
  439. raw_data += resp.read()
  440. raise ValueError('Data format not match(%s:%s)'%(self.url.url, raw_data))
  441. except Exception, e:
  442. resp.close()
  443. return -1, e
  444. def gaeproxy(*a, **kw):
  445. return _base_init(GAEHandler, *a, **kw)
  446. # =============================== plugins.forold ===============================
  447. class OldHandler(Handler):
  448. handler_name = 'OOLD'
  449. crypto = Crypto2('XOR--32')
  450. _unquote_map = {'0':'\x10', '1':'=', '2':'&'}
  451. def _quote(self, s):
  452. return str(s).replace('\x10', '\x100').replace('=','\x101').replace('&','\x102')
  453. def dump_data(self, dic):
  454. return zlib.compress('&'.join('%s=%s' % (self._quote(k),
  455. self._quote(v)) for k,v in dic.iteritems()))
  456. def _unquote(self, s):
  457. res = s.split('\x10')
  458. for i in xrange(1, len(res)):
  459. item = res[i]
  460. try:
  461. res[i] = self._unquote_map[item[0]] + item[1:]
  462. except KeyError:
  463. res[i] = '\x10' + item
  464. return ''.join(res)
  465. def load_data(self, qs):
  466. pairs = qs.split('&')
  467. dic = {}
  468. for name_value in pairs:
  469. if not name_value:
  470. continue
  471. nv = name_value.split('=', 1)
  472. if len(nv) != 2:
  473. continue
  474. if len(nv[1]):
  475. dic[self._unquote(nv[0])] = self._unquote(nv[1])
  476. return dic
  477. def __init__(self, config):
  478. if 'crypto' in config:
  479. self.__class__.crypto = Crypto2(config.pop('crypto'))
  480. Handler.__init__(self, config)
  481. def fetch(self, data):
  482. data, resp = self._fetch(data)
  483. if data == -1: return data, resp
  484. try:
  485. raw_data = resp.read(); resp.close()
  486. data = self.crypto.decrypt(raw_data, self.key)
  487. if data[0] == '0':
  488. data = data[1:]
  489. elif data[0] == '1':
  490. data = zlib.decompress(data[1:])
  491. else:
  492. return -1, 'Data format not match(%s:%s)' % (self.url.url,raw_data)
  493. code, hlen, clen = struct.unpack('>3I', data[:12])
  494. if len(data) != 12+hlen+clen:
  495. return -1, 'Data length not match'
  496. content = data[12+hlen:]
  497. if code == 555: #Urlfetch Failed
  498. return -1, 'Server: '+content
  499. headers = HeaderDict(self.load_data(data[12:12+hlen]))
  500. return 1, code, headers, content
  501. except Exception, e:
  502. return -1, e
  503. def forold(*a, **kw):
  504. return _base_init(OldHandler, *a, **kw)
  505. # =============================== plugins.goagent ==============================
  506. from binascii import a2b_hex, b2a_hex
  507. class GAHandler(OldHandler):
  508. handler_name = 'OGA'
  509. crypto = Crypto('XOR--0'); key = ''
  510. def dump_data(self, dic):
  511. return zlib.compress('&'.join('%s=%s' % (k,b2a_hex(str(v))) for k,v in dic.iteritems()))
  512. def load_data(self, qs):
  513. return dict((k,a2b_hex(v)) for k,v in (x.split('=') for x in qs.split('&')))
  514. def __init__(self, config):
  515. config.pop('crypto', None)
  516. self.password = config.pop('key', '')
  517. OldHandler.__init__(self, config)
  518. def process_request(self, req, force_range):
  519. request, rawrange = OldHandler.process_request(self, req, force_range)
  520. request['password'] = self.password
  521. return request, rawrange
  522. def goagent(*a, **kw):
  523. return _base_init(GAHandler, *a, **kw)
  524. # =============================== plugins.simple ===============================
  525. class SPHandler(GAEHandler):
  526. handler_name = 'OSP'
  527. def dump_data(self, dic):
  528. return zlib.compress('&'.join('%s=%s' % (k,b2a_hex(str(v))) for k,v in dic.iteritems()))
  529. def load_data(self, qs):
  530. return dict((k,a2b_hex(v)) for k,v in (x.split('=') for x in qs.split('&'))) if qs else {}
  531. process_request = Handler.process_request
  532. def simple(*a, **kw):
  533. return _base_init(SPHandler, *a, **kw)
  534. # =============================== plugins.simple2 ==============================
  535. import marshal
  536. class SP2Handler(Handler):
  537. handler_name = 'OSP2'
  538. def dump_data(self, data):
  539. return marshal.dumps(tuple((k,str(v)) for k,v in data.iteritems()))
  540. def load_data(self, data):
  541. return dict(marshal.loads(data))
  542. def fetch(self, data):
  543. data, resp = self._fetch(data)
  544. if data == -1: return data, resp
  545. crypto = self.crypto.getcrypto(self.key)
  546. try:
  547. raw_data = resp.read(7)
  548. mix, code, hlen = struct.unpack('>BHI', raw_data)
  549. if mix == 0:
  550. headers = self.crypto.unpaddata(crypto.decrypt(resp.read(hlen)))
  551. if code == 555:
  552. content = self.crypto.unpaddata(crypto.decrypt(resp.read()))
  553. raise ValueError('Server: '+content)
  554. headers = HeaderDict(headers)
  555. return 0, code, headers, (resp, crypto)
  556. elif mix == 1:
  557. data = self.crypto.unpaddata(crypto.decrypt(resp.read()))
  558. content = data[hlen:]
  559. if code == 555:
  560. raise ValueError('Server: '+content)
  561. headers = HeaderDict(data[:hlen])
  562. resp.close()
  563. return 1, code, headers, content
  564. else:
  565. raw_data += resp.read()
  566. raise ValueError('Data format not match(%s:%s)'%(self.url.url, raw_data))
  567. except Exception, e:
  568. resp.close()
  569. return -1, e
  570. def simple2(*a, **kw):
  571. return _base_init(SP2Handler, *a, **kw)
  572. # ==============================================================================
  573. globals().update(gaeproxy=gaeproxy, forold=forold,
  574. goagent=goagent, simple=simple, simple2=simple2)