/local/src.zip/old.py
Python | 638 lines | 631 code | 4 blank | 3 comment | 7 complexity | f1c0a979edf97b8ca07bafe12498e95f MD5 | raw file
1# -*- coding: utf-8 -*-
2
3def old():
4 import_from, global_proxy = config.import_from(config)
5
6 # ================================ util.crypto =================================
7 import hashlib, itertools
8
9 class XOR:
10 '''XOR with pure Python in case no PyCrypto'''
11 def __init__(self, key):
12 self.key = key
13
14 def encrypt(self, data):
15 xorsize = 1024
16 key = itertools.cycle(map(ord, self.key))
17 dr = xrange(0, len(data), xorsize)
18 ss = [None] * len(dr)
19 for i,j in enumerate(dr):
20 dd = [ord(d)^k for d,k in itertools.izip(data[j:j+xorsize], key)]
21 ss[i] = ''.join(map(chr, dd))
22 return ''.join(ss)
23 decrypt = encrypt
24
25 class NUL:
26 def encrypt(self, data):
27 return data
28 decrypt = encrypt
29
30 class Crypto:
31 _BlockSize = {'AES':16, 'ARC2':8, 'ARC4':1, 'Blowfish':8, 'CAST':8,
32 'DES':8, 'DES3':8, 'IDEA':8, 'RC5':8, 'XOR':1}
33 _Modes = ['ECB', 'CBC', 'CFB', 'OFB', 'PGP'] #CTR needs 4 args
34 _KeySize = {'AES':[16,24,32], 'CAST':xrange(5,17),
35 'DES':[8], 'DES3':[16,24], 'IDEA':[16]}
36
37 def __init__(self, mode='AES-CBC-32'):
38 mode = mode.split('-')
39 mode += [''] * (3 - len(mode))
40 #check cipher
41 self.cipher = mode[0] if mode[0] else 'AES'
42 if self.cipher not in self._BlockSize:
43 raise ValueError('Invalid cipher: '+self.cipher)
44 #check ciphermode
45 if self._BlockSize[self.cipher] == 1:
46 self.ciphermode = ''
47 else:
48 self.ciphermode = mode[1] if mode[1] in self._Modes else 'CBC'
49 #check keysize
50 try:
51 self.keysize = int(mode[2])
52 except ValueError:
53 self.keysize = 32
54 if self.keysize != 0:
55 if self.cipher in self._KeySize:
56 keysize = self._KeySize[self.cipher]
57 if self.keysize not in keysize:
58 self.keysize = keysize[-1]
59 #avoid Memmory Error
60 if self.cipher=='RC5' and self.keysize in (1, 57): self.keysize=32
61 #try to import Crypto.Cipher.xxxx
62 try:
63 cipherlib = __import__('Crypto.Cipher.'+self.cipher, fromlist='x')
64 self._newobj = cipherlib.new
65 if self._BlockSize[self.cipher] != 1:
66 self._ciphermode = getattr(cipherlib, 'MODE_'+self.ciphermode)
67 except ImportError:
68 if self.cipher == 'XOR': self._newobj = XOR
69 else: raise
70
71 def paddata(self, data):
72 blocksize = self._BlockSize[self.cipher]
73 if blocksize != 1:
74 padlen = (blocksize - len(data) - 1) % blocksize
75 data = '%s%s%s' % (chr(padlen), ' '*padlen, data)
76 return data
77
78 def unpaddata(self, data):
79 if self._BlockSize[self.cipher] != 1:
80 padlen = ord(data[0])
81 data = data[padlen+1:]
82 return data
83
84 def getcrypto(self, key):
85 if self.keysize==0 and key=='':
86 return NUL()
87 khash = hashlib.sha512(key).digest()
88 if self.keysize != 0:
89 key = khash[:self.keysize]
90 blocksize = self._BlockSize[self.cipher]
91 if blocksize == 1:
92 return self._newobj(key)
93 return self._newobj(key, self._ciphermode, khash[-blocksize:])
94
95 def encrypt(self, data, key):
96 crypto = self.getcrypto(key)
97 data = self.paddata(data)
98 return crypto.encrypt(data)
99
100 def decrypt(self, data, key):
101 crypto = self.getcrypto(key)
102 data = crypto.decrypt(data)
103 return self.unpaddata(data)
104
105 def getmode(self):
106 return '%s-%s-%d' % (self.cipher, self.ciphermode, self.keysize)
107
108 def __str__(self):
109 return '%s("%s")' % (self.__class__, self.getmode())
110
111 def getsize(self, size):
112 blocksize = self._BlockSize[self.cipher]
113 return (size + blocksize - 1) // blocksize * blocksize
114
115 class Crypto2(Crypto):
116 def paddata(self, data):
117 blocksize = self._BlockSize[self.cipher]
118 if blocksize != 1:
119 padlen = (blocksize - len(data) - 1) % blocksize
120 data = '%s%s%s' % (data, ' '*padlen, chr(padlen))
121 return data
122
123 def unpaddata(self, data):
124 if self._BlockSize[self.cipher] != 1:
125 padlen = ord(data[-1])
126 data = data[:-(padlen+1)]
127 return data
128
129 # =============================== plugins._base ================================
130 HeaderDict, Proxy, URLInfo, del_bad_hosts, start_new_server, unparse_netloc = import_from(utils)
131 import time, re, random, threading, socket, os, traceback
132
133 class Handler(object):
134 _dirty_headers = ('Connection', 'Proxy-Connection', 'Proxy-Authorization',
135 'Content-Length', 'Host', 'Vary', 'Via', 'X-Forwarded-For')
136 _range_re = re.compile(r'(\d+)?-(\d+)?')
137 _crange_re = re.compile(r'bytes\s+(\d+)-(\d+)/(\d+)')
138 crypto = Crypto('XOR--32'); key = ''
139 proxy = global_proxy
140 headers = HeaderDict('Content-Type: application/octet-stream')
141 range0 = 100000; range = 500000; max_threads = 10
142
143 def __init__(self, config):
144 dic = {'crypto': Crypto, 'key': lambda v:v, 'headers': HeaderDict,
145 'proxy': lambda v:global_proxy if v=='default' else Proxy(v),
146 'range0': lambda v:v if v>=10000 else self.__class__.range0,
147 'range': lambda v:v if v>=100000 else self.__class__.range,
148 'max_threads': lambda v:v if v>0 else self.__class__.max_threads,}
149 self.url = URLInfo(config['url'])
150 for k,v in dic.iteritems():
151 if k in config:
152 setattr(self.__class__, k, v(config[k]))
153 setattr(self, k, getattr(self.__class__, k))
154
155 def __str__(self):
156 return ' %s %s %d %d %d' % (self.url.url, self.crypto.getmode(),
157 self.range0, self.range, self.max_threads)
158
159 def dump_data(self, data):
160 raise NotImplementedError
161
162 def load_data(self, data):
163 raise NotImplementedError
164
165 def process_request(self, req, force_range):
166 data, headers = req.read_body(), req.headers
167 for k in self._dirty_headers:
168 del headers[k]
169 if req.command == 'GET':
170 rawrange, range = self._process_range(req.headers)
171 if force_range:
172 headers['Range'] = range
173 else:
174 rawrange, range = '', ''
175 request = {'url':req.url, 'method':req.command,
176 'headers':headers, 'payload':data, 'range':range}
177 return request, rawrange
178
179 def _process_range(self, headers):
180 range = headers.get('Range', '')
181 m = self._range_re.search(range)
182 if m:
183 m = m.groups()
184 if m[0] is None:
185 if m[1] is None: m = None
186 else:
187 m = 1, int(m[1])
188 if m[1] > self.range0: range = 'bytes=-1024'
189 else:
190 if m[1] is None:
191 m = 0, int(m[0])
192 range = 'bytes=%d-%d' % (m[1], m[1]+self.range0-1)
193 else:
194 m = 2, int(m[0]), int(m[1])
195 if m[2]-m[1]+1 > self.range0:
196 range = 'bytes=%d-%d' % (m[1], m[1]+self.range0-1)
197 if m is None:
198 range = 'bytes=0-%d' % (self.range0 - 1)
199 return m, range
200
201 def _fetch(self, data):
202 data = self.crypto.encrypt(data, self.key)
203 url = self.url
204 opener = self.proxy.get_opener(url)
205 try:
206 resp = opener.open(url, data, 'POST', self.headers, 0)
207 except Exception, e:
208 return -1, e
209 if resp.status != 200:
210 opener.close()
211 return -1, '%s: %s' % (resp.status, resp.reason)
212 return 0, resp
213
214 def fetch(self, data):
215 raise NotImplementedError
216
217 def read_data(self, type, data):
218 if type == 1: return data
219 resp, crypto = data
220 data = self.crypto.unpaddata(crypto.decrypt(resp.read()))
221 resp.close()
222 return data
223
224 def write_data(self, req, type, data):
225 sendall = req.socket.sendall
226 if type == 1:
227 sendall(data)
228 else:
229 resp, crypto = data
230 size = self.crypto.getsize(16384)
231 data = crypto.decrypt(resp.read(size))
232 sendall(self.crypto.unpaddata(data))
233 data = resp.read(size)
234 while data:
235 sendall(crypto.decrypt(data))
236 data = resp.read(size)
237 resp.close()
238
239 def _need_range_fetch(self, req, res, range):
240 headers = res[2]
241 m = self._crange_re.search(headers.get('Content-Range', ''))
242 if not m: return None
243 m = map(int, m.groups())#bytes %d-%d/%d
244 if range is None:
245 start=0; end=m[2]-1
246 code = 200
247 del headers['Content-Range']
248 else:
249 if range[0] == 0: #bytes=%d-
250 start=range[1]; end=m[2]-1
251 elif range[0] == 1: #bytes=-%d
252 start=m[2]-range[1]; end=m[2]-1
253 else: #bytes=%d-%d
254 start=range[1]; end=range[2]
255 code = 206
256 headers['Content-Range'] = 'bytes %d-%d/%d' % (start, end, m[2])
257 headers['Content-Length'] = str(end-start+1)
258 req.start_response(code, headers)
259 if start == m[0]: #Valid
260 self.write_data(req, res[0], res[3])
261 start = m[1] + 1
262 return start, end
263
264 def range_fetch(self, req, handler, request, start, end):
265 t = time.time()
266 if self._range_fetch(req, handler, request, start, end):
267 t = time.time() - t
268 t = (end - start + 1) / 1000.0 / t
269 print '>>>>>>>>>> Range Fetch ended (all @ %sKB/s)' % t
270 else:
271 req.close_connection = 1
272 print '>>>>>>>>>> Range Fetch failed'
273
274 def _range_fetch(self, req, handler, request, start, end):
275 request['range'] = '' # disable server auto-range-fetch
276 i, s, thread_size, tasks = 0, start, 10, []
277 while s <= end:
278 e = s + (i < thread_size and self.range0 or self.range) - 1
279 if e > end: e = end
280 tasks.append((i, s, e))
281 i += 1; s = e + 1
282 task_size = len(tasks)
283 thread_size = min(task_size, len(handler)*2, self.max_threads)
284 print ('>>>>>>>>>> Range Fetch started: threads=%d blocks=%d '
285 'bytes=%d-%d' % (thread_size, task_size, start, end))
286 if thread_size == 1:
287 return self._single_fetch(req, handler, request, tasks)
288 handler = list(handler); random.shuffle(handler)
289 if thread_size > len(handler): handler *= 2
290 results = [None] * task_size
291 mutex = threading.Lock()
292 threads = {}
293 for i in xrange(thread_size):
294 t = threading.Thread(target=handler[i]._range_thread,
295 args=(request, tasks, results, threads, mutex))
296 threads[t] = set()
297 t.setDaemon(True)
298 for t in threads: t.start()
299 i = 0; t = False
300 while i < task_size:
301 if results[i] is not None:
302 try:
303 self.write_data(req, 1, results[i])
304 results[i] = None
305 i += 1
306 continue
307 except:
308 mutex.acquire()
309 del tasks[:]
310 mutex.release()
311 break
312 if not threads: #All threads failed
313 if t: break
314 t = True; continue
315 time.sleep(1)
316 else:
317 return True
318 return False
319
320 def _single_fetch(self, req, handler, request, tasks):
321 try:
322 for task in tasks:
323 request['headers']['Range'] = 'bytes=%d-%d' % task[1:]
324 data = self.dump_data(request)
325 for i in xrange(3):
326 self = random.choice(handler)
327 res = self.fetch(data)
328 if res[0] == -1:
329 time.sleep(2)
330 elif res[1] == 206:
331 #print res[2]
332 print '>>>>>>>>>> block=%d bytes=%d-%d' % task
333 self.write_data(req, res[0], res[3])
334 break
335 else:
336 raise StopIteration('Failed')
337 except:
338 return False
339 return True
340
341 def _range_thread(self, request, tasks, results, threads, mutex):
342 ct = threading.current_thread()
343 while True:
344 mutex.acquire()
345 try:
346 if threads[ct].intersection(*threads.itervalues()):
347 raise StopIteration('All threads failed')
348 for i,task in enumerate(tasks):
349 if task[0] not in threads[ct]:
350 task = tasks.pop(i)
351 break
352 else:
353 raise StopIteration('No task for me')
354 request['headers']['Range'] = 'bytes=%d-%d' % task[1:]
355 data = self.dump_data(request)
356 except StopIteration, e:
357 #print '>>>>>>>>>> %s: %s' % (ct.name, e)
358 del threads[ct]
359 break
360 finally:
361 mutex.release()
362 success = False
363 for i in xrange(2):
364 res = self.fetch(data)
365 if res[0] == -1:
366 time.sleep(2)
367 elif res[1] == 206:
368 try: data = self.read_data(res[0], res[3])
369 except: continue
370 if len(data) == task[2]-task[1]+1:
371 success = True
372 break
373 mutex.acquire()
374 if success:
375 print '>>>>>>>>>> block=%d bytes=%d-%d'%task, len(data)
376 results[task[0]] = data
377 else:
378 threads[ct].add(task[0])
379 tasks.append(task)
380 tasks.sort(key=lambda x: x[0])
381 mutex.release()
382
383 def handle(self, handler, req, force_range):
384 req.handler_name = handler[0].handler_name
385 if len(handler) == 1:
386 handlers = handler[0], handler[0]
387 else:
388 handlers = random.sample(handler, 2)
389 request, range = self.process_request(req, force_range)
390 data = self.dump_data(request)
391 errors = []
392 for self in handlers:
393 res = self.fetch(data)
394 if res[0] != -1: break
395 e = res[1]; es = str(e); errors.append(es)
396 if not es.startswith('Server: '): del_bad_hosts()
397 else:
398 return req.send_error(502, str(errors))
399 if res[1]==206 and req.command=='GET':
400 data = self._need_range_fetch(req, res, range)
401 if data:
402 start, end = data
403 if start > end: return #end
404 return self.range_fetch(req, handler, request, start, end)
405 req.start_response(res[1], res[2])
406 self.write_data(req, res[0], res[3])
407
408 def _base_init(cls, config, listen=None):
409 name = cls.handler_name
410 print 'Initializing %s for old version.' % name
411 server = [None] * len(config)
412 for i,v in enumerate(config):
413 if isinstance(v, basestring):
414 v = {'url': v}
415 try:
416 server[i] = cls(v)
417 print server[i]
418 except:
419 traceback.print_exc()
420 def handler(req, force_range=False):
421 return server[0].handle(server, req, force_range)
422 if listen:
423 def find_handler(req):
424 if req.proxy_type.endswith('http'):
425 return handler
426 listen = data['%s_server'%name] = start_new_server(listen, find_handler)
427 print ' %s listen on: %s' % (name, unparse_netloc(listen.server_address[:2]))
428 return handler
429
430 # ============================== plugins.gaeproxy ==============================
431 import zlib, struct, cPickle as pickle
432
433 class GAEHandler(Handler):
434 handler_name = 'OGAE'
435 def dump_data(self, data):
436 return zlib.compress(pickle.dumps(data, 1))
437
438 def load_data(self, data):
439 return pickle.loads(data)
440
441 def process_request(self, req, force_range):
442 data, headers = req.read_body(), req.headers
443 for k in self._dirty_headers:
444 del headers[k]
445 if req.command == 'GET':
446 rawrange, range = self._process_range(req.headers)
447 if force_range:
448 headers['Range'] = range
449 else:
450 rawrange, range = '', ''
451 request = {'url':req.url, 'method':req.command, 'payload':data,
452 'headers':headers.__getstate__(), 'range':range}
453 return request, rawrange
454
455 def fetch(self, data):
456 data, resp = self._fetch(data)
457 if data == -1: return data, resp
458 crypto = self.crypto.getcrypto(self.key)
459 headers = HeaderDict()
460 try:
461 raw_data = resp.read(7)
462 zip, code, hlen = struct.unpack('>BHI', raw_data)
463 if zip == 1:
464 data = self.crypto.unpaddata(crypto.decrypt(resp.read()))
465 data = zlib.decompress(data)
466 content = data[hlen:]
467 if code == 555:
468 raise ValueError('Server: '+content)
469 headers.__setstate__(self.load_data(data[:hlen]))
470 resp.close()
471 return 1, code, headers, content
472 elif zip == 0:
473 h = crypto.decrypt(resp.read(hlen))
474 headers.__setstate__(self.load_data(self.crypto.unpaddata(h)))
475 if code == 555:
476 content = crypto.decrypt(resp.read())
477 raise ValueError('Server: '+self.crypto.unpaddata(content))
478 return 0, code, headers, (resp, crypto)
479 else:
480 raw_data += resp.read()
481 raise ValueError('Data format not match(%s:%s)'%(self.url.url, raw_data))
482 except Exception, e:
483 resp.close()
484 return -1, e
485
486 def gaeproxy(*a, **kw):
487 return _base_init(GAEHandler, *a, **kw)
488
489 # =============================== plugins.forold ===============================
490 class OldHandler(Handler):
491 handler_name = 'OOLD'
492 crypto = Crypto2('XOR--32')
493
494 _unquote_map = {'0':'\x10', '1':'=', '2':'&'}
495 def _quote(self, s):
496 return str(s).replace('\x10', '\x100').replace('=','\x101').replace('&','\x102')
497 def dump_data(self, dic):
498 return zlib.compress('&'.join('%s=%s' % (self._quote(k),
499 self._quote(v)) for k,v in dic.iteritems()))
500 def _unquote(self, s):
501 res = s.split('\x10')
502 for i in xrange(1, len(res)):
503 item = res[i]
504 try:
505 res[i] = self._unquote_map[item[0]] + item[1:]
506 except KeyError:
507 res[i] = '\x10' + item
508 return ''.join(res)
509 def load_data(self, qs):
510 pairs = qs.split('&')
511 dic = {}
512 for name_value in pairs:
513 if not name_value:
514 continue
515 nv = name_value.split('=', 1)
516 if len(nv) != 2:
517 continue
518 if len(nv[1]):
519 dic[self._unquote(nv[0])] = self._unquote(nv[1])
520 return dic
521
522 def __init__(self, config):
523 if 'crypto' in config:
524 self.__class__.crypto = Crypto2(config.pop('crypto'))
525 Handler.__init__(self, config)
526
527 def fetch(self, data):
528 data, resp = self._fetch(data)
529 if data == -1: return data, resp
530 try:
531 raw_data = resp.read(); resp.close()
532 data = self.crypto.decrypt(raw_data, self.key)
533 if data[0] == '0':
534 data = data[1:]
535 elif data[0] == '1':
536 data = zlib.decompress(data[1:])
537 else:
538 return -1, 'Data format not match(%s:%s)' % (self.url.url,raw_data)
539 code, hlen, clen = struct.unpack('>3I', data[:12])
540 if len(data) != 12+hlen+clen:
541 return -1, 'Data length not match'
542 content = data[12+hlen:]
543 if code == 555: #Urlfetch Failed
544 return -1, 'Server: '+content
545 headers = HeaderDict(self.load_data(data[12:12+hlen]))
546 return 1, code, headers, content
547 except Exception, e:
548 return -1, e
549
550 def forold(*a, **kw):
551 return _base_init(OldHandler, *a, **kw)
552
553 # =============================== plugins.goagent ==============================
554 from binascii import a2b_hex, b2a_hex
555
556 class GAHandler(OldHandler):
557 handler_name = 'OGA'
558 crypto = Crypto('XOR--0'); key = ''
559
560 def dump_data(self, dic):
561 return zlib.compress('&'.join('%s=%s' % (k,b2a_hex(str(v))) for k,v in dic.iteritems()))
562
563 def load_data(self, qs):
564 return dict((k,a2b_hex(v)) for k,v in (x.split('=') for x in qs.split('&')))
565
566 def __init__(self, config):
567 config.pop('crypto', None)
568 self.password = config.pop('key', '')
569 OldHandler.__init__(self, config)
570
571 def process_request(self, req, force_range):
572 request, rawrange = OldHandler.process_request(self, req, force_range)
573 request['password'] = self.password
574 return request, rawrange
575
576 def goagent(*a, **kw):
577 return _base_init(GAHandler, *a, **kw)
578
579 # =============================== plugins.simple ===============================
580 class SPHandler(GAEHandler):
581 handler_name = 'OSP'
582 def dump_data(self, dic):
583 return zlib.compress('&'.join('%s=%s' % (k,b2a_hex(str(v))) for k,v in dic.iteritems()))
584
585 def load_data(self, qs):
586 return dict((k,a2b_hex(v)) for k,v in (x.split('=') for x in qs.split('&'))) if qs else {}
587
588 process_request = Handler.process_request
589
590 def simple(*a, **kw):
591 return _base_init(SPHandler, *a, **kw)
592
593 # =============================== plugins.simple2 ==============================
594 import marshal
595
596 class SP2Handler(Handler):
597 handler_name = 'OSP2'
598 def dump_data(self, data):
599 return marshal.dumps(tuple((k,str(v)) for k,v in data.iteritems()))
600
601 def load_data(self, data):
602 return dict(marshal.loads(data))
603
604 def fetch(self, data):
605 data, resp = self._fetch(data)
606 if data == -1: return data, resp
607 crypto = self.crypto.getcrypto(self.key)
608 try:
609 raw_data = resp.read(7)
610 mix, code, hlen = struct.unpack('>BHI', raw_data)
611 if mix == 0:
612 headers = self.crypto.unpaddata(crypto.decrypt(resp.read(hlen)))
613 if code == 555:
614 content = self.crypto.unpaddata(crypto.decrypt(resp.read()))
615 raise ValueError('Server: '+content)
616 headers = HeaderDict(headers)
617 return 0, code, headers, (resp, crypto)
618 elif mix == 1:
619 data = self.crypto.unpaddata(crypto.decrypt(resp.read()))
620 content = data[hlen:]
621 if code == 555:
622 raise ValueError('Server: '+content)
623 headers = HeaderDict(data[:hlen])
624 resp.close()
625 return 1, code, headers, content
626 else:
627 raw_data += resp.read()
628 raise ValueError('Data format not match(%s:%s)'%(self.url.url, raw_data))
629 except Exception, e:
630 resp.close()
631 return -1, e
632
633 def simple2(*a, **kw):
634 return _base_init(SP2Handler, *a, **kw)
635
636 # ==============================================================================
637 globals().update(gaeproxy=gaeproxy, forold=forold,
638 goagent=goagent, simple=simple, simple2=simple2)