/assets/goagent.py
Python | 2417 lines | 2293 code | 65 blank | 59 comment | 108 complexity | 2ee13670943693f3914cc4607be18fb3 MD5 | raw file
Possible License(s): GPL-3.0
Large files files are truncated, but you can click here to view the full file
- #!/usr/bin/env python
- # coding:utf-8
- # Based on GAppProxy 2.0.0 by Du XiaoGang <dugang.2008@gmail.com>
- # Based on WallProxy 0.4.0 by Hust Moon <www.ehust@gmail.com>
- # Contributor:
- # Phus Lu <phus.lu@gmail.com>
- # Hewig Xu <hewigovens@gmail.com>
- # Ayanamist Yang <ayanamist@gmail.com>
- # V.E.O <V.E.O@tom.com>
- # Max Lv <max.c.lv@gmail.com>
- # AlsoTang <alsotang@gmail.com>
- # Christopher Meng <i@cicku.me>
- # Yonsm Guo <YonsmGuo@gmail.com>
- # Parkman <cseparkman@gmail.com>
- # Ming Bai <mbbill@gmail.com>
- # Bin Yu <yubinlove1991@gmail.com>
- # lileixuan <lileixuan@gmail.com>
- # Cong Ding <cong@cding.org>
- # Zhang Youfu <zhangyoufu@gmail.com>
- # Lu Wei <luwei@barfoo>
- # Harmony Meow <harmony.meow@gmail.com>
- # logostream <logostream@gmail.com>
- # Rui Wang <isnowfy@gmail.com>
- # Wang Wei Qiang <wwqgtxx@gmail.com>
- # Felix Yan <felixonmars@gmail.com>
- # Sui Feng <suifeng.me@qq.com>
- # QXO <qxodream@gmail.com>
- # Geek An <geekan@foxmail.com>
- # Poly Rabbit <mcx_221@foxmail.com>
- # oxnz <yunxinyi@gmail.com>
- # Shusen Liu <liushusen.smart@gmail.com>
- # Yad Smood <y.s.inside@gmail.com>
- # Chen Shuang <cs0x7f@gmail.com>
- # cnfuyu <cnfuyu@gmail.com>
- # cuixin <steven.cuixin@gmail.com>
- # s2marine0 <s2marine0@gmail.com>
- # Toshio Xiang <snachx@gmail.com>
- __version__ = '3.1.5'
- import sys
- import os
- import glob
- # GAEProxy Patch
- # The sys path in Android is set up outside.
- try:
- import gevent
- import gevent.socket
- import gevent.server
- import gevent.queue
- import gevent.monkey
- gevent.monkey.patch_all(subprocess=True)
- except ImportError:
- gevent = None
- except TypeError:
- gevent.monkey.patch_all()
- sys.stderr.write('\033[31m Warning: Please update gevent to the latest 1.0 version!\033[0m\n')
- import errno
- import binascii
- import time
- import struct
- import collections
- import zlib
- import functools
- import itertools
- import re
- import io
- import fnmatch
- import traceback
- import random
- import subprocess
- import base64
- import string
- import hashlib
- import threading
- import thread
- import socket
- import ssl
- import select
- import Queue
- import SocketServer
- import ConfigParser
- import BaseHTTPServer
- import httplib
- import urllib2
- import urlparse
- try:
- import dnslib
- except ImportError:
- dnslib = None
- try:
- import OpenSSL
- except ImportError:
- OpenSSL = None
- try:
- import pacparser
- except ImportError:
- pacparser = None
- # GAEProxy Patch
- class NullDevice():
- def write(self, s):
- pass
- sys.stdout = NullDevice()
- sys.stderr = sys.stdout
- class DNSCacheUtil(object):
- '''DNSCache module, integrated with GAEProxy'''
- cache = {"127.0.0.1": 'localhost'}
- @staticmethod
- def getHost(address):
- p = "(?:\d{1,3}\.){3}\d{1,3}"
- if re.match(p, address) is None:
- return
- if address in DNSCacheUtil.cache:
- return DNSCacheUtil.cache[address]
- host = None
- sock = None
- address_family = socket.AF_INET
- retry = 0
- while address not in DNSCacheUtil.cache:
- try:
- sock = socket.socket(family=address_family, type=socket.SOCK_STREAM)
- sock.settimeout(2)
- sock.connect(("127.0.0.1", 9090))
- sock.sendall(address + "\r\n")
- host = sock.recv(512)
- if host is not None and not host.startswith("null"):
- host = host.strip()
- DNSCacheUtil.cache[address] = host
- break
- else:
- if retry > 3:
- host = None
- break
- else:
- retry = retry + 1
- continue
- except socket.error as e:
- if e[0] in (10060, 'timed out'):
- continue
- except Exception, e:
- logging.error('reverse dns query exception: %s', e)
- break
- finally:
- if sock:
- sock.close()
- return host
- HAS_PYPY = hasattr(sys, 'pypy_version_info')
- NetWorkIOError = (socket.error, ssl.SSLError, OSError) if not OpenSSL else (socket.error, ssl.SSLError, OpenSSL.SSL.Error, OSError)
- class Logging(type(sys)):
- CRITICAL = 50
- FATAL = CRITICAL
- ERROR = 40
- WARNING = 30
- WARN = WARNING
- INFO = 20
- DEBUG = 10
- NOTSET = 0
- def __init__(self, *args, **kwargs):
- self.level = self.__class__.INFO
- #GAEProxy Patch
- @classmethod
- def getLogger(cls, *args, **kwargs):
- return cls(*args, **kwargs)
- def basicConfig(self, *args, **kwargs):
- self.level = int(kwargs.get('level', self.__class__.INFO))
- if self.level > self.__class__.DEBUG:
- self.debug = self.dummy
- def log(self, level, fmt, *args, **kwargs):
- sys.stderr.write('%s - [%s] %s\n' % (level, time.ctime()[4:-5], fmt % args))
- def dummy(self, *args, **kwargs):
- pass
- def debug(self, fmt, *args, **kwargs):
- self.log('DEBUG', fmt, *args, **kwargs)
- def info(self, fmt, *args, **kwargs):
- self.log('INFO', fmt, *args)
- def warning(self, fmt, *args, **kwargs):
- self.log('WARNING', fmt, *args, **kwargs)
- def warn(self, fmt, *args, **kwargs):
- self.warning(fmt, *args, **kwargs)
- def error(self, fmt, *args, **kwargs):
- self.log('ERROR', fmt, *args, **kwargs)
- def exception(self, fmt, *args, **kwargs):
- self.error(fmt, *args, **kwargs)
- sys.stderr.write(traceback.format_exc() + '\n')
- def critical(self, fmt, *args, **kwargs):
- self.log('CRITICAL', fmt, *args, **kwargs)
- logging = sys.modules['logging'] = Logging('logging')
- class LRUCache(object):
- """http://pypi.python.org/pypi/lru/"""
- def __init__(self, max_items=100):
- self.cache = {}
- self.key_order = []
- self.max_items = max_items
- def __setitem__(self, key, value):
- self.cache[key] = value
- self._mark(key)
- def __getitem__(self, key):
- value = self.cache[key]
- self._mark(key)
- return value
- def _mark(self, key):
- if key in self.key_order:
- self.key_order.remove(key)
- self.key_order.insert(0, key)
- if len(self.key_order) > self.max_items:
- remove = self.key_order[self.max_items]
- del self.cache[remove]
- self.key_order.pop(self.max_items)
- def clear(self):
- self.cache = {}
- self.key_order = []
- class CertUtil(object):
- """CertUtil module, based on mitmproxy"""
- ca_vendor = 'GoAgent'
- ca_keyfile = 'CA.crt'
- ca_certdir = 'certs'
- ca_lock = threading.Lock()
- @staticmethod
- def create_ca():
- key = OpenSSL.crypto.PKey()
- key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
- ca = OpenSSL.crypto.X509()
- ca.set_serial_number(0)
- ca.set_version(2)
- subj = ca.get_subject()
- subj.countryName = 'CN'
- subj.stateOrProvinceName = 'Internet'
- subj.localityName = 'Cernet'
- subj.organizationName = CertUtil.ca_vendor
- subj.organizationalUnitName = '%s Root' % CertUtil.ca_vendor
- subj.commonName = '%s CA' % CertUtil.ca_vendor
- ca.gmtime_adj_notBefore(0)
- ca.gmtime_adj_notAfter(24 * 60 * 60 * 3652)
- ca.set_issuer(ca.get_subject())
- ca.set_pubkey(key)
- ca.add_extensions([
- OpenSSL.crypto.X509Extension(b'basicConstraints', True, b'CA:TRUE'),
- OpenSSL.crypto.X509Extension(b'nsCertType', True, b'sslCA'),
- OpenSSL.crypto.X509Extension(b'extendedKeyUsage', True, b'serverAuth,clientAuth,emailProtection,timeStamping,msCodeInd,msCodeCom,msCTLSign,msSGC,msEFS,nsSGC'),
- OpenSSL.crypto.X509Extension(b'keyUsage', False, b'keyCertSign, cRLSign'),
- OpenSSL.crypto.X509Extension(b'subjectKeyIdentifier', False, b'hash', subject=ca), ])
- ca.sign(key, 'sha1')
- return key, ca
- @staticmethod
- def dump_ca():
- key, ca = CertUtil.create_ca()
- with open(CertUtil.ca_keyfile, 'wb') as fp:
- fp.write(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, ca))
- fp.write(OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key))
- @staticmethod
- def _get_cert(commonname, sans=()):
- with open(CertUtil.ca_keyfile, 'rb') as fp:
- content = fp.read()
- key = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, content)
- ca = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, content)
- pkey = OpenSSL.crypto.PKey()
- pkey.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
- req = OpenSSL.crypto.X509Req()
- subj = req.get_subject()
- subj.countryName = 'CN'
- subj.stateOrProvinceName = 'Internet'
- subj.localityName = 'Cernet'
- subj.organizationalUnitName = '%s Branch' % CertUtil.ca_vendor
- if commonname[0] == '.':
- subj.commonName = '*' + commonname
- subj.organizationName = '*' + commonname
- sans = ['*'+commonname] + [x for x in sans if x != '*'+commonname]
- else:
- subj.commonName = commonname
- subj.organizationName = commonname
- sans = [commonname] + [x for x in sans if x != commonname]
- # GAEProxy Patch
- req.add_extensions([OpenSSL.crypto.X509Extension(b'subjectAltName', True, ', '.join('DNS: %s' % x for x in sans))])
- req.set_pubkey(pkey)
- req.sign(pkey, 'sha1')
- cert = OpenSSL.crypto.X509()
- # GAEProxy Patch
- cert.set_version(3)
- try:
- cert.set_serial_number(int(hashlib.md5(commonname.encode('utf-8')).hexdigest(), 16))
- except OpenSSL.SSL.Error:
- cert.set_serial_number(int(time.time()*1000))
- cert.gmtime_adj_notBefore(0)
- cert.gmtime_adj_notAfter(60 * 60 * 24 * 3652)
- cert.set_issuer(ca.get_subject())
- cert.set_subject(req.get_subject())
- cert.set_pubkey(req.get_pubkey())
- if commonname[0] == '.':
- sans = ['*'+commonname] + [s for s in sans if s != '*'+commonname]
- else:
- sans = [commonname] + [s for s in sans if s != commonname]
- # GAEProxy Patch
- cert.add_extensions([OpenSSL.crypto.X509Extension(b'subjectAltName', True, ', '.join('DNS: %s' % x for x in sans))])
- cert.sign(key, 'sha1')
- certfile = os.path.join(CertUtil.ca_certdir, commonname + '.crt')
- with open(certfile, 'wb') as fp:
- fp.write(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert))
- fp.write(OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, pkey))
- return certfile
- @staticmethod
- def get_cert(commonname, sans=()):
- # GAEProxy Patch
- sans = ["*.akamaihd.net","*.fbcdn.net","*.google.com","*.appspot.com","*.googleapis.com","*.googlevideo.com","*.twitter.com","*.facebook.com","*.whatsapp.net"]
- if commonname.count('.') >= 2 and [len(x) for x in reversed(commonname.split('.'))] > [2, 4]:
- commonname = '.'+commonname.partition('.')[-1]
- certfile = os.path.join(CertUtil.ca_certdir, commonname + '.crt')
- if os.path.exists(certfile):
- return certfile
- elif OpenSSL is None:
- return CertUtil.ca_keyfile
- else:
- with CertUtil.ca_lock:
- if os.path.exists(certfile):
- return certfile
- return CertUtil._get_cert(commonname, sans)
- @staticmethod
- def import_ca(certfile):
- commonname = os.path.splitext(os.path.basename(certfile))[0]
- if OpenSSL:
- try:
- with open(certfile, 'rb') as fp:
- x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fp.read())
- commonname = next(v.decode() for k, v in x509.get_subject().get_components() if k == b'O')
- except Exception as e:
- logging.error('load_certificate(certfile=%r) failed:%s', certfile, e)
- # GAEProxy Patch
- return 0
- @staticmethod
- def check_ca():
- #Check CA exists
- capath = os.path.join(os.path.dirname(os.path.abspath(__file__)), CertUtil.ca_keyfile)
- certdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), CertUtil.ca_certdir)
- if not os.path.exists(capath):
- if not OpenSSL:
- logging.critical('CA.key is not exist and OpenSSL is disabled, ABORT!')
- sys.exit(-1)
- if os.path.exists(certdir):
- if os.path.isdir(certdir):
- any(os.remove(x) for x in glob.glob(certdir+'/*.crt')+glob.glob(certdir+'/.*.crt'))
- else:
- os.remove(certdir)
- os.mkdir(certdir)
- CertUtil.dump_ca()
- if glob.glob('%s/*.key' % CertUtil.ca_certdir):
- for filename in glob.glob('%s/*.key' % CertUtil.ca_certdir):
- try:
- os.remove(filename)
- os.remove(os.path.splitext(filename)[0]+'.crt')
- except EnvironmentError:
- pass
- #Check CA imported
- if CertUtil.import_ca(capath) != 0:
- logging.warning('install root certificate failed, Please run as administrator/root/sudo')
- #Check Certs Dir
- if not os.path.exists(certdir):
- os.makedirs(certdir)
- class SSLConnection(object):
- has_gevent = socket.socket is getattr(sys.modules.get('gevent.socket'), 'socket', None)
- def __init__(self, context, sock):
- self._context = context
- self._sock = sock
- self._connection = OpenSSL.SSL.Connection(context, sock)
- self._makefile_refs = 0
- if self.has_gevent:
- self._wait_read = gevent.socket.wait_read
- self._wait_write = gevent.socket.wait_write
- self._wait_readwrite = gevent.socket.wait_readwrite
- else:
- self._wait_read = lambda fd,t: select.select([fd], [], [fd], t)
- self._wait_write = lambda fd,t: select.select([], [fd], [fd], t)
- self._wait_readwrite = lambda fd,t: select.select([fd], [fd], [fd], t)
- def __getattr__(self, attr):
- if attr not in ('_context', '_sock', '_connection', '_makefile_refs'):
- return getattr(self._connection, attr)
- def accept(self):
- sock, addr = self._sock.accept()
- client = OpenSSL.SSL.Connection(sock._context, sock)
- return client, addr
- def do_handshake(self):
- timeout = self._sock.gettimeout()
- while True:
- try:
- self._connection.do_handshake()
- break
- except (OpenSSL.SSL.WantReadError, OpenSSL.SSL.WantX509LookupError, OpenSSL.SSL.WantWriteError):
- sys.exc_clear()
- self._wait_readwrite(self._sock.fileno(), timeout)
- def connect(self, *args, **kwargs):
- timeout = self._sock.gettimeout()
- while True:
- try:
- self._connection.connect(*args, **kwargs)
- break
- except (OpenSSL.SSL.WantReadError, OpenSSL.SSL.WantX509LookupError):
- sys.exc_clear()
- self._wait_read(self._sock.fileno(), timeout)
- except OpenSSL.SSL.WantWriteError:
- sys.exc_clear()
- self._wait_write(self._sock.fileno(), timeout)
- def send(self, data, flags=0):
- timeout = self._sock.gettimeout()
- while True:
- try:
- self._connection.send(data, flags)
- break
- except (OpenSSL.SSL.WantReadError, OpenSSL.SSL.WantX509LookupError):
- sys.exc_clear()
- self._wait_read(self._sock.fileno(), timeout)
- except OpenSSL.SSL.WantWriteError:
- sys.exc_clear()
- self._wait_write(self._sock.fileno(), timeout)
- except OpenSSL.SSL.SysCallError as e:
- if e[0] == -1 and not data:
- # errors when writing empty strings are expected and can be ignored
- return 0
- raise
- def recv(self, bufsiz, flags=0):
- timeout = self._sock.gettimeout()
- pending = self._connection.pending()
- if pending:
- return self._connection.recv(min(pending, bufsiz))
- while True:
- try:
- return self._connection.recv(bufsiz, flags)
- except (OpenSSL.SSL.WantReadError, OpenSSL.SSL.WantX509LookupError):
- sys.exc_clear()
- self._wait_read(self._sock.fileno(), timeout)
- except OpenSSL.SSL.WantWriteError:
- sys.exc_clear()
- self._wait_write(self._sock.fileno(), timeout)
- except OpenSSL.SSL.ZeroReturnError:
- return ''
- def read(self, bufsiz, flags=0):
- return self.recv(bufsiz, flags)
- def write(self, buf, flags=0):
- return self.sendall(buf, flags)
- def close(self):
- if self._makefile_refs < 1:
- self._connection = None
- if self._sock:
- socket.socket.close(self._sock)
- else:
- self._makefile_refs -= 1
- def makefile(self, mode='r', bufsize=-1):
- self._makefile_refs += 1
- return socket._fileobject(self, mode, bufsize, close=True)
- class ProxyUtil(object):
- """ProxyUtil module, based on urllib2"""
- @staticmethod
- def parse_proxy(proxy):
- return urllib2._parse_proxy(proxy)
- @staticmethod
- def get_system_proxy():
- proxies = urllib2.getproxies()
- return proxies.get('https') or proxies.get('http') or {}
- @staticmethod
- def get_listen_ip():
- sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
- sock.connect(('8.8.8.8', 53))
- listen_ip = sock.getsockname()[0]
- sock.close()
- return listen_ip
- # GAEProxy Patch
- # No PAC
- def dns_remote_resolve(qname, dnsservers, blacklist, timeout):
- """
- http://gfwrev.blogspot.com/2009/11/gfwdns.html
- http://zh.wikipedia.org/wiki/域名服务器缓存污染
- http://support.microsoft.com/kb/241352
- """
- query = dnslib.DNSRecord(q=dnslib.DNSQuestion(qname))
- query_data = query.pack()
- dns_v4_servers = [x for x in dnsservers if ':' not in x]
- dns_v6_servers = [x for x in dnsservers if ':' in x]
- sock_v4 = sock_v6 = None
- socks = []
- if dns_v4_servers:
- sock_v4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
- socks.append(sock_v4)
- if dns_v6_servers:
- sock_v6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
- socks.append(sock_v6)
- timeout_at = time.time() + timeout
- try:
- for _ in xrange(2):
- try:
- for dnsserver in dns_v4_servers:
- sock_v4.sendto(query_data, (dnsserver, 53))
- for dnsserver in dns_v6_servers:
- sock_v6.sendto(query_data, (dnsserver, 53))
- while time.time() < timeout_at:
- ins, _, _ = select.select(socks, [], [], 0.1)
- for sock in ins:
- reply_data, _ = sock.recvfrom(512)
- reply = dnslib.DNSRecord.parse(reply_data)
- rtypes = (1, 28) if sock is sock_v6 else (1,)
- iplist = [str(x.rdata) for x in reply.rr if x.rtype in rtypes]
- if any(x in blacklist for x in iplist):
- logging.warning('query qname=%r reply bad iplist=%r', qname, iplist)
- else:
- logging.debug('query qname=%r reply iplist=%s', qname, iplist)
- return iplist
- except socket.error as e:
- logging.warning('handle dns query=%s socket: %r', query, e)
- finally:
- for sock in socks:
- sock.close()
- def get_dnsserver_list():
- if os.name == 'nt':
- import ctypes, ctypes.wintypes, struct, socket
- DNS_CONFIG_DNS_SERVER_LIST = 6
- buf = ctypes.create_string_buffer(2048)
- ctypes.windll.dnsapi.DnsQueryConfig(DNS_CONFIG_DNS_SERVER_LIST, 0, None, None, ctypes.byref(buf), ctypes.byref(ctypes.wintypes.DWORD(len(buf))))
- ips = struct.unpack('I', buf[0:4])[0]
- out = []
- for i in xrange(ips):
- start = (i+1) * 4
- out.append(socket.inet_ntoa(buf[start:start+4]))
- return out
- elif os.path.isfile('/etc/resolv.conf'):
- with open('/etc/resolv.conf', 'rb') as fp:
- return re.findall(r'(?m)^nameserver\s+(\S+)', fp.read())
- else:
- logging.warning("get_dnsserver_list failed: unsupport platform '%s-%s'", sys.platform, os.name)
- return []
- def spawn_later(seconds, target, *args, **kwargs):
- def wrap(*args, **kwargs):
- __import__('time').sleep(seconds)
- return target(*args, **kwargs)
- return __import__('thread').start_new_thread(wrap, args, kwargs)
- class HTTPUtil(object):
- """HTTP Request Class"""
- MessageClass = dict
- protocol_version = 'HTTP/1.1'
- skip_headers = frozenset(['Vary', 'Via', 'X-Forwarded-For', 'Proxy-Authorization', 'Proxy-Connection', 'Upgrade', 'X-Chrome-Variations', 'Connection', 'Cache-Control'])
- ssl_validate = False
- ssl_obfuscate = False
- ssl_ciphers = ':'.join(['ECDHE-ECDSA-AES256-SHA',
- 'ECDHE-RSA-AES256-SHA',
- 'DHE-RSA-CAMELLIA256-SHA',
- 'DHE-DSS-CAMELLIA256-SHA',
- 'DHE-RSA-AES256-SHA',
- 'DHE-DSS-AES256-SHA',
- 'ECDH-RSA-AES256-SHA',
- 'ECDH-ECDSA-AES256-SHA',
- 'CAMELLIA256-SHA',
- 'AES256-SHA',
- 'ECDHE-ECDSA-RC4-SHA',
- 'ECDHE-ECDSA-AES128-SHA',
- 'ECDHE-RSA-RC4-SHA',
- 'ECDHE-RSA-AES128-SHA',
- 'DHE-RSA-CAMELLIA128-SHA',
- 'DHE-DSS-CAMELLIA128-SHA',
- 'DHE-RSA-AES128-SHA',
- 'DHE-DSS-AES128-SHA',
- 'ECDH-RSA-RC4-SHA',
- 'ECDH-RSA-AES128-SHA',
- 'ECDH-ECDSA-RC4-SHA',
- 'ECDH-ECDSA-AES128-SHA',
- 'SEED-SHA',
- 'CAMELLIA128-SHA',
- 'RC4-SHA',
- 'RC4-MD5',
- 'AES128-SHA',
- 'ECDHE-ECDSA-DES-CBC3-SHA',
- 'ECDHE-RSA-DES-CBC3-SHA',
- 'EDH-RSA-DES-CBC3-SHA',
- 'EDH-DSS-DES-CBC3-SHA',
- 'ECDH-RSA-DES-CBC3-SHA',
- 'ECDH-ECDSA-DES-CBC3-SHA',
- 'DES-CBC3-SHA',
- 'TLS_EMPTY_RENEGOTIATION_INFO_SCSV'])
- def __init__(self, max_window=4, max_timeout=8, max_retry=4, proxy='', dns_servers=[], dns_blacklist=set()):
- # http://docs.python.org/dev/library/ssl.html
- # http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html
- # http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c
- # http://www.openssl.org/docs/apps/ciphers.html
- # openssl s_server -accept 443 -key CA.crt -cert CA.crt
- # set_ciphers as Modern Browsers
- self.max_window = max_window
- self.max_retry = max_retry
- self.max_timeout = max_timeout
- self.tcp_connection_time = collections.defaultdict(float)
- self.tcp_connection_cache = collections.defaultdict(Queue.PriorityQueue)
- self.ssl_connection_time = collections.defaultdict(float)
- self.ssl_connection_cache = collections.defaultdict(Queue.PriorityQueue)
- self.dns = {}
- self.proxy = proxy
- self.openssl_context = None
- if self.proxy:
- self.dns_resolve = self.__dns_resolve_withproxy
- self.create_connection = self.__create_connection_withproxy
- self.create_ssl_connection = self.__create_ssl_connection_withproxy
- self.dns_servers = dns_servers
- self.dns_blacklist = dns_blacklist
- def set_openssl_option(self, validate=True, obfuscate=True):
- if self.openssl_context is None:
- self.openssl_context = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD)
- self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10)))
- if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'):
- self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH)
- if validate:
- self.openssl_context.load_verify_locations(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cacert.pem'))
- self.openssl_context.set_verify(OpenSSL.SSL.VERIFY_PEER, lambda c, x, e, d, ok: ok)
- if obfuscate:
- ssl_ciphers = ':'.join(x for x in self.ssl_ciphers.split(':') if random.random() > 0.5)
- self.openssl_context.set_cipher_list(ssl_ciphers)
- def dns_resolve(self, host, dnsservers=[], ipv4_only=True):
- iplist = self.dns.get(host)
- if not iplist:
- if not dnsservers:
- iplist = list(set(socket.gethostbyname_ex(host)[-1]) - self.dns_blacklist)
- else:
- iplist = dns_remote_resolve(host, dnsservers, self.dns_blacklist, timeout=2)
- if not iplist:
- iplist = dns_remote_resolve(host, self.dns_servers, self.dns_blacklist, timeout=2)
- if ipv4_only:
- iplist = [ip for ip in iplist if re.match(r'\d+\.\d+\.\d+\.\d+', ip)]
- self.dns[host] = iplist = list(set(iplist))
- return iplist
- def __dns_resolve_withproxy(self, host, dnsservers=[], ipv4_only=True):
- return [host]
- def create_connection(self, address, timeout=None, source_address=None, **kwargs):
- connection_cache_key = kwargs.get('cache_key')
- def _create_connection(ipaddr, timeout, queobj):
- sock = None
- try:
- # create a ipv4/ipv6 socket object
- sock = socket.socket(socket.AF_INET if ':' not in ipaddr[0] else socket.AF_INET6)
- # set reuseaddr option to avoid 10048 socket error
- sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- # resize socket recv buffer 8K->32K to improve browser releated application performance
- sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
- # disable nagle algorithm to send http request quickly.
- sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
- # set a short timeout to trigger timeout retry more quickly.
- sock.settimeout(timeout or self.max_timeout)
- # start connection time record
- start_time = time.time()
- # TCP connect
- sock.connect(ipaddr)
- # record TCP connection time
- self.tcp_connection_time[ipaddr] = time.time() - start_time
- # put ssl socket object to output queobj
- queobj.put(sock)
- except (socket.error, OSError) as e:
- # any socket.error, put Excpetions to output queobj.
- queobj.put(e)
- # reset a large and random timeout to the ipaddr
- self.tcp_connection_time[ipaddr] = self.max_timeout+random.random()
- # close tcp socket
- if sock:
- sock.close()
- def _close_connection(count, queobj):
- for i in range(count):
- sock = queobj.get()
- if sock and not isinstance(sock, Exception):
- if connection_cache_key and i == 0:
- self.tcp_connection_cache[connection_cache_key].put((time.time(), sock))
- else:
- sock.close()
- try:
- while connection_cache_key:
- ctime, sock = self.tcp_connection_cache[connection_cache_key].get_nowait()
- if time.time() - ctime < 30:
- return sock
- except Queue.Empty:
- pass
- host, port = address
- result = None
- addresses = [(x, port) for x in self.dns_resolve(host)]
- if port == 443:
- get_connection_time = lambda addr: self.ssl_connection_time.__getitem__(addr) or self.tcp_connection_time.__getitem__(addr)
- else:
- get_connection_time = self.tcp_connection_time.__getitem__
- for i in range(self.max_retry):
- window = min((self.max_window+1)//2 + min(i, 1), len(addresses))
- addresses.sort(key=get_connection_time)
- addrs = addresses[:window] + random.sample(addresses, min(len(addresses), window, self.max_window-window))
- queobj = Queue.Queue()
- for addr in addrs:
- thread.start_new_thread(_create_connection, (addr, timeout, queobj))
- for i in range(len(addrs)):
- result = queobj.get()
- if not isinstance(result, (socket.error, OSError)):
- thread.start_new_thread(_close_connection, (len(addrs)-i-1, queobj))
- return result
- else:
- if i == 0:
- # only output first error
- logging.warning('create_connection to %s return %r, try again.', addrs, result)
- def create_ssl_connection(self, address, timeout=None, source_address=None, **kwargs):
- connection_cache_key = kwargs.get('cache_key')
- validate = kwargs.get('validate')
- def _create_ssl_connection(ipaddr, timeout, queobj):
- sock = None
- ssl_sock = None
- try:
- # create a ipv4/ipv6 socket object
- sock = socket.socket(socket.AF_INET if ':' not in ipaddr[0] else socket.AF_INET6)
- # set reuseaddr option to avoid 10048 socket error
- sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- # resize socket recv buffer 8K->32K to improve browser releated application performance
- sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
- # disable negal algorithm to send http request quickly.
- sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
- # set a short timeout to trigger timeout retry more quickly.
- sock.settimeout(timeout or self.max_timeout)
- # pick up the certificate
- if not validate:
- ssl_sock = ssl.wrap_socket(sock, do_handshake_on_connect=False)
- else:
- ssl_sock = ssl.wrap_socket(sock, cert_reqs=ssl.CERT_REQUIRED, ca_certs=os.path.join(os.path.dirname(os.path.abspath(__file__)),'cacert.pem'), do_handshake_on_connect=False)
- ssl_sock.settimeout(timeout or self.max_timeout)
- # start connection time record
- start_time = time.time()
- # TCP connect
- ssl_sock.connect(ipaddr)
- connected_time = time.time()
- # SSL handshake
- ssl_sock.do_handshake()
- handshaked_time = time.time()
- # record TCP connection time
- self.tcp_connection_time[ipaddr] = ssl_sock.tcp_time = connected_time - start_time
- # record SSL connection time
- self.ssl_connection_time[ipaddr] = ssl_sock.ssl_time = handshaked_time - start_time
- ssl_sock.ssl_time = connected_time - start_time
- # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
- ssl_sock.sock = sock
- # verify SSL certificate.
- if validate and address[0].endswith('.appspot.com'):
- cert = ssl_sock.getpeercert()
- orgname = next((v for ((k, v),) in cert['subject'] if k == 'organizationName'))
- if not orgname.lower().startswith('google '):
- raise ssl.SSLError("%r certificate organizationName(%r) not startswith 'Google'" % (address[0], orgname))
- # put ssl socket object to output queobj
- queobj.put(ssl_sock)
- except (socket.error, ssl.SSLError, OSError) as e:
- # any socket.error, put Excpetions to output queobj.
- queobj.put(e)
- # reset a large and random timeout to the ipaddr
- self.ssl_connection_time[ipaddr] = self.max_timeout + random.random()
- # close ssl socket
- if ssl_sock:
- ssl_sock.close()
- # close tcp socket
- if sock:
- sock.close()
- def _create_openssl_connection(ipaddr, timeout, queobj):
- sock = None
- ssl_sock = None
- try:
- # create a ipv4/ipv6 socket object
- sock = socket.socket(socket.AF_INET if ':' not in ipaddr[0] else socket.AF_INET6)
- # set reuseaddr option to avoid 10048 socket error
- sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- # resize socket recv buffer 8K->32K to improve browser releated application performance
- sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
- # disable negal algorithm to send http request quickly.
- sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
- # set a short timeout to trigger timeout retry more quickly.
- sock.settimeout(timeout or self.max_timeout)
- # pick up the certificate
- server_hostname = b'www.google.com' if address[0].endswith('.appspot.com') else None
- ssl_sock = SSLConnection(self.openssl_context, sock)
- ssl_sock.set_connect_state()
- if server_hostname:
- ssl_sock.set_tlsext_host_name(server_hostname)
- # start connection time record
- start_time = time.time()
- # TCP connect
- ssl_sock.connect(ipaddr)
- connected_time = time.time()
- # SSL handshake
- ssl_sock.do_handshake()
- handshaked_time = time.time()
- # record TCP connection time
- self.tcp_connection_time[ipaddr] = ssl_sock.tcp_time = connected_time - start_time
- # record SSL connection time
- self.ssl_connection_time[ipaddr] = ssl_sock.ssl_time = handshaked_time - start_time
- # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
- ssl_sock.sock = sock
- # verify SSL certificate.
- if validate and address[0].endswith('.appspot.com'):
- cert = ssl_sock.get_peer_certificate()
- commonname = next((v for k, v in cert.get_subject().get_components() if k == 'CN'))
- if '.google' not in commonname and not commonname.endswith('.appspot.com'):
- raise socket.error("Host name '%s' doesn't match certificate host '%s'" % (address[0], commonname))
- # put ssl socket object to output queobj
- queobj.put(ssl_sock)
- except (socket.error, OpenSSL.SSL.Error, OSError) as e:
- # any socket.error, put Excpetions to output queobj.
- queobj.put(e)
- # reset a large and random timeout to the ipaddr
- self.ssl_connection_time[ipaddr] = self.max_timeout + random.random()
- # close ssl socket
- if ssl_sock:
- ssl_sock.close()
- # close tcp socket
- if sock:
- sock.close()
- def _close_ssl_connection(count, queobj, first_tcp_time, first_ssl_time):
- for i in range(count):
- sock = queobj.get()
- ssl_time_threshold = min(1, 1.5 * first_ssl_time)
- if sock and not isinstance(sock, Exception):
- if connection_cache_key and sock.ssl_time < ssl_time_threshold:
- self.ssl_connection_cache[connection_cache_key].put((time.time(), sock))
- else:
- sock.close()
- try:
- while connection_cache_key:
- ctime, sock = self.ssl_connection_cache[connection_cache_key].get_nowait()
- if time.time() - ctime < 30:
- return sock
- except Queue.Empty:
- pass
- host, port = address
- result = None
- # create_connection = _create_ssl_connection if not validate else _create_openssl_connection
- create_connection = _create_ssl_connection
- addresses = [(x, port) for x in self.dns_resolve(host)]
- for i in range(self.max_retry):
- window = min((self.max_window+1)//2 + min(i, 1), len(addresses))
- addresses.sort(key=self.ssl_connection_time.__getitem__)
- addrs = addresses[:window] + random.sample(addresses, min(len(addresses), window, self.max_window-window))
- queobj = Queue.Queue()
- for addr in addrs:
- thread.start_new_thread(create_connection, (addr, timeout, queobj))
- for i in range(len(addrs)):
- result = queobj.get()
- if not isinstance(result, Exception):
- thread.start_new_thread(_close_ssl_connection, (len(addrs)-i-1, queobj, result.tcp_time, result.ssl_time))
- return result
- else:
- if i == 0:
- # only output first error
- logging.warning('create_ssl_connection to %s return %r, try again.', addrs, result)
- def __create_connection_withproxy(self, address, timeout=None, source_address=None, **kwargs):
- host, port = address
- logging.debug('__create_connection_withproxy connect (%r, %r)', host, port)
- _, proxyuser, proxypass, proxyaddress = ProxyUtil.parse_proxy(self.proxy)
- try:
- try:
- self.dns_resolve(host)
- except (socket.error, OSError):
- pass
- proxyhost, _, proxyport = proxyaddress.rpartition(':')
- sock = socket.create_connection((proxyhost, int(proxyport)))
- if host in self.dns:
- hostname = random.choice(self.dns[host])
- elif host.endswith('.appspot.com'):
- hostname = 'www.google.com'
- else:
- hostname = host
- request_data = 'CONNECT %s:%s HTTP/1.1\r\n' % (hostname, port)
- if proxyuser and proxypass:
- request_data += 'Proxy-authorization: Basic %s\r\n' % base64.b64encode(('%s:%s' % (proxyuser, proxypass)).encode()).decode().strip()
- request_data += '\r\n'
- sock.sendall(request_data)
- response = httplib.HTTPResponse(sock)
- response.begin()
- if response.status >= 400:
- logging.error('__create_connection_withproxy return http error code %s', response.status)
- sock = None
- return sock
- except Exception as e:
- logging.error('__create_connection_withproxy error %s', e)
- raise
- def __create_ssl_connection_withproxy(self, address, timeout=None, source_address=None, **kwargs):
- host, port = address
- logging.debug('__create_ssl_connection_withproxy connect (%r, %r)', host, port)
- try:
- sock = self.__create_connection_withproxy(address, timeout, source_address)
- ssl_sock = ssl.wrap_socket(sock)
- ssl_sock.sock = sock
- return ssl_sock
- except Exception as e:
- logging.error('__create_ssl_connection_withproxy error %s', e)
- raise
- def forward_socket(self, local, remote, timeout=60, tick=2, bufsize=8192, maxping=None, maxpong=None):
- try:
- timecount = timeout
- while 1:
- timecount -= tick
- if timecount <= 0:
- break
- (ins, _, errors) = select.select([local, remote], [], [local, remote], tick)
- if errors:
- break
- if ins:
- for sock in ins:
- data = sock.recv(bufsize)
- if data:
- if sock is remote:
- local.sendall(data)
- timecount = maxpong or timeout
- else:
- remote.sendall(data)
- timecount = maxping or timeout
- else:
- return
- except NetWorkIOError as e:
- if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET, errno.ENOTCONN, errno.EPIPE):
- raise
- finally:
- if local:
- local.close()
- if remote:
- remote.close()
- def green_forward_socket(self, local, remote, timeout=60, tick=2, bufsize=8192, maxping=None, maxpong=None, pongcallback=None, bitmask=None):
- def io_copy(dest, source):
- try:
- dest.settimeout(timeout)
- source.settimeout(timeout)
- while 1:
- data = source.recv(bufsize)
- if not data:
- break
- if bitmask:
- data = ''.join(chr(ord(x) ^ bitmask) for x in data)
- dest.sendall(data)
- except NetWorkIOError as e:
- if e.args[0] not in ('timed out', errno.ECONNABORTED, errno.ECONNRESET, errno.EBADF, errno.EPIPE, errno.ENOTCONN, errno.ETIMEDOUT):
- raise
- finally:
- if local:
- local.close()
- if remote:
- remote.close()
- thread.start_new_thread(io_copy, (remote.dup(), local.dup()))
- io_copy(local, remote)
- def _request(self, sock, method, path, protocol_version, headers, payload, bufsize=8192, crlf=None, return_sock=None):
- skip_headers = self.skip_headers
- need_crlf = bool(crlf)
- if need_crlf:
- fakehost = 'www.' + ''.join(random.choice(('bcdfghjklmnpqrstvwxyz','aeiou')[x&1]) for x in xrange(random.randint(5,20))) + random.choice(['.net', '.com', '.org'])
- request_data = 'GET / HTTP/1.1\r\nHost: %s\r\n\r\n\r\n\r\r' % fakehost
- else:
- request_data = ''
- request_data += '%s %s %s\r\n' % (method, path, protocol_version)
- request_data += ''.join('%s: %s\r\n' % (k.title(), v) for k, v in headers.items() if k.title() not in skip_headers)
- if self.proxy:
- _, username, password, _ = ProxyUtil.parse_proxy(self.proxy)
- if username and password:
- request_data += 'Proxy-Authorization: Basic %s\r\n' % base64.b64encode(('%s:%s' % (username, password)).encode()).decode().strip()
- request_data += '\r\n'
- if isinstance(payload, bytes):
- sock.sendall(request_data.encode() + payload)
- elif hasattr(payload, 'read'):
- sock.sendall(request_data)
- while 1:
- data = payload.read(bufsize)
- if not data:
- break
- sock.sendall(data)
- else:
- raise TypeError('http_util.request(payload) must be a string or buffer, not %r' % type(payload))
- if need_crlf:
- try:
- response = httplib.HTTPResponse(sock)
- response.begin()
- response.read()
- except Exception:
- logging.exception('crlf skip read')
- return None
- if return_sock:
- return sock
- response = httplib.HTTPResponse(sock, buffering=True)
- try:
- response.begin()
- except httplib.BadStatusLine:
- response = None
- return response
- def request(self, method, url, payload=None, headers={}, realhost='', bufsize=8192, crlf=None, validate=None, return_sock=None, connection_cache_key=None):
- scheme, netloc, path, _, query, _ = urlparse.urlparse(url)
- if netloc.rfind(':') <= netloc.rfind(']'):
- # no port number
- host = netloc
- port = 443 if scheme == 'https' else 80
- else:
- host, _, port = netloc.rpartition(':')
- port = int(port)
- if query:
- path += '?' + query
- if 'Host' not in headers:
- headers['Host'] = host
- if payload and 'Content-Length' not in headers:
- headers['Content-Length'] = str(len(payload))
- for i in range(self.max_retry):
- sock = None
- ssl_sock = None
- try:
- if scheme == 'https':
- ssl_sock = self.create_ssl_connection((realhost or host, port), self.max_timeout, validate=validate, cache_key=connection_cache_key)
- if ssl_sock:
- sock = ssl_sock.sock
- del ssl_sock.sock
- else:
- raise socket.error('timed out', 'create_ssl_connection(%r,%r)' % (realhost or host, port))
- else:
- sock = self.create_connection((realhost or host, port), self.max_timeout, cache_key=connection_cache_key)
- if sock:
- if scheme == 'https':
- crlf = 0
- return self._request(ssl_sock or sock, method, path, self.protocol_version, headers, payload, bufsize=bufsize, crlf=crlf, return_sock=return_sock)
- except Exception as e:
- logging.debug('request "%s %s" failed:%s', method, url, e)
- if ssl_sock:
- ssl_sock.close()
- if sock:
- sock.close()
- if i == self.max_retry - 1:
- raise
- else:
- continue
- class Common(object):
- """Global Config Object"""
- ENV_CONFIG_PREFIX = 'GOAGENT_'
- def __init__(self):
- """load config from proxy.ini"""
- ConfigParser.RawConfigParser.OPTCRE = re.compile(r'(?P<option>[^=\s][^=]*)\s*(?P<vi>[=])\s*(?P<value>.*)$')
- self.CONFIG = ConfigParser.ConfigParser()
- # GAEProxy Patch
- self.CONFIG_FILENAME = '/data/data/org.gaeproxy/proxy.ini'
- self.CONFIG.read(self.CONFIG_FILENAME)
- self.LISTEN_IP = self.CONFIG.get('listen', 'ip')
- self.LISTEN_PORT = self.CONFIG.getint('listen', 'port')
- self.LISTEN_VISIBLE = self.CONFIG.getint('listen', 'visible')
- self.LISTEN_DEBUGINFO = self.CONFIG.getint('listen', 'debuginfo')
- self.GAE_APPIDS = re.findall(r'[\w\-\.]+', self.CONFIG.get('gae', 'appid').replace('.appspot.com', ''))
- self.GAE_PASSWORD = self.CONFIG.get('gae', 'password').strip()
- self.GAE_PATH = self.CONFIG.get('gae', 'path')
- self.GAE_MODE = self.CONFIG.get('gae', 'mode')
- self.GAE_PROFILE = self.CONFIG.get('gae', 'profile').strip()
- self.GAE_WINDOW = self.CONFIG.getint('gae', 'window')
- self.GAE_VALIDATE = self.CONFIG.getint('gae', 'validate')
- self.GAE_OBFUSCATE = self.CONFIG.getint('gae', 'obfuscate')
- self.GAE_OPTIONS = self.CONFIG.get('gae', 'options')
- hosts_section, http_section = '%s/hosts' % self.GAE_PROFILE, '%s/http' % self.GAE_PROFILE
- self.HOSTS_MAP = collections.OrderedDict((k, v or k) for k, v in self.CONFIG.items(hosts_section) if '\\' not in k and ':' not in k and not k.startswith('.'))
- self.HOSTS_POSTFIX_MAP = collections.OrderedDict((k, v) for k, v in self.CONFIG.items(hosts_section) if '\\' not in k and ':' not in k and k.startswith('.'))
- self.HOSTS_POSTFIX_ENDSWITH = tuple(self.HOSTS_POSTFIX_MAP)
- self.CONNECT_HOSTS_MAP = collections.OrderedDict((k, v) for k, v in self.CONFIG.items(hosts_section) if ':' in k and not k.startswith('.'))
- self.CONNECT_POSTFIX_MAP = collections.OrderedDict((k, v) for k, v in self.CONFIG.items(hosts_section) if ':' in k and k.startswith('.'))
- self.CONNECT_POSTFIX_ENDSWITH = tuple(self.CONNECT_POSTFIX_MAP)
- self.METHOD_REMATCH_MAP = collections.OrderedDict((re.compile(k).match, v) for k, v in self.CONFIG.items(hosts_section) if '\\' in k)
- self.METHOD_REMATCH_HAS_LOCALFILE = any(x.startswith('file://') for x in self.METHOD_REMATCH_MAP.values())
- self.HTTP_WITHGAE = tuple(self.CONFIG.get(http_section, 'withgae').split('|'))
- self.HTTP_CRLFSITES = tuple(self.CONFIG.get(http_section, 'crlfsites').split('|'))
- self.HTTP_FORCEHTTPS = set(self.CONFIG.get(http_section, 'forcehttps').split('|'))
- self.HTTP_FAKEHTTPS = set(self.CONFIG.get(http_section, 'fakehttps').split('|'))
- self.HTTP_DNS = self.CONFIG.get(http_section, 'dns').split('|') if self.CONFIG.has_o…
Large files files are truncated, but you can click here to view the full file