PageRenderTime 100ms CodeModel.GetById 5ms app.highlight 81ms RepoModel.GetById 1ms app.codeStats 1ms

/assets/goagent.py

https://github.com/yourmoonlight/gaeproxy
Python | 2417 lines | 2293 code | 65 blank | 59 comment | 84 complexity | 2ee13670943693f3914cc4607be18fb3 MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1#!/usr/bin/env python
   2# coding:utf-8
   3# Based on GAppProxy 2.0.0 by Du XiaoGang <dugang.2008@gmail.com>
   4# Based on WallProxy 0.4.0 by Hust Moon <www.ehust@gmail.com>
   5# Contributor:
   6#      Phus Lu           <phus.lu@gmail.com>
   7#      Hewig Xu          <hewigovens@gmail.com>
   8#      Ayanamist Yang    <ayanamist@gmail.com>
   9#      V.E.O             <V.E.O@tom.com>
  10#      Max Lv            <max.c.lv@gmail.com>
  11#      AlsoTang          <alsotang@gmail.com>
  12#      Christopher Meng  <i@cicku.me>
  13#      Yonsm Guo         <YonsmGuo@gmail.com>
  14#      Parkman           <cseparkman@gmail.com>
  15#      Ming Bai          <mbbill@gmail.com>
  16#      Bin Yu            <yubinlove1991@gmail.com>
  17#      lileixuan         <lileixuan@gmail.com>
  18#      Cong Ding         <cong@cding.org>
  19#      Zhang Youfu       <zhangyoufu@gmail.com>
  20#      Lu Wei            <luwei@barfoo>
  21#      Harmony Meow      <harmony.meow@gmail.com>
  22#      logostream        <logostream@gmail.com>
  23#      Rui Wang          <isnowfy@gmail.com>
  24#      Wang Wei Qiang    <wwqgtxx@gmail.com>
  25#      Felix Yan         <felixonmars@gmail.com>
  26#      Sui Feng          <suifeng.me@qq.com>
  27#      QXO               <qxodream@gmail.com>
  28#      Geek An           <geekan@foxmail.com>
  29#      Poly Rabbit       <mcx_221@foxmail.com>
  30#      oxnz              <yunxinyi@gmail.com>
  31#      Shusen Liu        <liushusen.smart@gmail.com>
  32#      Yad Smood         <y.s.inside@gmail.com>
  33#      Chen Shuang       <cs0x7f@gmail.com>
  34#      cnfuyu            <cnfuyu@gmail.com>
  35#      cuixin            <steven.cuixin@gmail.com>
  36#      s2marine0         <s2marine0@gmail.com>
  37#      Toshio Xiang      <snachx@gmail.com>
  38
  39__version__ = '3.1.5'
  40
  41import sys
  42import os
  43import glob
  44
  45# GAEProxy Patch
  46# The sys path in Android is set up outside.
  47
  48try:
  49    import gevent
  50    import gevent.socket
  51    import gevent.server
  52    import gevent.queue
  53    import gevent.monkey
  54    gevent.monkey.patch_all(subprocess=True)
  55except ImportError:
  56    gevent = None
  57except TypeError:
  58    gevent.monkey.patch_all()
  59    sys.stderr.write('\033[31m  Warning: Please update gevent to the latest 1.0 version!\033[0m\n')
  60
  61import errno
  62import binascii
  63import time
  64import struct
  65import collections
  66import zlib
  67import functools
  68import itertools
  69import re
  70import io
  71import fnmatch
  72import traceback
  73import random
  74import subprocess
  75import base64
  76import string
  77import hashlib
  78import threading
  79import thread
  80import socket
  81import ssl
  82import select
  83import Queue
  84import SocketServer
  85import ConfigParser
  86import BaseHTTPServer
  87import httplib
  88import urllib2
  89import urlparse
  90try:
  91    import dnslib
  92except ImportError:
  93    dnslib = None
  94try:
  95    import OpenSSL
  96except ImportError:
  97    OpenSSL = None
  98try:
  99    import pacparser
 100except ImportError:
 101    pacparser = None
 102
 103# GAEProxy Patch
 104class NullDevice():
 105    def write(self, s):
 106        pass
 107
 108sys.stdout = NullDevice()
 109sys.stderr = sys.stdout
 110
 111class DNSCacheUtil(object):
 112    '''DNSCache module, integrated with GAEProxy'''
 113
 114    cache = {"127.0.0.1": 'localhost'}
 115
 116    @staticmethod
 117    def getHost(address):
 118
 119        p = "(?:\d{1,3}\.){3}\d{1,3}"
 120
 121        if re.match(p, address) is None:
 122            return
 123
 124        if address in DNSCacheUtil.cache:
 125            return DNSCacheUtil.cache[address]
 126
 127        host = None
 128
 129        sock = None
 130        address_family = socket.AF_INET
 131        retry = 0
 132        while address not in DNSCacheUtil.cache:
 133            try:
 134                sock = socket.socket(family=address_family, type=socket.SOCK_STREAM)
 135                sock.settimeout(2)
 136                sock.connect(("127.0.0.1", 9090))
 137                sock.sendall(address + "\r\n")
 138                host = sock.recv(512)
 139                if host is not None and not host.startswith("null"):
 140                    host = host.strip()
 141                    DNSCacheUtil.cache[address] = host
 142                    break
 143                else:
 144                    if retry > 3:
 145                        host = None
 146                        break
 147                    else:
 148                        retry = retry + 1
 149                        continue
 150            except socket.error as e:
 151                if e[0] in (10060, 'timed out'):
 152                    continue
 153            except Exception, e:
 154                logging.error('reverse dns query exception: %s', e)
 155                break
 156            finally:
 157                if sock:
 158                    sock.close()
 159
 160        return host
 161
 162HAS_PYPY = hasattr(sys, 'pypy_version_info')
 163NetWorkIOError = (socket.error, ssl.SSLError, OSError) if not OpenSSL else (socket.error, ssl.SSLError, OpenSSL.SSL.Error, OSError)
 164
 165
 166class Logging(type(sys)):
 167    CRITICAL = 50
 168    FATAL = CRITICAL
 169    ERROR = 40
 170    WARNING = 30
 171    WARN = WARNING
 172    INFO = 20
 173    DEBUG = 10
 174    NOTSET = 0
 175
 176    def __init__(self, *args, **kwargs):
 177        self.level = self.__class__.INFO
 178        #GAEProxy Patch
 179
 180    @classmethod
 181    def getLogger(cls, *args, **kwargs):
 182        return cls(*args, **kwargs)
 183
 184    def basicConfig(self, *args, **kwargs):
 185        self.level = int(kwargs.get('level', self.__class__.INFO))
 186        if self.level > self.__class__.DEBUG:
 187            self.debug = self.dummy
 188
 189    def log(self, level, fmt, *args, **kwargs):
 190        sys.stderr.write('%s - [%s] %s\n' % (level, time.ctime()[4:-5], fmt % args))
 191
 192    def dummy(self, *args, **kwargs):
 193        pass
 194
 195    def debug(self, fmt, *args, **kwargs):
 196        self.log('DEBUG', fmt, *args, **kwargs)
 197
 198    def info(self, fmt, *args, **kwargs):
 199        self.log('INFO', fmt, *args)
 200
 201    def warning(self, fmt, *args, **kwargs):
 202        self.log('WARNING', fmt, *args, **kwargs)
 203
 204    def warn(self, fmt, *args, **kwargs):
 205        self.warning(fmt, *args, **kwargs)
 206
 207    def error(self, fmt, *args, **kwargs):
 208        self.log('ERROR', fmt, *args, **kwargs)
 209
 210    def exception(self, fmt, *args, **kwargs):
 211        self.error(fmt, *args, **kwargs)
 212        sys.stderr.write(traceback.format_exc() + '\n')
 213
 214    def critical(self, fmt, *args, **kwargs):
 215        self.log('CRITICAL', fmt, *args, **kwargs)
 216logging = sys.modules['logging'] = Logging('logging')
 217
 218
 219class LRUCache(object):
 220    """http://pypi.python.org/pypi/lru/"""
 221
 222    def __init__(self, max_items=100):
 223        self.cache = {}
 224        self.key_order = []
 225        self.max_items = max_items
 226
 227    def __setitem__(self, key, value):
 228        self.cache[key] = value
 229        self._mark(key)
 230
 231    def __getitem__(self, key):
 232        value = self.cache[key]
 233        self._mark(key)
 234        return value
 235
 236    def _mark(self, key):
 237        if key in self.key_order:
 238            self.key_order.remove(key)
 239        self.key_order.insert(0, key)
 240        if len(self.key_order) > self.max_items:
 241            remove = self.key_order[self.max_items]
 242            del self.cache[remove]
 243            self.key_order.pop(self.max_items)
 244
 245    def clear(self):
 246        self.cache = {}
 247        self.key_order = []
 248
 249
 250class CertUtil(object):
 251    """CertUtil module, based on mitmproxy"""
 252
 253    ca_vendor = 'GoAgent'
 254    ca_keyfile = 'CA.crt'
 255    ca_certdir = 'certs'
 256    ca_lock = threading.Lock()
 257
 258    @staticmethod
 259    def create_ca():
 260        key = OpenSSL.crypto.PKey()
 261        key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
 262        ca = OpenSSL.crypto.X509()
 263        ca.set_serial_number(0)
 264        ca.set_version(2)
 265        subj = ca.get_subject()
 266        subj.countryName = 'CN'
 267        subj.stateOrProvinceName = 'Internet'
 268        subj.localityName = 'Cernet'
 269        subj.organizationName = CertUtil.ca_vendor
 270        subj.organizationalUnitName = '%s Root' % CertUtil.ca_vendor
 271        subj.commonName = '%s CA' % CertUtil.ca_vendor
 272        ca.gmtime_adj_notBefore(0)
 273        ca.gmtime_adj_notAfter(24 * 60 * 60 * 3652)
 274        ca.set_issuer(ca.get_subject())
 275        ca.set_pubkey(key)
 276        ca.add_extensions([
 277            OpenSSL.crypto.X509Extension(b'basicConstraints', True, b'CA:TRUE'),
 278            OpenSSL.crypto.X509Extension(b'nsCertType', True, b'sslCA'),
 279            OpenSSL.crypto.X509Extension(b'extendedKeyUsage', True, b'serverAuth,clientAuth,emailProtection,timeStamping,msCodeInd,msCodeCom,msCTLSign,msSGC,msEFS,nsSGC'),
 280            OpenSSL.crypto.X509Extension(b'keyUsage', False, b'keyCertSign, cRLSign'),
 281            OpenSSL.crypto.X509Extension(b'subjectKeyIdentifier', False, b'hash', subject=ca), ])
 282        ca.sign(key, 'sha1')
 283        return key, ca
 284
 285    @staticmethod
 286    def dump_ca():
 287        key, ca = CertUtil.create_ca()
 288        with open(CertUtil.ca_keyfile, 'wb') as fp:
 289            fp.write(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, ca))
 290            fp.write(OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key))
 291
 292    @staticmethod
 293    def _get_cert(commonname, sans=()):
 294        with open(CertUtil.ca_keyfile, 'rb') as fp:
 295            content = fp.read()
 296            key = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, content)
 297            ca = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, content)
 298
 299        pkey = OpenSSL.crypto.PKey()
 300        pkey.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
 301
 302        req = OpenSSL.crypto.X509Req()
 303        subj = req.get_subject()
 304        subj.countryName = 'CN'
 305        subj.stateOrProvinceName = 'Internet'
 306        subj.localityName = 'Cernet'
 307        subj.organizationalUnitName = '%s Branch' % CertUtil.ca_vendor
 308        if commonname[0] == '.':
 309            subj.commonName = '*' + commonname
 310            subj.organizationName = '*' + commonname
 311            sans = ['*'+commonname] + [x for x in sans if x != '*'+commonname]
 312        else:
 313            subj.commonName = commonname
 314            subj.organizationName = commonname
 315            sans = [commonname] + [x for x in sans if x != commonname]
 316        # GAEProxy Patch
 317        req.add_extensions([OpenSSL.crypto.X509Extension(b'subjectAltName', True, ', '.join('DNS: %s' % x for x in sans))])
 318        req.set_pubkey(pkey)
 319        req.sign(pkey, 'sha1')
 320
 321        cert = OpenSSL.crypto.X509()
 322        # GAEProxy Patch
 323        cert.set_version(3)
 324        try:
 325            cert.set_serial_number(int(hashlib.md5(commonname.encode('utf-8')).hexdigest(), 16))
 326        except OpenSSL.SSL.Error:
 327            cert.set_serial_number(int(time.time()*1000))
 328        cert.gmtime_adj_notBefore(0)
 329        cert.gmtime_adj_notAfter(60 * 60 * 24 * 3652)
 330        cert.set_issuer(ca.get_subject())
 331        cert.set_subject(req.get_subject())
 332        cert.set_pubkey(req.get_pubkey())
 333        if commonname[0] == '.':
 334            sans = ['*'+commonname] + [s for s in sans if s != '*'+commonname]
 335        else:
 336            sans = [commonname] + [s for s in sans if s != commonname]
 337        # GAEProxy Patch
 338        cert.add_extensions([OpenSSL.crypto.X509Extension(b'subjectAltName', True, ', '.join('DNS: %s' % x for x in sans))])
 339        cert.sign(key, 'sha1')
 340
 341        certfile = os.path.join(CertUtil.ca_certdir, commonname + '.crt')
 342        with open(certfile, 'wb') as fp:
 343            fp.write(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert))
 344            fp.write(OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, pkey))
 345        return certfile
 346
 347    @staticmethod
 348    def get_cert(commonname, sans=()):
 349        # GAEProxy Patch
 350        sans = ["*.akamaihd.net","*.fbcdn.net","*.google.com","*.appspot.com","*.googleapis.com","*.googlevideo.com","*.twitter.com","*.facebook.com","*.whatsapp.net"]
 351        if commonname.count('.') >= 2 and [len(x) for x in reversed(commonname.split('.'))] > [2, 4]:
 352            commonname = '.'+commonname.partition('.')[-1]
 353        certfile = os.path.join(CertUtil.ca_certdir, commonname + '.crt')
 354        if os.path.exists(certfile):
 355            return certfile
 356        elif OpenSSL is None:
 357            return CertUtil.ca_keyfile
 358        else:
 359            with CertUtil.ca_lock:
 360                if os.path.exists(certfile):
 361                    return certfile
 362                return CertUtil._get_cert(commonname, sans)
 363
 364    @staticmethod
 365    def import_ca(certfile):
 366        commonname = os.path.splitext(os.path.basename(certfile))[0]
 367        if OpenSSL:
 368            try:
 369                with open(certfile, 'rb') as fp:
 370                    x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fp.read())
 371                    commonname = next(v.decode() for k, v in x509.get_subject().get_components() if k == b'O')
 372            except Exception as e:
 373                logging.error('load_certificate(certfile=%r) failed:%s', certfile, e)
 374        # GAEProxy Patch
 375        return 0
 376
 377    @staticmethod
 378    def check_ca():
 379        #Check CA exists
 380        capath = os.path.join(os.path.dirname(os.path.abspath(__file__)), CertUtil.ca_keyfile)
 381        certdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), CertUtil.ca_certdir)
 382        if not os.path.exists(capath):
 383            if not OpenSSL:
 384                logging.critical('CA.key is not exist and OpenSSL is disabled, ABORT!')
 385                sys.exit(-1)
 386            if os.path.exists(certdir):
 387                if os.path.isdir(certdir):
 388                    any(os.remove(x) for x in glob.glob(certdir+'/*.crt')+glob.glob(certdir+'/.*.crt'))
 389                else:
 390                    os.remove(certdir)
 391                    os.mkdir(certdir)
 392            CertUtil.dump_ca()
 393        if glob.glob('%s/*.key' % CertUtil.ca_certdir):
 394            for filename in glob.glob('%s/*.key' % CertUtil.ca_certdir):
 395                try:
 396                    os.remove(filename)
 397                    os.remove(os.path.splitext(filename)[0]+'.crt')
 398                except EnvironmentError:
 399                    pass
 400        #Check CA imported
 401        if CertUtil.import_ca(capath) != 0:
 402            logging.warning('install root certificate failed, Please run as administrator/root/sudo')
 403        #Check Certs Dir
 404        if not os.path.exists(certdir):
 405            os.makedirs(certdir)
 406
 407
 408class SSLConnection(object):
 409
 410    has_gevent = socket.socket is getattr(sys.modules.get('gevent.socket'), 'socket', None)
 411
 412    def __init__(self, context, sock):
 413        self._context = context
 414        self._sock = sock
 415        self._connection = OpenSSL.SSL.Connection(context, sock)
 416        self._makefile_refs = 0
 417        if self.has_gevent:
 418            self._wait_read = gevent.socket.wait_read
 419            self._wait_write = gevent.socket.wait_write
 420            self._wait_readwrite = gevent.socket.wait_readwrite
 421        else:
 422            self._wait_read = lambda fd,t: select.select([fd], [], [fd], t)
 423            self._wait_write = lambda fd,t: select.select([], [fd], [fd], t)
 424            self._wait_readwrite = lambda fd,t: select.select([fd], [fd], [fd], t)
 425
 426    def __getattr__(self, attr):
 427        if attr not in ('_context', '_sock', '_connection', '_makefile_refs'):
 428            return getattr(self._connection, attr)
 429
 430    def accept(self):
 431        sock, addr = self._sock.accept()
 432        client = OpenSSL.SSL.Connection(sock._context, sock)
 433        return client, addr
 434
 435    def do_handshake(self):
 436        timeout = self._sock.gettimeout()
 437        while True:
 438            try:
 439                self._connection.do_handshake()
 440                break
 441            except (OpenSSL.SSL.WantReadError, OpenSSL.SSL.WantX509LookupError, OpenSSL.SSL.WantWriteError):
 442                sys.exc_clear()
 443                self._wait_readwrite(self._sock.fileno(), timeout)
 444
 445    def connect(self, *args, **kwargs):
 446        timeout = self._sock.gettimeout()
 447        while True:
 448            try:
 449                self._connection.connect(*args, **kwargs)
 450                break
 451            except (OpenSSL.SSL.WantReadError, OpenSSL.SSL.WantX509LookupError):
 452                sys.exc_clear()
 453                self._wait_read(self._sock.fileno(), timeout)
 454            except OpenSSL.SSL.WantWriteError:
 455                sys.exc_clear()
 456                self._wait_write(self._sock.fileno(), timeout)
 457
 458    def send(self, data, flags=0):
 459        timeout = self._sock.gettimeout()
 460        while True:
 461            try:
 462                self._connection.send(data, flags)
 463                break
 464            except (OpenSSL.SSL.WantReadError, OpenSSL.SSL.WantX509LookupError):
 465                sys.exc_clear()
 466                self._wait_read(self._sock.fileno(), timeout)
 467            except OpenSSL.SSL.WantWriteError:
 468                sys.exc_clear()
 469                self._wait_write(self._sock.fileno(), timeout)
 470            except OpenSSL.SSL.SysCallError as e:
 471                if e[0] == -1 and not data:
 472                    # errors when writing empty strings are expected and can be ignored
 473                    return 0
 474                raise
 475
 476    def recv(self, bufsiz, flags=0):
 477        timeout = self._sock.gettimeout()
 478        pending = self._connection.pending()
 479        if pending:
 480            return self._connection.recv(min(pending, bufsiz))
 481        while True:
 482            try:
 483                return self._connection.recv(bufsiz, flags)
 484            except (OpenSSL.SSL.WantReadError, OpenSSL.SSL.WantX509LookupError):
 485                sys.exc_clear()
 486                self._wait_read(self._sock.fileno(), timeout)
 487            except OpenSSL.SSL.WantWriteError:
 488                sys.exc_clear()
 489                self._wait_write(self._sock.fileno(), timeout)
 490            except OpenSSL.SSL.ZeroReturnError:
 491                return ''
 492
 493    def read(self, bufsiz, flags=0):
 494        return self.recv(bufsiz, flags)
 495
 496    def write(self, buf, flags=0):
 497        return self.sendall(buf, flags)
 498
 499    def close(self):
 500        if self._makefile_refs < 1:
 501            self._connection = None
 502            if self._sock:
 503                socket.socket.close(self._sock)
 504        else:
 505            self._makefile_refs -= 1
 506
 507    def makefile(self, mode='r', bufsize=-1):
 508        self._makefile_refs += 1
 509        return socket._fileobject(self, mode, bufsize, close=True)
 510
 511
 512
 513class ProxyUtil(object):
 514    """ProxyUtil module, based on urllib2"""
 515
 516    @staticmethod
 517    def parse_proxy(proxy):
 518        return urllib2._parse_proxy(proxy)
 519
 520    @staticmethod
 521    def get_system_proxy():
 522        proxies = urllib2.getproxies()
 523        return proxies.get('https') or proxies.get('http') or {}
 524
 525    @staticmethod
 526    def get_listen_ip():
 527        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
 528        sock.connect(('8.8.8.8', 53))
 529        listen_ip = sock.getsockname()[0]
 530        sock.close()
 531        return listen_ip
 532
 533# GAEProxy Patch
 534# No PAC
 535
 536def dns_remote_resolve(qname, dnsservers, blacklist, timeout):
 537    """
 538    http://gfwrev.blogspot.com/2009/11/gfwdns.html
 539    http://zh.wikipedia.org/wiki/域名服务器缓存污染
 540    http://support.microsoft.com/kb/241352
 541    """
 542    query = dnslib.DNSRecord(q=dnslib.DNSQuestion(qname))
 543    query_data = query.pack()
 544    dns_v4_servers = [x for x in dnsservers if ':' not in x]
 545    dns_v6_servers = [x for x in dnsservers if ':' in x]
 546    sock_v4 = sock_v6 = None
 547    socks = []
 548    if dns_v4_servers:
 549        sock_v4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
 550        socks.append(sock_v4)
 551    if dns_v6_servers:
 552        sock_v6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
 553        socks.append(sock_v6)
 554    timeout_at = time.time() + timeout
 555    try:
 556        for _ in xrange(2):
 557            try:
 558                for dnsserver in dns_v4_servers:
 559                    sock_v4.sendto(query_data, (dnsserver, 53))
 560                for dnsserver in dns_v6_servers:
 561                    sock_v6.sendto(query_data, (dnsserver, 53))
 562                while time.time() < timeout_at:
 563                    ins, _, _ = select.select(socks, [], [], 0.1)
 564                    for sock in ins:
 565                        reply_data, _ = sock.recvfrom(512)
 566                        reply = dnslib.DNSRecord.parse(reply_data)
 567                        rtypes = (1, 28) if sock is sock_v6 else (1,)
 568                        iplist = [str(x.rdata) for x in reply.rr if x.rtype in rtypes]
 569                        if any(x in blacklist for x in iplist):
 570                            logging.warning('query qname=%r reply bad iplist=%r', qname, iplist)
 571                        else:
 572                            logging.debug('query qname=%r reply iplist=%s', qname, iplist)
 573                            return iplist
 574            except socket.error as e:
 575                logging.warning('handle dns query=%s socket: %r', query, e)
 576    finally:
 577        for sock in socks:
 578            sock.close()
 579
 580
 581def get_dnsserver_list():
 582    if os.name == 'nt':
 583        import ctypes, ctypes.wintypes, struct, socket
 584        DNS_CONFIG_DNS_SERVER_LIST = 6
 585        buf = ctypes.create_string_buffer(2048)
 586        ctypes.windll.dnsapi.DnsQueryConfig(DNS_CONFIG_DNS_SERVER_LIST, 0, None, None, ctypes.byref(buf), ctypes.byref(ctypes.wintypes.DWORD(len(buf))))
 587        ips = struct.unpack('I', buf[0:4])[0]
 588        out = []
 589        for i in xrange(ips):
 590            start = (i+1) * 4
 591            out.append(socket.inet_ntoa(buf[start:start+4]))
 592        return out
 593    elif os.path.isfile('/etc/resolv.conf'):
 594        with open('/etc/resolv.conf', 'rb') as fp:
 595            return re.findall(r'(?m)^nameserver\s+(\S+)', fp.read())
 596    else:
 597        logging.warning("get_dnsserver_list failed: unsupport platform '%s-%s'", sys.platform, os.name)
 598        return []
 599
 600
 601def spawn_later(seconds, target, *args, **kwargs):
 602    def wrap(*args, **kwargs):
 603        __import__('time').sleep(seconds)
 604        return target(*args, **kwargs)
 605    return __import__('thread').start_new_thread(wrap, args, kwargs)
 606
 607
 608class HTTPUtil(object):
 609    """HTTP Request Class"""
 610
 611    MessageClass = dict
 612    protocol_version = 'HTTP/1.1'
 613    skip_headers = frozenset(['Vary', 'Via', 'X-Forwarded-For', 'Proxy-Authorization', 'Proxy-Connection', 'Upgrade', 'X-Chrome-Variations', 'Connection', 'Cache-Control'])
 614    ssl_validate = False
 615    ssl_obfuscate = False
 616    ssl_ciphers = ':'.join(['ECDHE-ECDSA-AES256-SHA',
 617                            'ECDHE-RSA-AES256-SHA',
 618                            'DHE-RSA-CAMELLIA256-SHA',
 619                            'DHE-DSS-CAMELLIA256-SHA',
 620                            'DHE-RSA-AES256-SHA',
 621                            'DHE-DSS-AES256-SHA',
 622                            'ECDH-RSA-AES256-SHA',
 623                            'ECDH-ECDSA-AES256-SHA',
 624                            'CAMELLIA256-SHA',
 625                            'AES256-SHA',
 626                            'ECDHE-ECDSA-RC4-SHA',
 627                            'ECDHE-ECDSA-AES128-SHA',
 628                            'ECDHE-RSA-RC4-SHA',
 629                            'ECDHE-RSA-AES128-SHA',
 630                            'DHE-RSA-CAMELLIA128-SHA',
 631                            'DHE-DSS-CAMELLIA128-SHA',
 632                            'DHE-RSA-AES128-SHA',
 633                            'DHE-DSS-AES128-SHA',
 634                            'ECDH-RSA-RC4-SHA',
 635                            'ECDH-RSA-AES128-SHA',
 636                            'ECDH-ECDSA-RC4-SHA',
 637                            'ECDH-ECDSA-AES128-SHA',
 638                            'SEED-SHA',
 639                            'CAMELLIA128-SHA',
 640                            'RC4-SHA',
 641                            'RC4-MD5',
 642                            'AES128-SHA',
 643                            'ECDHE-ECDSA-DES-CBC3-SHA',
 644                            'ECDHE-RSA-DES-CBC3-SHA',
 645                            'EDH-RSA-DES-CBC3-SHA',
 646                            'EDH-DSS-DES-CBC3-SHA',
 647                            'ECDH-RSA-DES-CBC3-SHA',
 648                            'ECDH-ECDSA-DES-CBC3-SHA',
 649                            'DES-CBC3-SHA',
 650                            'TLS_EMPTY_RENEGOTIATION_INFO_SCSV'])
 651
 652    def __init__(self, max_window=4, max_timeout=8, max_retry=4, proxy='', dns_servers=[], dns_blacklist=set()):
 653        # http://docs.python.org/dev/library/ssl.html
 654        # http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html
 655        # http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c
 656        # http://www.openssl.org/docs/apps/ciphers.html
 657        # openssl s_server -accept 443 -key CA.crt -cert CA.crt
 658        # set_ciphers as Modern Browsers
 659        self.max_window = max_window
 660        self.max_retry = max_retry
 661        self.max_timeout = max_timeout
 662        self.tcp_connection_time = collections.defaultdict(float)
 663        self.tcp_connection_cache = collections.defaultdict(Queue.PriorityQueue)
 664        self.ssl_connection_time = collections.defaultdict(float)
 665        self.ssl_connection_cache = collections.defaultdict(Queue.PriorityQueue)
 666        self.dns = {}
 667        self.proxy = proxy
 668        self.openssl_context = None
 669        if self.proxy:
 670            self.dns_resolve = self.__dns_resolve_withproxy
 671            self.create_connection = self.__create_connection_withproxy
 672            self.create_ssl_connection = self.__create_ssl_connection_withproxy
 673        self.dns_servers = dns_servers
 674        self.dns_blacklist = dns_blacklist
 675
 676    def set_openssl_option(self, validate=True, obfuscate=True):
 677        if self.openssl_context is None:
 678            self.openssl_context = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD)
 679            self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10)))
 680            if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'):
 681                self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH)
 682        if validate:
 683            self.openssl_context.load_verify_locations(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cacert.pem'))
 684            self.openssl_context.set_verify(OpenSSL.SSL.VERIFY_PEER, lambda c, x, e, d, ok: ok)
 685        if obfuscate:
 686            ssl_ciphers = ':'.join(x for x in self.ssl_ciphers.split(':') if random.random() > 0.5)
 687            self.openssl_context.set_cipher_list(ssl_ciphers)
 688
 689    def dns_resolve(self, host, dnsservers=[], ipv4_only=True):
 690        iplist = self.dns.get(host)
 691        if not iplist:
 692            if not dnsservers:
 693                iplist = list(set(socket.gethostbyname_ex(host)[-1]) - self.dns_blacklist)
 694            else:
 695                iplist = dns_remote_resolve(host, dnsservers, self.dns_blacklist, timeout=2)
 696            if not iplist:
 697                iplist = dns_remote_resolve(host, self.dns_servers, self.dns_blacklist, timeout=2)
 698            if ipv4_only:
 699                iplist = [ip for ip in iplist if re.match(r'\d+\.\d+\.\d+\.\d+', ip)]
 700            self.dns[host] = iplist = list(set(iplist))
 701        return iplist
 702
 703    def __dns_resolve_withproxy(self, host, dnsservers=[], ipv4_only=True):
 704        return [host]
 705
 706    def create_connection(self, address, timeout=None, source_address=None, **kwargs):
 707        connection_cache_key = kwargs.get('cache_key')
 708        def _create_connection(ipaddr, timeout, queobj):
 709            sock = None
 710            try:
 711                # create a ipv4/ipv6 socket object
 712                sock = socket.socket(socket.AF_INET if ':' not in ipaddr[0] else socket.AF_INET6)
 713                # set reuseaddr option to avoid 10048 socket error
 714                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
 715                # resize socket recv buffer 8K->32K to improve browser releated application performance
 716                sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
 717                # disable nagle algorithm to send http request quickly.
 718                sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
 719                # set a short timeout to trigger timeout retry more quickly.
 720                sock.settimeout(timeout or self.max_timeout)
 721                # start connection time record
 722                start_time = time.time()
 723                # TCP connect
 724                sock.connect(ipaddr)
 725                # record TCP connection time
 726                self.tcp_connection_time[ipaddr] = time.time() - start_time
 727                # put ssl socket object to output queobj
 728                queobj.put(sock)
 729            except (socket.error, OSError) as e:
 730                # any socket.error, put Excpetions to output queobj.
 731                queobj.put(e)
 732                # reset a large and random timeout to the ipaddr
 733                self.tcp_connection_time[ipaddr] = self.max_timeout+random.random()
 734                # close tcp socket
 735                if sock:
 736                    sock.close()
 737        def _close_connection(count, queobj):
 738            for i in range(count):
 739                sock = queobj.get()
 740                if sock and not isinstance(sock, Exception):
 741                    if connection_cache_key and i == 0:
 742                        self.tcp_connection_cache[connection_cache_key].put((time.time(), sock))
 743                    else:
 744                        sock.close()
 745        try:
 746            while connection_cache_key:
 747                ctime, sock = self.tcp_connection_cache[connection_cache_key].get_nowait()
 748                if time.time() - ctime < 30:
 749                    return sock
 750        except Queue.Empty:
 751            pass
 752        host, port = address
 753        result = None
 754        addresses = [(x, port) for x in self.dns_resolve(host)]
 755        if port == 443:
 756            get_connection_time = lambda addr: self.ssl_connection_time.__getitem__(addr) or self.tcp_connection_time.__getitem__(addr)
 757        else:
 758            get_connection_time = self.tcp_connection_time.__getitem__
 759        for i in range(self.max_retry):
 760            window = min((self.max_window+1)//2 + min(i, 1), len(addresses))
 761            addresses.sort(key=get_connection_time)
 762            addrs = addresses[:window] + random.sample(addresses, min(len(addresses), window, self.max_window-window))
 763            queobj = Queue.Queue()
 764            for addr in addrs:
 765                thread.start_new_thread(_create_connection, (addr, timeout, queobj))
 766            for i in range(len(addrs)):
 767                result = queobj.get()
 768                if not isinstance(result, (socket.error, OSError)):
 769                    thread.start_new_thread(_close_connection, (len(addrs)-i-1, queobj))
 770                    return result
 771                else:
 772                    if i == 0:
 773                        # only output first error
 774                        logging.warning('create_connection to %s return %r, try again.', addrs, result)
 775
 776    def create_ssl_connection(self, address, timeout=None, source_address=None, **kwargs):
 777        connection_cache_key = kwargs.get('cache_key')
 778        validate = kwargs.get('validate')
 779        def _create_ssl_connection(ipaddr, timeout, queobj):
 780            sock = None
 781            ssl_sock = None
 782            try:
 783                # create a ipv4/ipv6 socket object
 784                sock = socket.socket(socket.AF_INET if ':' not in ipaddr[0] else socket.AF_INET6)
 785                # set reuseaddr option to avoid 10048 socket error
 786                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
 787                # resize socket recv buffer 8K->32K to improve browser releated application performance
 788                sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
 789                # disable negal algorithm to send http request quickly.
 790                sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
 791                # set a short timeout to trigger timeout retry more quickly.
 792                sock.settimeout(timeout or self.max_timeout)
 793                # pick up the certificate
 794                if not validate:
 795                    ssl_sock = ssl.wrap_socket(sock, do_handshake_on_connect=False)
 796                else:
 797                    ssl_sock = ssl.wrap_socket(sock, cert_reqs=ssl.CERT_REQUIRED, ca_certs=os.path.join(os.path.dirname(os.path.abspath(__file__)),'cacert.pem'), do_handshake_on_connect=False)
 798                ssl_sock.settimeout(timeout or self.max_timeout)
 799                # start connection time record
 800                start_time = time.time()
 801                # TCP connect
 802                ssl_sock.connect(ipaddr)
 803                connected_time = time.time()
 804                # SSL handshake
 805                ssl_sock.do_handshake()
 806                handshaked_time = time.time()
 807                # record TCP connection time
 808                self.tcp_connection_time[ipaddr] = ssl_sock.tcp_time = connected_time - start_time
 809                # record SSL connection time
 810                self.ssl_connection_time[ipaddr] = ssl_sock.ssl_time = handshaked_time - start_time
 811                ssl_sock.ssl_time = connected_time - start_time
 812                # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
 813                ssl_sock.sock = sock
 814                # verify SSL certificate.
 815                if validate and address[0].endswith('.appspot.com'):
 816                    cert = ssl_sock.getpeercert()
 817                    orgname = next((v for ((k, v),) in cert['subject'] if k == 'organizationName'))
 818                    if not orgname.lower().startswith('google '):
 819                        raise ssl.SSLError("%r certificate organizationName(%r) not startswith 'Google'" % (address[0], orgname))
 820                # put ssl socket object to output queobj
 821                queobj.put(ssl_sock)
 822            except (socket.error, ssl.SSLError, OSError) as e:
 823                # any socket.error, put Excpetions to output queobj.
 824                queobj.put(e)
 825                # reset a large and random timeout to the ipaddr
 826                self.ssl_connection_time[ipaddr] = self.max_timeout + random.random()
 827                # close ssl socket
 828                if ssl_sock:
 829                    ssl_sock.close()
 830                # close tcp socket
 831                if sock:
 832                    sock.close()
 833        def _create_openssl_connection(ipaddr, timeout, queobj):
 834            sock = None
 835            ssl_sock = None
 836            try:
 837                # create a ipv4/ipv6 socket object
 838                sock = socket.socket(socket.AF_INET if ':' not in ipaddr[0] else socket.AF_INET6)
 839                # set reuseaddr option to avoid 10048 socket error
 840                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
 841                # resize socket recv buffer 8K->32K to improve browser releated application performance
 842                sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
 843                # disable negal algorithm to send http request quickly.
 844                sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
 845                # set a short timeout to trigger timeout retry more quickly.
 846                sock.settimeout(timeout or self.max_timeout)
 847                # pick up the certificate
 848                server_hostname = b'www.google.com' if address[0].endswith('.appspot.com') else None
 849                ssl_sock = SSLConnection(self.openssl_context, sock)
 850                ssl_sock.set_connect_state()
 851                if server_hostname:
 852                    ssl_sock.set_tlsext_host_name(server_hostname)
 853                # start connection time record
 854                start_time = time.time()
 855                # TCP connect
 856                ssl_sock.connect(ipaddr)
 857                connected_time = time.time()
 858                # SSL handshake
 859                ssl_sock.do_handshake()
 860                handshaked_time = time.time()
 861                # record TCP connection time
 862                self.tcp_connection_time[ipaddr] = ssl_sock.tcp_time = connected_time - start_time
 863                # record SSL connection time
 864                self.ssl_connection_time[ipaddr] = ssl_sock.ssl_time = handshaked_time - start_time
 865                # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
 866                ssl_sock.sock = sock
 867                # verify SSL certificate.
 868                if validate and address[0].endswith('.appspot.com'):
 869                    cert = ssl_sock.get_peer_certificate()
 870                    commonname = next((v for k, v in cert.get_subject().get_components() if k == 'CN'))
 871                    if '.google' not in commonname and not commonname.endswith('.appspot.com'):
 872                        raise socket.error("Host name '%s' doesn't match certificate host '%s'" % (address[0], commonname))
 873                # put ssl socket object to output queobj
 874                queobj.put(ssl_sock)
 875            except (socket.error, OpenSSL.SSL.Error, OSError) as e:
 876                # any socket.error, put Excpetions to output queobj.
 877                queobj.put(e)
 878                # reset a large and random timeout to the ipaddr
 879                self.ssl_connection_time[ipaddr] = self.max_timeout + random.random()
 880                # close ssl socket
 881                if ssl_sock:
 882                    ssl_sock.close()
 883                # close tcp socket
 884                if sock:
 885                    sock.close()
 886        def _close_ssl_connection(count, queobj, first_tcp_time, first_ssl_time):
 887            for i in range(count):
 888                sock = queobj.get()
 889                ssl_time_threshold = min(1, 1.5 * first_ssl_time)
 890                if sock and not isinstance(sock, Exception):
 891                    if connection_cache_key and sock.ssl_time < ssl_time_threshold:
 892                        self.ssl_connection_cache[connection_cache_key].put((time.time(), sock))
 893                    else:
 894                        sock.close()
 895        try:
 896            while connection_cache_key:
 897                ctime, sock = self.ssl_connection_cache[connection_cache_key].get_nowait()
 898                if time.time() - ctime < 30:
 899                    return sock
 900        except Queue.Empty:
 901            pass
 902        host, port = address
 903        result = None
 904        # create_connection = _create_ssl_connection if not validate else _create_openssl_connection
 905        create_connection = _create_ssl_connection
 906        addresses = [(x, port) for x in self.dns_resolve(host)]
 907        for i in range(self.max_retry):
 908            window = min((self.max_window+1)//2 + min(i, 1), len(addresses))
 909            addresses.sort(key=self.ssl_connection_time.__getitem__)
 910            addrs = addresses[:window] + random.sample(addresses, min(len(addresses), window, self.max_window-window))
 911            queobj = Queue.Queue()
 912            for addr in addrs:
 913                thread.start_new_thread(create_connection, (addr, timeout, queobj))
 914            for i in range(len(addrs)):
 915                result = queobj.get()
 916                if not isinstance(result, Exception):
 917                    thread.start_new_thread(_close_ssl_connection, (len(addrs)-i-1, queobj, result.tcp_time, result.ssl_time))
 918                    return result
 919                else:
 920                    if i == 0:
 921                        # only output first error
 922                        logging.warning('create_ssl_connection to %s return %r, try again.', addrs, result)
 923
 924    def __create_connection_withproxy(self, address, timeout=None, source_address=None, **kwargs):
 925        host, port = address
 926        logging.debug('__create_connection_withproxy connect (%r, %r)', host, port)
 927        _, proxyuser, proxypass, proxyaddress = ProxyUtil.parse_proxy(self.proxy)
 928        try:
 929            try:
 930                self.dns_resolve(host)
 931            except (socket.error, OSError):
 932                pass
 933            proxyhost, _, proxyport = proxyaddress.rpartition(':')
 934            sock = socket.create_connection((proxyhost, int(proxyport)))
 935            if host in self.dns:
 936                hostname = random.choice(self.dns[host])
 937            elif host.endswith('.appspot.com'):
 938                hostname = 'www.google.com'
 939            else:
 940                hostname = host
 941            request_data = 'CONNECT %s:%s HTTP/1.1\r\n' % (hostname, port)
 942            if proxyuser and proxypass:
 943                request_data += 'Proxy-authorization: Basic %s\r\n' % base64.b64encode(('%s:%s' % (proxyuser, proxypass)).encode()).decode().strip()
 944            request_data += '\r\n'
 945            sock.sendall(request_data)
 946            response = httplib.HTTPResponse(sock)
 947            response.begin()
 948            if response.status >= 400:
 949                logging.error('__create_connection_withproxy return http error code %s', response.status)
 950                sock = None
 951            return sock
 952        except Exception as e:
 953            logging.error('__create_connection_withproxy error %s', e)
 954            raise
 955
 956    def __create_ssl_connection_withproxy(self, address, timeout=None, source_address=None, **kwargs):
 957        host, port = address
 958        logging.debug('__create_ssl_connection_withproxy connect (%r, %r)', host, port)
 959        try:
 960            sock = self.__create_connection_withproxy(address, timeout, source_address)
 961            ssl_sock = ssl.wrap_socket(sock)
 962            ssl_sock.sock = sock
 963            return ssl_sock
 964        except Exception as e:
 965            logging.error('__create_ssl_connection_withproxy error %s', e)
 966            raise
 967
 968    def forward_socket(self, local, remote, timeout=60, tick=2, bufsize=8192, maxping=None, maxpong=None):
 969        try:
 970            timecount = timeout
 971            while 1:
 972                timecount -= tick
 973                if timecount <= 0:
 974                    break
 975                (ins, _, errors) = select.select([local, remote], [], [local, remote], tick)
 976                if errors:
 977                    break
 978                if ins:
 979                    for sock in ins:
 980                        data = sock.recv(bufsize)
 981                        if data:
 982                            if sock is remote:
 983                                local.sendall(data)
 984                                timecount = maxpong or timeout
 985                            else:
 986                                remote.sendall(data)
 987                                timecount = maxping or timeout
 988                        else:
 989                            return
 990        except NetWorkIOError as e:
 991            if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET, errno.ENOTCONN, errno.EPIPE):
 992                raise
 993        finally:
 994            if local:
 995                local.close()
 996            if remote:
 997                remote.close()
 998
 999    def green_forward_socket(self, local, remote, timeout=60, tick=2, bufsize=8192, maxping=None, maxpong=None, pongcallback=None, bitmask=None):
1000        def io_copy(dest, source):
1001            try:
1002                dest.settimeout(timeout)
1003                source.settimeout(timeout)
1004                while 1:
1005                    data = source.recv(bufsize)
1006                    if not data:
1007                        break
1008                    if bitmask:
1009                        data = ''.join(chr(ord(x) ^ bitmask) for x in data)
1010                    dest.sendall(data)
1011            except NetWorkIOError as e:
1012                if e.args[0] not in ('timed out', errno.ECONNABORTED, errno.ECONNRESET, errno.EBADF, errno.EPIPE, errno.ENOTCONN, errno.ETIMEDOUT):
1013                    raise
1014            finally:
1015                if local:
1016                    local.close()
1017                if remote:
1018                    remote.close()
1019        thread.start_new_thread(io_copy, (remote.dup(), local.dup()))
1020        io_copy(local, remote)
1021
1022    def _request(self, sock, method, path, protocol_version, headers, payload, bufsize=8192, crlf=None, return_sock=None):
1023        skip_headers = self.skip_headers
1024        need_crlf = bool(crlf)
1025        if need_crlf:
1026            fakehost = 'www.' + ''.join(random.choice(('bcdfghjklmnpqrstvwxyz','aeiou')[x&1]) for x in xrange(random.randint(5,20))) + random.choice(['.net', '.com', '.org'])
1027            request_data = 'GET / HTTP/1.1\r\nHost: %s\r\n\r\n\r\n\r\r' % fakehost
1028        else:
1029            request_data = ''
1030        request_data += '%s %s %s\r\n' % (method, path, protocol_version)
1031        request_data += ''.join('%s: %s\r\n' % (k.title(), v) for k, v in headers.items() if k.title() not in skip_headers)
1032        if self.proxy:
1033            _, username, password, _ = ProxyUtil.parse_proxy(self.proxy)
1034            if username and password:
1035                request_data += 'Proxy-Authorization: Basic %s\r\n' % base64.b64encode(('%s:%s' % (username, password)).encode()).decode().strip()
1036        request_data += '\r\n'
1037
1038        if isinstance(payload, bytes):
1039            sock.sendall(request_data.encode() + payload)
1040        elif hasattr(payload, 'read'):
1041            sock.sendall(request_data)
1042            while 1:
1043                data = payload.read(bufsize)
1044                if not data:
1045                    break
1046                sock.sendall(data)
1047        else:
1048            raise TypeError('http_util.request(payload) must be a string or buffer, not %r' % type(payload))
1049
1050        if need_crlf:
1051            try:
1052                response = httplib.HTTPResponse(sock)
1053                response.begin()
1054                response.read()
1055            except Exception:
1056                logging.exception('crlf skip read')
1057                return None
1058
1059        if return_sock:
1060            return sock
1061
1062        response = httplib.HTTPResponse(sock, buffering=True)
1063        try:
1064            response.begin()
1065        except httplib.BadStatusLine:
1066            response = None
1067        return response
1068
1069    def request(self, method, url, payload=None, headers={}, realhost='', bufsize=8192, crlf=None, validate=None, return_sock=None, connection_cache_key=None):
1070        scheme, netloc, path, _, query, _ = urlparse.urlparse(url)
1071        if netloc.rfind(':') <= netloc.rfind(']'):
1072            # no port number
1073            host = netloc
1074            port = 443 if scheme == 'https' else 80
1075        else:
1076            host, _, port = netloc.rpartition(':')
1077            port = int(port)
1078        if query:
1079            path += '?' + query
1080
1081        if 'Host' not in headers:
1082            headers['Host'] = host
1083        if payload and 'Content-Length' not in headers:
1084            headers['Content-Length'] = str(len(payload))
1085
1086        for i in range(self.max_retry):
1087            sock = None
1088            ssl_sock = None
1089            try:
1090                if scheme == 'https':
1091                    ssl_sock = self.create_ssl_connection((realhost or host, port), self.max_timeout, validate=validate, cache_key=connection_cache_key)
1092                    if ssl_sock:
1093                        sock = ssl_sock.sock
1094                        del ssl_sock.sock
1095                    else:
1096                        raise socket.error('timed out', 'create_ssl_connection(%r,%r)' % (realhost or host, port))
1097                else:
1098                    sock = self.create_connection((realhost or host, port), self.max_timeout, cache_key=connection_cache_key)
1099                if sock:
1100                    if scheme == 'https':
1101                        crlf = 0
1102                    return self._request(ssl_sock or sock, method, path, self.protocol_version, headers, payload, bufsize=bufsize, crlf=crlf, return_sock=return_sock)
1103            except Exception as e:
1104                logging.debug('request "%s %s" failed:%s', method, url, e)
1105                if ssl_sock:
1106                    ssl_sock.close()
1107                if sock:
1108                    sock.close()
1109                if i == self.max_retry - 1:
1110                    raise
1111                else:
1112                    continue
1113
1114
1115class Common(object):
1116    """Global Config Object"""
1117
1118    ENV_CONFIG_PREFIX = 'GOAGENT_'
1119
1120    def __init__(self):
1121        """load config from proxy.ini"""
1122        ConfigParser.RawConfigParser.OPTCRE = re.compile(r'(?P<option>[^=\s][^=]*)\s*(?P<vi>[=])\s*(?P<value>.*)$')
1123        self.CONFIG = ConfigParser.ConfigParser()
1124        # GAEProxy Patch
1125        self.CONFIG_FILENAME = '/data/data/org.gaeproxy/proxy.ini'
1126        self.CONFIG.read(self.CONFIG_FILENAME)
1127
1128        self.LISTEN_IP = self.CONFIG.get('listen', 'ip')
1129        self.LISTEN_PORT = self.CONFIG.getint('listen', 'port')
1130        self.LISTEN_VISIBLE = self.CONFIG.getint('listen', 'visible')
1131        self.LISTEN_DEBUGINFO = self.CONFIG.getint('listen', 'debuginfo')
1132
1133        self.GAE_APPIDS = re.findall(r'[\w\-\.]+', self.CONFIG.get('gae', 'appid').replace('.appspot.com', ''))
1134        self.GAE_PASSWORD = self.CONFIG.get('gae', 'password').strip()
1135        self.GAE_PATH = self.CONFIG.get('gae', 'path')
1136        self.GAE_MODE = self.CONFIG.get('gae', 'mode')
1137        self.GAE_PROFILE = self.CONFIG.get('gae', 'profile').strip()
1138        self.GAE_WINDOW = self.CONFIG.getint('gae', 'window')
1139        self.GAE_VALIDATE = self.CONFIG.getint('gae', 'validate')
1140        self.GAE_OBFUSCATE = self.CONFIG.getint('gae', 'obfuscate')
1141        self.GAE_OPTIONS = self.CONFIG.get('gae', 'options')
1142
1143        hosts_section, http_section = '%s/hosts' % self.GAE_PROFILE, '%s/http' % self.GAE_PROFILE
1144        self.HOSTS_MAP = collections.OrderedDict((k, v or k) for k, v in self.CONFIG.items(hosts_section) if '\\' not in k and ':' not in k and not k.startswith('.'))
1145        self.HOSTS_POSTFIX_MAP = collections.OrderedDict((k, v) for k, v in self.CONFIG.items(hosts_section) if '\\' not in k and ':' not in k and k.startswith('.'))
1146        self.HOSTS_POSTFIX_ENDSWITH = tuple(self.HOSTS_POSTFIX_MAP)
1147
1148        self.CONNECT_HOSTS_MAP = collections.OrderedDict((k, v) for k, v in self.CONFIG.items(hosts_section) if ':' in k and not k.startswith('.'))
1149        self.CONNECT_POSTFIX_MAP = collections.OrderedDict((k, v) for k, v in self.CONFIG.items(hosts_section) if ':' in k and k.startswith('.'))
1150        self.CONNECT_POSTFIX_ENDSWITH = tuple(self.CONNECT_POSTFIX_MAP)
1151
1152        self.METHOD_REMATCH_MAP = collections.OrderedDict((re.compile(k).match, v) for k, v in self.CONFIG.items(hosts_section) if '\\' in k)
1153        self.METHOD_REMATCH_HAS_LOCALFILE = any(x.startswith('file://') for x in self.METHOD_REMATCH_MAP.values())
1154
1155        self.HTTP_WITHGAE = tuple(self.CONFIG.get(http_section, 'withgae').split('|'))
1156        self.HTTP_CRLFSITES = tuple(self.CONFIG.get(http_section, 'crlfsites').split('|'))
1157        self.HTTP_FORCEHTTPS = set(self.CONFIG.get(http_section, 'forcehttps').split('|'))
1158        self.HTTP_FAKEHTTPS = set(self.CONFIG.get(http_section, 'fakehttps').split('|'))
1159        self.HTTP_DNS = self.CONFIG.get(http_section, 'dns').split('|') if self.CONFIG.has_o

Large files files are truncated, but you can click here to view the full file