fix bug: fast lost ip, refactor xlog

gae_proxy/local/appids_manager.py

@@ -6,7 +6,8 @@
 import time
 
 from config import config
-from proxy import xlog
+from xlog import getLogger
+xlog = getLogger("gae_proxy")
 import check_ip
 
 

gae_proxy/local/cert_util.py

@@ -12,7 +12,8 @@
 import threading
 import subprocess
 
-from proxy import xlog
+from xlog import getLogger
+xlog = getLogger("gae_proxy")
 
 current_path = os.path.dirname(os.path.abspath(__file__))
 python_path = os.path.abspath( os.path.join(current_path, os.pardir, os.pardir, 'python27', '1.0'))

gae_proxy/local/check_ip.py

@@ -5,26 +5,42 @@
 import httplib
 import time
 import socket
-import threading
 import struct
-import socks
+import binascii
 
 current_path = os.path.dirname(os.path.abspath(__file__))
 
+if __name__ == "__main__":
+    python_path = os.path.abspath( os.path.join(current_path, os.pardir, os.pardir, 'python27', '1.0'))
+
+    noarch_lib = os.path.abspath( os.path.join(python_path, 'lib', 'noarch'))
+    sys.path.append(noarch_lib)
+
+    if sys.platform == "win32":
+        win32_lib = os.path.abspath( os.path.join(python_path, 'lib', 'win32'))
+        sys.path.append(win32_lib)
+    elif sys.platform.startswith("linux"):
+        linux_lib = os.path.abspath( os.path.join(python_path, 'lib', 'linux'))
+        sys.path.append(linux_lib)
+
 import OpenSSL
 SSLError = OpenSSL.SSL.WantReadError
 
-
+import socks
+import check_local_network
 from config import config
 import cert_util
 from openssl_wrap import SSLConnection
 
-from appids_manager import appid_manager
-from proxy import xlog
+from xlog import getLogger
+xlog = getLogger("gae_proxy")
 
 
 g_cacertfile = os.path.join(current_path, "cacert.pem")
-openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile) # check cacert cost too many cpu, 100 check thread cost 60%.
+openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile)
+openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10)))
+if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'):
+    openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH)
 
 max_timeout = 5
 
@@ -49,137 +65,6 @@ def load_proxy_config():
 load_proxy_config()
 
 
-#####################################
-#  Checking network ok
-
-checking_lock = threading.Lock()
-checking_num = 0
-network_stat = "unknown"
-last_check_time = 0
-continue_fail_count = 0
-
-
-def report_network_ok():
-    global network_stat, last_check_time, continue_fail_count
-    network_stat = "OK"
-    last_check_time = time.time()
-    continue_fail_count = 0
-
-def check_worker():
-    global checking_lock, checking_num, network_stat, last_check_time
-    time_now = time.time()
-    if config.PROXY_ENABLE:
-        socket.socket = socks.socksocket
-        xlog.debug("patch socks")
-
-    checking_lock.acquire()
-    checking_num += 1
-    checking_lock.release()
-    try:
-        conn = httplib.HTTPSConnection("github.com", 443, timeout=30)
-        header = {"user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
-                  "accept":"application/json, text/javascript, */*; q=0.01",
-                  "accept-encoding":"gzip, deflate, sdch",
-                  "accept-language":'en-US,en;q=0.8,ja;q=0.6,zh-CN;q=0.4,zh;q=0.2',
-                  "connection":"keep-alive"
-                  }
-        conn.request("HEAD", "/", headers=header)
-        response = conn.getresponse()
-        if response.status:
-            last_check_time = time.time()
-            report_network_ok()
-            xlog.debug("network is ok, cost:%d ms", 1000*(time.time() - time_now))
-            return True
-    except Exception as e:
-        xlog.warn("network fail:%r", e)
-        network_stat = "Fail"
-        last_check_time = time.time()
-        return False
-    finally:
-        checking_lock.acquire()
-        checking_num -= 1
-        checking_lock.release()
-
-        if config.PROXY_ENABLE:
-            socket.socket = default_socket
-            xlog.debug("restore socket")
-
-
-def simple_check_worker():
-    global checking_lock, checking_num, network_stat, last_check_time
-    time_now = time.time()
-    if config.PROXY_ENABLE:
-        socket.socket = socks.socksocket
-        xlog.debug("patch socks")
-
-    checking_lock.acquire()
-    checking_num += 1
-    checking_lock.release()
-    try:
-        conn = httplib.HTTPConnection("www.baidu.com", 80, timeout=3)
-        header = {"user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
-                  "accept":"application/json, text/javascript, */*; q=0.01",
-                  "accept-encoding":"gzip, deflate, sdch",
-                  "accept-language":'en-US,en;q=0.8,ja;q=0.6,zh-CN;q=0.4,zh;q=0.2',
-                  "connection":"keep-alive"
-                  }
-        conn.request("HEAD", "/", headers=header)
-        response = conn.getresponse()
-        if response.status:
-            last_check_time = time.time()
-            report_network_ok()
-            xlog.debug("network is ok, cost:%d ms", 1000*(time.time() - time_now))
-            return True
-    except Exception as e:
-        xlog.warn("network fail:%r", e)
-        network_stat = "Fail"
-        last_check_time = time.time()
-        return False
-    finally:
-        checking_lock.acquire()
-        checking_num -= 1
-        checking_lock.release()
-
-        if config.PROXY_ENABLE:
-            socket.socket = default_socket
-            xlog.debug("restore socket")
-
-simple_check_worker()
-
-
-def triger_check_network(force=False):
-    global checking_lock, checking_num, network_stat, last_check_time
-    time_now = time.time()
-    if not force:
-        if checking_num > 0:
-            return
-
-        if network_stat == "OK":
-            if time_now - last_check_time < 10:
-                return
-        else:
-            # Fail or unknown
-            if time_now - last_check_time < 3:
-                return
-
-    last_check_time = time_now
-    th = threading.Thread(target=simple_check_worker)
-    th.start()
-
-
-######################################
-# about ip connect time and handshake time
-# handshake time is double of connect time in common case.
-# after connect and handshaked, http get time is like connect time
-#
-# connect time is zero if you use socks proxy.
-#
-#
-# most case, connect time is 300ms - 600ms.
-# good case is 60ms
-# bad case is 1300ms and more.
-
-
 def connect_ssl(ip, port=443, timeout=5, openssl_context=None, check_cert=True):
     ip_port = (ip, port)
 
@@ -193,6 +78,8 @@ def connect_ssl(ip, port=443, timeout=5, openssl_context=None, check_cert=True):
     sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
     # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
     sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
+    # resize socket recv buffer 8K->32K to improve browser releated application performance
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
     sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
     sock.settimeout(timeout)
 
@@ -205,18 +92,19 @@ def connect_ssl(ip, port=443, timeout=5, openssl_context=None, check_cert=True):
     ssl_sock.do_handshake()
     time_handshaked = time.time()
 
-    #report_network_ok
-    global  network_stat, last_check_time, continue_fail_count
-    network_stat = "OK"
-    last_check_time = time_handshaked
-    continue_fail_count = 0
+    # report network ok
+    check_local_network.network_stat = "OK"
+    check_local_network.last_check_time = time_handshaked
+    check_local_network.continue_fail_count = 0
 
     cert = ssl_sock.get_peer_certificate()
     if not cert:
         raise socket.error(' certficate is none')
 
     if check_cert:
         issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
+        if __name__ == "__main__":
+            xlog.debug("issued by:%s", issuer_commonname)
         if not issuer_commonname.startswith('Google'):
             raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
 
@@ -225,7 +113,7 @@ def connect_ssl(ip, port=443, timeout=5, openssl_context=None, check_cert=True):
     #xlog.debug("conn: %d  handshake:%d", connct_time, handshake_time)
 
     # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
-    ssl_sock.sock = sock
+    ssl_sock._sock = sock
     ssl_sock.connct_time = connct_time
     ssl_sock.handshake_time = handshake_time
 
@@ -239,37 +127,47 @@ def get_ssl_cert_domain(ssl_sock):
 
     #issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
     ssl_cert = cert_util.SSLCert(cert)
-    #xlog.info("%s CN:%s", ip, ssl_cert.cn)
+    if __name__ == "__main__":
+        xlog.info("%s CN:%s", ip, ssl_cert.cn)
     ssl_sock.domain = ssl_cert.cn
 
 
-def check_appid(ssl_sock, appid, ip):
+def check_goagent(ssl_sock, appid):
     request_data = 'GET /_gh/ HTTP/1.1\r\nHost: %s.appspot.com\r\n\r\n' % appid
     ssl_sock.send(request_data.encode())
     response = httplib.HTTPResponse(ssl_sock, buffering=True)
 
     response.begin()
     if response.status == 404:
-        #xlog.warn("app check %s status:%d", appid, response.status)
+        if __name__ == "__main__":
+            xlog.warn("app check %s status:%d", appid, response.status)
         return False
 
     if response.status == 503:
         # out of quota
         server_type = response.getheader('Server', "")
         if "gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type:
+            if __name__ == "__main__":
+                xlog.warn("503 but server type:%s", server_type)
             return False
         else:
+            if __name__ == "__main__":
+                xlog.info("503 server type:%s", server_type)
             return True
 
     if response.status != 200:
-        #xlog.warn("app check %s ip:%s status:%d", appid, ip, response.status)
+        if __name__ == "__main__":
+            xlog.warn("app check %s ip:%s status:%d", appid, ip, response.status)
         return False
 
     content = response.read()
     if "GoAgent" not in content:
-        #xlog.warn("app check %s content:%s", appid, content)
+        if __name__ == "__main__":
+            xlog.warn("app check %s content:%s", appid, content)
         return False
 
+    if __name__ == "__main__":
+        xlog.info("check_goagent ok")
     return True
 
 
@@ -280,42 +178,25 @@ def test_gae_ip(ip, appid=None):
         get_ssl_cert_domain(ssl_sock)
 
         if not appid:
-            appid = appid_manager.get_appid()
-        if appid.startswith("xxnet-"):
-            appid = "xxnet-check"
-        if not check_appid(ssl_sock, appid, ip):
+            appid = "xxnet-1"
+        if not check_goagent(ssl_sock, appid):
             return False
 
         return ssl_sock
-    except Exception as e:
-        #xlog.exception("test_gae_ip %s e:%r",ip, e)
+    except socket.timeout:
+        if __name__ == "__main__":
+            xlog.warn("connect timeout")
         return False
-
-#===========================================
-
-
-def check_ipv6_host(host):
-    try:
-        conn = httplib.HTTPConnection(host, 80, timeout=5)
-        header = {"user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
-                  "accept":"application/json, text/javascript, */*; q=0.01",
-                  "accept-encoding":"gzip, deflate, sdch",
-                  "accept-language":'en-US,en;q=0.8,ja;q=0.6,zh-CN;q=0.4,zh;q=0.2',
-                  "connection":"keep-alive"
-                  }
-        conn.request("HEAD", "/", headers=header)
-        response = conn.getresponse()
-        if response.status:
-            return True
-        else:
-            return False
     except Exception as e:
+        xlog.exception("test_gae_ip %s e:%r",ip, e)
         return False
 
 
-def check_ipv6():
-    hosts = ["www.6rank.edu.cn", "v6.testmyipv6.com", ]
-    for host in hosts:
-        if check_ipv6_host(host):
-            return True
-    return False
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        ip = sys.argv[1]
+        xlog.info("test ip:%s", ip)
+        res = test_gae_ip(ip)
+        print res
+    else:
+        xlog.info("check_ip <ip>")