update connect rule and new release 2.3.0

README.md

@@ -3,13 +3,13 @@
 
 XX-Net
 ========
-项目状态：2015.9.1，各地情况不同，请尝试 2.0.5, 2.1.2, 2.2.0
+项目状态：2015.9.7  根据各地情况不同，启动后需要扫描ip 二十分钟到1个小时。
 
 
 下载链接：
 ==========
 测试版：
-https://codeload.github.com/XX-net/XX-Net/zip/2.1.2
+https://codeload.github.com/XX-net/XX-Net/zip/2.3.0
 
 稳定版：
 https://codeload.github.com/XX-net/XX-Net/zip/2.0.5

gae_proxy/local/check_ip.py

@@ -593,7 +593,7 @@ def test2(self):
     #test_gws("216.58.196.176") #gvs
     #result = test_gws("139.175.107.212")
     #print result
-    test('216.58.208.112', 1)
+    test('118.98.36.203', 1)
     #test("216.239.38.123")
     #     test_multi_thread_search_ip()
     #check_all_exist_ip()

gae_proxy/local/connect_control.py

@@ -131,7 +131,7 @@ def inactive_time():
     global last_request_time
     return time.time() - last_request_time
 
-def is_active(timeout=600):
+def is_active(timeout=60 * 30):
     if inactive_time() < timeout:
         return True
     else:

gae_proxy/local/connect_manager.py

@@ -174,7 +174,7 @@ def __init__(self):
         # http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c
         # openssl s_server -accept 443 -key CA.crt -cert CA.crt
 
-        self.timeout = 2
+        self.timeout = 4
         self.max_timeout = 15
         self.thread_num = 0
 
@@ -241,6 +241,7 @@ def head_request(self, ssl_sock):
                 raise Exception("app check fail")
             return True
         except httplib.BadStatusLine as e:
+            xlog.debug("head request BadStatusLine fail:%r", e)
             return False
         except Exception as e:
             xlog.debug("head request fail:%r", e)
@@ -254,6 +255,7 @@ def keep_alive_worker(self, sock):
             self.save_ssl_connection_for_reuse(sock)
         else:
             sock.close()
+            self.create_more_connection()
 
     def start_keep_alive(self, sock):
         work_thread = threading.Thread(target=self.keep_alive_worker, args=(sock,))
@@ -274,6 +276,7 @@ def keep_alive_thread(self):
                 inactive_time = time.time() - ssl_sock.last_use_time
                 if inactive_time > self.keep_alive:
                     ssl_sock.close()
+                    self.create_more_connection()
                 else:
                     self.start_keep_alive(ssl_sock)
 
@@ -310,7 +313,9 @@ def create_more_connection(self, type="gae"):
         #while self.thread_num < self.max_thread_num and connect_control.keep_running:
         for i in range(0, target_thread_num):
             if not connect_control.allow_connect():
-                break
+                xlog.warn("create more connect, control not allow")
+                time.sleep(10)
+                continue
             if self.thread_num > self.max_thread_num:
                 break
 
@@ -413,9 +418,11 @@ def create_connection_worker(self, type="gae"):
             while connect_control.keep_running:
                 if type == "gae":
                     if (self.new_conn_pool.qsize() + self.gae_conn_pool.qsize()) >= self.connection_pool_min_num:
+                        xlog.debug("get enough conn for gae")
                         break
                 else:
                     if self.new_conn_pool.qsize() >= self.connection_pool_min_num:
+                        xlog.debug("get enough conn for %s", type)
                         break
 
                 ip_str = google_ip.get_gws_ip()
@@ -432,7 +439,8 @@ def create_connection_worker(self, type="gae"):
                     ssl_sock.last_use_time = time.time()
                     self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock))
                 elif not connect_control.allow_connect():
-                    break
+                    xlog.debug("create_connection_worker, control not allow")
+                    time.sleep(10)
                 time.sleep(1)
         finally:
             self.thread_num_lock.acquire()

gae_proxy/local/direct_handler.py

@@ -88,7 +88,7 @@ def handler(method, host, url, headers, body, wfile):
         try:
             response = fetch(method, host, url, headers, body)
             if response:
-                if response.status == 404 or response.status == 400:
+                if response.status > 400:
                     server_type = response.getheader('Server', "")
                     if "gws" not in server_type:
                         xlog.warn("IP:%s not support GAE, server type:%s", response.ssl_sock.ip, server_type)

gae_proxy/local/gae_handler.py

@@ -305,12 +305,12 @@ def handler(method, url, headers, body, wfile):
             if response.app_status != 200:
                 xlog.warn("fetch gae status:%s url:%s", response.app_status, url)
 
-            server_type = response.getheader('Server', "")
-            if "gws" not in server_type:
-                xlog.warn("IP:%s not support GAE, server type:%s", response.ssl_sock.ip, server_type)
-                google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
-                response.close()
-                continue
+                server_type = response.getheader('Server', "")
+                if "gws" not in server_type:
+                    xlog.warn("IP:%s not support GAE, server type:%s", response.ssl_sock.ip, server_type)
+                    google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
+                    response.close()
+                    continue
 
             if response.app_status == 404:
                 xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)

gae_proxy/local/google_ip.py

@@ -257,12 +257,17 @@ def get_gws_ip(self):
                 if time.time() - get_time < self.ip_connect_interval:
                     self.gws_ip_pointer += 1
                     continue
-                handshake_time = self.ip_dict[ip_str]["handshake_time"]
+
+                if time.time() - self.ip_dict[ip_str]['success_time'] > 300: # 5 min
+                    fail_connect_interval = 1800 # 30 min
+                else:
+                    fail_connect_interval = 120 # 2 min
                 fail_time = self.ip_dict[ip_str]["fail_time"]
-                if time.time() - fail_time < 300:
+                if time.time() - fail_time < fail_connect_interval:
                     self.gws_ip_pointer += 1
                     continue
 
+                handshake_time = self.ip_dict[ip_str]["handshake_time"]
                 xlog.debug("get ip:%s t:%d", ip_str, handshake_time)
                 self.ip_dict[ip_str]['history'].append([time.time(), "get"])
                 self.ip_dict[ip_str]['get_time'] = time.time()
@@ -326,7 +331,7 @@ def add_ip(self, ip_str, handshake_time, domain=None, server=None):
 
             self.ip_dict[ip_str] = {'handshake_time':handshake_time, 'domain':domain, 'server':server,
                                     'timeout':0, "history":[[time.time(), handshake_time]], "fail_time":0,
-                                    "get_time":0}
+                                    "success_time":0, "get_time":0}
 
             if 'gws' in server:
                 self.gws_ip_list.append(ip_str)

gae_proxy/local/proxy.ini

@@ -84,24 +84,24 @@ use_ipv6 = 0
 auto_adjust_scan_ip_thread_num = 1
 max_scan_ip_thread_num = 20
 max_good_ip_num = 3000
-ip_connect_interval = 10
+ip_connect_interval = 5
 
 [connect_manager]
-https_max_connect_thread = 5
+https_max_connect_thread = 20
 https_connection_pool_min = 10
-https_connection_pool_max = 60
+https_connection_pool_max = 50
 
 ; keep connection pool until no active request timeout
-keep_active_timeout = 60
+keep_active_timeout = 600
 
 ; the time that HTTPS connection can reuse after last transmit, drop it or send new request to keep it alive
 https_keep_alive = 55
 
-forward_max_connect_thread = 5
+forward_max_connect_thread = 10
 
 ; limit interval for two tcp connect try.
 ; too frequently connect to google ip may make GFW block all google ip for you.
-connect_interval = 200
+connect_interval = 40
 
 [love]
 enable = 0

gae_proxy/local/proxy_handler.py

@@ -149,6 +149,7 @@ def get_crlf(rfile):
         gae_handler.handler(self.command, self.path, request_headers, payload, self.wfile)
 
     def do_CONNECT(self):
+        touch_active()
         host, _, port = self.path.rpartition(':')
 
         if host in config.HOSTS_GAE:
@@ -243,11 +244,13 @@ def do_CONNECT_AGENT(self):
                 self.request_version = ''
                 self.command = ''
                 self.send_error(414)
+                xlog.warn("read request line len:%d", len(self.raw_requestline))
                 return
             if not self.raw_requestline:
-                self.close_connection = 1
+                xlog.warn("read request line empty")
                 return
             if not self.parse_request():
+                xlog.warn("parse request fail:%s", self.raw_requestline)
                 return
         except NetWorkIOError as e:
             if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET, errno.EPIPE):