new release for scan ip problem

README.md

@@ -1,37 +1,33 @@
 
-叉叉翻墙
 
 XX-Net
 ========
-最新状态（2015.10.9）： 各地情况不同，启动后需要扫描ip 十分钟到1个小时。  
-  上海电信连接困难。  
-
+翻墙工具套件  
+* GAE proxy, 稳定、快速、易用
+* Web界面，人性化交互
+
 
 下载链接：
 ==========
 测试版：
-https://codeload.github.com/XX-net/XX-Net/zip/2.5.2
+https://codeload.github.com/XX-net/XX-Net/zip/2.5.4
 
 稳定版：
 https://codeload.github.com/XX-net/XX-Net/zip/2.5.1
 
 
-历史版本：https://github.com/XX-net/XX-Net/releases
-   
+使用方法：  
+https://github.com/XX-net/XX-Net/wiki/%E4%BD%BF%E7%94%A8%E6%96%B9%E6%B3%95
 
 
 
-主要特性
-========
-* 集成GoAgent、GoGoTest等自动扫描IP，省事省力
-* Web界面，人性化交互，傻瓜易用
-* 内置了公共 appid, 方便新手
-* 方便易用的细节、细节提示
+
 
 
 平台支持
 ================
 * Windows 7/8/10 （xp 需要 tcpip.sys 补丁, 比如用 tcp-z）
+  Win10 存在声音卡的问题 
 * Linux （Ubuntu不显示系统托盘）
 * Mac OS X(10.7; 10.8; 10.9; 10.10)
 
@@ -40,7 +36,6 @@ https://codeload.github.com/XX-net/XX-Net/zip/2.5.1
 ## 链接
 |   |   |
 | --------   | :----  |
-|使用方法|https://github.com/XX-net/XX-Net/wiki/%E4%BD%BF%E7%94%A8%E6%96%B9%E6%B3%95|
 |问题报告:  |https://github.com/XX-net/XX-Net/issues|
 |讨论群:  |https://groups.google.com/forum/#!forum/xx-net|
 

gae_proxy/local/check_ip.py

@@ -37,6 +37,7 @@
 import ip_utils
 from appids_manager import appid_manager
 
+
 if __name__ == "__main__":
     import xlog
 else:
@@ -55,12 +56,15 @@ def warn(fmt, *args, **kwargs):
         def exception(fmt, *args, **kwargs):
             pass
 
+
 g_cacertfile = os.path.join(current_path, "cacert.pem")
 max_timeout = 5000
 g_conn_timeout = 1
 g_handshake_timeout = 2
 
 default_socket = None
+
+
 def load_sock():
     global default_socket
     if config.PROXY_ENABLE:
@@ -79,6 +83,76 @@ def load_sock():
         default_socket = socket.socket
 load_sock()
 
+
+#####################################
+
+checking_lock = threading.Lock()
+checking_num = 0
+network_ok = True
+last_check_time = 0
+check_network_interval = 100
+
+
+def network_is_ok():
+    global checking_lock, checking_num, network_ok, last_check_time, check_network_interval
+    if time.time() - last_check_time < check_network_interval:
+        return network_ok
+
+    if checking_num > 0:
+        return network_ok
+
+    if config.PROXY_ENABLE:
+        socket.socket = socks.socksocket
+        xlog.debug("patch socks")
+
+    checking_lock.acquire()
+    checking_num += 1
+    checking_lock.release()
+    try:
+        conn = httplib.HTTPSConnection("github.com", 443, timeout=30)
+        header = {"user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
+                  "accept":"application/json, text/javascript, */*; q=0.01",
+                  "accept-encoding":"gzip, deflate, sdch",
+                  "accept-language":'en-US,en;q=0.8,ja;q=0.6,zh-CN;q=0.4,zh;q=0.2',
+                  "connection":"keep-alive"
+                  }
+        conn.request("HEAD", "/", headers=header)
+        response = conn.getresponse()
+        if response.status:
+            xlog.debug("network is ok")
+            network_ok = True
+            last_check_time = time.time()
+            return True
+    except:
+        pass
+    finally:
+        checking_lock.acquire()
+        checking_num -= 1
+        checking_lock.release()
+
+        if config.PROXY_ENABLE:
+            socket.socket = default_socket
+            xlog.debug("restore socket")
+
+    xlog.warn("network fail.")
+    network_ok = False
+    last_check_time = time.time()
+    return False
+
+######################################
+# about ip connect time and handshake time
+# handshake time is double of connect time in common case.
+# after connect and handshaked, http get time is like connect time
+#
+# connect time is zero if you use socks proxy.
+#
+#
+# most case, connect time is 300ms - 600ms.
+# good case is 60ms
+# bad case is 1300ms and more.
+
+
+
 def connect_ssl(ip, port=443, timeout=5, openssl_context=None):
     import struct
     ip_port = (ip, port)
@@ -123,6 +197,7 @@ def connect_ssl(ip, port=443, timeout=5, openssl_context=None):
     ssl_sock.sock = sock
     return ssl_sock, connct_time, handshake_time
 
+
 class Check_result():
     def __init__(self):
         self.domain = ""
@@ -131,6 +206,7 @@ def __init__(self):
         self.connect_time = max_timeout
         self.handshake_time = max_timeout
 
+
 class Check_frame(object):
     def __init__(self, ip, check_cert=True):
         self.result = Check_result()
@@ -244,6 +320,7 @@ def test_app_head(ssl_sock, ip):
     xlog.debug("app check time:%d", time_cost)
     return True
 
+
 def test_app_check(ssl_sock, ip):
     request_data = 'GET /check HTTP/1.1\r\nHost: xxnet-check.appspot.com\r\n\r\n'
     time_start = time.time()
@@ -266,69 +343,21 @@ def test_app_check(ssl_sock, ip):
     xlog.debug("app check time:%d", time_cost)
     return True
 
-checking_lock = threading.Lock()
-checking_num = 0
-network_ok = True
-last_check_time = 0
-check_network_interval = 100
-def network_is_ok():
-    global checking_lock, checking_num, network_ok, last_check_time, check_network_interval
-    if time.time() - last_check_time < check_network_interval:
-        return network_ok
-
-    if checking_num > 0:
-        return network_ok
-
-    if config.PROXY_ENABLE:
-        socket.socket = socks.socksocket
-        xlog.debug("patch socks")
-
-    checking_lock.acquire()
-    checking_num += 1
-    checking_lock.release()
-    try:
-        conn = httplib.HTTPSConnection("github.com", 443, timeout=30)
-        header = {"user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
-                  "accept":"application/json, text/javascript, */*; q=0.01",
-                  "accept-encoding":"gzip, deflate, sdch",
-                  "accept-language":'en-US,en;q=0.8,ja;q=0.6,zh-CN;q=0.4,zh;q=0.2',
-                  "connection":"keep-alive"
-                  }
-        conn.request("HEAD", "/", headers=header)
-        response = conn.getresponse()
-        if response.status:
-            xlog.debug("network is ok")
-            network_ok = True
-            last_check_time = time.time()
-            return True
-    except:
-        pass
-    finally:
-        checking_lock.acquire()
-        checking_num -= 1
-        checking_lock.release()
-
-        if config.PROXY_ENABLE:
-            socket.socket = default_socket
-            xlog.debug("restore socket")
-
-    xlog.warn("network fail.")
-    network_ok = False
-    last_check_time = time.time()
-    return False
-
 def test_gae(ip_str):
     xlog.info("==>%s", ip_str)
     check = Check_frame(ip_str)
 
     result = check.check(callback=test_app_head, check_ca=True)
     if not result:
         return False
+    else:
+        check.result.appspot_ok = result
 
     check.result.server_type = "gws"
 
     return check.result
 
+
 def test_gws(ip_str):
     xlog.info("==>%s", ip_str)
     check = Check_frame(ip_str)
@@ -342,6 +371,7 @@ def test_gws(ip_str):
 
     return check.result
 
+
 def test_gvs(ip_str):
     #logging.info("%s", ip_str)
     check = Check_frame(ip_str, check_cert=False)
@@ -363,6 +393,10 @@ def test_with_app(ip_str):
     xlog.info("test_with_app %s app %s", ip_str, result)
 
 
+#####################################################################################
+# Test function for develop, not working code
+#####################################################################################
+
 
 def test(ip_str, loop=1):
     xlog.info("==>%s", ip_str)
@@ -402,7 +436,7 @@ def test_main():
         ip_str = ip_utils.ip_num_to_string(ip_int)
         test(ip_str, 1)
 
-import threading
+
 class fast_search_ip():
     check_num = 0
     gws_num = 0
@@ -497,6 +531,7 @@ def test_keep_alive(ip_str, interval=5):
     #print result
     xlog.info("result:%r", result)
 
+
 def test_alive(ip_str="74.125.96.107", begin=50, end=60, interval=2):
 
     test_array = {}
@@ -582,23 +617,11 @@ def test2(self):
     #test_gws("216.58.196.176") #gvs
     #result = test_gws("139.175.107.212")
     #print result
-    test('1.255.22.210', 1)
+    result = test_gae('64.15.119.69')
     #test("216.239.38.123")
     #     test_multi_thread_search_ip()
     #check_all_exist_ip()
     #test_gws("74.125.216.36")
     #test = Test_cipher()
     #test.test2()
     pass
-
-# about ip connect time and handshake time
-# handshake time is double of connect time in common case.
-# after connect and handshaked, http get time is like connect time
-#
-# connect time is zero if you use socks proxy.
-#
-#
-# most case, connect time is 300ms - 600ms.
-# good case is 60ms
-# bad case is 1300ms and more.
-

gae_proxy/local/connect_manager.py

@@ -251,8 +251,9 @@ def head_request(self, ssl_sock):
                 response.close()
 
     def keep_alive_worker(self, sock):
+        call_time = time.time()
         if self.head_request(sock):
-            self.save_ssl_connection_for_reuse(sock)
+            self.save_ssl_connection_for_reuse(sock, call_time=call_time)
         else:
             sock.close()
             #self.create_more_connection()
@@ -430,8 +431,7 @@ def verify_SSL_certificate_issuer(ssl_sock):
 
                 issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
                 if not issuer_commonname.startswith('Google'):
-                    #google_ip.report_bad_ip(ssl_sock.ip)
-                    #connect_control.fall_into_honeypot()
+                    google_ip.report_connect_fail(ip, force_remove=True)
                     raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
 
             verify_SSL_certificate_issuer(ssl_sock)

gae_proxy/local/gae_handler.py

@@ -410,12 +410,17 @@ def handler(method, url, headers, body, wfile):
         body_length = end - start + 1
 
         last_read_time = time.time()
+        time_response = time.time()
         while True:
             if start > end:
+                time_finished = time.time()
+                speed = body_length / (time_finished - time_response)
                 response.ssl_sock.received_size += body_length
                 google_ip.report_ip_traffic(response.ssl_sock.ip, body_length)
                 https_manager.save_ssl_connection_for_reuse(response.ssl_sock, call_time=time_request)
-                xlog.info("GAE %d|%s|%d t:%d s:%d %d %s", response.ssl_sock.fd, response.ssl_sock.ip, response.ssl_sock.received_size, (time.time()-time_request)*1000, length, response.status, url)
+                xlog.info("GAE %d|%s|%d t:%d s:%d hs:%d Spd:%d %d %s", 
+                    response.ssl_sock.fd, response.ssl_sock.ip, response.ssl_sock.received_size, (time_finished-time_request)*1000, 
+                    length, response.ssl_sock.handshake_time, int(speed), response.status, url)
                 return
 
             data = response.read(config.AUTORANGE_BUFSIZE)