1.4.0

README.md

@@ -5,7 +5,7 @@
 
 下载链接：
 ==========
-https://codeload.github.com/XX-net/XX-Net/zip/1.3.6
+https://codeload.github.com/XX-net/XX-Net/zip/1.4.0
 
 版本历史： https://github.com/XX-net/XX-Net/releases
 
@@ -16,14 +16,15 @@ https://codeload.github.com/XX-net/XX-Net/zip/1.3.6
 * GoAgent 稳定快速
 * Web界面，傻瓜易用
 * 内置了公共 appid, 上千可用ip, 开箱即用
+* 自动导入证书
 
 平台支持情况
 ================
 * Windows XP （需要 tcpip.sys 补丁, 比如用 tcp-z）
 * Win7/8/10
 * Ubuntu （不显示系统托盘）
 * Debian
-* Mac OS X （不显示系统托盘）
+* Mac OS X
 
 ## 链接
 |   |   |
@@ -40,14 +41,16 @@ https://codeload.github.com/XX-net/XX-Net/zip/1.3.6
   - Win7/8/10：提示请求管理员权限, 安装CA证书。请点击同意。
   - 第一次启动, 会提示在桌面建立快捷方式,可根据自己需要选择。
   - 推荐用Chrome浏览器, 安装SwichySharp, 可在swichysharp目录下找到插件和配置文件
+  - Firefox 需手动导入证书 data/goagent/CA.crt 启动后生成
 * Linux下, 执行 start.sh
+  - 自动导入证书，需安装 libnss3-tools 包
   - 第一次启动, 请用sudo ./start.sh, 以安装CA证书
   - 配置http代理 localhost 8087, 勾选全部协议使用这个代理。
 * Mac下，双击 start.command
-  - 请导入data/goagent/CA.crt证书
+  - 会自动导入证书，如果还有提示非安全连接，请手动导入data/goagent/CA.crt证书
 * 服务端
   - 协议采用3.1的版本，请重新部署服务端，新版服务端兼容3.2的客户端
-  - 虽然系统内置了公共appid, 还是建议部署自己的appid
+  - 虽然系统内置了公共appid, 还是建议部署自己的appid，公共appid限制看视频
 
 感谢
 =========

data/launcher/config.yaml

@@ -1,5 +1,5 @@
 modules:
-  goagent: {auto_start: 1, current_version: 3.1.40, ignore_version: 3.1.38}
-  launcher: {current_version: 1.0.8, ignore_version: 1.0.5}
+  goagent: {auto_start: 1, current_version: 3.1.41, ignore_version: 3.1.38}
+  launcher: {current_version: 1.0.9, ignore_version: 1.0.5}
 update: {check_update: 1, last_path: /, node_id: '0',
   uuid: 0}

goagent/3.1.40/local/cert_util.py → goagent/3.1.41/local/cert_util.py

@@ -10,6 +10,7 @@
 import base64
 import hashlib
 import threading
+import subprocess
 
 import logging
 
@@ -29,6 +30,9 @@
     elif sys.platform == "linux" or sys.platform == "linux2":
         linux_lib = os.path.abspath( os.path.join(python_path, 'lib', 'linux'))
         sys.path.append(linux_lib)
+    elif sys.platform == "darwin":
+        darwin_lib = os.path.abspath( os.path.join(python_path, 'lib', 'darwin'))
+        sys.path.append(darwin_lib)
 
 import OpenSSL
 
@@ -154,7 +158,7 @@ def create_ca():
         subj.localityName = 'Cernet'
         subj.organizationName = CertUtil.ca_vendor
         subj.organizationalUnitName = '%s Root' % CertUtil.ca_vendor
-        subj.commonName = '%s CA' % CertUtil.ca_vendor
+        subj.commonName = '%s XX-Net' % CertUtil.ca_vendor
         req.set_pubkey(key)
         req.sign(key, 'sha1')
         ca = OpenSSL.crypto.X509()
@@ -417,13 +421,45 @@ def file_is_same(file1, file2):
         else:
             return True
 
+
+
+    @staticmethod
+    def import_mac_ca(common_name, certfile):
+        commonname = "GoAgent XX-Net"
+        ca_hash = CertUtil.ca_thumbprint.replace(':', '')
+
+        def get_exist_ca_sha1():
+            args = ['security', 'find-certificate', '-Z', '-a', '-c', commonname]
+            output = subprocess.check_output(args)
+            for line in output.splitlines(True):
+                if len(line) == 53 and line.startswith("SHA-1 hash:"):
+                    sha1_hash = line[12:52]
+                    return sha1_hash
+
+        exist_ca_sha1 = get_exist_ca_sha1()
+        if exist_ca_sha1 == ca_hash:
+            logging.info("GoAgent CA exist")
+            return
+
+        import_command = 'security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ../../../data/goagent/CA.crt'# % certfile.decode('utf-8')
+        if exist_ca_sha1:
+            delete_ca_command = 'security delete-certificate -Z %s' % exist_ca_sha1
+            exec_command = "%s;%s" % (delete_ca_command, import_command)
+        else:
+            exec_command = import_command
+
+        admin_command = """osascript -e 'do shell script "%s" with administrator privileges' """ % exec_command
+        cmd = admin_command.encode('utf-8')
+        logging.info("try auto import CA command:%s", cmd)
+        os.system(cmd)
+
     @staticmethod
     def import_ca(certfile):
-        commonname = "GoAgent CA - GoAgent"
+        commonname = "GoAgent XX-Net - GoAgent"
         if sys.platform.startswith('win'):
             CertUtil.import_windows_ca(commonname, certfile)
         elif sys.platform == 'darwin':
-            os.system(('security find-certificate -a -c "%s" | grep "%s" >/dev/null || security add-trusted-cert -d -r trustRoot -k "/Library/Keychains/System.keychain" "%s"' % (commonname, commonname, certfile.decode('utf-8'))).encode('utf-8'))
+            CertUtil.import_mac_ca(commonname, certfile)
         elif sys.platform.startswith('linux'):
             CertUtil.import_debian_ca(commonname, certfile)
             CertUtil.import_linux_firefox_ca(commonname, certfile)
@@ -468,6 +504,15 @@ def test_del_ca():
     cmd_line = 'certutil -L -d sql:$HOME/.pki/nssdb |grep "GoAgent" ||certutil -d sql:$HOME/.pki/nssdb -D -n "%s" ' % ( commonname)
     os.system(cmd_line)
 
+def test_cmd():
+    cmd = "pwd"
+    cmd = ['security', 'find-certificate', '-Z', '-a', '-c', 'GoAgent']
+    #cmd = ['security', 'find-certificate',]
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+    out = proc.communicate()
+    print out[0]
+
 if __name__ == '__main__':
     CertUtil.init_ca()
     #test_del_ca()
+    #test_cmd()

goagent/3.1.40/local/check_ip.py → goagent/3.1.41/local/check_ip.py

@@ -19,6 +19,7 @@
 import httplib
 import time
 import socket
+import threading
 
 current_path = os.path.dirname(os.path.abspath(__file__))
 
@@ -220,12 +221,22 @@ def test_app_check(ssl_sock, ip):
     logging.debug("app check time:%d", time_cost)
     return True
 
+checking_lock = threading.Lock()
+checking_num = 0
 def network_is_ok():
+    global checking_lock, checking_num
+    if checking_num > 0:
+        return False
+
     if config.PROXY_ENABLE:
         socket.socket = socks.socksocket
         logging.debug("patch socks")
+
+    checking_lock.acquire()
+    checking_num += 1
+    checking_lock.release()
     try:
-        conn = httplib.HTTPSConnection("github.com", 443)
+        conn = httplib.HTTPSConnection("www.baidu.com", 443, timeout=3)
         header = {"user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
                   "accept":"application/json, text/javascript, */*; q=0.01",
                   "accept-encoding":"gzip, deflate, sdch",
@@ -239,10 +250,15 @@ def network_is_ok():
     except:
         pass
     finally:
+        checking_lock.acquire()
+        checking_num -= 1
+        checking_lock.release()
+
         if config.PROXY_ENABLE:
             socket.socket = default_socket
             logging.debug("restore socket")
 
+    logging.warn("network check to github fail.")
     return False
 
 def test_gws(ip_str):
@@ -385,11 +401,11 @@ def check_all_exist_ip():
 if __name__ == "__main__":
     #print network_is_ok()
     #print network_is_ok()
-    #test("208.117.224.103", 10) #gws
+    test("216.58.220.86", 10) #gws
     #test('208.117.224.213', 10)
-    #test("218.176.242.24")
+    test("64.233.189.166")
     #test_main()
-    check_all_exist_ip()
+    #check_all_exist_ip()
 
 
 # about ip connect time and handshake time

goagent/3.1.40/local/connect_manager.py → goagent/3.1.41/local/connect_manager.py

@@ -148,15 +148,15 @@ def __init__(self):
         # openssl s_server -accept 443 -key CA.crt -cert CA.crt
 
         self.max_retry = 3
-        self.timeout = 3
+        self.timeout = 1.5
         self.max_timeout = 5
         self.thread_num = 0
         self.max_thread_num = config.CONFIG.getint("connect_manager", "https_max_connect_thread") #10
         self.connection_pool_num = config.CONFIG.getint("connect_manager", "https_connection_pool") #20/30
 
         self.conn_pool = Connect_pool() #Queue.PriorityQueue()
 
-        self.openssl_context = SSLConnection.context_builder(ssl_version="TLSv1")
+        self.openssl_context = SSLConnection.context_builder(ssl_version="TLSv1") #, ca_certs=g_cacertfile)
 
         # ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html
         self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10)))
@@ -171,9 +171,9 @@ def save_ssl_connection_for_reuse(self, ssl_sock):
             t, ssl_sock = self.conn_pool.get_slowest()
 
             if t < 500:
-                #self.conn_pool.put( (ssl_sock.handshake_time, ssl_sock) )
-                ssl_sock.close()
-                return
+                self.conn_pool.put( (ssl_sock.handshake_time, ssl_sock) )
+                #ssl_sock.close()
+                #return
             else:
                 ssl_sock.close()
 
@@ -284,7 +284,7 @@ def create_more_connection():
                 p = threading.Thread(target = connect_thread)
                 p.daemon = True
                 p.start()
-                time.sleep(0.1)
+                time.sleep(0.3)
 
 
         while True:
@@ -295,7 +295,7 @@ def create_more_connection():
                 ssl_sock = None
                 break
 
-            if time.time() - ssl_sock.last_use_time < 230: # gws ssl connection can keep for 230s after created
+            if time.time() - ssl_sock.last_use_time < 210: # gws ssl connection can keep for 230s after created
                 logging.debug("ssl_pool.get:%s handshake:%d", ssl_sock.ip, handshake_time)
                 break
             else:
@@ -335,7 +335,8 @@ def create_connection(self, host="", port=443, sock_life=5):
             logging.warn("forward port %d not supported.", port)
             return None
 
-        def _create_connection(ip_port):
+        def _create_connection(ip_port, delay=0):
+            time.sleep(delay)
             ip = ip_port[0]
             sock = None
             # start connection time record

goagent/3.1.40/local/google_ip.py → goagent/3.1.41/local/google_ip.py

@@ -294,8 +294,10 @@ def update_ip(self, ip_str, handshake_time):
     def report_connect_fail(self, ip_str, force_remove=False):
         # ignore if system network is disconnected.
         if time.time() - self.network_fail_time < 3:
+            #logging.debug("report_connect_fail network fail recently")
             return
         if not force_remove and not self.network_is_ok():
+            #logging.debug("report_connect_fail network fail")
             return
 
         self.ip_lock.acquire()

goagent/3.1.40/local/proxy.ini → goagent/3.1.41/local/proxy.ini

@@ -91,8 +91,8 @@ port = 8084
 
 [google_ip]
 max_check_ip_thread_num = 5
-max_good_ip_num = 4000
-ip_connect_interval = 5
+max_good_ip_num = 20000
+ip_connect_interval = 10
 
 [connect_manager]
 https_max_connect_thread = 10

goagent/3.1.40/local/proxy.py → goagent/3.1.41/local/proxy.py

@@ -51,7 +51,11 @@
 elif sys.platform == "linux" or sys.platform == "linux2":
     win32_lib = os.path.abspath( os.path.join(python_path, 'lib', 'linux'))
     sys.path.append(win32_lib)
-
+elif sys.platform == "darwin":
+    darwin_lib = os.path.abspath( os.path.join(python_path, 'lib', 'darwin'))
+    sys.path.append(darwin_lib)
+    extra_lib = "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python"
+    sys.path.append(extra_lib)
 
 import time
 import traceback

goagent/3.1.40/local/proxy_handler.py → goagent/3.1.41/local/proxy_handler.py

@@ -458,14 +458,6 @@ def do_AGENT(self):
                     else:
                         continue
 
-                # public gae appid deny this site request.
-                if response.app_status == 403:
-                    logging.warn("public appid deny this host:%s", host)
-                    html = generate_message_html('403 public Appid deny this host proxy', u'共用appid因为资源有限，限制观看视频和文件下载等消耗资源过多的访问，请使用自己的appid')
-                    self.wfile.write(b'HTTP/1.0 403\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8'))
-                    response.close()
-                    return
-
                 # appid not exists, try remove it from appid
                 if response.app_status == 404:
                     logging.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
@@ -496,21 +488,21 @@ def do_AGENT(self):
 
                 # 500 is Web server internal err
                 if response.app_status == 500 and range_in_query and special_range:
-
                     logging.warning('500 with range in query') #, need trying another APPID?
                     response.close()
                     continue
 
                 if response.app_status == 501:
                     deploy_url = "http://127.0.0.1:8085/?module=goagent&menu=deploy"
                     message = u'请重新部署服务端: <a href="%s">%s</a>' % (deploy_url, deploy_url)
-                    html = generate_message_html('Please deploy new server', message)
+                    html = generate_message_html('Please deploy your new server', message)
                     self.wfile.write(b'HTTP/1.0 501\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8'))
+                    logging.warning('501 Please deploy your new server') #, need trying another APPID?
                     response.close()
                     return
 
                 if response.app_status != 200 and retry == config.FETCHMAX_LOCAL-1:
-                    logging.info('GAE %s %s status:%s', self.command, self.path, response.status)
+                    logging.warn('GAE %s %s status:%s', self.command, self.path, response.status)
                     self.wfile.write(('HTTP/1.1 %s\r\n%s\r\n' % (response.status, ''.join('%s: %s\r\n' % (k.title(), v) for k, v in response.getheaders() if k.title() != 'Transfer-Encoding'))))
                     self.wfile.write(response.read())
                     response.close()
@@ -542,23 +534,33 @@ def do_AGENT(self):
                 else:
                     start, end, length = 0, content_length-1, content_length
 
+                if content_length == 0:
+                    https_manager.save_ssl_connection_for_reuse(response.ssl_sock)
+                    return
+
+                send_to_broswer = True
                 while True:
-                    data = response.read(8192)
+                    data = response.read(8192) #TODO: loop read until timeout or except.
                     if not data:
                         response.close()
                         return
                     data_len = len(data)
                     start += data_len
-                    ret = self.wfile.write(data)
-                    if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
-                        logging.debug("self.wfile.write ret:%d", ret)
-                        ret = self.wfile.write(data)
+                    if send_to_broswer:
+                        try:
+                            ret = self.wfile.write(data)
+                            if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
+                                logging.debug("self.wfile.write ret:%d", ret)
+                                ret = self.wfile.write(data)
+                        except Exception as e_b:
+                            logging.exception('GAEProxyHandler.do_METHOD_AGENT send to browser %r return %r', self.path, e_b)
+                            send_to_broswer = False
 
                     if start >= end:
-                        https_manager.save_ssl_connection_for_reuse(response.ssl_sock) #, response.connection_cache_key)
-                        #response.close()
+                        https_manager.save_ssl_connection_for_reuse(response.ssl_sock)
                         return
             except Exception as e:
+                logging.exception('GAEProxyHandler.do_METHOD_AGENT %r return %r', self.path, e)
                 errors.append(e)
                 if response:
                     response.close()
@@ -575,7 +577,6 @@ def do_AGENT(self):
                     return
                 else:
                     logging.exception('GAEProxyHandler.do_METHOD_AGENT %r return %r', self.path, e) #IOError(9, 'Bad file descriptor'), int(e.args[0])
-                    #traceback.print_exc()
 
     def do_CONNECT(self):
         """handle CONNECT cmmand, socket forward or deploy a fake cert"""

goagent/3.1.40/server/gae/gae.py → goagent/3.1.41/server/gae/gae.py

@@ -166,7 +166,7 @@ def application(environ, start_response):
     netloc = urlparse.urlparse(url).netloc
 
     if __hostsdeny__ and netloc.endswith(__hostsdeny__):
-        yield format_response(403, {'Content-Type': 'text/html; charset=utf-8'}, message_html('403 Hosts Deny', 'Hosts Deny(%r)' % netloc, detail='url=%r' % url))
+        yield format_response(403, {'Content-Type': 'text/html; charset=utf-8'}, message_html('403 Hosts Deny', 'Hosts Deny(%r)' % netloc, detail='共用appid因为资源有限，限制观看视频和文件下载等消耗资源过多的访问，请使用自己的appid <a href=" https://github.com/XX-net/XX-Net/wiki/Register-Google-appid" target="_blank">帮助</a> '))
         raise StopIteration
 
     if len(url) > MAX_URL_LENGTH: