Rev399, Urandom and Msgpack benchmark, Better random string generatio…

…n, Never render page on OPTIONS request, Fix for Chrome browser socket hang on zeronet version update

plugins/Stats/StatsPlugin.py

@@ -375,6 +375,7 @@ def benchmark(name, standard):
 
         t = time.time()
 
+        # CryptBitcoin
         yield "<br>CryptBitcoin:<br>"
         from Crypt import CryptBitcoin
 
@@ -417,19 +418,57 @@ def benchmark(name, standard):
             assert ok, "does not verify from %s" % address
         CryptBitcoin.opensslVerify = openssl_verify_bk
 
+        # CryptHash
         yield "<br>CryptHash:<br>"
         from Crypt import CryptHash
         from cStringIO import StringIO
 
         data = StringIO("Hello" * 1024 * 1024)  # 5m
-        with benchmark("sha512 x 10 000", 1):
+        with benchmark("sha512 x 100 000", 1):
             for i in range(10):
                 for y in range(10000):
                     hash = CryptHash.sha512sum(data)
                 yield "."
             valid = "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce"
             assert hash == valid, "%s != %s" % (hash, valid)
 
+        with benchmark("os.urandom(256) x 100 000", 0.65):
+            for i in range(10):
+                for y in range(10000):
+                    data = os.urandom(256)
+                yield "."
+
+        # Msgpack
+        yield "<br>Msgpack:<br>"
+        import msgpack
+        binary = 'fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv'
+        data = {"int": 1024*1024*1024, "float": 12345.67890, "text": "hello"*1024, "binary": binary}
+        with benchmark("pack 5K x 10 000", 0.78):
+            for i in range(10):
+                for y in range(1000):
+                    data_packed = msgpack.packb(data)
+                yield "."
+            valid = """\x84\xa3int\xce@\x00\x00\x00\xa4text\xda\x14\x00hellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohello\xa5float\xcb@\xc8\x1c\xd6\xe61\xf8\xa1\xa6binary\xda\x01\x00fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv"""
+            assert data_packed == valid, "%s<br>!=<br>%s" % (repr(data_packed), repr(valid))
+
+        with benchmark("unpack 5K x 10 000", 1.2):
+            for i in range(10):
+                for y in range(1000):
+                    data_unpacked = msgpack.unpackb(data_packed)
+                yield "."
+            assert data == data_unpacked, "%s != %s" % (data_unpack, data)
+
+        with benchmark("streaming unpack 5K x 10 000", 1.4):
+            for i in range(10):
+                unpacker = msgpack.Unpacker()
+                for y in range(1000):
+                    unpacker.feed(data_packed)
+                    for data_unpacked in unpacker:
+                        pass
+                yield "."
+            assert data == data_unpacked, "%s != %s" % (data_unpack, data)
+
+        # Db
         yield "<br>Db:<br>"
         from Db import Db
 

src/Config.py

@@ -8,7 +8,7 @@ class Config(object):
 
     def __init__(self, argv):
         self.version = "0.3.2"
-        self.rev = 396
+        self.rev = 399
         self.argv = argv
         self.action = None
         self.createParser()

src/Connection/ConnectionServer.py

@@ -13,6 +13,7 @@
 from Connection import Connection
 from Config import config
 from Crypt import CryptConnection
+from Crypt import CryptHash
 
 
 class ConnectionServer:
@@ -36,10 +37,7 @@ def __init__(self, ip=None, port=None, request_handler=None):
         self.bytes_sent = 0
 
         # Bittorrent style peerid
-        self.peer_id = "-ZN0%s-%s" % (
-            config.version.replace(".", ""),
-            ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12))
-        )
+        self.peer_id = "-ZN0%s-%s" % (config.version.replace(".", ""), CryptHash.random(12, "base64"))
 
         # Check msgpack version
         if msgpack.version[0] == 0 and msgpack.version[1] < 4:

src/Crypt/CryptHash.py

@@ -1,4 +1,6 @@
 import hashlib
+import os
+import base64
 
 
 def sha1sum(file, blocksize=65536):
@@ -19,6 +21,15 @@ def sha512sum(file, blocksize=65536):
     return hash.hexdigest()[0:64]  # Truncate to 256bits is good enough
 
 
+def random(length=64, encoding="hex"):
+    if encoding == "base64":  # Characters: A-Za-z0-9
+        hash = hashlib.sha512(os.urandom(256)).digest()
+        return base64.standard_b64encode(hash).replace("+", "").replace("/", "").replace("=", "")[0:length]
+    else:  # Characters: a-f0-9 (faster)
+        return hashlib.sha512(os.urandom(256)).hexdigest()[0:length]
+
+
+
 if __name__ == "__main__":
     import cStringIO as StringIO
     a = StringIO.StringIO()

src/Site/Site.py

@@ -24,6 +24,7 @@
 from Debug import Debug
 from Content import ContentManager
 from SiteStorage import SiteStorage
+from Crypt import CryptHash
 import SiteManager
 
 
@@ -50,16 +51,12 @@ def __init__(self, address, allow_create=True):
         self.content_manager = ContentManager(self)  # Load contents
 
         if not self.settings.get("auth_key"):  # To auth user in site (Obsolete, will be removed)
-            self.settings["auth_key"] = ''.join(
-                random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24)
-            )
+            self.settings["auth_key"] = CryptHash.random()
             self.log.debug("New auth key: %s" % self.settings["auth_key"])
             self.saveSettings()
 
         if not self.settings.get("wrapper_key"):  # To auth websocket permissions
-            self.settings["wrapper_key"] = ''.join(
-                random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12)
-            )
+            self.settings["wrapper_key"] = CryptHash.random()
             self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"])
             self.saveSettings()
 

src/Ui/UiRequest.py

@@ -12,6 +12,7 @@
 from User import UserManager
 from Plugin import PluginManager
 from Ui.UiWebsocket import UiWebsocket
+from Crypt import CryptHash
 
 status_texts = {
     200: "200 OK",
@@ -48,6 +49,10 @@ def route(self, path):
         path = re.sub("^http://zero[/]+", "/", path)  # Remove begining http://zero/ for chrome extension
         path = re.sub("^http://", "/", path)  # Remove begining http for chrome extension .bit access
 
+        if self.env["REQUEST_METHOD"] == "OPTIONS":
+            self.sendHeader()
+            return ""
+
         if path == "/":
             return self.actionIndex()
         elif path.endswith("favicon.ico"):
@@ -265,9 +270,7 @@ def renderWrapper(self, site, path, inner_path, title, extra_headers):
 
     # Create a new wrapper nonce that allows to get one html file without the wrapper
     def getWrapperNonce(self):
-        wrapper_nonce = ''.join(
-            random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24)
-        )
+        wrapper_nonce = CryptHash.random()
         self.server.wrapper_nonces.append(wrapper_nonce)
         return wrapper_nonce
 

src/Ui/UiServer.py

@@ -1,6 +1,7 @@
 import logging
 import time
 import cgi
+import socket
 
 from gevent.pywsgi import WSGIServer
 from gevent.pywsgi import WSGIHandler
@@ -22,7 +23,6 @@ def __init__(self, *args, **kwargs):
         self.kwargs = kwargs
 
     def run_application(self):
-        self.server.sockets[self.client_address] = self.socket
         if "HTTP_UPGRADE" in self.environ:  # Websocket request
             try:
                 ws_handler = WebSocketHandler(*self.args, **self.kwargs)
@@ -32,20 +32,21 @@ def run_application(self):
                 logging.error("UiWSGIHandler websocket error: %s" % Debug.formatException(err))
                 if config.debug:  # Allow websocket errors to appear on /Debug
                     import sys
-                    del self.server.sockets[self.client_address]
                     sys.modules["main"].DebugHook.handleError()
         else:  # Standard HTTP request
-            # print self.application.__class__.__name__
             try:
                 super(UiWSGIHandler, self).run_application()
             except Exception, err:
                 logging.error("UiWSGIHandler error: %s" % Debug.formatException(err))
                 if config.debug:  # Allow websocket errors to appear on /Debug
                     import sys
-                    del self.server.sockets[self.client_address]
                     sys.modules["main"].DebugHook.handleError()
-        if self.client_address in self.server.sockets:
-            del self.server.sockets[self.client_address]
+
+    def handle(self):
+        # Save socket to be able to close them properly on exit
+        self.server.sockets[self.client_address] = self.socket
+        super(UiWSGIHandler, self).handle()
+        del self.server.sockets[self.client_address]
 
 
 class UiServer:
@@ -131,12 +132,15 @@ def stop(self):
         sock_closed = 0
         for sock in self.server.sockets.values():
             try:
-                sock._sock.close()
-                sock.close()
+                sock.send("bye")
+                sock.shutdown(socket.SHUT_RDWR)
+                #sock._sock.close()
+                #sock.close()
                 sock_closed += 1
-            except Exception:
-                pass
+            except Exception, err:
+                self.log.debug("Http connection close error: %s" % err)
         self.log.debug("Socket closed: %s" % sock_closed)
+        time.sleep(0.1)
 
         self.server.socket.close()
         self.server.stop()

src/Ui/media/Wrapper.css

@@ -10,6 +10,7 @@ a { color: black }
 .button { padding: 5px 10px; margin-left: 10px; background-color: #FFF85F; border-bottom: 2px solid #CDBD1E; border-radius: 2px; text-decoration: none; transition: all 0.5s; background-position: left center; }
 .button:hover { background-color: #FFF400; border-bottom: 2px solid #4D4D4C; transition: none }
 .button:active { position: relative; top: 1px }
+.button:focus { outline: none }
 
 .button-Delete { background-color: #e74c3c; border-bottom-color: #c0392b; color: white }
 .button-Delete:hover { background-color: #FF5442; border-bottom-color: #8E2B21 }

src/Ui/media/all.css

@@ -15,6 +15,7 @@ a { color: black }
 .button { padding: 5px 10px; margin-left: 10px; background-color: #FFF85F; border-bottom: 2px solid #CDBD1E; -webkit-border-radius: 2px; -moz-border-radius: 2px; -o-border-radius: 2px; -ms-border-radius: 2px; border-radius: 2px ; text-decoration: none; -webkit-transition: all 0.5s; -moz-transition: all 0.5s; -o-transition: all 0.5s; -ms-transition: all 0.5s; transition: all 0.5s ; background-position: left center; }
 .button:hover { background-color: #FFF400; border-bottom: 2px solid #4D4D4C; -webkit-transition: none ; -moz-transition: none ; -o-transition: none ; -ms-transition: none ; transition: none  }
 .button:active { position: relative; top: 1px }
+.button:focus { outline: none }
 
 .button-Delete { background-color: #e74c3c; border-bottom-color: #c0392b; color: white }
 .button-Delete:hover { background-color: #FF5442; border-bottom-color: #8E2B21 }