Version 0.3.4, Rev656, CryptMessage plugin for AES and ECIES encrypti…

…on, Added pyelliptic lib for OpenSSSL based encryption methods, Test CryptMessage plugin, Force reload content.json before signing and after write, Escaped Sql IN queries support, Test Sql parameter escaping, ui_websocket Test fixture, Plugin testing support, Always return websocket errors as dict, Wait for file on weboscket fileGet command if its already in bad_files queue, PushState and ReplaceState url manipulation support in wrapper API, Per auth-address localstorage, Longer timeout for udp tracker query

.travis.yml

@@ -7,6 +7,7 @@ install:
 before_script:
   - openssl version -a
 script:
+ - python -m pytest plugins/CryptMessage/Test
  - python -m pytest src/Test --cov=src --cov-config src/Test/coverage.ini
 before_install:
   - pip install -U pytest mock pytest-cov

plugins/CryptMessage/CryptMessage.py

@@ -0,0 +1,53 @@
+from lib.pybitcointools import bitcoin as btctools
+import hashlib
+
+ecc_cache = {}
+
+
+def encrypt(data, pubkey, ephemcurve=None, ciphername='aes-256-cbc'):
+    from lib import pyelliptic
+    curve, pubkey_x, pubkey_y, i = pyelliptic.ECC._decode_pubkey(pubkey)
+    if ephemcurve is None:
+        ephemcurve = curve
+    ephem = pyelliptic.ECC(curve=ephemcurve)
+    key = hashlib.sha512(ephem.raw_get_ecdh_key(pubkey_x, pubkey_y)).digest()
+    key_e, key_m = key[:32], key[32:]
+    pubkey = ephem.get_pubkey()
+    iv = pyelliptic.OpenSSL.rand(pyelliptic.OpenSSL.get_cipher(ciphername).get_blocksize())
+    ctx = pyelliptic.Cipher(key_e, iv, 1, ciphername)
+    ciphertext = iv + pubkey + ctx.ciphering(data)
+    mac = pyelliptic.hmac_sha256(key_m, ciphertext)
+    return key_e, ciphertext + mac
+
+
+def split(encrypted):
+    iv = encrypted[0:16]
+    ciphertext = encrypted[16+70:-32]
+
+    return iv, ciphertext
+
+
+def getEcc(privatekey=None):
+    from lib import pyelliptic
+    global eccs
+    if privatekey not in ecc_cache:
+        if privatekey:
+            publickey_bin = btctools.encode_pubkey(btctools.privtopub(privatekey), "bin")
+            publickey_openssl = toOpensslPublickey(publickey_bin)
+            privatekey_openssl = toOpensslPrivatekey(privatekey)
+            ecc_cache[privatekey] = pyelliptic.ECC(curve='secp256k1', privkey=privatekey_openssl, pubkey=publickey_openssl)
+        else:
+            ecc_cache[None] = pyelliptic.ECC()
+    return ecc_cache[privatekey]
+
+
+def toOpensslPrivatekey(privatekey):
+    privatekey_bin = btctools.encode_privkey(privatekey, "bin")
+    return '\x02\xca\x00\x20' + privatekey_bin
+
+
+def toOpensslPublickey(publickey):
+    publickey_bin = btctools.encode_pubkey(publickey, "bin")
+    publickey_bin = publickey_bin[1:]
+    publickey_openssl = '\x02\xca\x00 ' + publickey_bin[:32] + '\x00 ' + publickey_bin[32:]
+    return publickey_openssl

plugins/CryptMessage/CryptMessagePlugin.py

@@ -0,0 +1,149 @@
+import base64
+import os
+
+from Plugin import PluginManager
+from Crypt import CryptBitcoin
+from lib.pybitcointools import bitcoin as btctools
+
+import CryptMessage
+
+
+@PluginManager.registerTo("UiWebsocket")
+class UiWebsocketPlugin(object):
+    def encrypt(self, text, publickey):
+        encrypted = CryptMessage.encrypt(text, CryptMessage.toOpensslPublickey(publickey))
+        return encrypted
+
+    def decrypt(self, encrypted, privatekey):
+        back = CryptMessage.getEcc(privatekey).decrypt(encrypted)
+        return back.decode("utf8")
+
+    # - Actions -
+
+    # Returns user's public key unique to site
+    # Return: Public key
+    def actionUserPublickey(self, to, index=0):
+        publickey = self.user.getEncryptPublickey(self.site.address, index)
+        self.response(to, publickey)
+
+    # Encrypt a text using the publickey or user's sites unique publickey
+    # Return: Encrypted text using base64 encoding
+    def actionEciesEncrypt(self, to, text, publickey=0, return_aes_key=False):
+        if type(publickey) is int:  # Encrypt using user's publickey
+            publickey = self.user.getEncryptPublickey(self.site.address, publickey)
+        aes_key, encrypted = self.encrypt(text.encode("utf8"), publickey.decode("base64"))
+        if return_aes_key:
+            self.response(to, [base64.b64encode(encrypted), base64.b64encode(aes_key)])
+        else:
+            self.response(to, base64.b64encode(encrypted))
+
+    # Decrypt a text using privatekey or the user's site unique private key
+    # Return: Decrypted text or list of decrypted texts
+    def actionEciesDecrypt(self, to, param, privatekey=0):
+        if type(privatekey) is int:  # Decrypt using user's privatekey
+            privatekey = self.user.getEncryptPrivatekey(self.site.address, privatekey)
+
+        if type(param) == list:
+            encrypted_texts = param
+        else:
+            encrypted_texts = [param]
+
+        texts = []  # Decoded texts
+        for encrypted_text in encrypted_texts:
+            try:
+                text = self.decrypt(encrypted_text.decode("base64"), privatekey)
+                texts.append(text)
+            except Exception, err:
+                texts.append(None)
+
+        if type(param) == list:
+            self.response(to, texts)
+        else:
+            self.response(to, texts[0])
+
+    # Encrypt a text using AES
+    # Return: Iv, AES key, Encrypted text
+    def actionAesEncrypt(self, to, text, key=None, iv=None):
+        from lib import pyelliptic
+
+        if key:
+            key = key.decode("base64")
+        else:
+            key = os.urandom(32)
+
+        if iv:  # Generate new AES key if not definied
+            iv = iv.decode("base64")
+        else:
+            iv = pyelliptic.Cipher.gen_IV('aes-256-cbc')
+
+        if text:
+            encrypted = pyelliptic.Cipher(key, iv, 1, ciphername='aes-256-cbc').ciphering(text.encode("utf8"))
+        else:
+            encrypted = ""
+
+        self.response(to, [base64.b64encode(key), base64.b64encode(iv), base64.b64encode(encrypted)])
+
+    # Decrypt a text using AES
+    # Return: Decrypted text
+    def actionAesDecrypt(self, to, *args):
+        from lib import pyelliptic
+
+        if len(args) == 3:  # Single decrypt
+            encrypted_texts = [(args[0], args[1])]
+            keys = [args[2]]
+        else:  # Batch decrypt
+            encrypted_texts, keys = args
+
+        texts = []  # Decoded texts
+        for iv, encrypted_text in encrypted_texts:
+            encrypted_text = encrypted_text.decode("base64")
+            iv = iv.decode("base64")
+            text = None
+            for key in keys:
+                ctx = pyelliptic.Cipher(key.decode("base64"), iv, 0, ciphername='aes-256-cbc')
+                try:
+                    decrypted = ctx.ciphering(encrypted_text)
+                    if decrypted and decrypted.decode("utf8"):  # Valid text decoded
+                        text = decrypted
+                except Exception, err:
+                    pass
+            texts.append(text)
+
+        if len(args) == 3:
+            self.response(to, texts[0])
+        else:
+            self.response(to, texts)
+
+
+@PluginManager.registerTo("User")
+class UserPlugin(object):
+    def getEncryptPrivatekey(self, address, param_index=0):
+        assert param_index >= 0 and param_index <= 1000
+        site_data = self.getSiteData(address)
+
+        if site_data.get("cert"):  # Different privatekey for different cert provider
+            index = param_index + self.getAddressAuthIndex(site_data["cert"])
+        else:
+            index = param_index
+
+        if "encrypt_privatekey_%s" % index not in site_data:
+            address_index = self.getAddressAuthIndex(address)
+            crypt_index = address_index + 1000 + index
+            site_data["encrypt_privatekey_%s" % index] = CryptBitcoin.hdPrivatekey(self.master_seed, crypt_index)
+            self.log.debug("New encrypt privatekey generated for %s:%s" % (address, index))
+        return site_data["encrypt_privatekey_%s" % index]
+
+    def getEncryptPublickey(self, address, param_index=0):
+        assert param_index >= 0 and param_index <= 1000
+        site_data = self.getSiteData(address)
+
+        if site_data.get("cert"):  # Different privatekey for different cert provider
+            index = param_index + self.getAddressAuthIndex(site_data["cert"])
+        else:
+            index = param_index
+
+        if "encrypt_publickey_%s" % index not in site_data:
+            privatekey = self.getEncryptPrivatekey(address, param_index)
+            publickey = btctools.encode_pubkey(btctools.privtopub(privatekey), "bin_compressed")
+            site_data["encrypt_publickey_%s" % index] = base64.b64encode(publickey)
+        return site_data["encrypt_publickey_%s" % index]

plugins/CryptMessage/Test/TestCrypt.py

@@ -0,0 +1,106 @@
+import pytest
+from CryptMessage import CryptMessage
+
+@pytest.mark.usefixtures("resetSettings")
+class TestCrypt:
+    def testPublickey(self, ui_websocket):
+        pub = ui_websocket.testAction("UserPublickey", 0)
+        assert len(pub) == 44  # Compressed, b64 encoded publickey
+
+        # Different pubkey for specificed index
+        assert ui_websocket.testAction("UserPublickey", 1) != ui_websocket.testAction("UserPublickey", 0)
+
+        # Same publickey for same index
+        assert ui_websocket.testAction("UserPublickey", 2) == ui_websocket.testAction("UserPublickey", 2)
+
+        # Different publickey for different cert
+        pub1 = ui_websocket.testAction("UserPublickey", 0)
+        site_data = ui_websocket.user.getSiteData(ui_websocket.site.address)
+        site_data["cert"] = "zeroid.bit"
+        pub2 = ui_websocket.testAction("UserPublickey", 0)
+        assert pub1 != pub2
+
+
+
+    def testEcies(self, ui_websocket):
+        ui_websocket.actionUserPublickey(0, 0)
+        pub = ui_websocket.ws.result
+
+        ui_websocket.actionEciesEncrypt(0, "hello", pub)
+        encrypted = ui_websocket.ws.result
+        assert len(encrypted) == 180
+
+        # Don't allow decrypt using other privatekey index
+        ui_websocket.actionEciesDecrypt(0, encrypted, 123)
+        decrypted = ui_websocket.ws.result
+        assert decrypted != "hello"
+
+        # Decrypt using correct privatekey
+        ui_websocket.actionEciesDecrypt(0, encrypted)
+        decrypted = ui_websocket.ws.result
+        assert decrypted == "hello"
+
+        # Decrypt batch
+        ui_websocket.actionEciesDecrypt(0, [encrypted, "baad", encrypted])
+        decrypted = ui_websocket.ws.result
+        assert decrypted == ["hello", None, "hello"]
+
+
+    def testEciesUtf8(self, ui_websocket):
+        # Utf8 test
+        utf8_text = u'\xc1rv\xedzt\xfbr\xf5t\xfck\xf6rf\xfar\xf3g\xe9p'
+        ui_websocket.actionEciesEncrypt(0, utf8_text)
+        encrypted = ui_websocket.ws.result
+
+        ui_websocket.actionEciesDecrypt(0, encrypted)
+        assert ui_websocket.ws.result == utf8_text
+
+
+    def testEciesAes(self, ui_websocket):
+        ui_websocket.actionEciesEncrypt(0, "hello", return_aes_key=True)
+        ecies_encrypted, aes_key = ui_websocket.ws.result
+
+        # Decrypt using Ecies
+        ui_websocket.actionEciesDecrypt(0, ecies_encrypted)
+        assert ui_websocket.ws.result == "hello"
+
+        # Decrypt using AES
+        aes_iv, aes_encrypted = CryptMessage.split(ecies_encrypted.decode("base64"))
+
+        ui_websocket.actionAesDecrypt(0, aes_iv.encode("base64"), aes_encrypted.encode("base64"), aes_key)
+        assert ui_websocket.ws.result == "hello"
+
+
+    def testAes(self, ui_websocket):
+        ui_websocket.actionAesEncrypt(0, "hello")
+        key, iv, encrypted = ui_websocket.ws.result
+
+        assert len(key) == 44
+        assert len(iv) == 24
+        assert len(encrypted) == 24
+
+        # Single decrypt
+        ui_websocket.actionAesDecrypt(0, iv, encrypted, key)
+        assert ui_websocket.ws.result == "hello"
+
+        # Batch decrypt
+        ui_websocket.actionAesEncrypt(0, "hello")
+        key2, iv2, encrypted2 = ui_websocket.ws.result
+
+        assert [key, iv, encrypted] != [key2, iv2, encrypted2]
+
+        # 2 correct key
+        ui_websocket.actionAesDecrypt(0, [[iv, encrypted], [iv, encrypted], [iv, "baad"], [iv2, encrypted2]], [key])
+        assert ui_websocket.ws.result == ["hello", "hello", None, None]
+
+        # 3 key
+        ui_websocket.actionAesDecrypt(0, [[iv, encrypted], [iv, encrypted], [iv, "baad"], [iv2, encrypted2]], [key, key2])
+        assert ui_websocket.ws.result == ["hello", "hello", None, "hello"]
+
+    def testAesUtf8(self, ui_websocket):
+        utf8_text = u'\xc1rv\xedzt\xfbr\xf5t\xfck\xf6rf\xfar\xf3g\xe9'
+        ui_websocket.actionAesEncrypt(0, utf8_text)
+        key, iv, encrypted = ui_websocket.ws.result
+
+        ui_websocket.actionAesDecrypt(0, iv, encrypted, key)
+        assert ui_websocket.ws.result == utf8_text

plugins/CryptMessage/Test/conftest.py

@@ -0,0 +1 @@
+from src.Test.conftest import *

plugins/CryptMessage/Test/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+python_files = Test*.py
+addopts = -rsxX -v --durations=6
+markers =
+    webtest: mark a test as a webtest.

plugins/CryptMessage/__init__.py

@@ -0,0 +1 @@
+import CryptMessagePlugin

src/Config.py

@@ -7,8 +7,8 @@
 class Config(object):
 
     def __init__(self, argv):
-        self.version = "0.3.3"
-        self.rev = 619
+        self.version = "0.3.4"
+        self.rev = 656
         self.argv = argv
         self.action = None
         self.createParser()

src/Content/ContentManager.py

@@ -26,7 +26,7 @@ def __init__(self, site):
 
     # Load content.json to self.content
     # Return: Changed files ["index.html", "data/messages.json"], Deleted files ["old.jpg"]
-    def loadContent(self, content_inner_path="content.json", add_bad_files=True, delete_removed_files=True, load_includes=True):
+    def loadContent(self, content_inner_path="content.json", add_bad_files=True, delete_removed_files=True, load_includes=True, force=False):
         content_inner_path = content_inner_path.strip("/")  # Remove / from begning
         old_content = self.contents.get(content_inner_path)
         content_path = self.site.storage.getPath(content_inner_path)
@@ -36,13 +36,14 @@ def loadContent(self, content_inner_path="content.json", add_bad_files=True, del
         if os.path.isfile(content_path):
             try:
                 # Check if file is newer than what we have
-                if old_content:
+                if not force and old_content and not self.site.settings.get("own"):
                     for line in open(content_path):
-                        if '"modified"' in line:
-                            match = re.search("([0-9\.]+),$", line.strip(" \r\n"))
-                            if match and float(match.group(1)) <= old_content.get("modified", 0):
-                                self.log.debug("loadContent same json file, skipping")
-                                return [], []
+                        if '"modified"' not in line:
+                            continue
+                        match = re.search("([0-9\.]+),$", line.strip(" \r\n"))
+                        if match and float(match.group(1)) <= old_content.get("modified", 0):
+                            self.log.debug("loadContent same json file, skipping")
+                            return [], []
 
                 new_content = json.load(open(content_path))
             except Exception, err:
@@ -427,6 +428,7 @@ def sign(self, inner_path="content.json", privatekey=None, filewrite=True, updat
         if filewrite:
             self.log.info("Saving to %s..." % inner_path)
             self.site.storage.writeJson(inner_path, new_content)
+            self.contents[inner_path] = new_content
 
         self.log.info("File %s signed!" % inner_path)