Skip to content

Commit

Permalink
version 0.2.1, better browser open, site size limit, save number of p…
Browse files Browse the repository at this point in the history
…eers to sites.json to faster warmup, silent wsgihandler error, siteSetLimit API comment, grant ADMIN permissions to wrapper, display site changetime from includes too, loading screen warning support
  • Loading branch information
shortcutme committed Feb 14, 2015
1 parent 8825779 commit fa7164a
Show file tree
Hide file tree
Showing 14 changed files with 243 additions and 48 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ Decentralized websites using Bitcoin crypto and BitTorrent network
- After starting `zeronet.py` you will be able to visit zeronet sites using http://127.0.0.1:43110/{zeronet_address} (eg. http://127.0.0.1:43110/1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr).
- When you visit a new zeronet site, it's trying to find peers using BitTorrent network and download the site files (html, css, js...) from them.
- Each visited sites become also served by You.
- Every site containing a `site.json` which holds all other files sha1 hash and a sign generated using site's private key.
- Every site containing a `site.json` which holds all other files sha512 hash and a sign generated using site's private key.
- If the site owner (who has the private key for the site address) modifies the site, then he/she signs the new `content.json` and publish it to the peers. After the peers have verified the `content.json` integrity (using the sign), they download the modified files and publish the new content to other peers.


Expand Down
4 changes: 3 additions & 1 deletion src/Config.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

class Config(object):
def __init__(self):
self.version = "0.2.0"
self.version = "0.2.1"
self.parser = self.createArguments()
argv = sys.argv[:] # Copy command line arguments
argv = self.parseConfig(argv) # Add arguments from config file
Expand Down Expand Up @@ -60,7 +60,9 @@ def createArguments(self):
parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='ip')
parser.add_argument('--ui_port', help='Web interface bind port', default=43110, type=int, metavar='port')
parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip')
parser.add_argument('--open_browser', help='Open homepage in web browser automatically', nargs='?', const="default_browser", metavar='browser_name')
parser.add_argument('--homepage', help='Web interface Homepage', default='1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr', metavar='address')
parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, metavar='size_limit')

parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip')
parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, type=int, metavar='port')
Expand Down
52 changes: 44 additions & 8 deletions src/Content/ContentManager.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ def __init__(self, site):
self.log = self.site.log
self.contents = {} # Known content.json (without files and includes)
self.loadContent(add_bad_files = False)
self.site.settings["size"] = self.getTotalSize()


# Load content.json to self.content
Expand Down Expand Up @@ -68,9 +69,24 @@ def loadContent(self, content_inner_path = "content.json", add_bad_files = True,
for inner_path in changed:
self.site.bad_files[inner_path] = True

if new_content["modified"] > self.site.settings.get("modified", 0):
self.site.settings["modified"] = new_content["modified"]

return changed


# Get total size of site
# Return: 32819 (size of files in kb)
def getTotalSize(self, ignore=None):
total_size = 0
for inner_path, content in self.contents.iteritems():
if inner_path == ignore: continue
total_size += os.path.getsize(self.site.getPath(inner_path)) # Size of content.json
for file, info in content.get("files", {}).iteritems():
total_size += info["size"]
return total_size


# Find the file info line from self.contents
# Return: { "sha512": "c29d73d30ee8c9c1b5600e8a84447a6de15a3c3db6869aca4a2a578c1721f518", "size": 41 , "content_inner_path": "content.json"}
def getFileInfo(self, inner_path):
Expand Down Expand Up @@ -216,30 +232,50 @@ def getSignsRequired(self, inner_path):
return 1 # Todo: Multisig


# Checks if the content.json content is valid
# Return: True or False
def validContent(self, inner_path, content):
if inner_path == "content.json": return True # Always ok
content_size = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()]) # Size of new content
site_size = self.getTotalSize(ignore=inner_path)+content_size # Site size without old content
if site_size > self.site.settings.get("size", 0): self.site.settings["size"] = site_size # Save to settings if larger

site_size_limit = self.site.getSizeLimit()*1024*1024

# Check total site size limit
if site_size > site_size_limit:
self.log.error("%s: Site too large %s > %s, aborting task..." % (inner_path, site_size, site_size_limit))
task = self.site.worker_manager.findTask(inner_path)
if task: # Dont try to download from other peers
self.site.worker_manager.failTask(task)
return False

if inner_path == "content.json": return True # Root content.json is passed

# Load include details
include_info = self.getIncludeInfo(inner_path)
if not include_info:
self.log.error("%s: No include info" % inner_path)
return False

if include_info.get("max_size"): # Size limit
total_size = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()])
if total_size > include_info["max_size"]:
self.log.error("%s: Too large %s > %s" % (inner_path, total_size, include_info["max_size"]))
# Check include size limit
if include_info.get("max_size"): # Include size limit
if content_size > include_info["max_size"]:
self.log.error("%s: Include too large %s > %s" % (inner_path, total_size, include_info["max_size"]))
return False

# Check if content includes allowed
if include_info.get("includes_allowed") == False and content.get("includes"):
self.log.error("%s: Includes not allowed" % inner_path)
return False # Includes not allowed

if include_info.get("files_allowed"): # Filename limit
# Filename limit
if include_info.get("files_allowed"):
for file_inner_path in content["files"].keys():
if not re.match("^%s$" % include_info["files_allowed"], file_inner_path):
self.log.error("%s: File not allowed: " % file_inner_path)
return False

return True
return True # All good



Expand Down Expand Up @@ -292,7 +328,7 @@ def verifyFile(self, inner_path, file, ignore_same = True):
self.log.error("Verify sign error: %s" % Debug.formatException(err))
return False

else: # Check using sha1 hash
else: # Check using sha512 hash
file_info = self.getFileInfo(inner_path)
if file_info:
if "sha512" in file_info:
Expand Down
4 changes: 3 additions & 1 deletion src/File/FileRequest.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,9 @@ def actionUpdate(self, params):
return False
if site.settings["own"] and params["inner_path"].endswith("content.json"):
self.log.debug("Someone trying to push a file to own site %s, reload local %s first" % (site.address, params["inner_path"]))
site.content_manager.loadContent(params["inner_path"])
changed = site.content_manager.loadContent(params["inner_path"], add_bad_files=False)
if changed: # Content.json changed locally
site.settings["size"] = site.content_manager.getTotalSize() # Update site size
buff = StringIO(params["body"])
valid = site.content_manager.verifyFile(params["inner_path"], buff)
if valid == True: # Valid and changed
Expand Down
28 changes: 26 additions & 2 deletions src/Site/Site.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ def __init__(self, address, allow_create=True):
self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self.page_requested = False # Page viewed in browser

self.content_manager = ContentManager(self) # Load contents
self.loadSettings() # Load settings from sites.json
self.content_manager = ContentManager(self) # Load contents

if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed)
self.settings["auth_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24))
Expand Down Expand Up @@ -74,6 +74,22 @@ def saveSettings(self):
return


# Max site size in MB
def getSizeLimit(self):
return self.settings.get("size_limit", config.size_limit)


# Next size limit based on current size
def getNextSizeLimit(self):
size_limits = [10,20,50,100,200,500,1000,2000,5000,10000,20000,50000,100000]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size*1.2 < size_limit*1024*1024:
return size_limit
return 999999



# Sercurity check and return path of site's file
def getPath(self, inner_path):
inner_path = inner_path.replace("\\", "/") # Windows separator fix
Expand Down Expand Up @@ -123,10 +139,14 @@ def downloadContent(self, inner_path, download_files=True, peer=None):

# Download all files of the site
@util.Noparallel(blocking=False)
def download(self):
def download(self, check_size=False):
self.log.debug("Start downloading...%s" % self.bad_files)
self.announce()
self.last_downloads = []
if check_size: # Check the size first
valid = downloadContent(download_files=False)
if not valid: return False # Cant download content.jsons or size is not fits

found = self.downloadContent("content.json")

return found
Expand All @@ -147,6 +167,8 @@ def update(self):
if not self.settings["own"]: self.checkFiles(quick_check=True) # Quick check files based on file size
if self.bad_files:
self.download()

self.settings["size"] = self.content_manager.getTotalSize() # Update site size
return changed


Expand Down Expand Up @@ -266,6 +288,8 @@ def announce(self, force=False):
if added:
self.worker_manager.onPeers()
self.updateWebsocket(peers_added=added)
self.settings["peers"] = len(peers)
self.saveSettings()
self.log.debug("Found %s peers, new: %s" % (len(peers), added))
else:
pass # TODO: http tracker support
Expand Down
2 changes: 2 additions & 0 deletions src/Site/SiteManager.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,15 @@ def isAddress(address):
# Return site and start download site files
def need(address, all_file=True):
from Site import Site
new = False
if address not in sites: # Site not exits yet
if not isAddress(address): return False # Not address: %s % address
logging.debug("Added new site: %s" % address)
sites[address] = Site(address)
if not sites[address].settings["serving"]: # Maybe it was deleted before
sites[address].settings["serving"] = True
sites[address].saveSettings()
new = True

site = sites[address]
if all_file: site.download()
Expand Down
13 changes: 12 additions & 1 deletion src/Ui/UiServer.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@ def run_application(self):
self.ws_handler.run_application()
else: # Standard HTTP request
#print self.application.__class__.__name__
return super(UiWSGIHandler, self).run_application()
try:
return super(UiWSGIHandler, self).run_application()
except Exception, err:
logging.debug("UiWSGIHandler error: %s" % err)


class UiServer:
Expand Down Expand Up @@ -77,5 +80,13 @@ def start(self):
self.log.info("Web interface: http://%s:%s/" % (config.ui_ip, config.ui_port))
self.log.info("--------------------------------------")

if config.open_browser:
logging.info("Opening browser: %s...", config.open_browser)
import webbrowser
if config.open_browser == "default_browser":
browser = webbrowser.get()
else:
browser = webbrowser.get(config.open_browser)
browser.open("http://%s:%s" % (config.ui_ip, config.ui_port), new=2)

WSGIServer((self.ip, self.port), handler, handler_class=UiWSGIHandler, log=self.log).serve_forever()
21 changes: 19 additions & 2 deletions src/Ui/UiWebsocket.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,9 @@ def handleRequest(self, data):
cmd = req.get("cmd")
params = req.get("params")
permissions = self.site.settings["permissions"]
if req["id"] >= 1000000: # Its a wrapper command, allow admin commands
permissions = permissions[:]
permissions.append("ADMIN")

if cmd == "response": # It's a response to a command
return self.actionResponse(req["to"], req["result"])
Expand Down Expand Up @@ -114,6 +117,8 @@ def handleRequest(self, data):
func = self.actionSiteDelete
elif cmd == "siteList" and "ADMIN" in permissions:
func = self.actionSiteList
elif cmd == "siteSetLimit" and "ADMIN" in permissions:
func = self.actionSiteSetLimit
elif cmd == "channelJoinAllsite" and "ADMIN" in permissions:
func = self.actionChannelJoinAllsite
# Unknown command
Expand Down Expand Up @@ -155,16 +160,21 @@ def formatSiteInfo(self, site):
if "sign" in content: del(content["sign"])
if "signs" in content: del(content["signs"])

settings = site.settings.copy()
del settings["wrapper_key"] # Dont expose wrapper key

ret = {
"auth_key": self.site.settings["auth_key"], # Obsolete, will be removed
"auth_key_sha512": hashlib.sha512(self.site.settings["auth_key"]).hexdigest()[0:64], # Obsolete, will be removed
"auth_address": self.user.getAuthAddress(site.address),
"address": site.address,
"settings": site.settings,
"settings": settings,
"content_updated": site.content_updated,
"bad_files": len(site.bad_files),
"size_limit": site.getSizeLimit(),
"next_size_limit": site.getNextSizeLimit(),
"last_downloads": len(site.last_downloads),
"peers": len(site.peers),
"peers": site.settings["peers"],
"tasks": len([task["inner_path"] for task in site.worker_manager.tasks]),
"content": content
}
Expand Down Expand Up @@ -344,3 +354,10 @@ def actionSiteDelete(self, to, address):
site.updateWebsocket()
else:
self.response(to, {"error": "Unknown site: %s" % address})


def actionSiteSetLimit(self, to, size_limit):
self.site.settings["size_limit"] = size_limit
self.site.saveSettings()
self.response(to, "Site size limit changed to %sMB" % size_limit)
self.site.download()
5 changes: 4 additions & 1 deletion src/Ui/media/Loading.coffee
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@ class Loading
if not @screen_visible then return false
$(".loadingscreen .console .cursor").remove() # Remove previous cursor
if type == "error" then text = "<span class='console-error'>#{text}</span>" else text = text+"<span class='cursor'> </span>"
$(".loadingscreen .console").append("<div class='console-line'>#{text}</div>")

line = $("<div class='console-line'>#{text}</div>").appendTo(".loadingscreen .console")
if type == "warning" then line.addClass("console-warning")
return line



Expand Down
Loading

0 comments on commit fa7164a

Please sign in to comment.