Skip to content

Commit

Permalink
Version 0.3.5, Rev830, Full Tor mode support with hidden services, On…
Browse files Browse the repository at this point in the history
…ion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
  • Loading branch information
shortcutme committed Jan 4, 2016
1 parent c9578e9 commit e9d2cdf
Show file tree
Hide file tree
Showing 99 changed files with 9,479 additions and 270 deletions.
5 changes: 4 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
language: python
cache: pip
python:
- 2.7
install:
- pip install -U pip wheel
- pip install -r requirements.txt
before_script:
- openssl version -a
Expand All @@ -16,3 +16,6 @@ before_install:
after_success:
- codecov
- coveralls --rcfile=src/Test/coverage.ini
cache:
directories:
- $HOME/.cache/pip
118 changes: 118 additions & 0 deletions plugins/AnnounceZero/AnnounceZeroPlugin.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
import hashlib
import time

from Plugin import PluginManager
from Peer import Peer
from util import helper
from Crypt import CryptRsa

allow_reload = False # No source reload supported in this plugin
time_full_announced = {} # Tracker address: Last announced all site to tracker
connection_pool = {} # Tracker address: Peer object


# Process result got back from tracker
def processPeerRes(site, peers):
added = 0
# Ip4
found_ip4 = 0
for packed_address in peers["ip4"]:
found_ip4 += 1
peer_ip, peer_port = helper.unpackAddress(packed_address)
if site.addPeer(peer_ip, peer_port):
added += 1
# Onion
found_onion = 0
for packed_address in peers["onion"]:
found_onion += 1
peer_onion, peer_port = helper.unpackOnionAddress(packed_address)
if site.addPeer(peer_onion, peer_port):
added += 1

if added:
site.worker_manager.onPeers()
site.updateWebsocket(peers_added=added)
site.log.debug("Found %s ip4, %s onion peers, new: %s" % (found_ip4, found_onion, added))


@PluginManager.registerTo("Site")
class SitePlugin(object):
def announceTracker(self, tracker_protocol, tracker_address, fileserver_port=0, add_types=[], my_peer_id="", mode="start"):
if tracker_protocol != "zero":
return super(SitePlugin, self).announceTracker(
tracker_protocol, tracker_address, fileserver_port, add_types, my_peer_id, mode
)

s = time.time()

need_types = ["ip4"]
if self.connection_server and self.connection_server.tor_manager.enabled:
need_types.append("onion")

if mode == "start" or mode == "more": # Single: Announce only this site
sites = [self]
full_announce = False
else: # Multi: Announce all currently serving site
full_announce = True
if time.time() - time_full_announced.get(tracker_address, 0) < 60 * 5: # No reannounce all sites within 5 minute
return True
time_full_announced[tracker_address] = time.time()
from Site import SiteManager
sites = [site for site in SiteManager.site_manager.sites.values() if site.settings["serving"]]

# Create request
request = {
"hashes": [], "onions": [], "port": fileserver_port, "need_types": need_types, "need_num": 20, "add": add_types
}
for site in sites:
if "onion" in add_types:
onion = self.connection_server.tor_manager.getOnion(site.address)
request["onions"].append(onion)
request["hashes"].append(hashlib.sha256(site.address).digest())

# Tracker can remove sites that we don't announce
if full_announce:
request["delete"] = True

# Sent request to tracker
tracker = connection_pool.get(tracker_address) # Re-use tracker connection if possible
if not tracker:
tracker_ip, tracker_port = tracker_address.split(":")
tracker = Peer(tracker_ip, tracker_port, connection_server=self.connection_server)
connection_pool[tracker_address] = tracker
res = tracker.request("announce", request)

if not res or "peers" not in res:
self.log.debug("Announce to %s failed: %s" % (tracker_address, res))
if full_announce:
time_full_announced[tracker_address] = 0
return False

# Add peers from response to site
site_index = 0
for site_res in res["peers"]:
site = sites[site_index]
processPeerRes(site, site_res)
site_index += 1

# Check if we need to sign prove the onion addresses
if "onion_sign_this" in res:
self.log.debug("Signing %s for %s to add %s onions" % (res["onion_sign_this"], tracker_address, len(sites)))
request["onion_signs"] = {}
request["onion_sign_this"] = res["onion_sign_this"]
request["need_num"] = 0
for site in sites:
onion = self.connection_server.tor_manager.getOnion(site.address)
sign = CryptRsa.sign(res["onion_sign_this"], self.connection_server.tor_manager.getPrivatekey(onion))
request["onion_signs"][self.connection_server.tor_manager.getPublickey(onion)] = sign
res = tracker.request("announce", request)
if not res or "onion_sign_this" in res:
self.log.debug("Announce onion address to %s failed: %s" % (tracker_address, res))
if full_announce:
time_full_announced[tracker_address] = 0
return False

if full_announce:
tracker.remove() # Close connection, we don't need it in next 5 minute

return time.time() - s
1 change: 1 addition & 0 deletions plugins/AnnounceZero/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
import AnnounceZeroPlugin
70 changes: 43 additions & 27 deletions plugins/Sidebar/SidebarPlugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ class UiWebsocketPlugin(object):
def sidebarRenderPeerStats(self, body, site):
connected = len([peer for peer in site.peers.values() if peer.connection and peer.connection.connected])
connectable = len([peer_id for peer_id in site.peers.keys() if not peer_id.endswith(":0")])
onion = len([peer_id for peer_id in site.peers.keys() if ".onion" in peer_id])
peers_total = len(site.peers)
if peers_total:
percent_connected = float(connected) / peers_total
Expand All @@ -77,6 +78,7 @@ def sidebarRenderPeerStats(self, body, site):
<ul class='graph-legend'>
<li class='color-green'><span>connected:</span><b>{connected}</b></li>
<li class='color-blue'><span>Connectable:</span><b>{connectable}</b></li>
<li class='color-purple'><span>Onion:</span><b>{onion}</b></li>
<li class='color-black'><span>Total:</span><b>{peers_total}</b></li>
</ul>
</li>
Expand Down Expand Up @@ -201,7 +203,6 @@ def sidebarRenderSizeLimit(self, body, site):
</li>
""".format(**locals()))


def sidebarRenderOptionalFileStats(self, body, site):
size_total = 0.0
size_downloaded = 0.0
Expand All @@ -213,7 +214,6 @@ def sidebarRenderOptionalFileStats(self, body, site):
if site.content_manager.hashfield.hasHash(file_details["sha512"]):
size_downloaded += file_details["size"]


if not size_total:
return False

Expand Down Expand Up @@ -365,30 +365,43 @@ def downloadGeoLiteDb(self, db_path):
import urllib
import gzip
import shutil
from util import helper

self.log.info("Downloading GeoLite2 City database...")
self.cmd("notification", ["geolite-info", "Downloading GeoLite2 City database (one time only, ~15MB)...", 0])
try:
# Download
file = urllib.urlopen("http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz")
data = StringIO.StringIO()
while True:
buff = file.read(1024 * 16)
if not buff:
break
data.write(buff)
self.log.info("GeoLite2 City database downloaded (%s bytes), unpacking..." % data.tell())
data.seek(0)

# Unpack
with gzip.GzipFile(fileobj=data) as gzip_file:
shutil.copyfileobj(gzip_file, open(db_path, "wb"))

self.cmd("notification", ["geolite-done", "GeoLite2 City database downloaded!", 5000])
time.sleep(2) # Wait for notify animation
except Exception, err:
self.cmd("notification", ["geolite-error", "GeoLite2 City database download error: %s!" % err, 0])
raise err
db_urls = [
"http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz",
"https://raw.githubusercontent.com/texnikru/GeoLite2-Database/master/GeoLite2-City.mmdb.gz"
]
for db_url in db_urls:
try:
# Download
response = helper.httpRequest(db_url)

data = StringIO.StringIO()
while True:
buff = response.read(1024 * 512)
if not buff:
break
data.write(buff)
self.log.info("GeoLite2 City database downloaded (%s bytes), unpacking..." % data.tell())
data.seek(0)

# Unpack
with gzip.GzipFile(fileobj=data) as gzip_file:
shutil.copyfileobj(gzip_file, open(db_path, "wb"))

self.cmd("notification", ["geolite-done", "GeoLite2 City database downloaded!", 5000])
time.sleep(2) # Wait for notify animation
return True
except Exception, err:
self.log.error("Error downloading %s: %s" % (db_url, err))
pass
self.cmd("notification", [
"geolite-error",
"GeoLite2 City database download error: %s!<br>Please download and unpack to data dir:<br>%s" % (err, db_urls[0]),
0
])

def actionSidebarGetPeers(self, to):
permissions = self.getPermissions(to)
Expand All @@ -397,8 +410,9 @@ def actionSidebarGetPeers(self, to):
try:
import maxminddb
db_path = config.data_dir + '/GeoLite2-City.mmdb'
if not os.path.isfile(db_path):
self.downloadGeoLiteDb(db_path)
if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0:
if not self.downloadGeoLiteDb(db_path):
return False
geodb = maxminddb.open_database(db_path)

peers = self.site.peers.values()
Expand Down Expand Up @@ -426,7 +440,10 @@ def actionSidebarGetPeers(self, to):
if peer.ip in loc_cache:
loc = loc_cache[peer.ip]
else:
loc = geodb.get(peer.ip)
try:
loc = geodb.get(peer.ip)
except:
loc = None
loc_cache[peer.ip] = loc
if not loc or "location" not in loc:
continue
Expand Down Expand Up @@ -458,7 +475,6 @@ def actionSiteSetOwned(self, to, owned):
return self.response(to, "You don't have permission to run this command")
self.site.settings["own"] = bool(owned)


def actionSiteSetAutodownloadoptional(self, to, owned):
permissions = self.getPermissions(to)
if "ADMIN" not in permissions:
Expand Down
45 changes: 31 additions & 14 deletions plugins/Stats/StatsPlugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ def actionStats(self):
<style>
* { font-family: monospace }
table td, table th { text-align: right; padding: 0px 10px }
.serving-False { color: gray }
</style>
"""

Expand Down Expand Up @@ -113,15 +114,20 @@ def actionStats(self):
])
yield "</table>"

# Tor hidden services
yield "<br><br><b>Tor hidden services (status: %s):</b><br>" % main.file_server.tor_manager.status
for site_address, onion in main.file_server.tor_manager.site_onions.items():
yield "- %-34s: %s<br>" % (site_address, onion)

# Sites
yield "<br><br><b>Sites</b>:"
yield "<table>"
yield "<tr><th>address</th> <th>connected</th> <th title='connected/good/total'>peers</th> <th>content.json</th> <th>out</th> <th>in</th> </tr>"
for site in self.server.sites.values():
for site in sorted(self.server.sites.values(), lambda a, b: cmp(a.address,b.address)):
yield self.formatTableRow([
(
"""<a href='#' onclick='document.getElementById("peers_%s").style.display="initial"; return false'>%s</a>""",
(site.address, site.address)
"""<a href='#' class='serving-%s' onclick='document.getElementById("peers_%s").style.display="initial"; return false'>%s</a>""",
(site.settings["serving"], site.address, site.address)
),
("%s", [peer.connection.id for peer in site.peers.values() if peer.connection and peer.connection.connected]),
("%s/%s/%s", (
Expand All @@ -133,10 +139,10 @@ def actionStats(self):
("%.0fkB", site.settings.get("bytes_sent", 0) / 1024),
("%.0fkB", site.settings.get("bytes_recv", 0) / 1024),
])
yield "<tr><td id='peers_%s' style='display: none; white-space: pre' colspan=2>" % site.address
yield "<tr><td id='peers_%s' style='display: none; white-space: pre' colspan=6>" % site.address
for key, peer in site.peers.items():
if peer.time_found:
time_found = int(time.time()-peer.time_found)/60
time_found = int(time.time() - peer.time_found) / 60
else:
time_found = "--"
if peer.connection:
Expand All @@ -145,7 +151,7 @@ def actionStats(self):
connection_id = None
if site.content_manager.hashfield:
yield "Optional files: %4s " % len(peer.hashfield)
yield "(#%4s, err: %s, found: %5s min ago) %22s -<br>" % (connection_id, peer.connection_error, time_found, key)
yield "(#%4s, err: %s, found: %5s min ago) %30s -<br>" % (connection_id, peer.connection_error, time_found, key)
yield "<br></td></tr>"
yield "</table>"

Expand All @@ -155,7 +161,6 @@ def actionStats(self):

# Object types


obj_count = {}
for obj in gc.get_objects():
obj_type = str(type(obj))
Expand Down Expand Up @@ -325,9 +330,12 @@ def actionListobj(self):
]
if not refs:
continue
yield "%.1fkb <span title=\"%s\">%s</span>... " % (
float(sys.getsizeof(obj)) / 1024, cgi.escape(str(obj)), cgi.escape(str(obj)[0:100].ljust(100))
)
try:
yield "%.1fkb <span title=\"%s\">%s</span>... " % (
float(sys.getsizeof(obj)) / 1024, cgi.escape(str(obj)), cgi.escape(str(obj)[0:100].ljust(100))
)
except:
continue
for ref in refs:
yield " ["
if "object at" in str(ref) or len(str(ref)) > 100:
Expand Down Expand Up @@ -445,12 +453,21 @@ def benchmark(name, standard):
from cStringIO import StringIO

data = StringIO("Hello" * 1024 * 1024) # 5m
with benchmark("sha512 x 100 000", 1):
with benchmark("sha256 5M x 10", 0.6):
for i in range(10):
for y in range(10000):
hash = CryptHash.sha512sum(data)
data.seek(0)
hash = CryptHash.sha256sum(data)
yield "."
valid = "8cd629d9d6aff6590da8b80782a5046d2673d5917b99d5603c3dcb4005c45ffa"
assert hash == valid, "%s != %s" % (hash, valid)

data = StringIO("Hello" * 1024 * 1024) # 5m
with benchmark("sha512 5M x 10", 0.6):
for i in range(10):
data.seek(0)
hash = CryptHash.sha512sum(data)
yield "."
valid = "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce"
valid = "9ca7e855d430964d5b55b114e95c6bbb114a6d478f6485df93044d87b108904d"
assert hash == valid, "%s != %s" % (hash, valid)

with benchmark("os.urandom(256) x 100 000", 0.65):
Expand Down
Loading

0 comments on commit e9d2cdf

Please sign in to comment.