rev 196, Configurable data dir, log dir and config file, fix for getF…

…ile unhandled exceptions, add http://zeronet.io to readme

README.md

@@ -1,6 +1,6 @@
 # ZeroNet
 
-Decentralized websites using Bitcoin crypto and the BitTorrent network
+Decentralized websites using Bitcoin crypto and the BitTorrent network - http://zeronet.io
 
 
 ## Why?

plugins/Stats/StatsPlugin.py

@@ -394,7 +394,7 @@ def benchmark(name, standard):
 
 		schema = {
 			"db_name": "TestDb",
-			"db_file": "data/benchmark.db",
+			"db_file": "%s/benchmark.db" % config.data_dir,
 			"maps": {
 				".*": {
 					"to_table": {
@@ -415,17 +415,17 @@ def benchmark(name, standard):
 			}
 		}
 
-		if os.path.isfile("data/benchmark.db"): os.unlink("data/benchmark.db") 
+		if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir) 
 
 		with benchmark("Open x 10", 0.13):
 			for i in range(10):
-				db = Db(schema, "data/benchmark.db")
+				db = Db(schema, "%s/benchmark.db" % config.data_dir)
 				db.checkTables() 
 				db.close()
 				yield "."
 
 
-		db = Db(schema, "data/benchmark.db")
+		db = Db(schema, "%s/benchmark.db" % config.data_dir)
 		db.checkTables() 
 		import json
 
@@ -434,9 +434,9 @@ def benchmark(name, standard):
 				data = {"test": []}
 				for i in range(1000): # 1000 line of data
 					data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
-				json.dump(data, open("data/test_%s.json" % u, "w"))
-				db.loadJson("data/test_%s.json" % u)
-				os.unlink("data/test_%s.json" % u)
+				json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w"))
+				db.loadJson("%s/test_%s.json" % (config.data_dir, u))
+				os.unlink("%s/test_%s.json" % (config.data_dir, u))
 				yield "."
 
 
@@ -448,9 +448,9 @@ def benchmark(name, standard):
 				data = {"test": []}
 				for i in range(100): # 1000 line of data
 					data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
-				json.dump(data, open("data/test_%s.json" % u, "w"))
-				db.loadJson("data/test_%s.json" % u, cur=cur)
-				os.unlink("data/test_%s.json" % u)
+				json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w"))
+				db.loadJson("%s/test_%s.json" % (config.data_dir, u), cur=cur)
+				os.unlink("%s/test_%s.json" % (config.data_dir, u))
 				if u%10 == 0: yield "."
 			cur.execute("COMMIT")
 
@@ -496,7 +496,7 @@ def benchmark(name, standard):
 
 
 		db.close()
-		if os.path.isfile("data/benchmark.db"): os.unlink("data/benchmark.db") 
+		if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir) 
 
 		gc.collect() # Implicit grabage collection
 

plugins/disabled-Dnschain/SiteManagerPlugin.py

@@ -11,7 +11,7 @@
 
 @PluginManager.registerTo("SiteManager")
 class SiteManagerPlugin(object):
-	dns_cache_path = "data/dns_cache.json"
+	dns_cache_path = "%s/dns_cache.json" % config.data_dir
 	dns_cache = None
 
 	# Checks if its a valid address

src/Config.py

@@ -4,7 +4,7 @@
 class Config(object):
 	def __init__(self):
 		self.version = "0.3.0"
-		self.rev = 194
+		self.rev = 196
 		self.parser = self.createArguments()
 		argv = sys.argv[:] # Copy command line arguments
 		argv = self.parseConfig(argv) # Add arguments from config file
@@ -103,12 +103,16 @@ def createArguments(self):
 		parser.add_argument('--debug', 			help='Debug mode', action='store_true')
 		parser.add_argument('--debug_socket', 	help='Debug socket connections', action='store_true')
 
+		parser.add_argument('--config_file', 	help='Path of config file', default="zeronet.conf", metavar="path")
+		parser.add_argument('--data_dir', 		help='Path of data directory', default="data", metavar="path")
+		parser.add_argument('--log_dir', 		help='Path of logging directory', default="log", metavar="path")
+
 		parser.add_argument('--ui_ip', 			help='Web interface bind address', default="127.0.0.1", metavar='ip')
 		parser.add_argument('--ui_port', 		help='Web interface bind port', default=43110, type=int, metavar='port')
 		parser.add_argument('--ui_restrict',	help='Restrict web access', default=False, metavar='ip', nargs='*')
 		parser.add_argument('--open_browser',	help='Open homepage in web browser automatically', nargs='?', const="default_browser", metavar='browser_name')
 		parser.add_argument('--homepage',		help='Web interface Homepage', default='1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr', metavar='address')
-		parser.add_argument('--size_limit',		help='Default site size limit in MB', default=10, metavar='size_limit')
+		parser.add_argument('--size_limit',		help='Default site size limit in MB', default=10, metavar='size')
 
 		parser.add_argument('--fileserver_ip', 	help='FileServer bind address', default="*", metavar='ip')
 		parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, type=int, metavar='port')
@@ -151,17 +155,19 @@ def parseCommandline(self, argv):
 		action = self.getAction(argv)
 		if len(argv) == 1 or not action: # If no action specificed set the main action
 			argv.append("main")
-		if "zeronet.py" in argv[0]:
-			self.arguments = self.parser.parse_args(argv[1:])
-		else: # Silent errors if not started with zeronet.py
-			self.arguments = self.parser.parse_args(argv[1:])
+		self.arguments = self.parser.parse_args(argv[1:])
 
 
 	# Parse config file
 	def parseConfig(self, argv):
-		if os.path.isfile("zeronet.conf"):
+		# Find config file path from parameters
+		config_file = "zeronet.conf"
+		if "--config_file" in argv:
+			config_file = argv[argv.index("--config_file")+1]
+		# Load config file
+		if os.path.isfile(config_file):
 			config = ConfigParser.ConfigParser(allow_no_value=True)
-			config.read('zeronet.conf')
+			config.read(config_file)
 			for section in config.sections():
 				for key, val in config.items(section):
 					if section != "global": # If not global prefix key with section

src/Debug/DebugReloader.py

@@ -35,7 +35,7 @@ def addWatcher(self, recursive=True):
 
 
 	def changed(self, evt):
-		if not evt.path or "data/" in evt.path or evt.path.endswith("pyc") or time.time()-self.last_chaged < 1: return False # Ignore *.pyc changes and no reload within 1 sec
+		if not evt.path or "%s/" % config.data_dir in evt.path or evt.path.endswith("pyc") or time.time()-self.last_chaged < 1: return False # Ignore *.pyc changes and no reload within 1 sec
 		#logging.debug("Changed: %s" % evt)
 		time.sleep(0.1) # Wait for lock release
 		self.callback()

src/File/FileRequest.py

@@ -154,7 +154,7 @@ def actionPex(self, params):
 		for peer in params["peers"]: # Add sent peers to site
 			address = self.unpackAddress(peer)
 			got_peer_keys.append("%s:%s" % address)
-			if (site.addPeer(*address)): added += 1
+			if site.addPeer(*address): added += 1
 		# Send back peers that is not in the sent list and connectable (not port 0)
 		packed_peers = [peer.packAddress() for peer in site.getConnectablePeers(params["need"], got_peer_keys)]
 		if added:

src/Site/Site.py

@@ -59,7 +59,7 @@ def __repr__(self):
 
 	# Load site settings from data/sites.json
 	def loadSettings(self):
-		sites_settings = json.load(open("data/sites.json"))
+		sites_settings = json.load(open("%s/sites.json" % config.data_dir))
 		if self.address in sites_settings:
 			self.settings = sites_settings[self.address]
 		else:
@@ -73,9 +73,9 @@ def loadSettings(self):
 
 	# Save site settings to data/sites.json
 	def saveSettings(self):
-		sites_settings = json.load(open("data/sites.json"))
+		sites_settings = json.load(open("%s/sites.json" % config.data_dir))
 		sites_settings[self.address] = self.settings
-		open("data/sites.json", "w").write(json.dumps(sites_settings, indent=2, sort_keys=True))
+		open("%s/sites.json" % config.data_dir, "w").write(json.dumps(sites_settings, indent=2, sort_keys=True))
 		return
 
 

src/Site/SiteManager.py

@@ -1,6 +1,7 @@
 import json, logging, time, re, os
 import gevent
 from Plugin import PluginManager
+from Config import config
 
 TRACKERS = [
 	("udp", "open.demonii.com", 1337),
@@ -33,8 +34,8 @@ def load(self):
 		address_found = []
 		added = 0
 		# Load new adresses
-		for address in json.load(open("data/sites.json")):
-			if address not in self.sites and os.path.isfile("data/%s/content.json" % address):
+		for address in json.load(open("%s/sites.json" % config.data_dir)):
+			if address not in self.sites and os.path.isfile("%s/%s/content.json" % (config.data_dir, address)):
 				self.sites[address] = Site(address)
 				added += 1
 			address_found.append(address)

src/Site/SiteStorage.py

@@ -2,12 +2,13 @@
 import gevent.event
 from Db import Db
 from Debug import Debug
+from Config import config
 
 
 class SiteStorage:
 	def __init__(self, site, allow_create=True):
 		self.site = site
-		self.directory = "data/%s" % self.site.address # Site data diretory
+		self.directory = "%s/%s" % (config.data_dir, self.site.address) # Site data diretory
 		self.log = site.log
 		self.db = None # Db class
 		self.db_checked = False # Checked db tables since startup

src/Test/test.py

@@ -120,11 +120,11 @@ def testTrackers(self):
 	def testDb(self):
 		print "Importing db..."
 		from Db import Db
-		for db_path in [os.path.abspath("data/test/zeronet.db"), "data/test/zeronet.db"]:
+		for db_path in [os.path.abspath("%s/test/zeronet.db" % config.data_dir), "%s/test/zeronet.db" % config.data_dir]:
 			print "Creating db using %s..." % db_path,
 			schema = {
 				"db_name": "TestDb",
-				"db_file": "data/test/zeronet.db",
+				"db_file": "%s/test/zeronet.db" % config.data_dir,
 				"map": {
 					"data.json": {
 						"to_table": {
@@ -144,14 +144,14 @@ def testDb(self):
 				}
 			}
 
-			if os.path.isfile("data/test/zeronet.db"): os.unlink("data/test/zeronet.db")
-			db = Db(schema, "data/test/zeronet.db")
+			if os.path.isfile("%s/test/zeronet.db" % config.data_dir): os.unlink("%s/test/zeronet.db" % config.data_dir)
+			db = Db(schema, "%s/test/zeronet.db" % config.data_dir)
 			db.checkTables()
 			db.close()
 
 			# Cleanup
-			os.unlink("data/test/zeronet.db")
-			os.rmdir("data/test/")
+			os.unlink("%s/test/zeronet.db" % config.data_dir)
+			os.rmdir("%s/test/" % config.data_dir)
 
 
 	def testContentManagerIncludes(self):

src/Ui/UiRequest.py

@@ -236,8 +236,8 @@ def actionSiteMedia(self, path):
 
 		if match: # Looks like a valid path
 			address = match.group("address")
-			file_path = "data/%s/%s" % (address, match.group("inner_path"))
-			allowed_dir = os.path.abspath("data/%s" % address) # Only files within data/sitehash allowed
+			file_path = "%s/%s/%s" % (config.data_dir, address, match.group("inner_path"))
+			allowed_dir = os.path.abspath("%s/%s" % (config.data_dir, address)) # Only files within data/sitehash allowed
 			data_dir = os.path.abspath("data") # No files from data/ allowed
 			if ".." in file_path or not os.path.dirname(os.path.abspath(file_path)).startswith(allowed_dir) or allowed_dir == data_dir: # File not in allowed path
 				return self.error403()

src/User/User.py

@@ -1,6 +1,8 @@
 import logging, json, time
 from Crypt import CryptBitcoin
 from Plugin import PluginManager
+from Config import config
+
 
 @PluginManager.acceptPlugins
 class User(object):
@@ -22,13 +24,13 @@ def __init__(self, master_address=None, master_seed=None, data={}):
 
 	# Save to data/users.json
 	def save(self):
-		users = json.load(open("data/users.json"))
+		users = json.load(open("%s/users.json" % config.data_dir))
 		if not self.master_address in users: users[self.master_address] = {} # Create if not exits
 		user_data = users[self.master_address]
 		if self.master_seed: user_data["master_seed"] = self.master_seed
 		user_data["sites"] = self.sites
 		user_data["certs"] = self.certs
-		open("data/users.json", "w").write(json.dumps(users, indent=2, sort_keys=True))
+		open("%s/users.json" % config.data_dir, "w").write(json.dumps(users, indent=2, sort_keys=True))
 		self.log.debug("Saved")
 
 

src/User/UserManager.py

@@ -1,6 +1,7 @@
 import json, logging, os
 from User import User
 from Plugin import PluginManager
+from Config import config
 
 
 @PluginManager.acceptPlugins
@@ -16,7 +17,7 @@ def load(self):
 		user_found = []
 		added = 0
 		# Load new users
-		for master_address, data in json.load(open("data/users.json")).items():
+		for master_address, data in json.load(open("%s/users.json" % config.data_dir)).items():
 			if master_address not in self.users:
 				user = User(master_address, data=data)
 				self.users[master_address] = user

src/Worker/Worker.py

@@ -40,7 +40,11 @@ def downloader(self):
 				self.task = task
 				site = task["site"]
 				task["workers_num"] += 1
-				buff = self.peer.getFile(site.address, task["inner_path"])
+				try:
+					buff = self.peer.getFile(site.address, task["inner_path"])
+				except Exception, err:
+					self.manager.log.debug("%s: getFile error: err" % (self.key, err))
+					buff = None
 				if self.running == False: # Worker no longer needed or got killed
 					self.manager.log.debug("%s: No longer needed, returning: %s" % (self.key, task["inner_path"]))
 					break

src/main.py

@@ -1,22 +1,23 @@
 import os, sys
 update_after_shutdown = False # If set True then update and restart zeronet after main loop ended
 
-# Create necessary files and dirs
-if not os.path.isdir("log"): os.mkdir("log")
-if not os.path.isdir("data"): os.mkdir("data")
-if not os.path.isfile("data/sites.json"): open("data/sites.json", "w").write("{}")
-if not os.path.isfile("data/users.json"): open("data/users.json", "w").write("{}")
-
 # Load config
 from Config import config
 
+# Create necessary files and dirs
+if not os.path.isdir(config.log_dir): os.mkdir(config.log_dir)
+if not os.path.isdir(config.data_dir): os.mkdir(config.data_dir)
+if not os.path.isfile("%s/sites.json" % config.data_dir): open("%s/sites.json" % config.data_dir, "w").write("{}")
+if not os.path.isfile("%s/users.json" % config.data_dir): open("%s/users.json" % config.data_dir, "w").write("{}")
+
+
 # Setup logging
 import logging
 if config.action == "main":
-	if os.path.isfile("log/debug.log"):  # Simple logrotate
-		if os.path.isfile("log/debug-last.log"): os.unlink("log/debug-last.log")
-		os.rename("log/debug.log", "log/debug-last.log")
-	logging.basicConfig(format='[%(asctime)s] %(levelname)-8s %(name)s %(message)s', level=logging.DEBUG, filename="log/debug.log")
+	if os.path.isfile("%s/debug.log" % config.log_dir):  # Simple logrotate
+		if os.path.isfile("%s/debug-last.log" % config.log_dir): os.unlink("%s/debug-last.log" % config.log_dir)
+		os.rename("%s/debug.log" % config.log_dir, "%s/debug-last.log" % config.log_dir)
+	logging.basicConfig(format='[%(asctime)s] %(levelname)-8s %(name)s %(message)s', level=logging.DEBUG, filename="%s/debug.log" % config.log_dir)
 else:
 	logging.basicConfig(level=logging.DEBUG, stream=open(os.devnull,"w")) # No file logging if action is not main
 
@@ -99,8 +100,8 @@ def siteCreate(self):
 
 		logging.info("Creating directory structure...")
 		from Site import Site
-		os.mkdir("data/%s" % address)
-		open("data/%s/index.html" % address, "w").write("Hello %s!" % address)
+		os.mkdir("%s/%s" % (config.data_dir, address))
+		open("%s/%s/index.html" % (config.data_dir, address), "w").write("Hello %s!" % address)
 
 		logging.info("Creating content.json...")
 		site = Site(address)