=

beaconDetectorThreads.rb

@@ -13,8 +13,7 @@ def is_1pixel_image?(url)
 #		return (isthere.to_s == "1")
 #	else	# no... wget it
 		begin
-			url="http://"+url if not url.include? "://"
-			pixels=FastImage.size(url, :timeout=>2)
+			pixels=FastImage.size("http://"+url, :timeout=>2)
 		    return pixels.to_s,url if pixels!=nil
 		rescue Exception => e  
 			if not e.message.include? "Network is unreachable"
@@ -32,99 +31,85 @@ def store(line,filename)
 	fw.close
 end
 
+def offload()
+	while @results.size>0
+		pixels,url=@results.pop
+		if pixels!=nil
+			@count+=1 if pixels=="[1, 1]"
+		else
+			pixels="-1"
+		end
+		res="lala"
+		res=@db.insertBEACON("beaconURLs",[pixels,url]) if url!=nil
+		@dictionary[url]=pixels
+	#	puts pixels+" "+url.to_s+" "+res.to_s
+	end
+end
 
+startScript=Time.now
 filename=ARGV[0]
 @defines=Defines.new(filename.gsub("BEACONS_",""))
 dbname=filename.rpartition("/")[0]+"beaconsDB.db"
 @db = Database.new(@defines,dbname)
-#if not File.file?(dbname)
-	@db.create("beaconURLs",'imageSize VARCHAR, url VARCHAR PRIMARY KEY')
-#end
-system("rm -f "+filename+"beaconDetectorTHREADS.csv")
+@db.create("beaconURLs",'imageSize VARCHAR, url VARCHAR PRIMARY KEY')
+results=@db.getAll("beaconURLs",nil,nil,nil,true)
+@dictionary=Hash.new
+for res in results
+	@dictionary[res["url"]]=res["imageSize"]
+end
+puts "Loaded previous snapshot of "+@dictionary.size.to_s+" elements"
+#system("rm -f "+filename+"beaconDetectorTHREADS.csv")
 f=File.new(filename)
-total=0
 totalLines= `wc -l "#{filename}"`.strip.split(' ')[0]
 @count=0
 h=Hash.new
-start=Time.now
-threads=[10]
-results=[10]
+threads=[]
+@results=[]
 while line=f.gets
-	begin
-		next	if h[line.chop]!=nil	
-		h[line.chop]=true
-		#puts line.chop+"\typarxei?\t"[email protected]("beaconURLs","imageSize","url",line.chop).to_s
-		next	if @db.get("beaconURLs","imageSize","url",line.chop)!=nil
-		threads[total]=Thread.new{ 
-			puts is_1pixel_image?(line.chop)[1]
-			Thread.exit}
-		total+=1
-		if threads.size==10
-			total=0
-			for thr in threads
-				thr.join
-for cell in results
-puts cell.to_s
+	next	if h[line.chop]!=nil	
+	h[line.chop]=true
 end
-			end
-
-
-
-abort
-
-
-
-
-
-
-
-
-
-
-
-
-
-			while threads.size>0
-				t=threads.pop		
-				pixels,url=t[:output]
-				if pixels!=nil
-					@count+=1 if pixels=="[1, 1]"
-				else
-					pixels="-1"
-				end
-				res="lala"
-				res=@db.insertBEACON("beaconURLs",[pixels,url]) if url!=nil
-				puts pixels+" "+url.to_s+" "+res.to_s
-			end
-			puts "\t"+total.to_s+"/"+totalLines+" lines so far... "+(Time.now - start).to_s+" seconds"
-			start=Time.now
-		end
+f.close
+total=0
+found=0
+start=Time.now
+h.keys.each{|url|
+	if @dictionary!=nil and @dictionary[url]!=nil
+		found+=1
+		next
+	end
+	begin
+	threads.push(Thread.new{  
+		@results.push(is_1pixel_image?(url))
+	})
 	rescue ThreadError => e
 		puts "ThreadError "+e.to_s+"\n"+total.to_s+" lines"
 		while threads.size>0
-			t=threads.pop
-			t.join
-			pixels,url=t[:output]
-			if pixels!=nil
-				@count+=1 if pixels=="[1, 1]"
-			else
-				pixels="-1"
-			end
-			@db.insertBEACON("beaconURLs",[pixels,url]) if url!=nil
+			thr=threads.pop
+			thr.join
 		end
+		#try again
+		threads.push(Thread.new{  
+			@results.push(is_1pixel_image?(url))
+		})		
 	end
-end
-while threads.size>0
-	t=threads.pop
-	t.join
-	pixels,url=t[:output]
-	if pixels!=nil
-		@count+=1 if pixels=="[1, 1]"
-	else
-		pixels="-1"
+	total+=1
+	if threads.size==10
+		puts "\t"+total.to_s+"/"+totalLines+" lines so far... "+(Time.now - start).to_s+" seconds" 			
+		while threads.size>0
+			thr=threads.pop
+			thr.join
+		end
+		start=Time.now
+		offload()
 	end
-	@db.insertBEACON("beaconURLs",[pixels,url]) if url!=nil
+}
+while threads.size>0
+	thr=threads.pop
+	thr.join
 end
-f.close
-puts "THREADS, I found "+@count.to_s+" Beacons out of "+total.to_s+" examined URLs for "+filename
+offload()
+puts "THREADS, I found "+@count.to_s+" Beacons from "+total.to_s+" examined URLs from "+totalLines.to_s+" lines of "+filename
+puts "Already found: "+found.to_s
 store(@count,filename)
+puts "Finished in "+(Time.now - startScript).to_s+" seconds"

config

@@ -21,7 +21,7 @@
                 "userResults": true,
                 "prices": false,
                 "traceResults": true,
-                "csyncResults": true,
+                "csyncResults": false,
                 "userVisits": false,
                 "userFiletypes": false,
                 "advertisers": true

core.rb

@@ -25,7 +25,7 @@ def makeDirsFiles()
 		@defines.puts "> Creating Directories..., "
 		Dir.mkdir @defines.dirs['rootDir'] unless File.exists?(@defines.dirs['rootDir'])
 		Dir.mkdir @defines.dirs['dataDir'] unless File.exists?(@defines.dirs['dataDir'])
-		Dir.mkdir @defines.dirs['adsDir'] unbess File.exists?(@defines.dirs['adsDir'])
+		Dir.mkdir @defines.dirs['adsDir'] unless File.exists?(@defines.dirs['adsDir'])
 		Dir.mkdir @defines.dirs['userDir'] unless File.exists?(@defines.dirs['userDir'])
 		Dir.mkdir @defines.dirs['timelines'] unless File.exists?(@defines.dirs['timelines'])	
 		@database=Database.new(@defines,nil)

define.rb

@@ -25,7 +25,7 @@ def initialize(filename)
 			"month_201512_20days_filtered_ES_sorted_uniq"=>3,"month_201512_30days_filtered_ES_sorted_uniq"=>3} 
 		filenames=["devices.csv","sizes3rd.csv","adParamsNum.csv","restParamsNum.csv","cmIDcount.csv","cmHosts.csv"]
 		@traceFile=filename
-		system("rm -f BEACONS_"+@traceFile)
+	#	system("rm -f BEACONS_"+@traceFile)
 		traceName=""
 		@siteFile="./sites.csv"
 		@resources="resources/"