From fef767d850acaeaf90707a039308ad6d672049a2 Mon Sep 17 00:00:00 2001 From: scudette Date: Sun, 18 Oct 2009 19:32:36 -0400 Subject: [PATCH] Fixed a bug in HTML periodic renderer. Ignore-this: 9da55b6606e0e74ac4c01c379c9a7197 darcs-hash:20091018233236-20ca2-f0d6945b7264264d0422cfad00ce1beedd20157c.gz --- config/Makefile.rules | 2 +- src/network/tcp.c | 8 ++-- src/plugins/NetworkForensics/Reassembler.py | 4 +- src/plugins/TableRenderers/HTMLBundle.py | 15 +++--- src/plugins/TableRenderers/PeriodicHTML.py | 51 +++++++++++---------- src/pyflag/Farm.py | 6 ++- tests/pyflag | 2 +- utilities/incremental_load.py | 25 ++++++---- 8 files changed, 63 insertions(+), 50 deletions(-) diff --git a/config/Makefile.rules b/config/Makefile.rules index fd1cf1bd..e27c97a9 100644 --- a/config/Makefile.rules +++ b/config/Makefile.rules @@ -8,7 +8,7 @@ %.so: %.c ## This compile line is for building shared objects under ## linux. The -fPIC seems to be required for 64 bit machines. - $(CC) $(CFLAGS) -shared -fPIC $(AM_CFLAGS) $(PYTHON_CPPFLAGS) -I. -I$(top_srcdir)/src/include $(PYTHON_LDFLAGS) -o $@ $? .libs/$*.a $(PYTHON_EXTRA_LIBS) $(AM_LDFLAGS) + $(CC) $(CFLAGS) -g -O0 -Wall -shared -fPIC $(AM_CFLAGS) $(PYTHON_CPPFLAGS) -I. -I$(top_srcdir)/src/include $(PYTHON_LDFLAGS) -o $@ $? .libs/$*.a $(PYTHON_EXTRA_LIBS) $(AM_LDFLAGS) ## $(STRIP) $@ clean-local: diff --git a/src/network/tcp.c b/src/network/tcp.c index e7067520..dae9055f 100644 --- a/src/network/tcp.c +++ b/src/network/tcp.c @@ -392,7 +392,7 @@ TCPStream TCPHashTable_find_stream(TCPHashTable self, IP ip) { if(!ip) return NULL; tcp =(TCP)ip->packet.payload; - + if(!tcp) return NULL; /** If we did not get a TCP packet, we fail */ /** The below should work but does not because __TCP is defined in 2 different shared objects reassemble.so and dissect.so. We are @@ -402,13 +402,13 @@ TCPStream TCPHashTable_find_stream(TCPHashTable self, IP ip) { FIXME: A possible optimization would be to create a class hash which we can use instead of a string comparison. - */ + */ if(ISNAMEINSTANCE(tcp,"TCP")) { tcp_packet = 1; } else if(ISNAMEINSTANCE(tcp,"UDP")) { udp_packet = 1; } else return NULL; - + forward.saddr = ip->packet.header.saddr; forward.daddr = ip->packet.header.daddr; forward.source = tcp->packet.header.source; @@ -437,7 +437,7 @@ TCPStream TCPHashTable_find_stream(TCPHashTable self, IP ip) { return i; }; }; - + reverse.saddr = ip->packet.header.daddr; reverse.daddr = ip->packet.header.saddr; reverse.source = tcp->packet.header.dest; diff --git a/src/plugins/NetworkForensics/Reassembler.py b/src/plugins/NetworkForensics/Reassembler.py index 67322a76..e1a89367 100644 --- a/src/plugins/NetworkForensics/Reassembler.py +++ b/src/plugins/NetworkForensics/Reassembler.py @@ -90,10 +90,10 @@ def __init__(self, case, fd, inode): self.inode_ids = [ int(x) for x in inode[1:].split("/")] ## Fill in some vital stats - dbh.execute("select inode.inode_id, reverse, src_ip, dest_ip, src_port, dest_port, ts_sec, type from `connection_details` join inode on inode.inode_id = connection_details.inode_id where inode.inode=%r limit 1", self.inode) + dbh.execute("select inode.inode_id, reverse, src_ip, dest_ip, src_port, dest_port, ts_sec from `connection_details` join inode on inode.inode_id = connection_details.inode_id where inode.inode=%r limit 1", self.inode) row=dbh.fetch() if not row: - dbh.execute("select inode_id,reverse, src_ip, dest_ip, src_port, dest_port, ts_sec, type from `connection_details` where inode_id = %r", self.inode_ids[0]) + dbh.execute("select inode_id,reverse, src_ip, dest_ip, src_port, dest_port, ts_sec from `connection_details` where inode_id = %r", self.inode_ids[0]) row = dbh.fetch() ## This updates our properties from the db diff --git a/src/plugins/TableRenderers/HTMLBundle.py b/src/plugins/TableRenderers/HTMLBundle.py index 35f071c3..1467ec01 100644 --- a/src/plugins/TableRenderers/HTMLBundle.py +++ b/src/plugins/TableRenderers/HTMLBundle.py @@ -337,7 +337,7 @@ def generator(query, result): self.add_file_from_string(page_name, page_data.encode("utf8")) - yield "Page %s\n" % page + yield "(%s) %s: Page %s\n" % (os.getpid(), self.name, page) page +=1 ## update the TOC page: @@ -419,19 +419,20 @@ def generate_rows(self, query, ordering=True): queries to maximise the chance of getting cache hits. """ dbh = DB.DBO(self.case) + print query, ordering + ordering = True self.sql = self._make_sql(query, ordering=ordering) - print self.sql - ## This allows pyflag to cache the resultset, needed to speed ## paging of slow queries. FIXME - implement - dbh.execute(self.sql + " limit %s,%s" % (self.limit,self.end_limit-self.limit)) + #dbh.execute(self.sql + " limit %s,%s" % (self.limit,self.end_limit-self.limit)) + dbh.execute(self.sql) self.count = 0 for row in dbh: self.count += 1 yield row - if self.end_limit > 0 \ - and self.count > self.end_limit: return + #if self.end_limit > 0 \ + # and self.count > self.end_limit: return def make_archive_filename(self, inode_id, directory = 'inodes/'): ## Add the inode to the exported bundle: @@ -586,8 +587,6 @@ def export(self, case, inode_id, table_renderer): ## Now explain this file: import pyflag.HTMLUI as HTMLUI - result = HTMLUI.HTMLUI(initial = True) - result.heading("How to derive inode %s" % fd.inode) filename = "inodes/%s_explain.html" % inode_id if not table_renderer.filename_in_archive(filename): diff --git a/src/plugins/TableRenderers/PeriodicHTML.py b/src/plugins/TableRenderers/PeriodicHTML.py index cd1d992e..0bac6d6c 100644 --- a/src/plugins/TableRenderers/PeriodicHTML.py +++ b/src/plugins/TableRenderers/PeriodicHTML.py @@ -29,7 +29,7 @@ """ import HTMLBundle -import cPickle +import cPickle,os import pyflag.DB as DB import pyflag.FlagFramework as FlagFramework import pyflag.pyflaglog as pyflaglog @@ -84,27 +84,28 @@ def real_render_table(self): ## Ok we need to figure out which pages need updating - we ## assume that data is only added to the tables not removed. self.limit = 0 - dbh = DB.DBO(self.case) - dbh.execute("select count(*) as total from reporting where " - " page_name like '%s%%'", self.page_name) - total = dbh.fetch()['total'] - - ## Now work out the limit of the last page - we redo the last - ## page because it may be more complete now. - dbh.execute("select * from reporting where " - " page_name like '%s%%' order by `limit` desc limit 1", - self.page_name) - row = dbh.fetch() - if row: - self.query.set("start_limit", row['limit']) - ## The initial page - page = total - else: - self.query.set("start_limit",0) - page = 1 - +# dbh = DB.DBO(self.case) +# dbh.execute("select count(*) as total from reporting where " +# " page_name like '%s%%'", self.page_name) +# total = dbh.fetch()['total'] + +# ## Now work out the limit of the last page - we redo the last +# ## page because it may be more complete now. +# dbh.execute("select * from reporting where " +# " page_name like '%s%%' order by `limit` desc limit 1", +# self.page_name) +# row = dbh.fetch() +# if row: +# self.query.set("start_limit", row['limit']) +# ## The initial page +# page = total +# else: +# self.query.set("start_limit",0) +# page = 1 + + page = 1 print "Doing page %s from %s" % (page, self.query['start_limit']) - self.parse_limits(self.query) +# self.parse_limits(self.query) g = self.generate_rows(self.query, ordering=False) self.add_constant_files() @@ -121,13 +122,14 @@ def real_render_table(self): while 1: page_name = "%s%03u.html" % (self.page_name, page) + start = self.count page_data = self.render_page(page_name, page, elements, g) if self.row_count ==0: break self.add_file_from_string(page_name, page_data.encode("utf8")) - print "Page %s\n" % page + print "(%s): Page %s %s-%s\n" % (os.getpid(), page, start, self.count) page +=1 ## update the TOC page: @@ -165,9 +167,12 @@ def periodic(self, dbh, case): if tables != new_table: pyflaglog.log(pyflaglog.DEBUG, "Re-exporting HTML Table %s" % renderer.page_name) try: + import pdb renderer.real_render_table() except Exception,e: - print e + print "Exception %s" % e + import pdb + pdb.post_mortem() pass dbh2.execute("update reporting_jobs set tables = %r where id=%r", cPickle.dumps(new_table), row['id']) diff --git a/src/pyflag/Farm.py b/src/pyflag/Farm.py index e646abb1..0f298eea 100755 --- a/src/pyflag/Farm.py +++ b/src/pyflag/Farm.py @@ -259,7 +259,11 @@ def nanny(cb, keepalive=None, *args, **kwargs): quits we restart it. """ if config.DISABLE_NANNY: - cb(*args, **kwargs) + try: + cb(*args, **kwargs) + except Exception,e: + import pdb + pdb.post_mortem() atexit.register(terminate_children) signal.signal(signal.SIGABRT, handler) diff --git a/tests/pyflag b/tests/pyflag index f3d74ff5..041219d2 100755 --- a/tests/pyflag +++ b/tests/pyflag @@ -1,7 +1,7 @@ #!/bin/bash ## This is a version of the launcher script which run on the src directory - paths are hard coded atm. Configure below: -SRC_DIR=~/pyflag/pyflag-mod-raid +SRC_DIR=~/pyflag/ INSTALL_DIR=/var/tmp/build/pyflag/ ######### END CONFIGURATION #################### diff --git a/utilities/incremental_load.py b/utilities/incremental_load.py index dfd3bba9..03f2cd5e 100755 --- a/utilities/incremental_load.py +++ b/utilities/incremental_load.py @@ -213,6 +213,18 @@ def load_file(filename, processor, pcap_dbh): ## Start up some workers if needed: Farm.start_workers() +def update_files(files_we_have): + try: + log_fd = open(config.log) + print "Reading log file" + for l in log_fd: + files_we_have.add(l.strip()) + print "Done - added %s files from log" % len(files_we_have) + log_fd.close() + except IOError: + pass + + def run(keepalive=None): global last_mtime, offset, output_fd @@ -235,15 +247,7 @@ def run(keepalive=None): last_time = 0 files_we_have = set() - try: - log_fd = open(config.log) - print "Reading log file" - for l in log_fd: - files_we_have.add(l.strip()) - print "Done - added %s files from log" % len(files_we_have) - except IOError: - pass - + update_files(files_we_have) log_fd = open(config.log, "a") last_mtime = os.stat(directory).st_mtime @@ -277,12 +281,13 @@ def run(keepalive=None): Farm.check_mem(finish) filename = "%s/%s" % (directory,f) - load_file(filename, processor, pcap_dbh) if config.log: log_fd.write(f+"\n") log_fd.flush() files_we_have.add(f) + load_file(filename, processor, pcap_dbh) + last_time = time.time() else: print "Lock file found"