Small bug fixes. Split HTTP export into two levels.

darcs-hash:20090429051525-20ca2-7025686beedd381e3b5f32c6ee86f3d4fb4ceb93.gz

src/include/packet.h

@@ -111,7 +111,7 @@ struct struct_property_t {
   /** This represents the number of bytes from the begining of the
       struct where this item may be found 
   */
-  uint64_t item;
+  int item;
 
   int size;
 

src/plugins/LogAnalysis/Whois.py

@@ -154,7 +154,7 @@ def insert_whois_cache(sql_ip, id, ipinfo):
                id = id,
 
                _geoip_city = DB.expand("(select id from geoip_city where city=%r "
-                                       "limit 1)", (ipinfo.get('city','Unknown'),)),    
+                                       "limit 1)", (ipinfo.get('city','Unknown'),)) or '',    
                _geoip_country = DB.expand("(select id from geoip_country where country"
                                           "=%r limit 1)", (ipinfo.get("country_code3","---"),)),
 

src/plugins/NetworkForensics/ProtocolHandlers/LiveCom.py

@@ -87,6 +87,42 @@
 import pyflag.Graph as Graph
 import pyflag.Time as Time
 
+class HTMLStringType(StringType):
+    """ A ColumnType which sanitises its input for HTML.
+    We also fetch images etc from the db if available.
+    """
+    def xxxdisplay(self, value, row, result):
+        parser = HTML.HTMLParser(tag_class = HTML.SanitizingTag)
+        parser.feed(value)
+        parser.close()
+
+        return parser.root.innerHTML()
+
+    def render_html(self, value, table_renderer):
+        import plugins.TableRenderers.HTMLBundle as HTMLBundle
+
+        parser = HTML.HTMLParser(tag_class = HTML.TextTag)
+
+        parser.feed(value or '')
+        parser.close()
+
+        text = parser.root.innerHTML()
+
+        ## Make sure its wrapped:
+        ui = HTMLUI.HTMLUI(initial=True)
+        ui.text(text, wrap ='full', font='typewriter')
+        return ui.__str__()
+
+    def display(self, value, row, result):
+        parser = HTML.HTMLParser(tag_class = HTML.TextTag)
+        parser.feed(value or '')
+        parser.close()
+
+        value = parser.root.innerHTML()
+
+	result.text(value, wrap='full', font='typewriter')
+
+
 class WebMailTable(FlagFramework.CaseTable):
     """ Table to store Web mail related information """
     name = 'webmail_messages'
@@ -95,12 +131,12 @@ class WebMailTable(FlagFramework.CaseTable):
         [ InodeIDType, dict(column = 'parent_inode_id')],
         [ StringType, dict(name="Service", column='service')],
         [ StringType, dict(name='Type', column='type')],
-        [ StringType, dict(name='From', column='From')],
+        [ HTMLStringType, dict(name='From', column='From')],
         [ StringType, dict(name='To', column='To')],
         [ StringType, dict(name='CC', column='CC')],
         [ StringType, dict(name='BCC', column='BCC')],
         [ StringType, dict(name='Subject', column='subject')],
-        [ StringType, dict(name='Message', column='message', text=True)],
+        [ HTMLStringType, dict(name='Message', column='message', text=True)],
         [ StringType, dict(name='Identifier', column='message_id')],
         [ TimestampType, dict(name='Sent', column='sent')],
         ]
@@ -497,45 +533,6 @@ def periodic(self, dbh, case):
                                         inode_id = row3['inode_id'],
                                         attachment = attachment)
 
-class HTMLStringType(StringType):
-    """ A ColumnType which sanitises its input for HTML.
-    We also fetch images etc from the db if available.
-    """
-    def xxxdisplay(self, value, row, result):
-        parser = HTML.HTMLParser(tag_class = HTML.SanitizingTag)
-        parser.feed(value)
-        parser.close()
-
-        return parser.root.innerHTML()
-
-    def render_html(self, value, table_renderer):
-        import plugins.TableRenderers.HTMLBundle as HTMLBundle
-#        parser = HTML.HTMLParser(tag_class = HTML.SanitizingTag2)
-        parser = HTML.HTMLParser(tag_class = HTML.TextTag)
-#        parser = HTML.HTMLParser(tag_class = \
-#                                 FlagFramework.Curry(HTMLBundle.BundleResolvingHTMLTag,
-#                                                     table_renderer = table_renderer,
-#                                                     inode_id = '',
-#                                                     prefix = "inodes/"
-#                                                     ))
-        parser.feed(value or '')
-        parser.close()
-
-        text = parser.root.innerHTML()
-
-        ## Make sure its wrapped:
-        ui = HTMLUI.HTMLUI(initial=True)
-        ui.text(text, wrap ='full', font='typewriter')
-        return ui.__str__()
-
-    def display(self, value, row, result):
-        parser = HTML.HTMLParser(tag_class = HTML.TextTag)
-        parser.feed(value or '')
-        parser.close()
-
-        value = parser.root.innerHTML()
-
-	result.text(value, wrap='full', font='typewriter')
 
 class AttachmentColumn(InodeIDType):
     """ Displays the attachments related to the webmail message """

src/plugins/NetworkForensics/Reassembler.py

@@ -431,10 +431,23 @@ class ViewConnections(Reports.PreCannedCaseTableReports):
 
 
 import pyflag.tests
+import pyflag.pyflagsh as pyflagsh
 
 class NetworkForensicTests2(pyflag.tests.ScannerTest):
-    """ Tests Reassembler with difficult to reassemble streams """
+    """ Tests Reassembler x with difficult to reassemble streams """
     test_case = "PyFlagTestCase"
+    #test_file = "full_dump.pcap"
     test_file = "stdcapture_0.4.pcap.e01"
-    subsystem = "EWF"
+    #subsystem = "Standard"
+    #subsystem = "EWF"
+
     fstype = 'PCAP Filesystem'
+
+    def test01(self):
+        """ Test Reassebler """
+        env = pyflagsh.environment(case=self.test_case)
+        pyflagsh.shell_execv(env=env,
+                             command="scan",
+                             argv=["*",                   ## Inodes (All)
+                                   "NetworkScanners",
+                                   ])                   ## List of Scanners

src/plugins/TableRenderers/HTMLBundle.py

@@ -38,6 +38,7 @@
 import pyflag.pyflaglog as pyflaglog
 import pyflag.HTMLUI as HTMLUI
 from pyflag.DB import expand
+import re
 
 config.add_option("REPORTING_DIR", default=config.RESULTDIR + "/Reports",
                   help = "Directory to emit reports into.")
@@ -249,6 +250,8 @@ def navigation_buttons(self, page_number):
             </abbr>
             </a>''' % (self.page_name, page_number+1,page_number+1)
 
+        result += '''<a href="toc.html"><abbr title="Home"><img border="0" src="images/home.png" /></abbr></a>'''
+
         return result
 
     def add_constant_files(self):
@@ -268,6 +271,7 @@ def add_constant_files(self):
                                 ('images/toolbar-bg.gif',None),
                                 ('images/question.png',None),
                                 ('images/browse.png',None),
+                                ('images/home.png', None),
                                 ('javascript/functions.js', None),
                                 ('javascript/html_render.js', "inodes/javascript/html_render.js"),
                                 ]:
@@ -342,26 +346,7 @@ def generator(query, result):
         result.generator.generator = generator(query,result)
 
     def toc(self):
-        result = HTMLUI.HTMLUI(initial = True)
-        result.heading("Case %s" % self.case)
-
-        result.start_table(**{'class': 'PyFlagTable'})
-        result.raw("<thead><th>Filename</th><th>Description</th><th>From</th><th>To</th></thead>")
-        dbh = DB.DBO(self.case)
-        dbh.execute("select * from reporting order by page_name")
-        for row in dbh:
-            if row['start_value'] == 'None': continue
-
-            result.row(expand("<a href=%r>%s</a>", (row['page_name'],row['page_name'])),
-                       row['description'],
-                       row['start_value'],
-                       row['end_value'],
-                       **{'class':'hoverRow'})
-        result.end_table()
-
-        result.raw("<p><p>\n<font size='-5' color=red>Report Produced using PyFlag Ver. %s</font>" % config.VERSION)
-
-        page = u'''<html><head><link media="all" href="images/pyflag.css" type="text/css" rel="stylesheet">
+        page_template = u'''<html><head><link media="all" href="images/pyflag.css" type="text/css" rel="stylesheet">
         <title>Table of Content</title>
         <style>
         body {
@@ -372,17 +357,63 @@ def toc(self):
         div.PyFlagPage {
         overflow: visible;
         width: 100%%;
-	}
+        }
         </style>
         </head>
         <body>
         <div id="PyFlagPage" class="PyFlagPage">
         %s
         </div>
-        </body></html>''' % result
+        </body></html>'''
 
+
+        toc = HTMLUI.HTMLUI(initial = True)
+        toc.heading("Case %s" % self.case)
+
+        toc.start_table(**{'class': 'PyFlagTable'})
+        toc.raw("<thead><th>Filename</th><th>Description</th></thead>")
+        dbh = DB.DBO(self.case)
+        pages = dict()
+        dbh.execute("select * from reporting order by page_name")
+        for row in dbh:
+            m=re.match("^([^\d]+)(\d+).html", row['page_name'])
+            pages[m.group(1)] = row['description']
+
+        page_names = pages.keys()
+        page_names.sort()
+
+        for page_name in page_names:
+            toc.row(expand("<a href='%s_toc.html'>%s</a>", (page_name,page_name)))
+
+        page = page_template % toc
         self.add_file_from_string("toc.html", page.encode("utf8"))
 
+        for page_name in page_names:
+            dbh = DB.DBO(self.case)
+            dbh.execute("select * from reporting where page_name like '%s%%' order by page_name", page_name)
+            result = HTMLUI.HTMLUI(initial = True)
+            result.heading("Case %s" % self.case)
+
+            result.start_table(**{'class': 'PyFlagTable'})
+            result.raw("<thead><th>Filename</th><th>Description</th><th>From</th><th>To</th></thead>")
+
+            for row in dbh:
+                if row['start_value'] == 'None': continue
+
+                result.row(expand("<a href=%r>%s</a>", (row['page_name'],row['page_name'])),
+                           row['description'],
+                           row['start_value'],
+                           row['end_value'],
+                           **{'class':'hoverRow'})
+
+            result.end_table()
+
+            result.raw("<p><p>\n<font size='-5' color=red>Report Produced using PyFlag Ver. %s</font>" % config.VERSION)
+
+            page = page_template % result
+            m=re.match("^([^\d]+)(\d+).html", row['page_name'])            
+            self.add_file_from_string("%s_toc.html" % m.group(1), page.encode("utf8"))
+
     def generate_rows(self, query, ordering=True):
         """ This implementation gets all the rows, but makes small
         queries to maximise the chance of getting cache hits.

utilities/incremental_load.py

@@ -147,14 +147,15 @@ def create_output_file():
         FlagFramework.print_bt_string()
 
 
-pcap_id = 0
-
 def load_file(filename, processor,  pcap_dbh):
-    global pcap_id
     global offset
 
     pyflaglog.log(pyflaglog.INFO, "%s: Processing %s" % (time.ctime(),filename))
 
+    pcap_dbh.execute("select max(id) as m from pcap")
+    row = pcap_dbh.fetch()
+    pcap_id = row['m'] or 0
+
     try:
         input_file = pypcap.PyPCAP(open(filename), output='little')
     except IOError,e:
@@ -168,7 +169,7 @@ def load_file(filename, processor,  pcap_dbh):
             pcap_id += 1
         except StopIteration:
             break
-        
+
         pcap_dbh.mass_insert(
             id = pcap_id,
             iosource = config.iosource,
@@ -212,6 +213,7 @@ def run(keepalive=None):
 
     create_output_file()
 
+    print "Created output_fd"
     ## Get the PCAPFS class and instantiate it:
     pcapfs = Registry.FILESYSTEMS.dispatch("PCAP Filesystem")(config.case)
     pcapfs.mount_point = config.mountpoint