Fixed more registry bugs.

darcs-hash:20090125114931-f1522-1786ff2f0e1d615b5d0a89d675023b431edf2df2.gz

config/Makefile.rules

@@ -8,5 +8,5 @@
 %.so: %.c
         ## This compile line is for building shared objects under
         ## linux. The -fPIC seems to be required for 64 bit machines.
-	$(CC) -shared -fPIC $(AM_CFLAGS) $(PYTHON_CPPFLAGS) -I. -I$(top_srcdir)/src/include $(PYTHON_LDFLAGS) -o $@ $? .libs/$*.a $(PYTHON_EXTRA_LIBS) $(AM_LDFLAGS)
+	$(CC) $(CFLAGS) -shared -fPIC $(AM_CFLAGS) $(PYTHON_CPPFLAGS) -I. -I$(top_srcdir)/src/include $(PYTHON_LDFLAGS) -o $@ $? .libs/$*.a $(PYTHON_EXTRA_LIBS) $(AM_LDFLAGS)
 	## $(STRIP) $@

src/FileFormats/HTML.py

@@ -477,6 +477,11 @@ def make_reference_to_inode(self, inode_id, hint=None):
 
         return '"f?%s"' % result
 
+    def follow_redirect(self, dbh, inode_id):
+        """ Follows a redirect in inode_id and return the new inode_id """
+        ## FIXME implement this
+        return inode_id
+
     def resolve_reference(self, reference, hint='', build_reference=True):
         original_reference = reference
 
@@ -514,14 +519,19 @@ def resolve_reference(self, reference, hint='', build_reference=True):
                     "isnull(http.inode_id) and size > 0 limit 1", reference)
         row = dbh.fetch()
         if row and row['inode_id']:
+            ## If the target was redirected - take care of that:
+            ## (DANGER - a circular redirection could be problematic)
+            ## FIXME - do this (we need to store the location header)
+            if row['status'] == 302:
+                inode_id = self.follow_redirect(dbh, row['inode_id'])
+            else:
+                inode_id = row['inode_id']
+
             ## This is needed to stop dbh leaks due to the highly
             ## recursive nature of this function.
             del dbh
 
-            ## If the target was redirected - take care of that:
-            ## (DANGER - a circular redirection could be problematic)
-            ## FIXME - do this (we need to store the location header)
-            result = self.make_reference_to_inode(row['inode_id'], hint)
+            result = self.make_reference_to_inode(inode_id, hint)
 
             if build_reference:
                 result += " reference=\"%s\" " % reference

src/plugins/DiskForensics/LogicalIndex.py

@@ -454,19 +454,12 @@ def form(self, query, result):
         result.textfield('Inode ID', 'inode_id')
         result.textfield('Word ID', 'word_id')
 
-    def analyse(self, query):
-        ## Check to see if the inode is up to date
-        #count, size = Indexing.count_outdated_inodes(
-        #    query['case'],
-        #    "from inode where inode_id=%s" % query['inode_id'],
-        #    unique = False)
-
+    def display(self,query,result):
         ## This indexing will be done in process (i.e. not
         ## distributable) because its exactly one job:
         task = Index()
         task.run(query['case'], query['inode_id'], 2**30 + int(query['word_id']))
 
-    def display(self,query,result):
         case = query['case']
         result.table(
             elements = [ InodeIDType(case=case),
@@ -615,7 +608,7 @@ def reindex():
                 word = row['word'].decode("UTF-8").lower()
                 for e in config.INDEX_ENCODINGS.split(","):
                     w = word.encode(e)
-                    if len(w)>3:
+                    if len(w)>=3:
                         INDEX.add_word(w,id, index.WORD_ENGLISH)
             except UnicodeDecodeError,error:
                 pyflaglog.log(pyflaglog.ERROR, "Unable to encode in encoding %e: %s" % (e,error))
@@ -652,7 +645,7 @@ def form(self, query,result):
         result.const_selector('Type:','type',('word','literal','regex'),('Word','Literal','RegEx'))
 
         result.hidden("cookie", time.time(), exclusive=True)
-        if query.has_key("class_override"):
+        if query.has_key("class_override") and len(query['class_override'])>2:
             query['class'] = query['class_override']
             ## Refresh ourselves to update to the new class name
             result.refresh(0, query)
@@ -1020,3 +1013,11 @@ def test01timing_tests(self):
 
             print "Indexed file in %s seconds (%s hits)" % (time.time() - new_t, count)
 
+if __name__=="__main__":
+    import pyflag.Registry as Registry
+    Registry.Init()
+    config.parse_options(True)
+
+    i=Index()
+    i.run("PyFlagTestCase", 330, 71186 + 2**30)
+

src/plugins/Flash/AdvancedCommands.py

@@ -11,6 +11,7 @@
 import time, types
 import pyflag.pyflaglog as pyflaglog
 import BasicCommands
+import pyflag.ScannerUtils as ScannerUtils
 
 class scan_path(pyflagsh.command):
     """ This takes a path as an argument and runs the specified scanner on the path
@@ -127,6 +128,8 @@ def execute(self):
         for i in range(1,len(self.args)):
             scanners.extend(fnmatch.filter(Registry.SCANNERS.scanners, self.args[i]))
 
+        scanners = ScannerUtils.fill_in_dependancies(scanners)
+
         for row in dbh:
             inode = row['inode']
             pdbh.mass_insert(

src/plugins/Flash/BasicCommands.py

@@ -318,7 +318,6 @@ def execute(self):
         for arg in args:
             ## Glob the inodes:
             dbh = DB.DBO(self.environment._CASE)
-            print arg
             if arg[0]=='/':
                 dbh.execute("select inode from inode where inode rlike %r", arg[1:-1])
             else:
@@ -377,7 +376,6 @@ def execute(self):
 class icp(iless):
     """ Copy Inodes from the VFS to the file system """
     def execute(self):
-        print self.args
         ## check that last arg is a dir
         mode = "file"
         if len(self.args)>2 and not os.isdir(self.args[-1]):
@@ -386,7 +384,6 @@ def execute(self):
             mode = "directory"
 
         for inode in self.args[:-1]:
-            print inode
             fd=self.environment._FS.open(inode=inode)
             if mode =='directory':
                 output_filename = inode.replace("/","_")
@@ -406,7 +403,6 @@ def execute(self):
 class iicp(iless):
     """ Copy Inodes from the VFS to the file system """
     def execute(self):
-        print self.args
         ## check that last arg is a dir
         mode = "file"
         if len(self.args)>2 and not os.isdir(self.args[-1]):

src/plugins/LoadData.py

@@ -404,7 +404,6 @@ def analyse(self,query):
         pyflaglog.log(pyflaglog.VERBOSE_DEBUG, "Asking pyflash to scan the inode: %s with scanners %s" % (query['inode'], scanner_names))
 
         #Use pyflash to do all the work
-        print scanner_names
         env = pyflagsh.environment(case=query['case'])
         pyflagsh.shell_execv(env=env, command="scan",
                              argv=[query['inode'],] + scanner_names)

src/plugins/NetworkForensics/ProtocolHandlers/HTTP.py

@@ -572,7 +572,7 @@ class BrowseHTTPRequests(Reports.CaseTableReports):
     name = "Browse HTTP Requests"
     family = "Network Forensics"
     default_table = 'HTTPCaseTable'
-    columns = ['InodeTable.Modified', 'Request Packet', 'Method', 'URL', 'Content Type' ]
+    columns = ['InodeTable.Modified', "InodeTable.Inode", 'Request Packet', 'Method', 'URL', 'Content Type' ]
 
 class BrowseHTTPRequestsXXX(Reports.report):
     """
@@ -636,7 +636,7 @@ def tabular_view(query,result):
                              #TimestampType(name='Date',column='date'),
                              PacketType(name='Request Packet',column='request_packet',
                                         case=query['case']),
-                             InodeIDType(case=query['case']),
+                             InodeIDType(),
                              StringType('Method','method'),
                              StringType('URL','url'),
                              StringType('Content Type','content_type') ],

src/pyflag/Farm.py

@@ -208,7 +208,8 @@ def terminate_children():
 
     for pid in children:
         try:
-            os.kill(pid, signal.SIGABRT)
+            #print "Terminating child %s" % pid
+            os.kill(pid, signal.SIGINT)
         except: pass
 
         ## Stop our logging thread

src/pyflag/Indexing.py

@@ -114,7 +114,7 @@ def insert_dictionary_word(word, word_type, classification='', binary=False):
             pdbh.insert('dictionary',
                         **{'__word': word,
                            'type': word_type,
-                           'class': classification,
+                           'class': classification or "English",
                            ## Cant be fast here - cache must be updated
                            ##'_fast': True,
                            })

src/pyflag/Registry.py

@@ -221,6 +221,7 @@ def import_module(self,name=None,load_as=None):
     def get_name(self, cls):
         try:
             name=cls.name
+            return name
         except AttributeError:
             name = ("%s" % cls).split(".")[-1]
             cls.name = name
@@ -307,7 +308,7 @@ def sort_function(x,y):
             return 1
 
         self.classes.sort(sort_function)
-        self.class_names = [ self.get_class_name(i) for i in self.classes ]
+        self.class_names = [ self.get_name(i) for i in self.classes ]
         self.scanners = self.class_names
 
     def get_class_name(self, cls):
@@ -338,10 +339,6 @@ def get_name(self, cls):
         cls.name = name
         return name
 
-class CaseTableRegistry(OrderedRegistry):
-    def get_name(self, cls):
-        return cls.name
-
 class FileHandlerRegistry(OrderedRegistry):
     def __init__(self, ParentClass):
         Registry.__init__(self, ParentClass)
@@ -588,7 +585,7 @@ def Init():
 
     ## Register Case Tables for dynamic schema
     global CASE_TABLES
-    CASE_TABLES = CaseTableRegistry(FlagFramework.CaseTable)
+    CASE_TABLES = OrderedRegistry(FlagFramework.CaseTable)
 
     global MAGIC_HANDLERS
     import pyflag.Magic as Magic