Some bug fixes - the nanny can now be disabled.

darcs-hash:20090427095816-f1522-547f8a9e5893a417b5fa2cabcd4c7e4e0dda5730.gz

src/FileFormats/HTML.py

@@ -595,7 +595,7 @@ class HTMLParser(lexer.Lexer):
         [ "CDATA", "<", "TAG_START", "TAG" ],
 
         ## Skip white spaces within a TAG
-        [ "TAG", " +", "SPACE", "TAG" ],
+        [ "TAG", "(?sm)\s+", "SPACE", "TAG" ],
         [ "TAG", ">", "END_TAG", "CDATA" ],
 
         ## Scripts can actually contain lots of <> which confuse us so
@@ -612,14 +612,14 @@ class HTMLParser(lexer.Lexer):
         [ "TAG", "/", "CLOSING_TAG", "TAG" ],
 
         ## Identify the tag name
-        [ "TAG", "[^ /<>]+", "TAG_NAME", "ATTRIBUTE LIST"],
+        [ "TAG", "(?sm)[^\s/<>]+", "TAG_NAME", "ATTRIBUTE LIST"],
 
         ## An attribute list is a list of key=value pairs within a tag
         [ "ATTRIBUTE LIST", "([-a-z0-9A-Z_]+)\s*=", "ATTRIBUTE_NAME", "ATTRIBUTE VALUE"],
         [ "ATTRIBUTE LIST", ">", "END_TAG", "CDATA"],
 
         ## Swallow spaces
-        [ "ATTRIBUTE LIST", "(?ms)\s+", "SPACE", "ATTRIBUTE LIST"],
+        [ "ATTRIBUTE LIST", r"(?ms)[\s\r\n]+", "SPACE", "ATTRIBUTE LIST"],
 
         ## End tag:
         [ "ATTRIBUTE LIST", "/>", "SELF_CLOSING_TAG,END_TAG", "CDATA" ],
@@ -631,7 +631,7 @@ class HTMLParser(lexer.Lexer):
         [ "ATTRIBUTE LIST", r"([-a-z0=9A-Z]+)(?=( [^\s]|[/>]))", "ATTRIBUTE_NAME", "ATTRIBUTE LIST"],
 
         ## Quoted attribute values
-        [ "ATTRIBUTE VALUE", "(?ms)'([^']*)'|\"([^\"]*)\"", "ATTRIBUTE_VALUE", "ATTRIBUTE LIST" ],
+        [ "ATTRIBUTE VALUE", "(?ms)'([^'#]*)'|\"([^\"]*)\"", "ATTRIBUTE_VALUE", "ATTRIBUTE LIST" ],
 
         ## Non quoted attribute value
         [ "ATTRIBUTE VALUE", " *([^ <>\"\']+) ?", "ATTRIBUTE_VALUE", "ATTRIBUTE LIST" ],

src/include/pcap.h

@@ -12,7 +12,8 @@ dependancy on libpcap. (We dont actually link against it). */
 #define DLT_NULL	0	/* BSD loopback encapsulation */
 #define DLT_EN10MB	1	/* Ethernet (10Mb) */
 #define DLT_EN3MB	2	/* Experimental Ethernet (3Mb) */
-#define DLT_AX25	3	/* Amateur Radio AX.25 */#define DLT_PRONET	4	/* Proteon ProNET Token Ring */
+#define DLT_AX25	3	/* Amateur Radio AX.25 */
+#define DLT_PRONET	4	/* Proteon ProNET Token Ring */
 #define DLT_CHAOS	5	/* Chaos */
 #define DLT_IEEE802	6	/* IEEE 802 Networks */
 #define DLT_ARCNET	7	/* ARCNET, with BSD-style header */

src/network/tcp.c

@@ -373,11 +373,11 @@ TCPHashTable TCPHashTable_Con(TCPHashTable self, int initial_con_id) {
 };
 
 static u_int32_t mkhash (const struct tuple4 *addr) {
-  int *data = (int *)addr;
+  u_int32_t *data = (u_int32_t *)addr;
   u_int32_t res=0;
   int i;
 
-  for (i = 0; i < sizeof(struct tuple4) / sizeof(int); i++)
+  for (i = 0; i < sizeof(struct tuple4) / sizeof(u_int32_t); i++)
     res += data[i];
 
   return res % (TCP_STREAM_TABLE_SIZE);

src/plugins/Configuration.py

@@ -66,6 +66,7 @@ def display(self, query, result):
                 return
 
             ## Force a re-read of the configuration file:
+            print "Forcing reread of %r" % config
             config.add_file(config.filename)
 
             result.refresh(0, query.__class__())

src/plugins/DiskForensics/FileHandlers/SQLite.py

@@ -252,7 +252,7 @@ class SQLiteScannerTest(pyflag.tests.ScannerTest):
 #    test_file = "pyflag_stdimage_0.5.dd"
 #    subsystem = 'Standard'
 
-    offset = "16128s"
+#    offset = "16128s"
 
     def test01RunScanner(self):
         """ Test scanner handling of SQLite files """

src/plugins/MemoryForensics/Volatility-1.3_Linux_rc.1/forensics/win32/crash_addrspace.py

@@ -36,48 +36,6 @@
 page_shift = 12
 
 debug_types = { \
-    '_PHYSICAL_MEMORY_RUN' : [ 0x8, { \
-    'BasePage' : [ 0x0, ['unsigned long']], \
-    'PageCount' : [ 0x4, ['unsigned long']], \
-} ], \
-    '_PHYSICAL_MEMORY_DESCRIPTOR' : [ 0x10, { \
-    'NumberOfRuns' : [ 0x0, ['unsigned long']], \
-    'NumberOfPages' : [ 0x4, ['unsigned long']], \
-    'Run' : [ 0x8, ['array', 1,['_PHYSICAL_MEMORY_RUN']]], \
-} ], \
-  '_DMP_HEADER' : [ 0x1000, { \
-    'Signature' : [ 0x0, ['array', 4,['unsigned char']]], \
-    'ValidDump' : [ 0x4, ['array', 4,['unsigned char']]], \
-    'MajorVersion' : [ 0x8, ['unsigned long']], \
-    'MinorVersion' : [ 0xc, ['unsigned long']], \
-    'DirectoryTableBase' : [ 0x10, ['unsigned long']], \
-    'PfnDataBase' : [ 0x14, ['unsigned long']], \
-    'PsLoadedModuleList' : [ 0x18, ['unsigned long']], \
-    'PsActiveProcessHead' : [ 0x1c, ['unsigned long']], \
-    'MachineImageType' : [ 0x20, ['unsigned long']], \
-    'NumberProcessors' : [ 0x24, ['unsigned long']], \
-    'BugCheckCode' : [ 0x28, ['unsigned long']], \
-    'BugCheckCodeParameter' : [ 0x2c, ['array', 4,['unsigned long']]], \
-    'VersionUser' : [ 0x3c, ['array', 32,['unsigned char']]], \
-    'PaeEnabled' : [ 0x5c, ['unsigned char']], \
-    'KdSecondaryVersion' : [ 0x5d, ['unsigned char']], \
-    'VersionUser' : [ 0x5e, ['array', 2,['unsigned char']]], \
-    'KdDebuggerDataBlock' : [ 0x60, ['unsigned long']], \
-    'PhysicalMemoryBlockBuffer' : [ 0x64, ['_PHYSICAL_MEMORY_DESCRIPTOR']], \
-    'ContextRecord' : [ 0x320, ['array', 1200,['unsigned char']]], \
-    'Exception' : [ 0x7d0, ['_EXCEPTION_RECORD32']], \
-    'Comment' : [ 0x820, ['array', 128,['unsigned char']]], \
-    'DumpType' : [ 0xf88, ['unsigned long']], \
-    'MiniDumpFields' : [ 0xf8c, ['unsigned long']], \
-    'SecondaryDataState' : [ 0xf90, ['unsigned long']], \
-    'ProductType' : [ 0xf94, ['unsigned long']], \
-    'SuiteMask' : [ 0xf98, ['unsigned long']], \
-    'WriterStatus' : [ 0xf9c, ['unsigned long']], \
-    'RequiredDumpSpace' : [ 0xfa0, ['unsigned __int64']], \
-    'SystemUpTime' : [ 0xfb8, ['unsigned __int64']], \
-    'SystemTime' : [ 0xfc0, ['unsigned __int64']], \
-    'reserved3' : [ 0xfc8, ['array', 56,['unsigned char']]], \
-} ], \
 }
 
 

src/plugins/MemoryForensics/Volatility-1.3_Linux_rc.1/memory_plugins/example3.py

@@ -32,7 +32,7 @@ class pslist_ex_3(forensics.commands.command):
 
     # Declare meta information associated with this plugin
 
-    meta_info = forensics.commands.command.meta_info 
+    meta_info = {} 
     meta_info['author'] = 'AAron Walters'
     meta_info['copyright'] = 'Copyright (c) 2007,2008 AAron Walters'
     meta_info['contact'] = '[email protected]'

src/plugins/MemoryForensics/Volatility-1.3_Linux_rc.1/vmodules.py

@@ -249,7 +249,7 @@ def get_dlllist(cmdname, argv):
             op.error("Unable to open image file %s" %(filename))
 
         directory_table_base = process_dtb(flat_address_space, types, offset)
-
+        print "Process directory_table_base %X" % directory_table_base
         process_address_space = create_addr_space(addr_space, directory_table_base)
 
         if process_address_space is None:
@@ -529,7 +529,8 @@ def get_open_files(cmdname, argv):
 
 
         table_levels = handle_table_levels(addr_space, types, table)
-
+        print "Levels %s" % table_levels
+        print hex(table)
         if table_levels == 0:
             num_entries = handle_num_entries(addr_space, types, table)
 
@@ -541,6 +542,7 @@ def get_open_files(cmdname, argv):
         elif table_levels == 1:
             for i in range(0, 0x200):
                 L1_entry = handle_table_L1_entry(addr_space, types, table, i)
+                print i,"L1_entry %X" % L1_entry
                 if not L1_entry is None:
                     L1_table = handle_entry_object(addr_space, types, L1_entry)
 

src/plugins/MemoryForensics/Volatility-1.3_Linux_rc.1/volatility.py

@@ -216,7 +216,11 @@ def main(argv=sys.argv):
 
 
 if __name__ == "__main__":
-    main()
-
-1
+    if 1:
+        main()
+    else:
+        import hotshot
+        prof = hotshot.Profile("hotshot_edi_stats")
+        prof.runcall(main)
+        prof.close()
 

src/plugins/MemoryForensics/VolatilityLinux.py

@@ -561,7 +561,7 @@ class VolatilityTests(pyflag.tests.ScannerTest):
 
     def test00preLoadCase(self):
         """ Load Memory image """
-        pyflag.tests.ScannerTest.test00preLoadCase(self)
+        #pyflag.tests.ScannerTest.test00preLoadCase(self)
         pyflagsh.shell_execv(command="execute",
                              argv=["Load Data.Load Filesystem image",'case=%s' % self.test_case,
                                    "iosource=test",

src/pyflag/Farm.py

@@ -250,10 +250,17 @@ class Task:
 config.add_option("JOB_QUEUE_POLL", default=10, type='int',
                   help='Number of seconds to wait between worker queue polls')
 
+config.add_option("DISABLE_NANNY", default=False,
+                  action = 'store_true',
+                  help = "Disables the use of a nanny. Useful for debugging")
+
 def nanny(cb, keepalive=None, *args, **kwargs):
     """ Runs cb in another process persistently. If the child process
     quits we restart it.
     """
+    if config.DISABLE_NANNY:
+        cb(*args, **kwargs)
+
     atexit.register(terminate_children)
     signal.signal(signal.SIGABRT, handler)
     signal.signal(signal.SIGUSR1, handler)
@@ -335,8 +342,10 @@ def worker_run(keepalive=None):
      while 1:
          ## Ping the parent
          try:
-             os.write(keepalive,"Checking")
-         except:
+             if keepalive:
+                 os.write(keepalive,"Checking")
+         except Exception, e:
+             print e
              pyflaglog.log(pyflaglog.WARNING,"Our nanny died - quitting")
              os._exit(1)
 
@@ -401,7 +410,8 @@ def worker_run(keepalive=None):
 
              finally:
                  try:
-                     os.write(keepalive, " ".join(row))
+                     if keepalive:
+                         os.write(keepalive, " ".join(row))
                  except:
                      pyflaglog.log(pyflaglog.WARNING,"Our nanny died - quitting")
                      os._exit(1)

src/pyflag/HTMLUI.py

@@ -158,7 +158,10 @@ def __str__(self):
         return self.result
 
     def __unicode__(self):
-        return self.__str__().decode("utf8",'ignore')
+        try:
+            return self.__str__().decode("utf8",'ignore')
+        except:
+            return self.__str__()
 
     def heading(self,string):
         """ Place string as a heading """