Merge branch 'diffbot-testing' into testing

File.cpp

@@ -1056,6 +1056,18 @@ time_t File::getLastModifiedTime ( ) {
 	return 0;
 }
 
+bool doesFileExist ( char *filename ) {
+	// allow the substitution of another filename
+        struct stat stats;
+	// return true if it exists
+        if ( stat ( filename , &stats ) == 0 ) return true;
+        // return 0 if it just does not exist and reset g_errno
+        if ( errno == ENOENT ) return false;
+        // resource temporarily unavailable (for newer libc)
+        if ( errno == EAGAIN ) return false;
+	// error
+        return false;
+}
 
 // . returns -1 on error
 // . returns  0 if does not exist

File.h

@@ -34,6 +34,8 @@
 #include "Loop.h"            // for g_loop.setNonBlocking(int fd)
 #include "SafeBuf.h"
 
+bool doesFileExist ( char *filename ) ;
+
 int64_t getFileSize ( char *filename ) ;
 
 int64_t getFileSize_cygwin ( char *filename ) ;

Process.cpp

@@ -1777,11 +1777,34 @@ bool Process::shutdown2 ( ) {
 		log(LOG_INFO,"gb: still has hdtemp thread");
 
 
-	log("gb. EXITING.");
+	log("gb. EXITING GRACEFULLY.");
+
+	// from main.cpp:
+	// extern SafeBuf g_pidFileName;
+	// extern bool g_createdPidFile;
+	// // first remove the pid file on graceful exit
+	// // remove pid file if we created it
+	// // take from main.cpp
+	// if ( g_createdPidFile && g_pidFileName.length() )
+	// 	::unlink ( g_pidFileName.getBufStart() );
+
+	// make a file called 'cleanexit' so bash keep alive loop will stop
+	// because bash does not get the correct exit code, 0 in this case,
+	// even though we explicitly say 'exit(0)' !!!! poop
+	char tmp[128];
+	SafeBuf cleanFileName(tmp,128);
+	cleanFileName.safePrintf("%s/cleanexit",g_hostdb.m_dir);
+	SafeBuf nothing;
+	// returns # of bytes written, -1 if could not create file
+	if ( nothing.save ( cleanFileName.getBufStart() ) == -1 )
+		log("gb: could not create %s",cleanFileName.getBufStart());
+
 
 	// exit abruptly
 	exit(0);
 
+	// let's return control to Loop.cpp?
+
 	// keep compiler happy
 	return true;
 }

Rdb.cpp

@@ -1746,7 +1746,7 @@ void Rdb::doneDumping ( ) {
 	// give the token back so someone else can dump or merge
 	//g_msg35.releaseToken();
 	// free mem in the primary buffer
-	if ( ! m_dumpErrno ) m_mem.freeDumpedMem();
+	if ( ! m_dumpErrno ) m_mem.freeDumpedMem( &m_tree );
 	// . tell RdbDump it is done
 	// . we have to set this here otherwise RdbMem's memory ring buffer
 	//   will think the dumping is no longer going on and use the primary

RdbDump.cpp

@@ -411,8 +411,26 @@ bool RdbDump::dumpTree ( bool recall ) {
 		     g_conf.m_verifyDumpedLists ) {
 			char *s = "none";
 			if ( m_rdb ) s = getDbnameFromId(m_rdb->m_rdbId);
+			char *ks1 = "";
+			char *ks2 = "";
+			char tmp1[32];
+			char tmp2[32];
+			if ( m_firstKeyInQueue ) {
+				strcpy ( tmp1 , 
+					 KEYSTR(m_firstKeyInQueue,
+						m_list->m_ks));
+				ks1 = tmp1;
+			}
+			if ( m_lastKeyInQueue ) {
+				strcpy ( tmp2 , 
+					 KEYSTR(m_lastKeyInQueue,
+						m_list->m_ks));
+				ks2 = tmp2;
+			}
+
 			log("dump: verifying list before dumping (rdb=%s "
-			    "collnum=%i)",s,(int)m_collnum);
+			    "collnum=%i k1=%s k2=%s)",s,
+			    (int)m_collnum,ks1,ks2);
 			m_list->checkList_r ( false , // removeNegRecs?
 					      false , // sleep on problem?
 					      m_rdb->m_rdbId );
@@ -460,6 +478,10 @@ bool RdbDump::dumpTree ( bool recall ) {
 	// . this doesn't work if you're doing an unordered dump, but we should
 	//   not allow adds when closing
 	m_lastKeyInQueue  = m_list->getLastKey();
+
+	// ensure we are getting the first key of the list
+	m_list->resetListPtr();
+
 	//m_firstKeyInQueue = m_list->getCurrentKey();
 	m_list->getCurrentKey(m_firstKeyInQueue);
 	// . write this list to disk

RdbList.cpp

@@ -742,7 +742,11 @@ bool RdbList::checkList_r ( bool removeNegRecs , bool sleepOnProblem ,
 			if ( data && 
 			     (*(int32_t *)data < 0 || 
 			      *(int32_t *)data > 100000000 ) ) {
-				char *xx = NULL; *xx = 0; }
+				log("rdblist: bad titlerec data for docid "
+				    "%"INT64,
+				    g_titledb.getDocIdFromKey((key_t *)k));
+				char *xx = NULL; *xx = 0; 
+			}
 		}		
 		// tagrec?
 		if ( rdbId == RDB_TAGDB && ! KEYNEG(k) ) {

RdbMem.cpp

@@ -111,6 +111,15 @@ void *RdbMem::allocData ( char *key , int32_t dataSize , collnum_t collnum ) {
 			// don't allow ptrs to equal each other because
 			// we know which way they're growing based on order
 			if ( m_ptr2 - dataSize <= m_ptr1 ) return NULL;
+
+		// debug why recs added during dump aren't going into
+		// secondary mem
+		// log("rdbmem: allocating %i bytes for rec in %s (cn=%i) "
+		//     "ptr1=%"PTRFMT" --ptr2=%"PTRFMT" mem=%"PTRFMT,
+		//     (int)dataSize,m_rdb->m_dbname,(int)collnum,
+		//     (PTRTYPE)m_ptr1,(PTRTYPE)m_ptr2,(PTRTYPE)m_mem);
+
+
 			// otherwise, grow downward
 			m_ptr2 -= dataSize;
 			// note it
@@ -122,6 +131,14 @@ void *RdbMem::allocData ( char *key , int32_t dataSize , collnum_t collnum ) {
 		// . if it's growing up...
 		// . return NULL if it would breech
 		if ( m_ptr2 + dataSize >= m_ptr1 ) return NULL;
+
+		// debug why recs added during dump aren't going into
+		// secondary mem
+		// log("rdbmem: allocating %i bytes for rec in %s (cn=%i) "
+		//     "ptr1=%"PTRFMT" ++ptr2=%"PTRFMT" mem=%"PTRFMT,
+		//     (int)dataSize,m_rdb->m_dbname,(int)collnum,
+		//     (PTRTYPE)m_ptr1,(PTRTYPE)m_ptr2,(PTRTYPE)m_mem);
+
 		// otherwise, grow downward
 		m_ptr2 += dataSize;
 		// note it
@@ -159,11 +176,99 @@ void *RdbMem::allocData ( char *key , int32_t dataSize , collnum_t collnum ) {
 	return m_ptr1 - dataSize;
 }
 
+#include "Threads.h"
+
 // . when a dump completes we free the primary mem space and make
 //   the secondary mem space the new primary mem space
-void RdbMem::freeDumpedMem() {
+void RdbMem::freeDumpedMem( RdbTree *tree ) {
 	// bail if we have no mem
 	if ( m_memSize == 0 ) return;
+
+	log("rdbmem: start freeing dumped mem");
+
+	//char *memEnd = m_mem + m_memSize;
+
+	// this should still be true so allocData() returns m_ptr2 ptrs
+	if ( ! m_rdb->m_inDumpLoop ) { char *xx=NULL;*xx=0; }
+
+	// count how many data nodes we had to move to avoid corruption
+	int32_t count = 0;
+	int32_t scanned = 0;
+	for ( int32_t i = 0 ; i < tree->m_minUnusedNode ; i++ ) {
+		// give up control to handle search query stuff of niceness 0
+		QUICKPOLL ( MAX_NICENESS );
+		// skip node if parents is -2 (unoccupied)
+		if ( tree->m_parents[i] == -2 ) continue;
+		scanned++;
+		// get the ptr
+		char *data = tree->m_data[i];
+		if ( ! data ) continue;
+		// how could it's data not be stored in here?
+		// if ( data < m_mem ) {
+		// 	log("rdbmem: bad data1");
+		// 	continue;
+		// }
+		// if ( data >= memEnd ) {
+		// 	log("rdbmem: bad data2");
+		// 	continue;
+		// }
+		// is it in primary mem? m_ptr1 mem was just dump
+		// if growing upward
+		bool needsMove = false;
+		// if the primary mem (that was dumped) is
+		// growing upwards
+		if ( m_ptr1 < m_ptr2 ) {
+		     // and the node data is in it...
+			if ( data < m_ptr1 ) 
+				needsMove = true;
+		}
+		// growing downward otherwise
+		else if ( data >= m_ptr1 ) {
+			needsMove = true;
+		}
+		if ( ! needsMove ) continue;
+		// move it. m_inDumpLoop should still 
+		// be true so we will get added to
+		// m_ptr2
+		int32_t size;
+		if ( tree->m_sizes ) size = tree->m_sizes[i];
+		else size = tree->m_fixedDataSize;
+		if ( size < 0 ) { char *xx=NULL;*xx=0; }
+		if ( size == 0 ) continue;
+		// m_inDumpLoop is still true at this point so 
+		// so allocData should return m_ptr2 guys
+		char *newData = (char *)allocData(NULL,size,0);
+		if ( ! newData ) {
+			log("rdbmem: failed to alloc %i "
+			    "bytes node %i",(int)size,(int)i);
+			continue;
+		}
+		// debug test
+		bool stillNeedsMove = false;
+		if ( m_ptr1 < m_ptr2 ) {
+		     // and the node data is in it...
+			if ( newData < m_ptr1 ) 
+				stillNeedsMove = true;
+		}
+		// growing downward otherwise
+		else if ( newData >= m_ptr1 ) {
+			stillNeedsMove = true;
+		}
+		if ( stillNeedsMove ) {// this should never happen!!
+			log("rdbmem: olddata=0x%"PTRFMT" newdata=0x%"PTRFMT,
+			    (PTRTYPE)data, (PTRTYPE)newData);
+			log("rdbmem: still needs move!");
+		}
+		count++;
+		gbmemcpy(newData,data,size);
+		tree->m_data[i] = newData;
+	}
+	if ( count > 0 )
+		log("rdbmem: moved %i tree nodes for %s",(int)count,
+		    m_rdb->m_dbname);
+
+	log("rdbmem: stop freeing dumped mem. scanned %i nodes.",(int)scanned);
+
 	// save primary ptr
 	char *tmp = m_ptr1;
 	// debug

RdbMem.h

@@ -52,7 +52,7 @@ class RdbMem {
 
 	// . when a dump completes we free the primary mem space and make
 	//   the secondary mem space the new primary mem space
-	void  freeDumpedMem();
+	void  freeDumpedMem( class RdbTree *tree );
 
 	// Rdb which contains this class calls this to prevent swap-out once
 	// per minute or so

RdbTree.cpp

@@ -1187,18 +1187,19 @@ bool RdbTree::fixTree ( ) {
 		// skip if empty
 		if ( m_parents[i] <= -2 ) continue;
 
-
+		char *key = &m_keys[i*m_ks];
 		if ( isTitledb && m_data[i] ) {
 			char *data = m_data[i];
 			int32_t ucompSize = *(int32_t *)data;
 			if ( ucompSize < 0 || ucompSize > 100000000 ) {
 				log("db: removing titlerec with uncompressed "
-				     "size of %i from tree",(int)ucompSize);
+				     "size of %i from tree (docid=%"INT64"",
+				    (int)ucompSize,
+				    g_titledb.getDocIdFromKey((key_t *)key));
 				continue;
 			}
 		}
 
-		char *key = &m_keys[i*m_ks];
 		if ( isSpiderdb && m_data[i] &&
 		     g_spiderdb.isSpiderRequest ( (SPIDERDBKEY *)key ) ) {
 			char *data = m_data[i];

XmlDoc.cpp

@@ -4659,6 +4659,11 @@ int32_t *XmlDoc::getIndexCode2 ( ) {
 		if ( *ch32 == od->m_contentHash32 ) {
 			m_indexCode = EDOCUNCHANGED;
 			m_indexCodeValid = true;
+			// hack these values on or off.
+			// really should be function calls.
+			// but it never gets set when it should if the
+			// doc is unchanged.
+			m_sentToDiffbot = od->m_sentToDiffbot;
 			return &m_indexCode;
 		}
 	}