add macro for enable / disable sniffer inline functions

add macro for enable workarround for read pcap via tcpreplay
debugging tcpreasembly & http
 - decrease stream delay interval from 30 to 5s
 - improve debug logs

Author: rbucek

http.cpp

@@ -270,6 +270,7 @@ void HttpData::processData(u_int32_t ip_src, u_int32_t ip_dst,
 		}
 	}
 	delete data;
+	this->cache.cleanup(false);
 }
 
 string HttpData::getUri(string &request) {
@@ -393,10 +394,15 @@ string HttpData::getJsonValue(string &data, const char *valueName) {
 	return("");
 }
 
+void HttpData::printContentSummary() {
+	cout << "HTTP CACHE: " << this->cache.getSize() << endl;
+	this->cache.cleanup(true);
+}
+
 
 HttpCache::HttpCache() {
 	this->cleanupCounter = 0;
-	this->lastAddTimestamp = 0;
+	this->lastAddTimestamp = 0;	
 }
 
 HttpDataCache HttpCache::get(u_int32_t ip_src, u_int32_t ip_dst,
@@ -420,15 +426,17 @@ void HttpCache::add(u_int32_t ip_src, u_int32_t ip_dst,
 	HttpDataCache_id idc(ip_src, ip_dst, port_src, port_dst, http, body, http_master, body_master);
 	this->cache[idc] = HttpDataCache(id, timestamp);
 	this->lastAddTimestamp = timestamp;
-	this->cleanup();
 }
 
-void HttpCache::cleanup() {
+void HttpCache::cleanup(bool force) {
 	++this->cleanupCounter;
-	if(!(this->cleanupCounter % 100)) {
+	if(force ||
+	   !(this->cleanupCounter % 100)) {
+		u_int64_t clock = getTimeMS()/1000;
 		map<HttpDataCache_id, HttpDataCache>::iterator iter;
 		for(iter = this->cache.begin(); iter != this->cache.end(); ) {
-			if(iter->second.timestamp < this->lastAddTimestamp - 120) {
+			if(iter->second.timestamp < this->lastAddTimestamp - 120 ||
+			   iter->second.timestamp_clock < clock - 120) {
 				this->cache.erase(iter++);
 			} else {
 				++iter;

http.h

@@ -51,9 +51,11 @@ struct HttpDataCache {
 	HttpDataCache(uint32_t id = 0, u_int64_t timestamp = 0) {
 		this->id = id;
 		this->timestamp = timestamp;
+		this->timestamp_clock = getTimeMS()/1000;
 	}
 	uint32_t id;
 	u_int64_t timestamp;
+	u_int64_t timestamp_clock;
 };
 
 class HttpCache {
@@ -68,8 +70,11 @@ class HttpCache {
 		 string *http, string *body,
 		 string *http_master, string *body_master,
 		 u_int32_t id, u_int64_t timestamp);
-	void cleanup();
+	void cleanup(bool force = false);
 	void clear();
+	u_int32_t getSize() {
+		return(this->cache.size());
+	}
 private:
 	map<HttpDataCache_id, HttpDataCache> cache;
 	u_int64_t cleanupCounter;
@@ -89,6 +94,7 @@ class HttpData : public TcpReassemblyProcessData {
 	string getUriPathValue(string &uri, const char *valueName);
 	string getTag(string &data, const char *tag);
 	string getJsonValue(string &data, const char *valueName);
+	void printContentSummary();
 private:
 	unsigned int counterProcessData;
 	HttpCache cache;

sniff_inline.cpp

@@ -30,7 +30,10 @@ extern TcpReassembly *tcpReassembly;
 extern unsigned int duplicate_counter;
 
 
-inline iphdr2 *convertHeaderIP_GRE(iphdr2 *header_ip) {
+#if SNIFFER_INLINE_FUNCTIONS
+inline 
+#endif
+iphdr2 *convertHeaderIP_GRE(iphdr2 *header_ip) {
 	char gre[8];
 	uint16_t a, b;
 	// if anyone know how to make network to hostbyte nicely, redesign this
@@ -65,7 +68,10 @@ inline iphdr2 *convertHeaderIP_GRE(iphdr2 *header_ip) {
 	return(header_ip);
 }
 
-inline int pcapProcess(pcap_pkthdr** header, u_char** packet, bool *destroy,
+#if SNIFFER_INLINE_FUNCTIONS
+inline 
+#endif
+int pcapProcess(pcap_pkthdr** header, u_char** packet, bool *destroy,
 		       bool enableDefrag, bool enableCalcMD5, bool enableDedup, bool enableDump,
 		       pcapProcessData *ppd, int pcapLinklayerHeaderType, pcap_dumper_t *pcapDumpHandle, const char *interfaceName) {
 	*destroy = false;
@@ -110,10 +116,13 @@ inline int pcapProcess(pcap_pkthdr** header, u_char** packet, bool *destroy,
 	}
 
 	if(ppd->protocol != 8) {
-		/* if(ppd->protocol == 0) { // workarround for read via tcpreplay
+		#if TCPREPLAY_WORKARROUND
+		if(ppd->protocol == 0) {
 			ppd->header_ip_offset += 2;
 			ppd->protocol = 8;
-		} else */ {
+		} else 
+		#endif
+		{
 			// not ipv4 
 			return(0);
 		}

sniff_inline.h

@@ -4,14 +4,14 @@
 
 #include "pcap_queue.h"
 
-
+#if SNIFFER_INLINE_FUNCTIONS
 #include "sniff_inline.cpp"
-/*
+#else
 iphdr2 *convertHeaderIP_GRE(iphdr2 *header_ip);
 int pcapProcess(pcap_pkthdr** header, u_char** packet, bool *destroy,
 		bool enableDefrag, bool enableCalcMD5, bool enableDedup, bool enableDump,
 		pcapProcessData *ppd, int pcapLinklayerHeaderType, pcap_dumper_t *pcapDumpHandle, const char *interfaceName);
-*/
+#endif
 
 
 #endif

tcpreassembly.cpp

@@ -29,7 +29,9 @@ bool debug_data = globalDebug && true;
 bool debug_check_ok = globalDebug && true;
 bool debug_check_ok_process = globalDebug && true;
 bool debug_save = globalDebug && true;
-bool debug_print_content = globalDebug && true;
+bool debug_cleanup = globalDebug && true;
+bool debug_print_content_summary = globalDebug && true;
+bool debug_print_content = globalDebug && false;
 u_int16_t debug_counter = 0;
 u_int16_t debug_limit_counter = 0;
 u_int16_t debug_port = 0;
@@ -548,7 +550,9 @@ bool TcpReassemblyLink::streamIterator::nextAckByMaxSeqInReverseDirection() {
 }
 
 void TcpReassemblyLink::streamIterator::print() {
-	cout << "iterator";
+	cout << "iterator " 
+	     << inet_ntostring(htonl(this->link->ip_src)) << " / " << this->link->port_src << " -> "
+	     << inet_ntostring(htonl(this->link->ip_dst)) << " / " << this->link->port_dst << " ";
 	if(this->stream) {
 		cout << "  ack: " << this->stream->ack
 		     << "  state: " << this->state;
@@ -976,7 +980,7 @@ void TcpReassemblyLink::cleanup(u_int64_t act_time) {
 	map<uint32_t, TcpReassemblyStream*>::iterator iter;
 	for(iter = this->queue_by_ack.begin(); iter != this->queue_by_ack.end(); ) {
 		if(iter->second->queue.size() > 500) {
-			if(this->reassembly->isActiveLog()) {
+			if(this->reassembly->isActiveLog() || debug_cleanup) {
 				in_addr ip;
 				ip.s_addr = this->ip_src;
 				string ip_src = inet_ntoa(ip);
@@ -989,6 +993,9 @@ void TcpReassemblyLink::cleanup(u_int64_t act_time) {
 				       << setw(15) << ip_src << "/" << setw(6) << this->port_src
 				       << " -> " 
 				       << setw(15) << ip_dst << "/" << setw(6) << this->port_dst;
+				if(debug_cleanup) {
+					cout << outStr.str() << endl;
+				}
 				this->reassembly->addLog(outStr.str().c_str());
 			}
 			delete iter->second;
@@ -1425,7 +1432,10 @@ void TcpReassemblyLink::complete_crazy(bool final, bool eraseCompletedStreams, b
 					if(i == 0 || i == countRequest) {
 						cout << endl << endl;
 					}
-					cout << "  ack: " << this->ok_streams[skip_offset + i]->ack << endl << endl;
+					cout << "  ack: " << this->ok_streams[skip_offset + i]->ack << "  "
+					     << inet_ntostring(htonl(this->ip_src)) << " / " << this->port_src << " -> "
+					     << inet_ntostring(htonl(this->ip_dst)) << " / " << this->port_dst << " "
+					     << endl << endl;
 					cout << data << endl << endl;
 				}
 				if(i < countRequest) {
@@ -1843,9 +1853,10 @@ void TcpReassembly::cleanup(bool all) {
 	}
 	this->unlock_links();
 
-	u_int64_t act_time = this->act_time_from_header + getTimeMS() - this->last_time;
-	
 	while(true) {
+	 
+		u_int64_t act_time = this->act_time_from_header + getTimeMS() - this->last_time;
+	 
 		TcpReassemblyLink *link = NULL;
 		this->lock_links();
 		for(iter = this->links.begin(); iter != this->links.end(); iter++) {
@@ -1857,7 +1868,7 @@ void TcpReassembly::cleanup(bool all) {
 			}
 		}
 		if(link && link->queue_by_ack.size() > 500) {
-			if(this->isActiveLog()) {
+			if(this->isActiveLog() || debug_cleanup) {
 				in_addr ip;
 				ip.s_addr = link->ip_src;
 				string ip_src = inet_ntoa(ip);
@@ -1870,6 +1881,9 @@ void TcpReassembly::cleanup(bool all) {
 				       << setw(15) << ip_src << "/" << setw(6) << link->port_src
 				       << " -> "
 				       << setw(15) << ip_dst << "/" << setw(6) << link->port_dst;
+				if(debug_cleanup) {
+					cout << outStr.str() << endl;
+				}
 				this->addLog(outStr.str().c_str());
 			}
 			link->unlock_queue();
@@ -1886,7 +1900,7 @@ void TcpReassembly::cleanup(bool all) {
 		bool final = act_time > link->last_packet_at_from_header + 2 * 60 * 1000;
 		if((all || final ||
 		    (link->last_packet_at_from_header &&
-		     act_time > link->last_packet_at_from_header + 30 * 1000 &&
+		     act_time > link->last_packet_at_from_header + 5 * 1000 &&
 		     link->last_packet_at_from_header > link->last_packet_process_cleanup_at)) &&
 		   (link->link_is_ok < 2 || opt_tcpreassembly_thread)) {
 
@@ -1960,6 +1974,9 @@ void TcpReassembly::cleanup(bool all) {
 		}
 		this->doPrintContent = false;
 	}
+	if(debug_print_content_summary) {
+		this->printContentSummary();
+	}
 }
 
 /*
@@ -1977,3 +1994,8 @@ void TcpReassembly::printContent() {
 		iter->second->printContent(1);
 	}
 }
+
+void TcpReassembly::printContentSummary() {
+	cout << "LINKS: " << this->links.size() << endl;
+	this->dataCallback->printContentSummary();
+}

tcpreassembly.h

@@ -151,6 +151,7 @@ class TcpReassemblyProcessData {
 				 u_int16_t port_src, u_int16_t port_dst,
 				 TcpReassemblyData *data,
 				 bool debugSave) = 0;
+	virtual void printContentSummary() {}
 };
 
 struct TcpReassemblyLink_id {
@@ -706,6 +707,7 @@ class TcpReassembly {
 	bool enableStop();
 	*/
 	void printContent();
+	void printContentSummary();
 	void setDoPrintContent() {
 		this->doPrintContent = true;
 	}

voipmonitor.cpp

@@ -3916,10 +3916,8 @@ void test_parsepacket() {
 	ParsePacket pp;
 	pp.setStdParse();
 
-	char *str = (char*)"REGISTER sip:mx.com SIP/2.0\r\nv: SIP/2.0/UDP 1.2.3.4:5080;branch=asf4aas-5454sadfasfasdf545fsd454asfd46saf;nat=true\r\nv: SIP/2.0/UDP 5.6.7.8:5060;rport=5060;branch=asdf4as54f65as4df5sdaffds\r\nRecord-Route: <sip:1.2.3.4:5080;transport=udp;dest=5.6.7.8-5060;to-tag=6546565654;lr=1>\r\nf: <sip:[email protected]>;tag=6546565654\r\nt: <sip:[email protected]>\r\ni: [email protected]\r\nCSeq: 128752 REGISTER\r\nMax-Forwards: 10\r\nAccept-Encoding: identity\r\nAccept: application/sdp, multipart/mixed\r\nAllow: INVITE,ACK,OPTIONS,CANCEL,BYE,UPDATE,PRACK,INFO,SUBSCRIBE,NOTIFY,REFER,MESSAGE,PUBLISH\r\nAllow-Events: telephone-event,refer,reg\r\nSupported: 100rel,replaces\r\nl: 0\r\n\r\n";
-	for(int i = 0; i < 100000; i++) {
-		pp.parseData(str, strlen(str), true);
-	}
+	char *str = (char*)"";
+	cout << pp.parseData(str, strlen(str), true) << endl;
 
 	pp.debugData();
 }

voipmonitor.h

@@ -34,6 +34,8 @@
 #define GRAPH_VERSION 4294967294 
 #define GRAPH_MARK 4294967293 
 
+#define SNIFFER_INLINE_FUNCTIONS true
+#define TCPREPLAY_WORKARROUND false
 
 
 /* choose what method wil be used to synchronize threads. NONBLOCK is the fastest. Do not enable both at once */