Removed GLib dependency and refactored storage functionality.

By removing this dependency, GoAccess is able to store data on a more
efficient manner. For instance, it avoids storing integer data as
void* (generic typing), thus improving memory consumption for integers
and gaining some performance too.

Makefile.am

@@ -57,8 +57,9 @@ goaccess_SOURCES += \
    src/tcbtdb.h
 else
 goaccess_SOURCES += \
-   src/glibht.c     \
-   src/glibht.h
+   src/khash.h      \
+   src/gkhash.c     \
+   src/gkhash.h
 endif
 
 if GEOLOCATION
@@ -77,7 +78,6 @@ if WITH_RDYNAMIC
 AM_LDFLAGS = -rdynamic
 endif
 
-AM_CFLAGS += @GLIB2_CFLAGS@
 AM_CFLAGS += -Wno-long-long -Wall -W -Wnested-externs -Wformat=2
 AM_CFLAGS += -Wmissing-prototypes -Wstrict-prototypes -Wmissing-declarations
 AM_CFLAGS += -Wwrite-strings -Wshadow -Wpointer-arith -Wsign-compare

configure.ac

@@ -154,36 +154,6 @@ if test "$WITH_TC" = "yes"; then
        CFLAGS="$CFLAGS -ltokyocabinet -lrt -lc"
     ;;
   esac
-
-# GLib otherwise
-else
-  # Check for pkg-config program, used for configuring some libraries.
-  m4_define_default([PKG_PROG_PKG_CONFIG],
-    [AC_MSG_CHECKING([pkg-config])
-    AC_MSG_RESULT([no])])
-
-  PKG_PROG_PKG_CONFIG
-
-  # If pkg-config autoconf support isn't installed, define its autoconf macro.
-  m4_define_default([PKG_CHECK_MODULES],
-    [AC_MSG_CHECKING([$1])
-    AC_MSG_RESULT([no])
-    $4])
-
-  AC_PATH_PROG([PKG_CONFIG], [pkg-config], [no])
-
-  AS_IF([test "x$PKG_CONFIG" = "xno"],[
-     AC_MSG_ERROR([
-     *** pkg-config script could not be found. Make sure it is
-     *** in your path, or set the PKG_CONFIG environment variable
-     *** to the full path to pkg-config. Otherwise, reinstall glib2
-     *** development files (libglib2.0-dev)])
-  ])
-
-  PKG_CHECK_MODULES(GLIB2, glib-2.0, [], AC_MSG_ERROR([*** Missing development libraries for GLib]))
-  AC_SUBST(GLIB2_CFLAGS)
-  AC_SUBST(GLIB2_LIBS)
-  AC_CHECK_LIB([glib-2.0], [g_list_append], [], [AC_MSG_ERROR([*** Missing development libraries for GLib])])
 fi
 AM_CONDITIONAL([TCB], [test "$WITH_TC" = "yes"])
 
@@ -192,7 +162,7 @@ if test "$tcb" = "memhash"; then
 elif test "$tcb" = "btree"; then
   storage="On-disk B+ Tree Database (Tokyo Cabinet)"
 else
-  storage="On-memory Hash Database (GLib)"
+  storage="On-memory Hash Database (Default)"
 fi
 
 # Solaris

src/commons.c

@@ -76,7 +76,7 @@ display_storage (void)
 #elif TCB_MEMHASH
   fprintf (stdout, "Built using Tokyo Cabinet On-Memory Hash database.\n");
 #else
-  fprintf (stdout, "Built using GLib On-Memory Hash database.\n");
+  fprintf (stdout, "Built using the default On-Memory Hash database.\n");
 #endif
 }
 

src/csv.c

@@ -38,7 +38,7 @@
 #ifdef HAVE_LIBTOKYOCABINET
 #include "tcabdb.h"
 #else
-#include "glibht.h"
+#include "gkhash.h"
 #endif
 
 #include "ui.h"
@@ -228,27 +228,27 @@ print_csv_summary (FILE * fp, GLog * logger)
 
   /* visitors */
   fmt = "\"%d\",,\"%s\",,,,,,,,\"%d\",\"%s\"\r\n";
-  total = get_ht_size_by_metric (VISITORS, MTRC_UNIQMAP);
+  total = ht_get_size_uniqmap (VISITORS);
   fprintf (fp, fmt, i++, GENER_ID, total, OVERALL_VISITORS);
 
   /* files */
-  total = get_ht_size_by_metric (REQUESTS, MTRC_DATAMAP);
+  total = ht_get_size_datamap (REQUESTS);
   fprintf (fp, fmt, i++, GENER_ID, total, OVERALL_FILES);
 
   /* excluded hits */
   total = logger->excluded_ip;
   fprintf (fp, fmt, i++, GENER_ID, total, OVERALL_EXCL_HITS);
 
   /* referrers */
-  total = get_ht_size_by_metric (REFERRERS, MTRC_DATAMAP);
+  total = ht_get_size_datamap (REFERRERS);
   fprintf (fp, fmt, i++, GENER_ID, total, OVERALL_REF);
 
   /* not found */
-  total = get_ht_size_by_metric (NOT_FOUND, MTRC_DATAMAP);
+  total = ht_get_size_datamap (NOT_FOUND);
   fprintf (fp, fmt, i++, GENER_ID, total, OVERALL_NOTFOUND);
 
   /* static files */
-  total = get_ht_size_by_metric (REQUESTS_STATIC, MTRC_DATAMAP);
+  total = ht_get_size_datamap (REQUESTS_STATIC);
   fprintf (fp, fmt, i++, GENER_ID, total, OVERALL_STATIC);
 
   /* log size */
@@ -271,7 +271,7 @@ print_csv_summary (FILE * fp, GLog * logger)
 
 #pragma GCC diagnostic warning "-Wformat-nonliteral"
 
-/* entry point to generate a a csv report writing it to the fp */
+/* Entry point to generate a a csv report writing it to the fp */
 void
 output_csv (GLog * logger, GHolder * holder)
 {

src/gdashboard.c

@@ -31,14 +31,9 @@
 
 #include "gdashboard.h"
 
-#ifdef HAVE_LIBTOKYOCABINET
-#include "tcabdb.h"
-#else
-#include "glibht.h"
-#endif
-
-#include "error.h"
 #include "color.h"
+#include "error.h"
+#include "gstorage.h"
 #include "util.h"
 #include "xmalloc.h"
 

src/gdns.c

@@ -48,7 +48,7 @@
 #ifdef HAVE_LIBTOKYOCABINET
 #include "tcabdb.h"
 #else
-#include "glibht.h"
+#include "gkhash.h"
 #endif
 
 #include "error.h"
@@ -182,10 +182,9 @@ void
 dns_resolver (char *addr)
 {
   pthread_mutex_lock (&gdns_thread.mutex);
+  /* queue is not full and the IP address is not in the queue */
   if (!gqueue_full (gdns_queue) && !gqueue_find (gdns_queue, addr)) {
-#ifndef HAVE_LIBTOKYOCABINET
-    g_hash_table_replace (ht_hostnames, g_strdup (addr), NULL);
-#endif
+    /* add the IP to the queue */
     gqueue_enqueue (gdns_queue, addr);
     pthread_cond_broadcast (&gdns_thread.not_empty);
   }
@@ -215,13 +214,12 @@ dns_worker (void GO_UNUSED (*ptr_data))
         free (host);
       break;
     }
-#ifdef HAVE_LIBTOKYOCABINET
-    tcadbput2 (ht_hostnames, ip, host);
-    free (host);
-#else
-    if (host != NULL && active_gdns)
-      g_hash_table_replace (ht_hostnames, g_strdup (ip), host);
-#endif
+
+    /* insert the corresponding IP -> hostname map */
+    if (host != NULL && active_gdns) {
+      ht_insert_hostname (ip, host);
+      free (host);
+    }
 
     pthread_cond_signal (&gdns_thread.not_full);
     pthread_mutex_unlock (&gdns_thread.mutex);

src/gholder.c

@@ -24,12 +24,18 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "gholder.h"
+#ifdef HAVE_LIBTOKYOCABINET
+#include "tcabdb.h"
+#else
+#include "gkhash.h"
+#endif
 
 #ifdef HAVE_LIBGEOIP
 #include "geolocation.h"
 #endif
 
+#include "gholder.h"
+
 #include "error.h"
 #include "gdns.h"
 #include "util.h"
@@ -71,7 +77,6 @@ static GPanel paneling[] = {
   {STATUS_CODES    , add_root_to_holder, NULL} ,
   {VISIT_TIMES     , add_data_to_holder, NULL} ,
 };
-
 /* *INDENT-ON* */
 
 static GPanel *
@@ -376,20 +381,20 @@ add_host_child_to_holder (GHolder * h)
   GMetrics *nmetrics;
   GSubList *sub_list = new_gsublist ();
 
-  char *host = h->items[h->idx].metrics->data;
+  char *ip = h->items[h->idx].metrics->data;
   char *hostname = NULL;
   int n = h->sub_items_size;
 
   /* add child nodes */
   set_host_sub_list (h, sub_list);
 
   pthread_mutex_lock (&gdns_thread.mutex);
-  hostname = get_hostname (host);
+  hostname = ht_get_hostname (ip);
   pthread_mutex_unlock (&gdns_thread.mutex);
 
-  /* hostname */
+  /* determine if we have the IP's hostname */
   if (!hostname) {
-    dns_resolver (host);
+    dns_resolver (ip);
   } else if (hostname) {
     set_host_child_metrics (hostname, MTRC_ID_HOSTNAME, &nmetrics);
     add_sub_item_back (sub_list, h->module, nmetrics);
@@ -412,40 +417,34 @@ add_host_to_holder (GRawDataItem item, GHolder * h, const GPanel * panel)
 static void
 add_data_to_holder (GRawDataItem item, GHolder * h, const GPanel * panel)
 {
-  GDataMap *map;
   char *data = NULL, *method = NULL, *protocol = NULL;
-  int data_nkey = 0, visitors = 0;
+  int visitors = 0;
   uint64_t bw = 0, cumts = 0, maxts = 0;
 
-  data_nkey = (*(int *) item.key);
-  map = (GDataMap *) item.value;
-  if (map == NULL)
-    return;
-
-  if (!(data = get_node_from_key (data_nkey, h->module, MTRC_DATAMAP)))
+  if (!(data = ht_get_datamap (h->module, item.key)))
     return;
 
-  bw = get_cumulative_from_key (data_nkey, h->module, MTRC_BW);
-  cumts = get_cumulative_from_key (data_nkey, h->module, MTRC_CUMTS);
-  maxts = get_cumulative_from_key (data_nkey, h->module, MTRC_MAXTS);
-  visitors = get_num_from_key (data_nkey, h->module, MTRC_VISITORS);
+  bw = ht_get_bw (h->module, item.key);
+  cumts = ht_get_cumts (h->module, item.key);
+  maxts = ht_get_maxts (h->module, item.key);
+  visitors = ht_get_visitors (h->module, item.key);
 
   h->items[h->idx].metrics = new_gmetrics ();
-  h->items[h->idx].metrics->hits = map->data;
+  h->items[h->idx].metrics->hits = item.value;
   h->items[h->idx].metrics->visitors = visitors;
   h->items[h->idx].metrics->data = data;
   h->items[h->idx].metrics->bw.nbw = bw;
-  h->items[h->idx].metrics->avgts.nts = cumts / map->data;;
+  h->items[h->idx].metrics->avgts.nts = cumts / item.value;
   h->items[h->idx].metrics->cumts.nts = cumts;
   h->items[h->idx].metrics->maxts.nts = maxts;
 
   if (conf.append_method) {
-    method = get_node_from_key (data_nkey, h->module, MTRC_METHODS);
+    method = ht_get_method (h->module, item.key);
     h->items[h->idx].metrics->method = method;
   }
 
   if (conf.append_protocol) {
-    protocol = get_node_from_key (data_nkey, h->module, MTRC_PROTOCOLS);
+    protocol = ht_get_protocol (h->module, item.key);
     h->items[h->idx].metrics->protocol = protocol;
   }
 
@@ -456,29 +455,28 @@ add_data_to_holder (GRawDataItem item, GHolder * h, const GPanel * panel)
 }
 
 static int
-set_root_metrics (int data_nkey, GDataMap * map, GModule module,
-                  GMetrics ** nmetrics)
+set_root_metrics (int key, int hits, GModule module, GMetrics ** nmetrics)
 {
   GMetrics *metrics;
   char *data = NULL;
   uint64_t bw = 0, cumts = 0, maxts = 0;
   int visitors = 0;
 
-  if (!(data = get_node_from_key (data_nkey, module, MTRC_DATAMAP)))
+  if (!(data = ht_get_datamap (module, key)))
     return 1;
 
-  bw = get_cumulative_from_key (data_nkey, module, MTRC_BW);
-  cumts = get_cumulative_from_key (data_nkey, module, MTRC_CUMTS);
-  maxts = get_cumulative_from_key (data_nkey, module, MTRC_MAXTS);
-  visitors = get_num_from_key (data_nkey, module, MTRC_VISITORS);
+  bw = ht_get_bw (module, key);
+  cumts = ht_get_cumts (module, key);
+  maxts = ht_get_maxts (module, key);
+  visitors = ht_get_visitors (module, key);
 
   metrics = new_gmetrics ();
-  metrics->avgts.nts = cumts / map->data;
+  metrics->avgts.nts = cumts / hits;
   metrics->cumts.nts = cumts;
   metrics->maxts.nts = maxts;
   metrics->bw.nbw = bw;
   metrics->data = data;
-  metrics->hits = map->data;
+  metrics->hits = hits;
   metrics->visitors = visitors;
   *nmetrics = metrics;
 
@@ -489,21 +487,15 @@ static void
 add_root_to_holder (GRawDataItem item, GHolder * h,
                     GO_UNUSED const GPanel * panel)
 {
-  GDataMap *map;
   GSubList *sub_list;
   GMetrics *metrics, *nmetrics;
   char *root = NULL;
-  int data_nkey = 0, root_idx = KEY_NOT_FOUND, idx = 0;
-
-  data_nkey = (*(int *) item.key);
-  map = (GDataMap *) item.value;
-  if (map == NULL)
-    return;
+  int root_idx = KEY_NOT_FOUND, idx = 0;
 
-  if (set_root_metrics (data_nkey, map, h->module, &nmetrics) == 1)
+  if (set_root_metrics (item.key, item.value, h->module, &nmetrics) == 1)
     return;
 
-  if (!(root = (get_root_from_key (map->root, h->module))))
+  if (!(root = (ht_get_root (h->module, item.key))))
     return;
 
   /* add data as a child node into holder */

src/gholder.h

@@ -27,12 +27,6 @@
 #include "commons.h"
 #include "sort.h"
 
-#ifdef HAVE_LIBTOKYOCABINET
-#include "tcabdb.h"
-#else
-#include "glibht.h"
-#endif
-
 /* Function Prototypes */
 GHolder *new_gholder (uint32_t size);
 void *add_hostname_node (void *ptr_holder);