GeoIP ./configure FLAG & Pipeline implementation

ChangeLog

@@ -1,10 +1,11 @@
 Changes to GoAccess 0.4.1 - Monday, December 13, 2010
 
-   * Ability to pass a flag to ./configure so it can disable GeoIP
+   * Ability to pass a flag to ./configure so GeoIP can be enabled if
+     needed.
    * More flexibility when resizing the terminal. Should work fine with
      the standard 80x24.
-   * Implemented a pipeline from stdin, so the input is not necessarily
-     only a filename.
+   * Implemented a pipeline from stdin, so the input doesn't have to be
+     only a file.
 
 Changes to GoAccess 0.4 - Tuesday, November 30, 2010
 

Makefile.am

@@ -3,7 +3,7 @@ bin_PROGRAMS = goaccess
 goaccess_SOURCES = settings.c settings.h error.c error.h alloc.h  commons.c  commons.h  goaccess.c  parser.c  parser.h  ui.c  ui.h  util.c  util.h goaccess.h
 
 AM_CFLAGS = -g -Wall @GLIB2_CFLAGS@
-AM_LDFLAGS = -lmenu -lcurses -lGeoIP -lm
+AM_LDFLAGS = -lmenu -lcurses -lm
 LIBS = @GLIB2_LIBS@ -lrt
 dist_man_MANS = goaccess.1
 

NEWS

@@ -2,6 +2,10 @@ Copyright (C) 2010
 Gerardo Orellana <[email protected]>
 
 * Version history:
+    - 0.4.1  [Monday, December 13, 2010]
+        . GoAccess 0.4 Released. See ChangeLog for new features/bug-fixes.
+    - 0.4  [Tuesday, November 30, 2010]
+        . GoAccess 0.4 Released. See ChangeLog for new features/bug-fixes.
     - 0.3.3  [Monday, September 27, 2010]
         . GoAccess 0.3.3 Released. See ChangeLog for new features/bug-fixes.
     - 0.3.2  [Thursday, September 09 2010]

TODO

@@ -13,7 +13,7 @@ email [email protected].
  - Increase the performance when parsing the log file.
  - Resize child windows when term is resized.
  - Add command-line parameters.
- - Filtering data by date.
+ - Filtering data by date. (Done, see `man goaccess`)
  - Add more reports to it.
 
 * Documentation:

commons.c

@@ -64,6 +64,7 @@ char *ignore_host;
 int bandwidth_flag = 0;
 int host_agents_list_flag = 0;
 int http_status_code_flag = 0;
+int piping = 0;
 
 /* hash iteration */
 int iter_ctr = 0;
@@ -248,6 +249,8 @@ char *help_main[] = {
    " ^t^     [detail view] move to the first item",
    " ^b^     [detail view] move to the last item",
    "",
+   "Examples can be found by running `man goaccess`.",
+   "",
    "If you believe you have found a bug, please drop me",
    "an email with details.",
    "",

commons.h

@@ -150,6 +150,7 @@ extern char *ignore_host;
 extern int bandwidth_flag;
 extern int host_agents_list_flag;
 extern int http_status_code_flag;
+extern int piping;
 
 /* iteration */
 extern int iter_ctr;

configure.ac

@@ -16,16 +16,21 @@ PKG_CHECK_MODULES(GLIB2, glib-2.0, [], AC_MSG_ERROR([*** glib2 not found!]))
 AC_SUBST(GLIB2_CFLAGS)
 AC_SUBST(GLIB2_LIBS)
 
+AC_ARG_ENABLE(geoip,--enable-geoip Enable GeoIP country lookup, USE_GEOIP="yes")
+
+if test "$USE_GEOIP" = "yes"; then
+  AC_CHECK_LIB([GeoIP], [GeoIP_new], [], [AC_MSG_ERROR([libgeoip-dev is missing])])
+  CFLAGS="-lGeoIP"
+fi
+
 # Checks for libraries.
 AC_CHECK_LIB([ncurses], [refresh], [], [AC_MSG_ERROR([libncurses5-dev is missing])])
 AC_CHECK_LIB([menu], [new_menu], [], [AC_MSG_ERROR([libncurses5-dev is missing])])
 AC_CHECK_LIB([glib-2.0], [g_free], [], [AC_MSG_ERROR([glib-2.x is missing])])
-AC_CHECK_LIB([GeoIP], [GeoIP_new], [], [AC_MSG_ERROR([libgeoip-dev is missing])])
 
 # Checks for header files.
 AC_HEADER_STDC
-AC_CHECK_HEADERS([locale.h arpa/inet.h netdb.h stdint.h stdlib.h string.h unistd.h])
-AC_CHECK_FUNCS([setlocale])
+AC_CHECK_HEADERS([arpa/inet.h locale.h netdb.h stdint.h stdlib.h string.h unistd.h])
 
 # Checks for typedefs, structures, and compiler characteristics.
 AC_C_CONST
@@ -35,13 +40,13 @@ AC_STRUCT_TM
 AC_CHECK_TYPES([ptrdiff_t])
 
 # Checks for library functions.
+AC_FUNC_FSEEKO
 AC_FUNC_MALLOC
-AC_FUNC_REALLOC
 AC_FUNC_MEMCMP
+AC_FUNC_REALLOC
 AC_FUNC_STAT
 AC_FUNC_STRFTIME
-AC_FUNC_FSEEKO
-AC_CHECK_FUNCS([floor gethostbyaddr gethostbyname strchr strdup strerror strrchr strstr strtol memset])
+AC_CHECK_FUNCS([floor gethostbyaddr gethostbyname memset setlocale strchr strdup strerror strrchr strstr strtol])
 
 AC_CONFIG_FILES([Makefile])
 AC_OUTPUT

goaccess.1

@@ -1,11 +1,11 @@
-.TH goaccess 1 "NOVEMBER 2010" Linux "User Manuals"
+.TH goaccess 1 "DECEMBER 2010" Linux "User Manuals"
 .SH NAME
 goaccess \- fast log analyzer and interactive viewer for the Apache web server
 .SH SYNOPSIS
 .B goaccess [-f
 .I input-file
 .B ]
-.I [-b] [-s] [-e]
+.I [-b] [-s] [-e] [-a]
 .SH DESCRIPTION
 .B goaccess
 is a free (GPL) real-time Apache web log analyzer and interactive 
@@ -137,8 +137,66 @@ Find the position of the next occurrence on any {detail view} window.
 Move to the first item on any {detail view} window.
 .IP "b"
 Move to the last item on any {detail view} window.
+.SH EXAMPLES
+The simplest and fastest usage would be:
+
+# goaccess -f access.log
+
+That will generate an interactive text-only output.
+
+To generate full statistics we can run GoAccess as:
+
+# goaccess -f access.log -a -s -b
+
+The 
+.I -a 
+flag indicates that we want to process an agent-list for every host parsed.
+The 
+.I -s 
+flag tells GoAccess go get every HTTP status code.
+The 
+.I -b 
+flag will process the total bandwidth consumption for files, hosts, and dates.
+
+Now if we want to add more flexibility to GoAccess, we can do a series of  pipes.
+For instance:
+
+If we would like to process all
+.I access.log.*.gz
+we can do:
+
+#  zcat access.log.*.gz | goaccess
+
+OR
+
+#  zcat -f access.log* | goaccess
+
+Another useful pipe would be filtering dates out of the Apache's access log
+
+The following will get all HTTP requests starting on 05/Dec/2010 until 
+the end of the file.
+
+# sed -n '/05\\/Dec\\/2010/,$ p' access.log | goaccess -s -b
+
+If we want to parse only a certain time-frame from DATE a to DATE b, we can do:
+
+sed -n '/5\\/Nov\\/2010/,/5\\/Dec\\/2010/ p' access.log | goaccess -s -b
+
+.I Note that this could take longer time to parse depending on the speed of sed.
+
+.B Also,
+it is worth pointing out that if we want to run GoAccess at lower priority, we
+can run it as:
+
+# nice -n 19 goaccess -f access.log -s -a -b
+
 .SH NOTES
 On each {detail view} window, the total number of items is 300.
+
+Piping a log to GoAccess will disable the real-time functionality. This is due to the
+portability issue on determining the actual size of STDIN. However, a future release *might* 
+include this feature.
+
 .SH BUGS
 If you think you have found a bug, please send me an email to 
 .I [email protected]

goaccess.c

@@ -35,6 +35,7 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
+#include <errno.h>
 
 #include "error.h"
 #include "alloc.h"
@@ -93,6 +94,7 @@ cmd_help (void)
    printf ("                 For faster parsing, don't enable this flag.\n");
    printf (" -e <argument> - Exclude an IP from being counted under the\n");
    printf ("                 HOST module. Disabled by default.\n\n");
+   printf ("Examples can be found by running `man goaccess`.\n\n");
    printf ("For more details visit: http://goaccess.prosoftcorp.com\n");
    printf ("GoAccess Copyright (C) 2009-2010 GNU GPL'd, by Gerardo Orellana");
    printf ("\n\n");
@@ -239,10 +241,11 @@ get_keys (void)
    int y, x, c;
 
    char buf[BUFFER];
-   FILE *fp;
-   unsigned long long size1, size2;
+   FILE *fp = NULL;
+   unsigned long long size1 = 0, size2 = 0;
 
-   size1 = file_size (ifile);
+   if (!piping)
+      size1 = file_size (ifile);
    while (((c = wgetch (stdscr)) != 'q')) {
       getmaxyx (main_win, y, x);
       switch (c) {
@@ -377,6 +380,8 @@ get_keys (void)
           render_screens ();
           break;
        default:
+          if (piping)
+             break;
           size2 = file_size (ifile);
           /* file has changed */
           if (size2 != size1) {
@@ -408,9 +413,6 @@ main (int argc, char *argv[])
    extern int optind, optopt, opterr;
    int row, col, o, index;
 
-   if (argc < 2)
-      cmd_help ();
-
    opterr = 0;
    while ((o = getopt (argc, argv, "f:e:sba")) != -1) {
       switch (o) {
@@ -441,7 +443,9 @@ main (int argc, char *argv[])
           abort ();
       }
    }
-   if (!ifile)
+   if (ifile != NULL && !isatty (STDIN_FILENO))
+      cmd_help ();
+   if (ifile == NULL && isatty (STDIN_FILENO))
       cmd_help ();
    for (index = optind; index < argc; index++)
       cmd_help ();

parser.c

@@ -575,23 +575,33 @@ process_log (struct logger *logger, char *line)
 int
 parse_log (struct logger *logger, char *filename, char *tail)
 {
-   FILE *fp;
+   FILE *fp = NULL;
    char line[BUFFER];
 
    if (tail != NULL) {
       if (process_log (logger, tail))
          return 1;
       return 0;
    }
-   if ((fp = fopen (filename, "r")) == NULL)
+   if (filename == NULL) {
+      fp = stdin;
+      piping = 1;
+   }
+   if (!piping && (fp = fopen (filename, "r")) == NULL)
       error_handler (__PRETTY_FUNCTION__, __FILE__, __LINE__,
                      "An error has occurred while opening the log file. Make sure it exists.");
+
    while (fgets (line, BUFFER, fp) != NULL) {
       if (process_log (logger, line)) {
-         fclose (fp);
+         if (!piping)
+            fclose (fp);
          return 1;
       }
    }
+   /* definitely not portable! */
+   if (piping)
+      stdin = freopen ("/dev/tty", "r", stdin);
+
    return 0;
 }
 

ui.c

@@ -30,9 +30,17 @@
 #define _XOPEN_SOURCE 700
 #define STDIN_FILENO  0
 
+#if HAVE_CONFIG_H
+#include <config.h>
+#endif
+
 #include <ctype.h>
 #include <curses.h>
+
+#ifdef HAVE_LIBGEOIP
 #include <GeoIP.h>
+#endif
+
 #include <glib.h>
 #include <math.h>
 #include <menu.h>
@@ -165,14 +173,18 @@ display_general (WINDOW * header_win, struct logger *logger, char *ifile)
 {
    int row, col;
    char *head_desc =
-      " General Summary - Analysed Log Statistics - Unique totals";
+      " General Summary - Overall Analysed Requests - Unique totals";
    getmaxyx (stdscr, row, col);
    draw_header (header_win, head_desc, 0, 0, col, 1);
 
    /* general stats */
    char *bw;
-   off_t log_size = file_size (ifile);
-   char *size = filesize_str (log_size);
+   char *size;
+   if (!piping) {
+      off_t log_size = file_size (ifile);
+      size = filesize_str (log_size);
+   } else
+      size = alloc_string ("N/A");
 
    if (bandwidth_flag)
       bw = filesize_str ((float) req_size);
@@ -184,6 +196,9 @@ display_general (WINDOW * header_win, struct logger *logger, char *ifile)
    const char *format_line2 = "%-15s %-9lu %-15s %-9d [%s";
    size_t len1, len2, len3;
 
+   if (ifile == NULL)
+      ifile = "STDIN";
+
    len1 =
       snprintf (NULL, 0, format_line1, T_REQUESTS, logger->total_process,
                 T_UNIQUE_VIS, g_hash_table_size (ht_unique_visitors),
@@ -744,11 +759,15 @@ static void
 load_reverse_dns_popup (WINDOW * ip_detail_win, char *addr)
 {
    char *my_addr = NULL;
+
+#ifdef HAVE_LIBGEOIP
    const char *location = NULL;
+#endif
+
    int y, x, c, quit = 1;
    struct scrolling scrolling;
    struct struct_agents *s_agents;
-   WINDOW *inner_win;
+   WINDOW *inner_win = NULL;
 
    /* make compiler happy */
    memset (&s_agents, 0, sizeof (s_agents));
@@ -762,6 +781,7 @@ load_reverse_dns_popup (WINDOW * ip_detail_win, char *addr)
    mvwprintw (ip_detail_win, 4, 2, "%s", my_addr);
    free (my_addr);
 
+#ifdef HAVE_LIBGEOIP
    /* Geolocation data */
    GeoIP *gi;
    gi = GeoIP_new (GEOIP_STANDARD);
@@ -772,12 +792,13 @@ load_reverse_dns_popup (WINDOW * ip_detail_win, char *addr)
    mvwprintw (ip_detail_win, 5, 2, "Country: %s", location);
    if (gi != NULL)
       GeoIP_delete (gi);
+#endif
 
    char *ptr_value;
    gpointer value_ptr = NULL;
 
    int i, m, delims = 0;
-   size_t alloc = 0, width_max = 0, inner_y, inner_x;
+   size_t alloc = 0, width_max = 0, inner_y = 0, inner_x = 0;
 
    /* agents' inner win */
    if (y < 10)

util.c

@@ -151,6 +151,19 @@ file_size (const char *filename)
    return -1;
 }
 
+off_t
+stdin_size (FILE * stdin_stream)
+{
+   struct stat st;
+
+   if (fstat (fileno (stdin_stream), &st) == 0)
+      return st.st_size;
+
+   fprintf (stderr, "Cannot determine size of STDIN: %s\n", strerror (errno));
+
+   return -1;
+}
+
 char *
 verify_os (char *str)
 {