Initial logging packets and data structures

.gitignore

@@ -28,6 +28,7 @@ build/
 bindings/python/pywatchy.c
 *.log
 src/test
+src/log
 src/watcher
 src/config.h.in~
 libtool

WatchyServer/StatsAggregator.py

@@ -6,14 +6,15 @@
 import traceback
 import ServerUtil
 
+from StatsServer import StatSession_Logs
 from StatsServer import StatSession_Hosts
 from StatsServer import StatSession_Metrics
 
-def consume (func):
+def consumer (func):
     def decorated (*args, **kwargs):
         if hasattr (args [0], 'backend'):
-            # TODO consume to backend
-            pass
+            if args [0].backend is not None:
+                args [0].backend.consume (*args, **kwargs)
         return func (*args, **kwargs)
     return decorated
 
@@ -31,34 +32,41 @@ def __init__ (self, host='localhost', port=8080, climit=20, backend=None):
         self.serverSocket.setblocking (0)
         threading.Thread.__init__ (self)
 
-    @consume
+    @consumer
     def consumeMetric (self, key, data):
         if key not in StatSession_Metrics:
             StatSession_Metrics [key] = []
         if len (StatSession_Metrics [key]) >= self.climit:
             StatSession_Metrics [key] = StatSession_Metrics [key][1:]
         StatSession_Metrics [key].append (data)
 
-    @consume
+    @consumer
     def consumeHost (self, key, data):
         if key not in StatSession_Hosts:
             StatSession_Hosts [key] = []
         if len (StatSession_Hosts [key]) >= self.climit:
             StatSession_Hosts [key] = StatSession_Hosts [key][1:]
         StatSession_Hosts [key].append (data)
 
+    @consumer
+    def consumeLog (self, key, data):
+        if key not in StatSession_Logs:
+            StatSession_Logs [key] = []
+        if len (StatSession_Logs [key]) >= self.climit:
+            StatSession_Logs [key] = StatSession_Logs [key][1:]
+        StatSession_Logs [key].append (data)
+
     def consume (self, data):
-        try:
-            key = data ['name']
-            which = data ['type']
-            if which == 'host':
-                self.consumeHost (key, data)
-            elif which == 'metric':
-                self.consumeMetric (key, data)
-            else:
-                ServerUtil.warning ('Invalid type [%s]' % which)
-        except:
-            pass
+        key = data ['name']
+        which = data ['type']
+        if which == 'host':
+            self.consumeHost (key, data)
+        elif which == 'metric':
+            self.consumeMetric (key, data)
+        elif which == 'log':
+            self.consumeLog (key, data)
+        else:
+            ServerUtil.warning ('Invalid type [%s]' % which)
 
     def run (self):
         ServerUtil.info ("Starting StatsAggregator on %s:%s" % (self.host, self.port))
@@ -78,8 +86,9 @@ def run (self):
                         sobject ['host'] = { 'host': rdata [1][0],
                                              'port': rdata [1][1]
                                          }
-                        ServerUtil.info ('Recieved stats from [%s:%i] for [%s]' \
-                                         % (sobject ['host']['host'],
+                        ServerUtil.info ('Recieved [%s] from [%s:%i] for [%s]' \
+                                         % (sobject ['type'],
+                                            sobject ['host']['host'],
                                             sobject ['host']['port'],
                                             sobject ['name']))
                         self.consume (sobject)

WatchyServer/StatsServer.py

@@ -14,8 +14,10 @@
 from gevent.pywsgi import WSGIServer
 from geventwebsocket.handler import WebSocketHandler
 
+StatSession_Logs = { }
 StatSession_Hosts = { }
 StatSession_Metrics = { }
+
 tfolder = os.path.join (os.path.dirname (os.path.abspath (__file__)), 'templates')
 sfolder = os.path.join (os.path.dirname (os.path.abspath (__file__)), 'static')
 app = Flask ('WatchyServer', template_folder=tfolder, static_folder=sfolder)
@@ -35,16 +37,25 @@ def metricsGraph (key):
 def hostsGraph (key):
     return render_template ('host.html', node=key)
 
+@app.route ("/logs/<path:key>")
+def logsView (key):
+    return render_template ('logs.html', node=key)
+
 @app.route ("/api/hosts/keys")
 def getHosts ():
     nodes = StatSession_Hosts.keys ()
     return jsonify ({'keys':nodes, 'len':len (nodes)})
 
 @app.route ("/api/metrics/keys")
-def getKeys ():
+def getMetrics ():
     nodes = StatSession_Metrics.keys ()
     return jsonify ({'keys':nodes, 'len':len (nodes)})
 
+@app.route ("/api/logs/keys")
+def getLogs ():
+    nodes = StatSession_Logs.keys ()
+    return jsonify ({'keys':nodes, 'len':len (nodes)})
+
 @app.route ("/api/metrics/graph/<path:key>")
 def getMetricGraph (key):
     if key not in StatSession_Metrics:
@@ -81,6 +92,12 @@ def getHostData (key):
         return jsonify ({'data':None, 'len':0})
     return jsonify ({'data':StatSession_Hosts [key], 'len':len (StatSession_Hosts [key])})
 
+@app.route ("/api/logs/data/<path:key>")
+def getLogData (key):
+    if key not in StatSession_Logs:
+        return jsonify ({'data':None, 'len':0})
+    return jsonify ({'data':StatSession_Logs [key], 'len':len (StatSession_Logs [key])})
+
 @app.route ("/deps/<path:path>")
 def statics (path):
     return app.send_static_file (path)

WatchyServer/templates/index.html

@@ -11,7 +11,6 @@
 
     <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>
     <script src="//netdna.bootstrapcdn.com/bootstrap/3.1.1/js/bootstrap.min.js"></script>
-    <script src="//fgnass.github.io/spin.js/jquery.spin.js"></script>
 
     <script type="text/javascript">
      $(function() {
@@ -108,6 +107,50 @@
 	 hostsWrapper ()
 	 timer = setInterval (hostsWrapper, 8000)
        }
+
+       function fillSideBar_Logs () {
+	 function logsWrapper () {
+	   var ovr = "<li class=\"active\"><a href=\"/\">Overview - Logs</a></li>"
+	   $('.table.table-body.logs').empty ()
+	   $('.nav.nav-sidebar').empty ()
+	   $('.nav.nav-sidebar').append (ovr)
+
+	   function generateTableRow (data){
+	     var lelem = data ['data'][data ['len'] - 1]
+	     var tmpl = "<tr><td>__key__</td><td>__ts__</td></tr>"
+	     var host = lelem ['host'] ['host']
+	     var key = lelem ['name']
+	     tmpl = tmpl.replace ("__key__", key)
+	     tmpl = tmpl.replace ("__ts__", Date (lelem ['timeStamp']))
+	     $('.table.table-body.logs').append (tmpl)
+	   }
+
+	   function keyFetch (keys) {
+	     for (var i in keys ['keys']) {
+	       var key = keys ['keys'][i]
+	       var tab = "<li><a href=\"/logs/__key__\">__key__</a></li>"
+	       tab = tab.replace ("__key__", key)
+	       tab = tab.replace ("__key__", key)
+	       $('.nav.nav-sidebar').append (tab)
+	       $.ajax({
+		 url: "/api/logs/data/" + key,
+		 type: "GET",
+		 dataType: "json",
+		 success: generateTableRow
+	       })
+	     }
+	   }
+
+	   $.ajax({
+	     url: "/api/logs/keys",
+	     type: "GET",
+	     dataType: "json",
+	     success: keyFetch
+	   })
+	 }
+	 logsWrapper ()
+	 timer = setInterval (logsWrapper, 8000)
+       }
 
        // initially we are on index screen
        fillSideBar_Metrics ()
@@ -121,6 +164,9 @@
 	   case "Hosts":
 	     func = fillSideBar_Hosts
 	     break;
+	   case "Logs":
+	     func = fillSideBar_Logs
+	     break;
 	   default:
 	     break;
 	 }
@@ -157,6 +203,7 @@ <h1 class="page-header">Dashboard</h1>
 	  <ul class="nav nav-tabs">
 	    <li class="active"><a href="#metrics" data-toggle="tab">Metrics</a></li>
 	    <li><a href="#hosts" data-toggle="tab">Hosts</a></li>
+	    <li><a href="#logs" data-toggle="tab">Logs</a></li>
 	  </ul>
 
 	  <!-- Tab panes -->
@@ -192,6 +239,19 @@ <h1 class="page-header">Dashboard</h1>
 	      </table>
 	    </div>
 
+	    <div class="tab-pane fade in" id="logs">
+	      <table class="table table-striped">
+		<thead>
+		  <tr>
+                    <th>Node</th>
+		    <th>Time Stamp</th>
+		  </tr>
+		</thead>
+		<tbody class="table table-body logs">
+		</tbody>
+	      </table>
+	    </div>
+
 	  </div>	  
         </div>
       </div>

WatchyServer/templates/logs.html

@@ -0,0 +1,85 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>Stats Node {{ node }}</title>
+
+    <link href="//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
+    <link href="/deps/css/index.css" rel="stylesheet" type="text/css">
+    <link href="/deps/css/graph.css" rel="stylesheet" type="text/css">
+    <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>
+    <script src="//netdna.bootstrapcdn.com/bootstrap/3.1.1/js/bootstrap.min.js"></script>
+
+    <script type="text/javascript">
+     $(function () {
+       var gkey = '{{ node }}'
+       function fillSideBar () {
+	 function keyFetch (keys) {
+	   $('.nav.nav-sidebar').empty ()
+	   var ovr = "<li><a href=\"/\">Overview</a></li>"
+	   $('.nav.nav-sidebar').append (ovr)
+
+	   for (var i in keys ["keys"]) {
+	     var ikey = keys ["keys"] [i]
+	     var tab = ""
+	     if (ikey == gkey) {
+	       tab = "<li class=\"active\"><a href=\"__key__\">__key__</a></li>"
+	     } else {
+	       tab = "<li><a href=\"__key__\">__key__</a></li>"
+	     }
+	     tab = tab.replace ("__key__", ikey)
+	     tab = tab.replace ("__key__", ikey)
+	     $('.nav.nav-sidebar').append (tab)
+	   }
+	 }
+	  $.ajax({
+	   url: "/api/logs/keys",
+	   type: "GET",
+	   dataType: "json",
+	   success: keyFetch
+	 });
+       }
+
+       fillSideBar ();
+       setInterval (fillSideBar, 10000)
+     });
+    </script>
+
+  </head>
+
+  <body>
+
+     <div class="navbar navbar-inverse navbar-fixed-top" role="navigation">
+      <div class="container-fluid">
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+            <span class="sr-only">Toggle navigation</span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <a class="navbar-brand" href="/">Watchy Stats Server</a>
+        </div>
+      </div>
+    </div>
+
+    <div class="container-fluid">
+      <div class="row">
+        <div class="col-sm-3 col-md-2 sidebar">
+          <ul class="nav nav-sidebar">
+
+          </ul>
+        </div>
+        <div class="col-sm-9 col-sm-offset-3 col-md-10 col-md-offset-2 main">
+	  <h1 class="page-header">Dashboard {{ node }}</h1>
+
+	  DO SOMETHING HERE
+
+        </div>
+      </div>
+    </div>
+
+  </body>
+</html>

src/Makefile.am

@@ -1,4 +1,4 @@
-bin_PROGRAMS = test$(EXEEXT) watcher$(EXEEXT)
+bin_PROGRAMS = test$(EXEEXT) log$(EXEEXT) watcher$(EXEEXT)
 lib_LTLIBRARIES = libwatchy.la
 
 pkgconfigdir = $(libdir)/pkgconfig
@@ -16,3 +16,6 @@ watcher_LDADD = libwatchy.la
 
 test_SOURCES = test.c
 test_LDADD = libwatchy.la
+
+log_SOURCES = log.c
+log_LDADD = libwatchy.la