Move hbase and hadoop to http /jmx collection

* remove the old java jars
* Create an http base class for hadoop metrics2
* Create hadoop namenode and datanode subclasses
* create hbase master and regionserver subclasses
* refactor some functions into utils.

Signed-off-by: Benoit Sigoure <[email protected]>

collectors/0/dfstat.py

@@ -46,17 +46,12 @@
   "rootfs",
 ])
 
-
-def err(msg):
-  print >> sys.stderr, msg
-
-
 def main():
   """dfstats main loop"""
   try:
     f_mounts = open("/proc/mounts", "r")
   except IOError, e:
-    err("error: can't open /proc/mounts: %s" % e)
+    utils.err("error: can't open /proc/mounts: %s" % e)
     return 13 # Ask tcollector to not respawn us
 
   utils.drop_privileges()
@@ -77,7 +72,7 @@ def main():
       try:
         fs_spec, fs_file, fs_vfstype, fs_mntops, fs_freq, fs_passno = line.split(None)
       except ValueError, e:
-        err("error: can't parse line at /proc/mounts: %s" % e)
+        utils.err("error: can't parse line at /proc/mounts: %s" % e)
         continue
 
       if fs_spec == "none":
@@ -107,7 +102,7 @@ def main():
       try:
         r = os.statvfs(fs_file)
       except OSError, e:
-        err("error: can't get info for mount point: %s" % fs_file)
+        utils.err("error: can't get info for mount point: %s" % fs_file)
         continue
 
       used = r.f_blocks - r.f_bfree

collectors/0/elasticsearch.py

@@ -44,11 +44,6 @@
   "red": 2,
 }
 
-
-def is_numeric(value):
-  return isinstance(value, (int, long, float))
-
-
 def err(msg):
   print >>sys.stderr, msg
 
@@ -145,7 +140,7 @@ def printmetric(metric, value, **tags):
       for stat, value in cstats.iteritems():
         if stat == "status":
           value = STATUS_MAP.get(value, -1)
-        elif not is_numeric(value):
+        elif not utils.is_numeric(value):
           continue
         printmetric("cluster." + stat, value)
 
@@ -248,14 +243,14 @@ def printmetric(metric, value, **tags):
        del d
     if "network" in nstats:
        for stat, value in nstats["network"]["tcp"].iteritems():
-         if is_numeric(value):
+         if utils.is_numeric(value):
            printmetric("network.tcp." + stat, value)
        for stat, value in nstats["transport"].iteritems():
-         if is_numeric(value):
+         if utils.is_numeric(value):
            printmetric("transport." + stat, value)
     # New in ES 0.17:
     for stat, value in nstats.get("http", {}).iteritems():
-      if is_numeric(value):
+      if utils.is_numeric(value):
         printmetric("http." + stat, value)
     del nstats
     time.sleep(COLLECTION_INTERVAL)

collectors/0/hadoop_datanode.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python
+# This file is part of tcollector.
+# Copyright (C) 2010  The tcollector Authors.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.  This program is distributed in the hope that it
+# will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
+# General Public License for more details.  You should have received a copy
+# of the GNU Lesser General Public License along with this program.  If not,
+# see <http://www.gnu.org/licenses/>.
+
+import sys
+import time
+
+try:
+    import json
+except ImportError:
+    json = None
+
+from collectors.lib import utils
+from collectors.lib.hadoop_http import HadoopHttp
+
+
+REPLACEMENTS = {
+    "datanodeactivity-": ["activity"],
+    "fsdatasetstate-ds-": ["fs_data_set_state"],
+    "rpcdetailedactivityforport": ["rpc_activity"],
+    "rpcactivityforport": ["rpc_activity"]
+}
+
+
+class HadoopDataNode(HadoopHttp):
+    """
+    Class that will retrieve metrics from an Apache Hadoop DataNode's jmx page.
+
+    This requires Apache Hadoop 1.0+ or Hadoop 2.0+.
+    Anything that has the jmx page will work but the best results will com from Hadoop 2.1.0+
+    """
+
+    def __init__(self):
+        super(HadoopDataNode, self).__init__('hadoop', 'datanode', 'localhost', 50075)
+
+    def emit(self):
+        current_time = int(time.time())
+        metrics = self.poll()
+        for context, metric_name, value in metrics:
+            for k, v in REPLACEMENTS.iteritems():
+                if any(c.startswith(k) for c in context):
+                    context = v
+            self.emit_metric(context, current_time, metric_name, value)
+
+
+def main(args):
+    utils.drop_privileges()
+    if json is None:
+        utils.err("This collector requires the `json' Python module.")
+        return 13  # Ask tcollector not to respawn us
+    datanode_service = HadoopDataNode()
+    while True:
+        datanode_service.emit()
+        time.sleep(15)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))
+