Update python reader (LLNL#375)

* python reader: Add Attribute::metadata()

* Add Attribute::is_global()

* Add python reader API in sphinx doc

* Small documentation update

* Documentation updates

doc/sphinx/conf.py.in

@@ -21,6 +21,8 @@ import shlex
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #sys.path.insert(0, os.path.abspath('.'))
 
+sys.path.insert(0, os.path.abspath('@CMAKE_SOURCE_DIR@/python/caliper-reader'))
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -30,7 +32,7 @@ import shlex
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'breathe',
+    'breathe', 'sphinx.ext.autodoc'
 ]
 
 # Import doxygen docs in sphinx
@@ -56,7 +58,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Caliper'
-copyright = u'2015-2016, Lawrence Livermore National Laboratory'
+copyright = u'2015-2021, Lawrence Livermore National Laboratory'
 author = u'David Boehme'
 
 # The version info for the project you're documenting, acts as replacement for

doc/sphinx/index.rst

@@ -82,6 +82,7 @@ aspects in detail:
    calql
    OutputFormats
    tools
+   pythonreader
    DeveloperGuide
 
 Materials

doc/sphinx/pythonreader.rst

@@ -0,0 +1,66 @@
+Python reader API
+===============================================
+
+The caliper-reader is a Python API for reading Caliper's .cali file 
+format. You can install it via pip::
+
+    $ pip install caliper-reader
+
+or add the ``python/caliper-reader`` directory in the cloned Caliper
+repository to ``PYTHONPATH``.
+
+Usage
+---------------------------------------
+
+The :py:class:`caliperreader.CaliperReader` class reads a Caliper file and then provides its
+contents in the `records` and `globals` class members, where `records`
+is a Python list-of-dicts containing the recorded performance data
+and `globals` is a Python dict with program metadata about the run.
+The dicts represent Caliper attribute:value records: the key is the
+Caliper attribute name; the value is a string or list of strings.
+The example below prints the `avg#inclusive#sum#time.duration` metric
+for every region path in the provided example profile data file::
+
+    import caliperreader as cr
+
+    r = cr.CaliperReader()
+    r.read('example-profile.cali')
+
+    metric = 'avg#inclusive#sum#time.duration'
+
+    for rec in r.records:
+        path = rec['path'] if 'path' in rec else 'UNKNOWN'
+        time = rec[metric] if metric in rec else '0'
+
+        if (isinstance(path, list)):
+            path = "/".join(path)
+
+        print("{0}: {1}".format(path, time))
+
+The CaliperReader `attributes()` function returns the list of Caliper
+attributes. The `attribute()` function returns an `Attribute` object
+to query metadata for a given Caliper attribute name::
+
+    a = r.attribute('avg#inclusive#sum#time.duration')
+    a.get('attribute.unit')
+    'sec'
+
+You can use the :py:func:`caliperreader.read_caliper_contents` function as a
+shortcut to read Caliper data without creating a `CaliperReader` object::
+
+    (records,globals) = cr.read_caliper_contents('example-profile.cali')
+
+API reference
+------------------------------------------
+
+.. automodule:: caliperreader
+    :members: read_caliper_contents, read_caliper_globals
+
+.. autoclass:: caliperreader.CaliperReader
+    :members:
+
+.. autoclass:: caliperreader.CaliperStreamReader
+    :members:
+
+.. autoclass:: caliperreader.metadatadb.Attribute
+    :members:

python/caliper-reader/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2015-2020, Lawrence Livermore National Security, LLC.
+Copyright (c) 2015-2021, Lawrence Livermore National Security, LLC.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

python/caliper-reader/caliperreader/caliperreader.py

@@ -51,7 +51,7 @@ def read(self, filename_or_stream):
 
 
     def attributes(self):
-        """ Return the attribute keys.
+        """ Return the list of attribute names.
 
         A file must have been read with read().
 

python/caliper-reader/caliperreader/caliperstreamreader.py

@@ -50,7 +50,7 @@ def read(self, filename_or_stream, process_record_fn = None):
 
 
     def attributes(self):
-        """ Return the attribute keys.
+        """ Return the list of attribute names.
 
         A file must have been read with read().
 

python/caliper-reader/caliperreader/metadatadb.py

@@ -63,6 +63,7 @@ class Attribute:
 
     CALI_ATTR_ASVALUE =   1
     CALI_ATTR_NESTED  = 256
+    CALI_ATTR_GLOBAL  = 512
 
     def __init__(self, node):
         self.node = node
@@ -114,6 +115,33 @@ def is_value(self):
         prop = self.node.get(self.prop_attribute_id)
         return (int(prop) & self.CALI_ATTR_ASVALUE) != 0
 
+    def is_global(self):
+        """ Is this a global attribute (run metadata)?
+
+        Attributes with the "global" property describe run metadata
+        like the execution environment or program configuration.
+        """
+
+        prop = self.node.get(self.prop_attribute_id)
+        return (int(prop) & self.CALI_ATTR_GLOBAL) != 0
+
+    def metadata(self):
+        """ Return a dict with all metadata entries for this attribute
+        """
+
+        node = self.node
+        result = {
+            "is_global" : self.is_global() ,
+            "is_value"  : self.is_value()  ,
+            "is_nested" : self.is_nested()
+        }
+
+        while node is not None:
+            result[node.attribute().name()] = node.data
+            node = node.parent
+
+        return result
+
 
 class MetadataDB:
     """ The Caliper metadata tree """

python/caliper-reader/caliperreader/version.py

@@ -3,5 +3,5 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-__version_info__ = ("0", "1", "1")
+__version_info__ = ("0", "2", "0")
 __version__ = ".".join(__version_info__)

python/caliper-reader/tests/test_reader.py

@@ -28,13 +28,23 @@ def test_reader_class(self):
         self.assertTrue( r.attribute('function').is_nested())
         self.assertFalse(r.attribute('function').is_value() )
 
-        self.assertFalse(r.attribute('avg#inclusive#sum#time.duration').is_nested())
-        self.assertTrue( r.attribute('avg#inclusive#sum#time.duration').is_value() )
+        attr = r.attribute("avg#inclusive#sum#time.duration")
+
+        self.assertFalse(attr.is_nested())
+        self.assertFalse(attr.is_global())
+        self.assertTrue( attr.is_value() )
+
+        self.assertIsNotNone(attr.get('attribute.unit'))
 
         self.assertEqual(r.attribute('function').attribute_type(), 'string')
         self.assertEqual(r.attribute('figure_of_merit').get('adiak.type'), 'double')
 
-        self.assertIsNotNone(r.attribute('avg#inclusive#sum#time.duration').get('attribute.unit'))
+        meta = r.attribute('figure_of_merit').metadata()
+
+        self.assertEqual( meta['adiak.type'], 'double' )
+        self.assertEqual( meta['is_value'  ], False    )
+        self.assertEqual( meta['is_global' ], True     )
+
         self.assertIsNone(r.attribute('function').get('attribute.unit'))
         self.assertIsNone(r.attribute('function').get('DOES NOT EXIST'))