Create directories for file output streams (LLNL#241)

* Add Log::perror()

* Create directories for file output streams

* Add directory create and file I/O test

include/caliper/common/Log.h

@@ -1,7 +1,7 @@
 // Copyright (c) 2019, Lawrence Livermore National Security, LLC.
 // See top-level LICENSE file for details.
 
-/// @file Log.h
+/// \file Log.h
 /// Log class declaration
 
 #ifndef CALI_LOG_H
@@ -13,6 +13,7 @@
 namespace cali
 {
 
+/// \brief A logging stream
 class Log
 {
     std::ofstream m_nullstream;
@@ -33,6 +34,25 @@ class Log
         return get_stream();
     }
 
+    /// \brief Print error message for a C/POSIX errno on the log stream
+    ///
+    /// Prints an error message for an \a errno value set by a POSIX call.
+    /// Does not append a newline, users should add a line break explicitly if
+    /// needed. Example:
+    ///
+    /// \code
+    ///    const char* filename = "/usr/bin/ls";
+    ///    if (open(filename, O_RDWD) == -1) {
+    ///        Log(0).perror(errno, "open: ") << ": " << filename << std::endl;
+    ///        // possible output: "open: Permission denied: /usr/bin/ls"
+    ///    }
+    /// \endcode
+    ///
+    /// \param \errnum The errno value
+    /// \param \msg Optional prefix message
+    /// \return The log stream's \a std::ostream object
+    std::ostream& perror(int errnum, const char* msg = "");
+
     Log(unsigned level = 1)
         : m_level { level }
         { }

src/common/Log.cpp

@@ -7,6 +7,7 @@
 #include "caliper/common/Log.h"
 #include "caliper/common/RuntimeConfig.h"
 
+#include <cstring>
 #include <memory>
 #include <map>
 
@@ -38,7 +39,7 @@ struct LogImpl
     void init_stream() {
         string name = m_config.get("logfile").to_string();
 
-        const map<string, Stream> strmap { 
+        const map<string, Stream> strmap {
             { "none",   Stream::None   },
             { "stdout", Stream::StdOut },
             { "stderr", Stream::StdErr } };
@@ -58,7 +59,7 @@ struct LogImpl
 
     // --- interface
 
-    LogImpl() 
+    LogImpl()
         : m_config { RuntimeConfig::get_default_config().init("log", s_configdata) },
           m_prefix { s_prefix }
     {
@@ -95,7 +96,7 @@ const ConfigSet::Entry LogImpl::s_configdata[] = {
       "Verbosity level.\n"
       "  0: no output\n"
       "  1: basic informational runtime output\n"
-      "  2: debug output" 
+      "  2: debug output"
     },
     { "logfile",   CALI_TYPE_STRING, "stderr",
       "Log file name",
@@ -105,33 +106,47 @@ const ConfigSet::Entry LogImpl::s_configdata[] = {
       "   none:   No output,\n"
       " or a log file name."
     },
-    ConfigSet::Terminator 
+    ConfigSet::Terminator
 };
 
 
 //
 // --- Log public interface
 //
 
-ostream& 
+ostream&
 Log::get_stream()
 {
     return (LogImpl::instance()->get_stream() << LogImpl::instance()->m_prefix);
 }
 
-unsigned 
+ostream&
+Log::perror(int errnum, const char* msg)
+{
+    if (verbosity() < m_level)
+        return m_nullstream;
+
+#ifdef _GLIBCXX_HAVE_STRERROR_R
+    char buf[120];
+    return get_stream() << msg << strerror_r(errnum, buf, sizeof(buf));
+#else
+    return get_stream() << msg << strerror(errnum);
+#endif
+}
+
+unsigned
 Log::verbosity()
 {
     return LogImpl::instance()->m_verbosity;
 }
 
-void 
+void
 Log::set_verbosity(unsigned v)
 {
     LogImpl::instance()->m_verbosity = v;
 }
 
-void 
+void
 Log::add_prefix(const std::string& prefix)
 {
     LogImpl::instance()->m_prefix += prefix;

src/common/OutputStream.cpp

@@ -6,48 +6,93 @@
 #include "caliper/common/Log.h"
 #include "caliper/common/SnapshotTextFormatter.h"
 
+#include <errno.h>
+#include <sys/stat.h>
+
 #include <cstring>
 #include <fstream>
 #include <mutex>
 #include <sstream>
 
 using namespace cali;
 
+namespace
+{
+
+bool check_and_create_directory(const std::string& filepath)
+{
+    auto pos = filepath.find_last_of('/');
+
+    if (pos == 0 || pos == std::string::npos)
+        return true;
+
+    // Check and create parent directories
+    std::string dir = filepath.substr(0, pos);
+
+    // Check if the directory exists
+    struct stat sb;
+    if (stat(dir.c_str(), &sb) == -1) {
+        // Doesn't exist - recursively descend and create the path
+        if (errno == ENOENT) {
+            if (!check_and_create_directory(dir))
+                return false;
+
+            Log(2).stream() << "OutputStream: creating directory " << dir << std::endl;
+
+            if (mkdir(dir.c_str(), 0755) == -1) {
+                Log(0).perror(errno, "OutputStream: mkdir: ") << ": " << dir << std::endl;
+                return false;
+            }
+        } else {
+            Log(0).perror(errno, "OutputStream: stat: ") << ": " << dir << std::endl;
+            return false;
+        }
+    } else if (!S_ISDIR(sb.st_mode)) {
+        Log(0).stream() << "OutputStream: " << dir << " is not a directory" << std::endl;
+        return false;
+    }
+
+    return true;
+}
+
+}
+
 struct OutputStream::OutputStreamImpl
 {
     StreamType    type;
 
     bool          is_initialized;
     std::mutex    init_mutex;
-    
+
     std::string   filename;
     std::ofstream fs;
 
     std::ostream* user_os;
 
-    void init() {        
+    void init() {
         if (is_initialized)
             return;
 
         std::lock_guard<std::mutex>
             g(init_mutex);
-        
+
         is_initialized = true;
 
         if (type == StreamType::File) {
+            check_and_create_directory(filename);
             fs.open(filename);
 
             if (!fs.is_open()) {
                 type = StreamType::None;
-                
+
                 Log(0).stream() << "Could not open output stream " << filename << std::endl;
             }
         }
     }
 
     std::ostream* stream() {
         init();
-    
+
         switch (type) {
         case None:
             return &fs;
@@ -71,7 +116,7 @@ struct OutputStream::OutputStreamImpl
         type = StreamType::None;
         is_initialized = false;
     }
-    
+
     OutputStreamImpl()
         : type(StreamType::None), is_initialized(false), user_os(nullptr)
     { }
@@ -118,7 +163,7 @@ void
 OutputStream::set_stream(std::ostream* os)
 {
     mP->reset();
-    
+
     mP->type    = StreamType::User;
     mP->user_os = os;
 }
@@ -136,7 +181,7 @@ void
 OutputStream::set_filename(const char* formatstr, const CaliperMetadataAccessInterface& db, const std::vector<Entry>& rec)
 {
     mP->reset();
-    
+
     if      (strcmp(formatstr, "stdout") == 0)
         mP->type = StreamType::StdOut;
     else if (strcmp(formatstr, "stderr") == 0)
@@ -146,7 +191,7 @@ OutputStream::set_filename(const char* formatstr, const CaliperMetadataAccessInt
         std::ostringstream    fnamestr;
 
         formatter.print(fnamestr, db, rec);
-        
+
         mP->filename = fnamestr.str();
         mP->type     = StreamType::File;
     }

test/ci_app_tests/CMakeLists.txt

@@ -44,6 +44,7 @@ set(PYTHON_SCRIPTS
   test_aggregate.py
   test_basictrace.py
   test_c_api.py
+  test_file_io.py
   test_json.py
   test_log.py
   test_multichannel.py

test/ci_app_tests/test_file_io.py

@@ -0,0 +1,37 @@
+# File and directory creation test cases
+
+import os
+import unittest
+
+import calipertest as cat
+
+class CaliperFileIOTest(unittest.TestCase):
+    """ Caliper file and directory creation test case """
+
+    def test_createdir(self):
+        target_cmd = [ './ci_test_macros', '0', 'hatchet-region-profile,output.format=json,output=foo/bar/test.json' ]
+
+        caliper_config = {
+            'CALI_LOG_VERBOSITY'     : '0'
+        }
+
+        if os.path.exists('foo/bar/test.json'):
+            os.remove('foo/bar/test.json')
+        if os.path.exists('foo/bar'):
+            os.removedirs('foo/bar')
+
+        self.assertFalse(os.path.exists('foo/bar'))
+
+        cat.run_test(target_cmd, caliper_config)
+
+        self.assertTrue(os.path.isdir('foo/bar'))
+        self.assertTrue(os.path.isfile('foo/bar/test.json'))
+
+        if os.path.exists('foo/bar/test.json'):
+            os.remove('foo/bar/test.json')
+        if os.path.exists('foo/bar'):
+            os.removedirs('foo/bar')
+
+
+if __name__ == "__main__":
+    unittest.main()