Increase max_log_size in FlushJob to 1024 bytes (facebook#6258)

Summary: When measure_io_stats_ is enabled, the volume of logging is beyond the default limit of 512 size. The patch allows the EventLoggerStream to change the limit, and also sets it to 1024 for FlushJob. Pull Request resolved: facebook#6258 Differential Revision: D19279269 Pulled By: maysamyabandeh fbshipit-source-id: 3fb5d468dad488f289ac99d713378177eb7504d6
dmitrip · Jan 6, 2020 · 28e5a9a · 28e5a9a
1 parent 83957dc
commit 28e5a9a
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 14 deletions.
diff --git a/db/flush_job.cc b/db/flush_job.cc
@@ -249,7 +249,8 @@ Status FlushJob::Run(LogsWithPrepTracker* prep_tracker,
   }
   RecordFlushIOStats();
 
-  auto stream = event_logger_->LogToBuffer(log_buffer_);
+  // When measure_io_stats_ is true, the default 512 bytes is not enough.
+  auto stream = event_logger_->LogToBuffer(log_buffer_, 1024);
   stream << "job" << job_context_->job_id << "event"
          << "flush_finished";
   stream << "output_compression"

diff --git a/logging/event_logger.cc b/logging/event_logger.cc
@@ -15,12 +15,18 @@
 
 namespace rocksdb {
 
-
 EventLoggerStream::EventLoggerStream(Logger* logger)
-    : logger_(logger), log_buffer_(nullptr), json_writer_(nullptr) {}
+    : logger_(logger),
+      log_buffer_(nullptr),
+      max_log_size_(0),
+      json_writer_(nullptr) {}
 
-EventLoggerStream::EventLoggerStream(LogBuffer* log_buffer)
-    : logger_(nullptr), log_buffer_(log_buffer), json_writer_(nullptr) {}
+EventLoggerStream::EventLoggerStream(LogBuffer* log_buffer,
+                                     const size_t max_log_size)
+    : logger_(nullptr),
+      log_buffer_(log_buffer),
+      max_log_size_(max_log_size),
+      json_writer_(nullptr) {}
 
 EventLoggerStream::~EventLoggerStream() {
   if (json_writer_) {
@@ -31,7 +37,8 @@ EventLoggerStream::~EventLoggerStream() {
     if (logger_) {
       EventLogger::Log(logger_, *json_writer_);
     } else if (log_buffer_) {
-      EventLogger::LogToBuffer(log_buffer_, *json_writer_);
+      assert(max_log_size_);
+      EventLogger::LogToBuffer(log_buffer_, *json_writer_, max_log_size_);
     }
 #endif
     delete json_writer_;
@@ -50,13 +57,14 @@ void EventLogger::Log(Logger* logger, const JSONWriter& jwriter) {
 #endif
 }
 
-void EventLogger::LogToBuffer(
-    LogBuffer* log_buffer, const JSONWriter& jwriter) {
+void EventLogger::LogToBuffer(LogBuffer* log_buffer, const JSONWriter& jwriter,
+                              const size_t max_log_size) {
 #ifdef ROCKSDB_PRINT_EVENTS_TO_STDOUT
   printf("%s\n", jwriter.Get().c_str());
 #else
   assert(log_buffer);
-  rocksdb::LogToBuffer(log_buffer, "%s %s", Prefix(), jwriter.Get().c_str());
+  rocksdb::LogToBuffer(log_buffer, max_log_size, "%s %s", Prefix(),
+                       jwriter.Get().c_str());
 #endif
 }
 

diff --git a/logging/event_logger.h b/logging/event_logger.h
@@ -162,10 +162,11 @@ class EventLoggerStream {
   }
   friend class EventLogger;
   explicit EventLoggerStream(Logger* logger);
-  explicit EventLoggerStream(LogBuffer* log_buffer);
+  explicit EventLoggerStream(LogBuffer* log_buffer, const size_t max_log_size);
   // exactly one is non-nullptr
   Logger* const logger_;
   LogBuffer* const log_buffer_;
+  const size_t max_log_size_;  // used only for log_buffer_
   // ownership
   JSONWriter* json_writer_;
 };
@@ -183,11 +184,17 @@ class EventLogger {
   explicit EventLogger(Logger* logger) : logger_(logger) {}
   EventLoggerStream Log() { return EventLoggerStream(logger_); }
   EventLoggerStream LogToBuffer(LogBuffer* log_buffer) {
-    return EventLoggerStream(log_buffer);
+    return EventLoggerStream(log_buffer, LogBuffer::kDefaultMaxLogSize);
+  }
+  EventLoggerStream LogToBuffer(LogBuffer* log_buffer,
+                                const size_t max_log_size) {
+    return EventLoggerStream(log_buffer, max_log_size);
   }
   void Log(const JSONWriter& jwriter);
   static void Log(Logger* logger, const JSONWriter& jwriter);
-  static void LogToBuffer(LogBuffer* log_buffer, const JSONWriter& jwriter);
+  static void LogToBuffer(
+      LogBuffer* log_buffer, const JSONWriter& jwriter,
+      const size_t max_log_size = LogBuffer::kDefaultMaxLogSize);
 
  private:
   Logger* logger_;

diff --git a/logging/log_buffer.cc b/logging/log_buffer.cc
@@ -81,11 +81,10 @@ void LogToBuffer(LogBuffer* log_buffer, size_t max_log_size, const char* format,
 }
 
 void LogToBuffer(LogBuffer* log_buffer, const char* format, ...) {
-  const size_t kDefaultMaxLogSize = 512;
   if (log_buffer != nullptr) {
     va_list ap;
     va_start(ap, format);
-    log_buffer->AddLogToBuffer(kDefaultMaxLogSize, format, ap);
+    log_buffer->AddLogToBuffer(LogBuffer::kDefaultMaxLogSize, format, ap);
     va_end(ap);
   }
 }

diff --git a/logging/log_buffer.h b/logging/log_buffer.h
@@ -30,6 +30,7 @@ class LogBuffer {
 
   // Flush all buffered log to the info log.
   void FlushBufferToLog();
+  static const size_t kDefaultMaxLogSize = 512;
 
  private:
   // One log entry with its timestamp