Add file name info to SequentialFileReader. (facebook#4026)

Summary: We potentially need this information for tracing, profiling and diagnosis. Closes facebook#4026 Differential Revision: D8555214 Pulled By: riversand963 fbshipit-source-id: 4263e06c00b6d5410b46aa46eb4e358ff2161dd2
xunnanxu · Jun 21, 2018 · 524c6e6 · 524c6e6
1 parent 14cee19
commit 524c6e6
Show file tree

Hide file tree

Showing 15 changed files with 40 additions and 32 deletions.
diff --git a/db/db_impl.cc b/db/db_impl.cc
@@ -2368,7 +2368,8 @@ Status DBImpl::GetDbIdentity(std::string& identity) const {
     if (!s.ok()) {
       return s;
     }
-    id_file_reader.reset(new SequentialFileReader(std::move(idfile)));
+    id_file_reader.reset(
+        new SequentialFileReader(std::move(idfile), idfilename));
   }
 
   uint64_t file_size;

diff --git a/db/db_impl_open.cc b/db/db_impl_open.cc
@@ -577,7 +577,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
           continue;
         }
       }
-      file_reader.reset(new SequentialFileReader(std::move(file)));
+      file_reader.reset(new SequentialFileReader(std::move(file), fname));
     }
 
     // Create the log reader.

diff --git a/db/log_test.cc b/db/log_test.cc
@@ -160,8 +160,8 @@ class LogTest : public ::testing::TestWithParam<int> {
       : reader_contents_(),
         dest_holder_(test::GetWritableFileWriter(
             new test::StringSink(&reader_contents_))),
-        source_holder_(
-            test::GetSequentialFileReader(new StringSource(reader_contents_))),
+        source_holder_(test::GetSequentialFileReader(
+            new StringSource(reader_contents_), "" /* file name */)),
         writer_(std::move(dest_holder_), 123, GetParam()),
         reader_(nullptr, std::move(source_holder_), &report_, true /*checksum*/,
                 0 /*initial_offset*/, 123) {
@@ -268,8 +268,8 @@ class LogTest : public ::testing::TestWithParam<int> {
 
   void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) {
     WriteInitialOffsetLog();
-    unique_ptr<SequentialFileReader> file_reader(
-        test::GetSequentialFileReader(new StringSource(reader_contents_)));
+    unique_ptr<SequentialFileReader> file_reader(test::GetSequentialFileReader(
+        new StringSource(reader_contents_), "" /* fname */));
     unique_ptr<Reader> offset_reader(
         new Reader(nullptr, std::move(file_reader), &report_,
                    true /*checksum*/, WrittenBytes() + offset_past_end, 123));
@@ -281,8 +281,8 @@ class LogTest : public ::testing::TestWithParam<int> {
   void CheckInitialOffsetRecord(uint64_t initial_offset,
                                 int expected_record_offset) {
     WriteInitialOffsetLog();
-    unique_ptr<SequentialFileReader> file_reader(
-        test::GetSequentialFileReader(new StringSource(reader_contents_)));
+    unique_ptr<SequentialFileReader> file_reader(test::GetSequentialFileReader(
+        new StringSource(reader_contents_), "" /* fname */));
     unique_ptr<Reader> offset_reader(
         new Reader(nullptr, std::move(file_reader), &report_,
                    true /*checksum*/, initial_offset, 123));

diff --git a/db/repair.cc b/db/repair.cc
@@ -341,7 +341,7 @@ class Repairer {
       return status;
     }
     unique_ptr<SequentialFileReader> lfile_reader(
-        new SequentialFileReader(std::move(lfile)));
+        new SequentialFileReader(std::move(lfile), logname));
 
     // Create the log reader.
     LogReporter reporter;

diff --git a/db/transaction_log_impl.cc b/db/transaction_log_impl.cc
@@ -46,13 +46,14 @@ Status TransactionLogIteratorImpl::OpenLogFile(
     const LogFile* logFile, unique_ptr<SequentialFileReader>* file_reader) {
   Env* env = options_->env;
   unique_ptr<SequentialFile> file;
+  std::string fname;
   Status s;
   EnvOptions optimized_env_options = env->OptimizeForLogRead(soptions_);
   if (logFile->Type() == kArchivedLogFile) {
-    std::string fname = ArchivedLogFileName(dir_, logFile->LogNumber());
+    fname = ArchivedLogFileName(dir_, logFile->LogNumber());
     s = env->NewSequentialFile(fname, &file, optimized_env_options);
   } else {
-    std::string fname = LogFileName(dir_, logFile->LogNumber());
+    fname = LogFileName(dir_, logFile->LogNumber());
     s = env->NewSequentialFile(fname, &file, optimized_env_options);
     if (!s.ok()) {
       //  If cannot open file in DB directory.
@@ -62,7 +63,7 @@ Status TransactionLogIteratorImpl::OpenLogFile(
     }
   }
   if (s.ok()) {
-    file_reader->reset(new SequentialFileReader(std::move(file)));
+    file_reader->reset(new SequentialFileReader(std::move(file), fname));
   }
   return s;
 }

diff --git a/db/version_set.cc b/db/version_set.cc
@@ -3106,7 +3106,7 @@ Status VersionSet::Recover(
       return s;
     }
     manifest_file_reader.reset(
-        new SequentialFileReader(std::move(manifest_file)));
+        new SequentialFileReader(std::move(manifest_file), manifest_filename));
   }
   uint64_t current_manifest_file_size;
   s = env_->GetFileSize(manifest_filename, &current_manifest_file_size);
@@ -3416,7 +3416,7 @@ Status VersionSet::ListColumnFamilies(std::vector<std::string>* column_families,
   if (!s.ok()) {
     return s;
   }
-  file_reader.reset(new SequentialFileReader(std::move(file)));
+  file_reader.reset(new SequentialFileReader(std::move(file), dscname));
   }
 
   std::map<uint32_t, std::string> column_family_names;
@@ -3560,7 +3560,7 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
     if (!s.ok()) {
       return s;
     }
-    file_reader.reset(new SequentialFileReader(std::move(file)));
+    file_reader.reset(new SequentialFileReader(std::move(file), dscname));
   }
 
   bool have_prev_log_number = false;

diff --git a/db/wal_manager.cc b/db/wal_manager.cc
@@ -444,7 +444,7 @@ Status WalManager::ReadFirstLine(const std::string& fname,
   Status status = env_->NewSequentialFile(
       fname, &file, env_->OptimizeForLogRead(env_options_));
   unique_ptr<SequentialFileReader> file_reader(
-      new SequentialFileReader(std::move(file)));
+      new SequentialFileReader(std::move(file), fname));
 
   if (!status.ok()) {
     return status;

diff --git a/port/win/win_jemalloc.cc b/port/win/win_jemalloc.cc
@@ -37,12 +37,10 @@ ZSTD_customMem GetJeZstdAllocationOverrides() {
 // Global operators to be replaced by a linker when this file is
 // a part of the build
 
-void* jemalloc_aligned_alloc( size_t size, size_t alignment) ROCKSDB_NOEXCEPT {
+void* jemalloc_aligned_alloc(size_t size, size_t alignment) ROCKSDB_NOEXCEPT {
   return je_aligned_alloc(alignment, size);
 }
-void jemalloc_aligned_free(void* p) ROCKSDB_NOEXCEPT {
-  je_free(p);
-}
+void jemalloc_aligned_free(void* p) ROCKSDB_NOEXCEPT { je_free(p); }
 
 void* operator new(size_t size) {
   void* p = je_malloc(size);

diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc
@@ -1962,7 +1962,8 @@ void DumpWalFile(std::string wal_file, bool print_header, bool print_values,
     unique_ptr<SequentialFile> file;
     status = env_->NewSequentialFile(wal_file, &file, soptions);
     if (status.ok()) {
-      wal_file_reader.reset(new SequentialFileReader(std::move(file)));
+      wal_file_reader.reset(
+          new SequentialFileReader(std::move(file), wal_file));
     }
   }
   if (!status.ok()) {

diff --git a/util/file_reader_writer.h b/util/file_reader_writer.h
@@ -26,11 +26,13 @@ std::unique_ptr<RandomAccessFile> NewReadaheadRandomAccessFile(
 class SequentialFileReader {
  private:
   std::unique_ptr<SequentialFile> file_;
+  std::string file_name_;
   std::atomic<size_t> offset_;  // read offset
 
  public:
-  explicit SequentialFileReader(std::unique_ptr<SequentialFile>&& _file)
-      : file_(std::move(_file)), offset_(0) {}
+  explicit SequentialFileReader(std::unique_ptr<SequentialFile>&& _file,
+                                const std::string& _file_name)
+      : file_(std::move(_file)), file_name_(_file_name), offset_(0) {}
 
   SequentialFileReader(SequentialFileReader&& o) ROCKSDB_NOEXCEPT {
     *this = std::move(o);
@@ -52,6 +54,8 @@ class SequentialFileReader {
 
   SequentialFile* file() { return file_.get(); }
 
+  std::string file_name() { return file_name_; }
+
   bool use_direct_io() const { return file_->use_direct_io(); }
 };
 

diff --git a/util/file_util.cc b/util/file_util.cc
@@ -41,7 +41,7 @@ Status CopyFile(Env* env, const std::string& source,
         return s;
       }
     }
-    src_reader.reset(new SequentialFileReader(std::move(srcfile)));
+    src_reader.reset(new SequentialFileReader(std::move(srcfile), source));
     dest_writer.reset(new WritableFileWriter(std::move(destfile), soptions));
   }
 

diff --git a/util/testutil.cc b/util/testutil.cc
@@ -135,9 +135,10 @@ RandomAccessFileReader* GetRandomAccessFileReader(RandomAccessFile* raf) {
                                     "[test RandomAccessFileReader]");
 }
 
-SequentialFileReader* GetSequentialFileReader(SequentialFile* se) {
+SequentialFileReader* GetSequentialFileReader(SequentialFile* se,
+                                              const std::string& fname) {
   unique_ptr<SequentialFile> file(se);
-  return new SequentialFileReader(std::move(file));
+  return new SequentialFileReader(std::move(file), fname);
 }
 
 void CorruptKeyType(InternalKey* ikey) {

diff --git a/util/testutil.h b/util/testutil.h
@@ -187,7 +187,8 @@ extern WritableFileWriter* GetWritableFileWriter(WritableFile* wf);
 
 extern RandomAccessFileReader* GetRandomAccessFileReader(RandomAccessFile* raf);
 
-extern SequentialFileReader* GetSequentialFileReader(SequentialFile* se);
+extern SequentialFileReader* GetSequentialFileReader(SequentialFile* se,
+                                                     const std::string& fname);
 
 class StringSink: public WritableFile {
  public:

diff --git a/utilities/backupable/backupable_db.cc b/utilities/backupable/backupable_db.cc
@@ -1218,7 +1218,7 @@ Status BackupEngineImpl::CopyOrCreateFile(
   unique_ptr<SequentialFileReader> src_reader;
   unique_ptr<char[]> buf;
   if (!src.empty()) {
-    src_reader.reset(new SequentialFileReader(std::move(src_file)));
+    src_reader.reset(new SequentialFileReader(std::move(src_file), src));
     buf.reset(new char[copy_file_buffer_size_]);
   }
 
@@ -1417,7 +1417,7 @@ Status BackupEngineImpl::CalculateChecksum(const std::string& src, Env* src_env,
   }
 
   unique_ptr<SequentialFileReader> src_reader(
-      new SequentialFileReader(std::move(src_file)));
+      new SequentialFileReader(std::move(src_file), src));
   std::unique_ptr<char[]> buf(new char[copy_file_buffer_size_]);
   Slice data;
 
@@ -1641,7 +1641,7 @@ Status BackupEngineImpl::BackupMeta::LoadFromFile(
   }
 
   unique_ptr<SequentialFileReader> backup_meta_reader(
-      new SequentialFileReader(std::move(backup_meta_file)));
+      new SequentialFileReader(std::move(backup_meta_file), meta_filename_));
   unique_ptr<char[]> buf(new char[max_backup_meta_file_size_ + 1]);
   Slice data;
   s = backup_meta_reader->Read(max_backup_meta_file_size_, &data, buf.get());

diff --git a/utilities/blob_db/blob_file.cc b/utilities/blob_db/blob_file.cc
@@ -84,14 +84,15 @@ std::shared_ptr<Reader> BlobFile::OpenSequentialReader(
     Env* env, const DBOptions& db_options,
     const EnvOptions& env_options) const {
   std::unique_ptr<SequentialFile> sfile;
-  Status s = env->NewSequentialFile(PathName(), &sfile, env_options);
+  std::string path_name(PathName());
+  Status s = env->NewSequentialFile(path_name, &sfile, env_options);
   if (!s.ok()) {
     // report something here.
     return nullptr;
   }
 
   std::unique_ptr<SequentialFileReader> sfile_reader;
-  sfile_reader.reset(new SequentialFileReader(std::move(sfile)));
+  sfile_reader.reset(new SequentialFileReader(std::move(sfile), path_name));
 
   std::shared_ptr<Reader> log_reader = std::make_shared<Reader>(
       std::move(sfile_reader), db_options.env, db_options.statistics.get());