Embed column family name in SST file

Summary:
Added the column family name to the properties block. This property
is omitted only if the property is unavailable, such as when RepairDB()
writes SST files.

In a next diff, I will change RepairDB to use this new property for
deciding to which column family an existing SST file belongs. If this
property is missing, it will add it to the "unknown" column family (same
as its existing behavior).

Test Plan:
New unit test:

  $ ./db_table_properties_test --gtest_filter=DBTablePropertiesTest.GetColumnFamilyNameProperty

Reviewers: IslamAbdelRahman, yhchiang, sdong

Reviewed By: sdong

Subscribers: andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D55605

db/builder.cc

@@ -41,13 +41,17 @@ TableBuilder* NewTableBuilder(
     const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
-    uint32_t column_family_id, WritableFileWriter* file,
-    const CompressionType compression_type,
+    uint32_t column_family_id, const std::string& column_family_name,
+    WritableFileWriter* file, const CompressionType compression_type,
     const CompressionOptions& compression_opts, const bool skip_filters) {
+  assert((column_family_id ==
+          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
+         column_family_name.empty());
   return ioptions.table_factory->NewTableBuilder(
       TableBuilderOptions(ioptions, internal_comparator,
                           int_tbl_prop_collector_factories, compression_type,
-                          compression_opts, skip_filters),
+                          compression_opts, skip_filters,
+                          column_family_name),
       column_family_id, file);
 }
 
@@ -58,12 +62,16 @@ Status BuildTable(
     const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
-    uint32_t column_family_id, std::vector<SequenceNumber> snapshots,
+    uint32_t column_family_id, const std::string& column_family_name,
+    std::vector<SequenceNumber> snapshots,
     SequenceNumber earliest_write_conflict_snapshot,
     const CompressionType compression,
     const CompressionOptions& compression_opts, bool paranoid_file_checks,
     InternalStats* internal_stats, const Env::IOPriority io_priority,
     TableProperties* table_properties, int level) {
+  assert((column_family_id ==
+          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
+         column_family_name.empty());
   // Reports the IOStats for flush for every following bytes.
   const size_t kReportFlushIOStatsEvery = 1048576;
   Status s;
@@ -87,7 +95,8 @@ Status BuildTable(
 
       builder = NewTableBuilder(
           ioptions, internal_comparator, int_tbl_prop_collector_factories,
-          column_family_id, file_writer.get(), compression, compression_opts);
+          column_family_id, column_family_name, file_writer.get(), compression,
+          compression_opts);
     }
 
     MergeHelper merge(env, internal_comparator.user_comparator(),

db/builder.h

@@ -33,13 +33,16 @@ class WritableFileWriter;
 class InternalStats;
 class InternalIterator;
 
+// @param column_family_name Name of the column family that is also identified
+//    by column_family_id, or empty string if unknown. It must outlive the
+//    TableBuilder returned by this function.
 TableBuilder* NewTableBuilder(
     const ImmutableCFOptions& options,
     const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
-    uint32_t column_family_id, WritableFileWriter* file,
-    const CompressionType compression_type,
+    uint32_t column_family_id, const std::string& column_family_name,
+    WritableFileWriter* file, const CompressionType compression_type,
     const CompressionOptions& compression_opts,
     const bool skip_filters = false);
 
@@ -48,14 +51,18 @@ TableBuilder* NewTableBuilder(
 // *meta will be filled with metadata about the generated table.
 // If no data is present in *iter, meta->file_size will be set to
 // zero, and no Table file will be produced.
+//
+// @param column_family_name Name of the column family that is also identified
+//    by column_family_id, or empty string if unknown.
 extern Status BuildTable(
     const std::string& dbname, Env* env, const ImmutableCFOptions& options,
     const EnvOptions& env_options, TableCache* table_cache,
     InternalIterator* iter, FileMetaData* meta,
     const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
-    uint32_t column_family_id, std::vector<SequenceNumber> snapshots,
+    uint32_t column_family_id, const std::string& column_family_name,
+    std::vector<SequenceNumber> snapshots,
     SequenceNumber earliest_write_conflict_snapshot,
     const CompressionType compression,
     const CompressionOptions& compression_opts, bool paranoid_file_checks,

db/compaction_job.cc

@@ -1019,7 +1019,7 @@ Status CompactionJob::OpenCompactionOutputFile(
       cfd->ioptions()->optimize_filters_for_hits && bottommost_level_;
   sub_compact->builder.reset(NewTableBuilder(
       *cfd->ioptions(), cfd->internal_comparator(),
-      cfd->int_tbl_prop_collector_factories(), cfd->GetID(),
+      cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
       sub_compact->outfile.get(), sub_compact->compaction->output_compression(),
       cfd->ioptions()->compression_opts, skip_filters));
   LogFlush(db_options_.info_log);

db/db_impl.cc

@@ -1452,8 +1452,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
       s = BuildTable(
           dbname_, env_, *cfd->ioptions(), env_options_, cfd->table_cache(),
           iter.get(), &meta, cfd->internal_comparator(),
-          cfd->int_tbl_prop_collector_factories(), cfd->GetID(), snapshot_seqs,
-          earliest_write_conflict_snapshot,
+          cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
+          snapshot_seqs, earliest_write_conflict_snapshot,
           GetCompressionFlush(*cfd->ioptions()),
           cfd->ioptions()->compression_opts, paranoid_file_checks,
           cfd->internal_stats(), Env::IO_HIGH, &info.table_properties);

db/db_table_properties_test.cc

@@ -215,6 +215,34 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfTablesInRange) {
     TestGetPropertiesOfTablesInRange(std::move(ranges));
   }
 }
+
+TEST_F(DBTablePropertiesTest, GetColumnFamilyNameProperty) {
+  std::string kExtraCfName = "pikachu";
+  CreateAndReopenWithCF({kExtraCfName}, CurrentOptions());
+
+  // Create one table per CF, then verify it was created with the column family
+  // name property.
+  for (int cf = 0; cf < 2; ++cf) {
+    Put(cf, "key", "val");
+    Flush(cf);
+
+    TablePropertiesCollection fname_to_props;
+    ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[cf], &fname_to_props));
+    ASSERT_EQ(1U, fname_to_props.size());
+
+    std::string expected_cf_name;
+    if (cf > 0) {
+      expected_cf_name = kExtraCfName;
+    } else {
+      expected_cf_name = kDefaultColumnFamilyName;
+    }
+    ASSERT_EQ(expected_cf_name,
+              fname_to_props.begin()->second->column_family_name);
+    ASSERT_EQ(cf, static_cast<uint32_t>(
+                      fname_to_props.begin()->second->column_family_id));
+  }
+}
+
 }  // namespace rocksdb
 
 #endif  // ROCKSDB_LITE

db/flush_job.cc

@@ -238,8 +238,9 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
           dbname_, db_options_.env, *cfd_->ioptions(), env_options_,
           cfd_->table_cache(), iter.get(), meta, cfd_->internal_comparator(),
           cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(),
-          existing_snapshots_, earliest_write_conflict_snapshot_,
-          output_compression_, cfd_->ioptions()->compression_opts,
+          cfd_->GetName(), existing_snapshots_,
+          earliest_write_conflict_snapshot_, output_compression_,
+          cfd_->ioptions()->compression_opts,
           mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(),
           Env::IO_HIGH, &table_properties_, 0 /* level */);
       info.table_properties = table_properties_;

db/plain_table_db_test.cc

@@ -263,7 +263,9 @@ class TestPlainTableReader : public PlainTableReader {
                        const TableProperties* table_properties,
                        unique_ptr<RandomAccessFileReader>&& file,
                        const ImmutableCFOptions& ioptions,
-                       bool* expect_bloom_not_match, bool store_index_in_file)
+                       bool* expect_bloom_not_match, bool store_index_in_file,
+                       uint32_t column_family_id,
+                       const std::string& column_family_name)
       : PlainTableReader(ioptions, std::move(file), env_options, icomparator,
                          encoding_type, file_size, table_properties),
         expect_bloom_not_match_(expect_bloom_not_match) {
@@ -276,6 +278,8 @@ class TestPlainTableReader : public PlainTableReader {
     EXPECT_TRUE(s.ok());
 
     TableProperties* props = const_cast<TableProperties*>(table_properties);
+    EXPECT_EQ(column_family_id, static_cast<uint32_t>(props->column_family_id));
+    EXPECT_EQ(column_family_name, props->column_family_name);
     if (store_index_in_file) {
       auto bloom_version_ptr = props->user_collected_properties.find(
           PlainTablePropertyNames::kBloomVersion);
@@ -308,13 +312,17 @@ extern const uint64_t kPlainTableMagicNumber;
 class TestPlainTableFactory : public PlainTableFactory {
  public:
   explicit TestPlainTableFactory(bool* expect_bloom_not_match,
-                                 const PlainTableOptions& options)
+                                 const PlainTableOptions& options,
+                                 uint32_t column_family_id,
+                                 std::string column_family_name)
       : PlainTableFactory(options),
         bloom_bits_per_key_(options.bloom_bits_per_key),
         hash_table_ratio_(options.hash_table_ratio),
         index_sparseness_(options.index_sparseness),
         store_index_in_file_(options.store_index_in_file),
-        expect_bloom_not_match_(expect_bloom_not_match) {}
+        expect_bloom_not_match_(expect_bloom_not_match),
+        column_family_id_(column_family_id),
+        column_family_name_(std::move(column_family_name)) {}
 
   Status NewTableReader(const TableReaderOptions& table_reader_options,
                         unique_ptr<RandomAccessFileReader>&& file,
@@ -354,7 +362,7 @@ class TestPlainTableFactory : public PlainTableFactory {
         table_reader_options.internal_comparator, encoding_type, file_size,
         bloom_bits_per_key_, hash_table_ratio_, index_sparseness_, props,
         std::move(file), table_reader_options.ioptions, expect_bloom_not_match_,
-        store_index_in_file_));
+        store_index_in_file_, column_family_id_, column_family_name_));
 
     *table = std::move(new_reader);
     return s;
@@ -366,6 +374,8 @@ class TestPlainTableFactory : public PlainTableFactory {
   size_t index_sparseness_;
   bool store_index_in_file_;
   bool* expect_bloom_not_match_;
+  const uint32_t column_family_id_;
+  const std::string column_family_name_;
 };
 
 TEST_P(PlainTableDBTest, Flush) {
@@ -492,7 +502,8 @@ TEST_P(PlainTableDBTest, Flush2) {
         plain_table_options.encoding_type = encoding_type;
         plain_table_options.store_index_in_file = store_index_in_file;
         options.table_factory.reset(new TestPlainTableFactory(
-            &expect_bloom_not_match, plain_table_options));
+            &expect_bloom_not_match, plain_table_options,
+            0 /* column_family_id */, kDefaultColumnFamilyName));
 
         DestroyAndReopen(&options);
         ASSERT_OK(Put("0000000000000bar", "b"));
@@ -561,7 +572,8 @@ TEST_P(PlainTableDBTest, Iterator) {
           plain_table_options.encoding_type = encoding_type;
 
           options.table_factory.reset(new TestPlainTableFactory(
-              &expect_bloom_not_match, plain_table_options));
+              &expect_bloom_not_match, plain_table_options,
+              0 /* column_family_id */, kDefaultColumnFamilyName));
         } else {
           PlainTableOptions plain_table_options;
           plain_table_options.user_key_len = 16;
@@ -572,7 +584,8 @@ TEST_P(PlainTableDBTest, Iterator) {
           plain_table_options.encoding_type = encoding_type;
 
           options.table_factory.reset(new TestPlainTableFactory(
-              &expect_bloom_not_match, plain_table_options));
+              &expect_bloom_not_match, plain_table_options,
+              0 /* column_family_id */, kDefaultColumnFamilyName));
         }
         DestroyAndReopen(&options);
 

db/repair.cc

@@ -294,9 +294,10 @@ class Repairer {
       status = BuildTable(
           dbname_, env_, ioptions_, env_options_, table_cache_, iter.get(),
           &meta, icmp_, &int_tbl_prop_collector_factories_,
-          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, {},
-          kMaxSequenceNumber, kNoCompression, CompressionOptions(), false,
-          nullptr);
+          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
+          std::string() /* column_family_name */, {}, kMaxSequenceNumber,
+          kNoCompression, CompressionOptions(), false,
+          nullptr /* internal_stats */);
     }
     delete mem->Unref();
     delete cf_mems_default;

db/table_properties_collector_test.cc

@@ -36,6 +36,7 @@ class TablePropertiesTest : public testing::Test,
 // Utilities test functions
 namespace {
 static const uint32_t kTestColumnFamilyId = 66;
+static const std::string kTestColumnFamilyName = "test_column_fam";
 
 void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions,
                  const InternalKeyComparator& internal_comparator,
@@ -48,8 +49,8 @@ void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions,
 
   builder->reset(NewTableBuilder(
       ioptions, internal_comparator, int_tbl_prop_collector_factories,
-      kTestColumnFamilyId /* column_family_id */, writable->get(),
-      options.compression, options.compression_opts));
+      kTestColumnFamilyId, kTestColumnFamilyName,
+      writable->get(), options.compression, options.compression_opts));
 }
 }  // namespace
 

include/rocksdb/table_properties.h

@@ -27,47 +27,6 @@ namespace rocksdb {
 // }
 typedef std::map<std::string, std::string> UserCollectedProperties;
 
-// TableProperties contains a bunch of read-only properties of its associated
-// table.
-struct TableProperties {
- public:
-  // the total size of all data blocks.
-  uint64_t data_size = 0;
-  // the size of index block.
-  uint64_t index_size = 0;
-  // the size of filter block.
-  uint64_t filter_size = 0;
-  // total raw key size
-  uint64_t raw_key_size = 0;
-  // total raw value size
-  uint64_t raw_value_size = 0;
-  // the number of blocks in this table
-  uint64_t num_data_blocks = 0;
-  // the number of entries in this table
-  uint64_t num_entries = 0;
-  // format version, reserved for backward compatibility
-  uint64_t format_version = 0;
-  // If 0, key is variable length. Otherwise number of bytes for each key.
-  uint64_t fixed_key_len = 0;
-
-  // The name of the filter policy used in this table.
-  // If no filter policy is used, `filter_policy_name` will be an empty string.
-  std::string filter_policy_name;
-
-  // user collected properties
-  UserCollectedProperties user_collected_properties;
-  UserCollectedProperties readable_properties;
-
-  // convert this object to a human readable form
-  //   @prop_delim: delimiter for each property.
-  std::string ToString(const std::string& prop_delim = "; ",
-                       const std::string& kv_delim = "=") const;
-
-  // Aggregate the numerical member variables of the specified
-  // TableProperties.
-  void Add(const TableProperties& tp);
-};
-
 // table properties' human-readable names in the property block.
 struct TablePropertiesNames {
   static const std::string kDataSize;
@@ -80,6 +39,8 @@ struct TablePropertiesNames {
   static const std::string kFormatVersion;
   static const std::string kFixedKeyLen;
   static const std::string kFilterPolicy;
+  static const std::string kColumnFamilyName;
+  static const std::string kColumnFamilyId;
 };
 
 extern const std::string kPropertiesBlock;
@@ -158,6 +119,55 @@ class TablePropertiesCollectorFactory {
   virtual const char* Name() const = 0;
 };
 
+// TableProperties contains a bunch of read-only properties of its associated
+// table.
+struct TableProperties {
+ public:
+  // the total size of all data blocks.
+  uint64_t data_size = 0;
+  // the size of index block.
+  uint64_t index_size = 0;
+  // the size of filter block.
+  uint64_t filter_size = 0;
+  // total raw key size
+  uint64_t raw_key_size = 0;
+  // total raw value size
+  uint64_t raw_value_size = 0;
+  // the number of blocks in this table
+  uint64_t num_data_blocks = 0;
+  // the number of entries in this table
+  uint64_t num_entries = 0;
+  // format version, reserved for backward compatibility
+  uint64_t format_version = 0;
+  // If 0, key is variable length. Otherwise number of bytes for each key.
+  uint64_t fixed_key_len = 0;
+  // ID of column family for this SST file, corresponding to the CF identified
+  // by column_family_name.
+  uint64_t column_family_id =
+      rocksdb::TablePropertiesCollectorFactory::Context::kUnknownColumnFamily;
+
+  // Name of the column family with which this SST file is associated.
+  // If column family is unknown, `column_family_name` will be an empty string.
+  std::string column_family_name;
+
+  // The name of the filter policy used in this table.
+  // If no filter policy is used, `filter_policy_name` will be an empty string.
+  std::string filter_policy_name;
+
+  // user collected properties
+  UserCollectedProperties user_collected_properties;
+  UserCollectedProperties readable_properties;
+
+  // convert this object to a human readable form
+  //   @prop_delim: delimiter for each property.
+  std::string ToString(const std::string& prop_delim = "; ",
+                       const std::string& kv_delim = "=") const;
+
+  // Aggregate the numerical member variables of the specified
+  // TableProperties.
+  void Add(const TableProperties& tp);
+};
+
 // Extra properties
 // Below is a list of non-basic properties that are collected by database
 // itself. Especially some properties regarding to the internal keys (which