data_dirs: clarify error message when all data dirs are full or failed

The existing error message suggested that we tried (and failed) to add a data directory to the tablet's data dir group when we did no such thing[1]. 1. In the future we could do this; see KUDU-2907 for details. Change-Id: I07fd860dcecf4f255f418b2063c1850eecbc6e65 Reviewed-on: http://gerrit.cloudera.org:8080/13969 Reviewed-by: Andrew Wong <[email protected]> Tested-by: Kudu Jenkins
limshu · Jul 31, 2019 · 536b3fe · 536b3fe
1 parent de49f8e
commit 536b3fe
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 13 deletions.
diff --git a/src/kudu/fs/block_manager-test.cc b/src/kudu/fs/block_manager-test.cc
@@ -851,8 +851,8 @@ TYPED_TEST(BlockManagerTest, TestDiskSpaceCheck) {
           // The dir was previously observed as full, so CreateBlock() checked
           // fullness again and failed.
           ASSERT_TRUE(s.IsIOError()) << s.ToString();
-          ASSERT_STR_CONTAINS(s.ToString(), "No directories available to add to test_tablet's "
-                                            "directory group");
+          ASSERT_STR_CONTAINS(
+              s.ToString(), "No directories available in test_tablet's directory group");
         } else {
           ASSERT_OK(s);
           ASSERT_OK(writer->Append("test data"));

diff --git a/src/kudu/fs/data_dirs.cc b/src/kudu/fs/data_dirs.cc
@@ -19,6 +19,7 @@
 
 #include <algorithm>
 #include <cerrno>
+#include <cstddef>
 #include <cstdint>
 #include <iterator>
 #include <memory>
@@ -949,9 +950,10 @@ Status DataDirManager::GetNextDataDir(const CreateBlockOptions& opts, DataDir**
   shared_lock<rw_spinlock> lock(dir_group_lock_.get_lock());
   const vector<int>* group_uuid_indices;
   vector<int> valid_uuid_indices;
+  DataDirGroup* group = nullptr;
   if (PREDICT_TRUE(!opts.tablet_id.empty())) {
     // Get the data dir group for the tablet.
-    DataDirGroup* group = FindOrNull(group_by_tablet_map_, opts.tablet_id);
+    group = FindOrNull(group_by_tablet_map_, opts.tablet_id);
     if (group == nullptr) {
       return Status::NotFound("Tried to get directory but no directory group "
                               "registered for tablet", opts.tablet_id);
@@ -988,18 +990,24 @@ Status DataDirManager::GetNextDataDir(const CreateBlockOptions& opts, DataDir**
       return Status::OK();
     }
   }
-  string tablet_id_str = "";
+
+  // All healthy directories are full. Return an error.
   if (PREDICT_TRUE(!opts.tablet_id.empty())) {
-    tablet_id_str = Substitute("$0's ", opts.tablet_id);
-  }
-  string dirs_state_str = Substitute("$0 failed", failed_data_dirs_.size());
-  if (metrics_) {
-    dirs_state_str = Substitute("$0 full, $1",
-                                metrics_->data_dirs_full->value(), dirs_state_str);
+    DCHECK(group);
+    size_t num_total = group->uuid_indices().size();
+    size_t num_full = valid_uuid_indices.size();
+    size_t num_failed = num_total - num_full;
+    return Status::IOError(
+        Substitute("No directories available in $0's directory group ($1 dirs "
+                   "total, $2 failed, $3 full)",
+                   opts.tablet_id, num_total, num_failed, num_full),
+        "", ENOSPC);
   }
-  return Status::IOError(Substitute("No directories available to add to $0directory group ($1 "
-                         "dirs total, $2).", tablet_id_str, data_dirs_.size(), dirs_state_str),
-                         "", ENOSPC);
+
+  return Status::IOError(
+      Substitute("No directories available ($0 dirs, all of which are full)",
+                 valid_uuid_indices.size()),
+      "", ENOSPC);
 }
 
 void DataDirManager::DeleteDataDirGroup(const std::string& tablet_id) {