KUDU-3223: Management of per-table level limit

When the table's size or row exceeds a threshold, the insert and update
privilege will be removed from the authz token, only scan and delete
privilege are allowed.

Admin is responsible to change the on_disk_size_limit or
row_count_limiti per-table. -1 means no limit.

This feature depends on authz token expiration, and it caused a delay
for write forbidden. Anyway, it provides an approximately management for
write.

Change-Id: I2dbf365ad59f17c0a4e2e7ea6a5afaa7680724b0
Reviewed-on: http://gerrit.cloudera.org:8080/17273
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong <[email protected]>

src/kudu/client/client.cc

@@ -65,6 +65,7 @@
 #include "kudu/common/partition.h"
 #include "kudu/common/partition_pruner.h"
 #include "kudu/common/row_operations.h"
+#include "kudu/common/row_operations.pb.h"
 #include "kudu/common/scan_spec.h"
 #include "kudu/common/schema.h"
 #include "kudu/common/txn_id.h"
@@ -690,7 +691,9 @@ Status KuduClient::GetTableStatistics(const string& table_name,
   unique_ptr<KuduTableStatistics> table_statistics(new KuduTableStatistics);
   table_statistics->data_ = new KuduTableStatistics::Data(
       resp.has_on_disk_size() ? boost::optional<int64_t>(resp.on_disk_size()) : boost::none,
-      resp.has_live_row_count() ? boost::optional<int64_t>(resp.live_row_count()) : boost::none);
+      resp.has_live_row_count() ? boost::optional<int64_t>(resp.live_row_count()) : boost::none,
+      resp.has_disk_size_limit() ? boost::optional<int64_t>(resp.disk_size_limit()) : boost::none,
+      resp.has_row_count_limit() ? boost::optional<int64_t>(resp.row_count_limit()) : boost::none);
 
   *statistics = table_statistics.release();
   return Status::OK();
@@ -993,6 +996,14 @@ int64_t KuduTableStatistics::live_row_count() const {
   return data_->live_row_count_ ? *data_->live_row_count_ : -1;
 }
 
+int64_t KuduTableStatistics::on_disk_size_limit() const {
+  return data_->on_disk_size_limit_ ? *data_->on_disk_size_limit_ : -1;
+}
+
+int64_t KuduTableStatistics::live_row_count_limit() const {
+  return data_->live_row_count_limit_ ? *data_->live_row_count_limit_ : -1;
+}
+
 std::string KuduTableStatistics::ToString() const {
   return data_->ToString();
 }
@@ -1467,6 +1478,16 @@ KuduTableAlterer* KuduTableAlterer::AlterExtraConfig(const map<string, string>&
   return this;
 }
 
+KuduTableAlterer* KuduTableAlterer::SetTableDiskSizeLimit(int64_t disk_size_limit) {
+  data_->disk_size_limit_ = disk_size_limit;
+  return this;
+}
+
+KuduTableAlterer* KuduTableAlterer::SetTableRowCountLimit(int64_t row_count_limit) {
+  data_->row_count_limit_ = row_count_limit;
+  return this;
+}
+
 KuduTableAlterer* KuduTableAlterer::timeout(const MonoDelta& timeout) {
   data_->timeout_ = timeout;
   return this;

src/kudu/client/client.h

@@ -55,6 +55,7 @@ namespace kudu {
 
 class AuthzTokenTest;
 class ClientStressTest_TestUniqueClientIds_Test;
+class DisableWriteWhenExceedingQuotaTest;
 class KuduPartialRow;
 class MonoDelta;
 class Partition;
@@ -921,6 +922,7 @@ class KUDU_EXPORT KuduClient : public sp::enable_shared_from_this<KuduClient> {
   friend class internal::RetrieveAuthzTokenRpc;
   friend class internal::WriteRpc;
   friend class kudu::AuthzTokenTest;
+  friend class kudu::DisableWriteWhenExceedingQuotaTest;
   friend class kudu::SecurityUnknownTskTest;
   friend class transactions::CoordinatorRpc;
   friend class transactions::ParticipantRpc;
@@ -1306,6 +1308,22 @@ class KUDU_EXPORT KuduTableStatistics {
   /// @note This statistic is pre-replication.
   int64_t live_row_count() const;
 
+  /// @return The table's on disk size limit.
+  ///  -1 is returned if there is no disk size limit on this table.
+  ///
+  /// @note It is experimental and may change or disappear in future.
+  /// This feature currently applies size limit on a single table, but
+  /// it should also support database level size limit.
+  int64_t on_disk_size_limit() const;
+
+  /// @return The table's live row count limit.
+  ///  -1 is returned if there is no row count limit on this table.
+  ///
+  /// @note It is experimental and may change or disappear in future.
+  /// This feature currently applies row count limit on a single table,
+  /// but it should also support database level row count limit.
+  int64_t live_row_count_limit() const;
+
   /// Stringify this Statistics.
   ///
   /// @return A string describing this statistics
@@ -1785,6 +1803,34 @@ class KUDU_EXPORT KuduTableAlterer {
   /// @return Raw pointer to this alterer object.
   KuduTableAlterer* AlterExtraConfig(const std::map<std::string, std::string>& extra_configs);
 
+  /// Set the disk size limit of the table by the super user.
+  ///
+  /// @note The table limit alterations, including disk_size_limit and row_count_limit,
+  /// cannot be changed in the same alteration request with other alterations, because the
+  /// table 'limit' alteration needs the super user permission.
+  ///
+  /// @note It is experimental and may change or disappear in future.
+  /// This feature currently applies size limit on a single table.
+  ///
+  /// @param [in] disk_size_limit
+  ///   The max table disk size, -1 is for no limit
+  /// @return Raw pointer to this alterer object.
+  KuduTableAlterer* SetTableDiskSizeLimit(int64_t disk_size_limit);
+
+  /// Set the row count limit of the table by the super user.
+  ///
+  /// @note The table limit alterations, including disk_size_limit and row_count_limit,
+  /// cannot be changed in the same alteration request with other alterations, because the
+  /// table 'limit' alteration needs the super user permission.
+  ///
+  /// @note It is experimental and may change or disappear in future.
+  /// This feature currently applies row count limit on a single table.
+  ///
+  /// @param [in] row_count_limit
+  ///   The max row count of the table, -1 is for no limit
+  /// @return Raw pointer to this alterer object.
+  KuduTableAlterer* SetTableRowCountLimit(int64_t row_count_limit);
+
   /// Set a timeout for the alteration operation.
   ///
   /// This includes any waiting after the alter has been submitted

src/kudu/client/table_alterer-internal.cc

@@ -27,9 +27,9 @@
 #include "kudu/client/schema-internal.h"
 #include "kudu/client/schema.h"
 #include "kudu/common/row_operations.h"
+#include "kudu/common/row_operations.pb.h"
 #include "kudu/common/schema.h"
 #include "kudu/common/wire_protocol.h"
-#include "kudu/common/wire_protocol.pb.h"
 #include "kudu/master/master.pb.h"
 
 using std::string;
@@ -60,6 +60,8 @@ Status KuduTableAlterer::Data::ToRequest(AlterTableRequestPB* req) {
   if (!rename_to_.is_initialized() &&
       !new_extra_configs_ &&
       !set_owner_to_.is_initialized() &&
+      !disk_size_limit_ &&
+      !row_count_limit_ &&
       steps_.empty()) {
     return Status::InvalidArgument("No alter steps provided");
   }
@@ -85,6 +87,13 @@ Status KuduTableAlterer::Data::ToRequest(AlterTableRequestPB* req) {
                              SCHEMA_PB_WITHOUT_COMMENT));
   }
 
+  if (disk_size_limit_) {
+    req->set_disk_size_limit(disk_size_limit_.get());
+  }
+  if (row_count_limit_) {
+    req->set_row_count_limit(row_count_limit_.get());
+  }
+
   for (const Step& s : steps_) {
     AlterTableRequestPB::Step* pb_step = req->add_alter_schema_steps();
     pb_step->set_type(s.step_type);

src/kudu/client/table_alterer-internal.h

@@ -17,6 +17,7 @@
 #ifndef KUDU_CLIENT_TABLE_ALTERER_INTERNAL_H
 #define KUDU_CLIENT_TABLE_ALTERER_INTERNAL_H
 
+#include <cstdint>
 #include <map>
 #include <memory>
 #include <string>
@@ -78,6 +79,9 @@ class KuduTableAlterer::Data {
 
   boost::optional<std::map<std::string, std::string>> new_extra_configs_;
 
+  boost::optional<int64_t> disk_size_limit_;
+  boost::optional<int64_t> row_count_limit_;
+
   // Set to true if there are alter partition steps.
   bool has_alter_partitioning_steps = false;
 

src/kudu/client/table_statistics-internal.h

@@ -33,23 +33,35 @@ using strings::Substitute;
 
 class KuduTableStatistics::Data {
  public:
-  Data(boost::optional<int64_t> on_disk_size, boost::optional<int64_t> live_row_count)
-      : on_disk_size_(std::move(on_disk_size)),
-        live_row_count_(std::move(live_row_count)) {
+  Data(boost::optional<int64_t> on_disk_size,
+       boost::optional<int64_t> live_row_count,
+       boost::optional<int64_t> on_disk_size_limit,
+       boost::optional<int64_t> live_row_count_limit)
+      : on_disk_size_(on_disk_size),
+        live_row_count_(live_row_count),
+        on_disk_size_limit_(on_disk_size_limit),
+        live_row_count_limit_(live_row_count_limit) {
   }
 
   ~Data() {
   }
 
   string ToString() const {
     return Substitute("on disk size: $0\n"
-                      "live row count: $1\n",
+                      "live row count: $1\n"
+                      "on disk size limit: $2\n"
+                      "live row count limit: $3\n",
                       on_disk_size_ ? std::to_string(*on_disk_size_) : "N/A",
-                      live_row_count_ ? std::to_string(*live_row_count_) : "N/A");
+                      live_row_count_ ? std::to_string(*live_row_count_) : "N/A",
+                      on_disk_size_limit_ ? std::to_string(*on_disk_size_limit_) : "N/A",
+                      live_row_count_limit_ ? std::to_string(*live_row_count_limit_) : "N/A");
+
   }
 
   const boost::optional<int64_t> on_disk_size_;
   const boost::optional<int64_t> live_row_count_;
+  const boost::optional<int64_t> on_disk_size_limit_;
+  const boost::optional<int64_t> live_row_count_limit_;
 
  private:
   DISALLOW_COPY_AND_ASSIGN(Data);

src/kudu/integration-tests/CMakeLists.txt

@@ -145,6 +145,7 @@ ADD_KUDU_TEST(ts_tablet_manager-itest)
 ADD_KUDU_TEST(update_scan_delta_compact-test RUN_SERIAL true)
 ADD_KUDU_TEST(webserver-crawl-itest LABELS no_dist_test)
 ADD_KUDU_TEST(webserver-stress-itest RUN_SERIAL true)
+ADD_KUDU_TEST(write_limit-itest)
 ADD_KUDU_TEST(write_throttling-itest)
 
 if (NOT APPLE)