From e6221d0370611c4c4e0557b0ee57172255eceec3 Mon Sep 17 00:00:00 2001 From: Vinay Bhat Date: Sat, 19 Jun 2021 13:20:21 -0700 Subject: [PATCH] [kuduraft] add truncation counter in raft consensus Summary: The counter is incremented whenever ops are truncated from raft log as a result of a new leader overwriting log entries Test Plan: test in fbcode and with new rpm Reviewers: arahut Subscribers: Tasks: Tags: --- src/kudu/consensus/raft_consensus.cc | 17 ++++++++++++++++- src/kudu/consensus/raft_consensus.h | 3 +++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/kudu/consensus/raft_consensus.cc b/src/kudu/consensus/raft_consensus.cc index 60241a8ab3..d8a6caea29 100644 --- a/src/kudu/consensus/raft_consensus.cc +++ b/src/kudu/consensus/raft_consensus.cc @@ -187,6 +187,12 @@ DEFINE_bool(track_removed_peers, true, // Metrics // --------- +METRIC_DEFINE_counter(server, raft_log_truncation_counter, + "Log truncation count", + kudu::MetricUnit::kRequests, + "Number of times ops written to raft log were truncated " + "as a result of the new leader overwriting ops"); + METRIC_DEFINE_counter(server, follower_memory_pressure_rejections, "Follower Memory Pressure Rejections", kudu::MetricUnit::kRequests, @@ -344,6 +350,9 @@ Status RaftConsensus::Start(const ConsensusBootstrapInfo& info, DCHECK(log_ != NULL); DCHECK(time_manager_ != NULL); + raft_log_truncation_counter_ = + metric_entity->FindOrCreateCounter(&METRIC_raft_log_truncation_counter); + term_metric_ = metric_entity->FindOrCreateGauge(&METRIC_raft_term, CurrentTerm()); follower_memory_pressure_rejections_ = metric_entity->FindOrCreateCounter(&METRIC_follower_memory_pressure_rejections); @@ -937,7 +946,13 @@ Status RaftConsensus::TruncateCallbackWithRaftLock(int64_t *index_if_truncated) // We pass -1 to TruncateOpsAfter in the log abstraction // It is the responsibility of the derived log to truncate from // the cached truncation index and clear it. - return log_->TruncateOpsAfter(-1, index_if_truncated); + RETURN_NOT_OK(log_->TruncateOpsAfter(-1, index_if_truncated)); + + if (index_if_truncated && *index_if_truncated != -1) { + raft_log_truncation_counter_->Increment(); + } + + return Status::OK(); } Status RaftConsensus::CheckLeadershipAndBindTerm(const scoped_refptr& round) { diff --git a/src/kudu/consensus/raft_consensus.h b/src/kudu/consensus/raft_consensus.h index 293c933e29..eb7bdfd700 100644 --- a/src/kudu/consensus/raft_consensus.h +++ b/src/kudu/consensus/raft_consensus.h @@ -1194,6 +1194,9 @@ class RaftConsensus : public std::enable_shared_from_this, // restarts bool new_leader_detected_failsafe_; + // Number of times ops in raft log were truncated as a result of new leader + // overwriting the log + scoped_refptr raft_log_truncation_counter_; // Proxy metrics. scoped_refptr raft_proxy_num_requests_received_;