Merge pull request ClickHouse#56354 from chhetripradeep/pchhetri/yiel…

…d-leadership-keeper

Add 4-letter command for yielding/resigning leadership

docs/en/operations/utilities/clickhouse-keeper-client.md

@@ -12,6 +12,7 @@ A client application to interact with clickhouse-keeper by its native protocol.
 -   `-q QUERY`, `--query=QUERY` — Query to execute. If this parameter is not passed, `clickhouse-keeper-client` will start in interactive mode.
 -   `-h HOST`, `--host=HOST` — Server host. Default value: `localhost`.
 -   `-p N`, `--port=N` — Server port. Default value: 9181
+-   `-c FILE_PATH`, `--config-file=FILE_PATH` — Set path of config file to get the connection string. Default value: `config.xml`.
 -   `--connection-timeout=TIMEOUT` — Set connection timeout in seconds. Default value: 10s.
 -   `--session-timeout=TIMEOUT` — Set session timeout in seconds. Default value: 10s.
 -   `--operation-timeout=TIMEOUT` — Set operation timeout in seconds. Default value: 10s.

src/Coordination/CoordinationSettings.cpp

@@ -36,7 +36,7 @@ void CoordinationSettings::loadFromConfig(const String & config_elem, const Poco
 }
 
 
-const String KeeperConfigurationAndSettings::DEFAULT_FOUR_LETTER_WORD_CMD = "conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl";
+const String KeeperConfigurationAndSettings::DEFAULT_FOUR_LETTER_WORD_CMD = "conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl,ydld";
 
 KeeperConfigurationAndSettings::KeeperConfigurationAndSettings()
     : server_id(NOT_EXIST)

src/Coordination/FourLetterCommand.cpp

@@ -172,6 +172,9 @@ void FourLetterCommandFactory::registerCommands(KeeperDispatcher & keeper_dispat
         FourLetterCommandPtr feature_flags_command = std::make_shared<FeatureFlagsCommand>(keeper_dispatcher);
         factory.registerCommand(feature_flags_command);
 
+        FourLetterCommandPtr yield_leadership_command = std::make_shared<YieldLeadershipCommand>(keeper_dispatcher);
+        factory.registerCommand(yield_leadership_command);
+
         factory.initializeAllowList(keeper_dispatcher);
         factory.setInitialize(true);
     }
@@ -579,4 +582,10 @@ String FeatureFlagsCommand::run()
     return ret.str();
 }
 
+String YieldLeadershipCommand::run()
+{
+    keeper_dispatcher.yieldLeadership();
+    return "Sent yield leadership request to leader.";
+}
+
 }

src/Coordination/FourLetterCommand.h

@@ -415,4 +415,17 @@ struct FeatureFlagsCommand : public IFourLetterCommand
     ~FeatureFlagsCommand() override = default;
 };
 
+/// Yield leadership and become follower.
+struct YieldLeadershipCommand : public IFourLetterCommand
+{
+    explicit YieldLeadershipCommand(KeeperDispatcher & keeper_dispatcher_)
+        : IFourLetterCommand(keeper_dispatcher_)
+    {
+    }
+
+    String name() override { return "ydld"; }
+    String run() override;
+    ~YieldLeadershipCommand() override = default;
+};
+
 }

src/Coordination/KeeperDispatcher.h

@@ -237,6 +237,12 @@ class KeeperDispatcher
         return server->requestLeader();
     }
 
+    /// Yield leadership and become follower.
+    void yieldLeadership()
+    {
+        return server->yieldLeadership();
+    }
+
     void recalculateStorageStats()
     {
         return server->recalculateStorageStats();

src/Coordination/KeeperServer.cpp

@@ -1087,6 +1087,12 @@ bool KeeperServer::requestLeader()
     return isLeader() || raft_instance->request_leadership();
 }
 
+void KeeperServer::yieldLeadership()
+{
+    if (isLeader())
+        raft_instance->yield_leadership();
+}
+
 void KeeperServer::recalculateStorageStats()
 {
     state_machine->recalculateStorageStats();

src/Coordination/KeeperServer.h

@@ -141,6 +141,8 @@ class KeeperServer
 
     bool requestLeader();
 
+    void yieldLeadership();
+
     void recalculateStorageStats();
 };
 

tests/integration/helpers/keeper_utils.py

@@ -248,6 +248,11 @@ def is_leader(cluster, node, port=9181):
     return "Mode: leader" in stat
 
 
+def is_follower(cluster, node, port=9181):
+    stat = send_4lw_cmd(cluster, node, "stat", port)
+    return "Mode: follower" in stat
+
+
 def get_leader(cluster, nodes):
     for node in nodes:
         if is_leader(cluster, node):

tests/integration/test_keeper_four_word_command/test.py

@@ -718,3 +718,26 @@ def get_memory_purges():
 
     finally:
         destroy_zk_client(zk)
+
+
+def test_cmd_ydld(started_cluster):
+    wait_nodes()
+    for node in [node1, node3]:
+        data = keeper_utils.send_4lw_cmd(cluster, node, cmd="ydld")
+        assert data == "Sent yield leadership request to leader."
+
+        print("ydld output -------------------------------------")
+        print(data)
+
+        if keeper_utils.is_leader(cluster, node):
+            # wait for it to yield leadership
+            retry = 0
+            while keeper_utils.is_leader(cluster, node) and retry < 30:
+                time.sleep(1)
+                retry += 1
+            if retry == 30:
+                print(
+                    node.name
+                    + " did not yield leadership after 30s, maybe there is something wrong."
+                )
+        assert keeper_utils.is_follower(cluster, node)