From 3df0e77ea3d5f03823f6f4e4b2a42296eb4afcb5 Mon Sep 17 00:00:00 2001 From: guoxiaolong <467887319@qq.com> Date: Wed, 11 Oct 2023 10:28:28 +0800 Subject: [PATCH 01/10] keeper_client add get_all_children_number command --- programs/keeper-client/Commands.cpp | 41 +++++++++++++++++++++++++ programs/keeper-client/Commands.h | 14 +++++++++ programs/keeper-client/KeeperClient.cpp | 2 ++ 3 files changed, 57 insertions(+) diff --git a/programs/keeper-client/Commands.cpp b/programs/keeper-client/Commands.cpp index 42057cf202c5..68313ae0e54e 100644 --- a/programs/keeper-client/Commands.cpp +++ b/programs/keeper-client/Commands.cpp @@ -475,4 +475,45 @@ void FourLetterWordCommand::execute(const ASTKeeperQuery * query, KeeperClient * std::cout << client->executeFourLetterCommand(query->args[0].safeGet()) << "\n"; } +bool GetAllChildrenNumberCommand::parse(IParser::Pos & pos, std::shared_ptr & node, Expected & expected) const +{ + String path; + if (!parseKeeperPath(pos, expected, path)) + path = "."; + + node->args.push_back(std::move(path)); + + return true; +} + +void GetAllChildrenNumberCommand::execute(const ASTKeeperQuery * query, KeeperClient * client) const +{ + auto path = client->getAbsolutePath(query->args[0].safeGet()); + + std::queue queue; + queue.push(path); + Coordination::Stat stat; + client->zookeeper->get(path, &stat); + + int totalNumChildren = stat.numChildren; + while (!queue.empty()) + { + auto next_path = queue.front(); + queue.pop(); + + auto children = client->zookeeper->getChildren(next_path); + std::transform(children.cbegin(), children.cend(), children.begin(), [&](const String & child) { return next_path / child; }); + + auto response = client->zookeeper->get(children); + + for (size_t i = 0; i < response.size(); ++i) + { + totalNumChildren += response[i].stat.numChildren; + queue.push(children[i]); + } + } + + std::cout << totalNumChildren << "\n"; +} + } diff --git a/programs/keeper-client/Commands.h b/programs/keeper-client/Commands.h index a58a63e8a0af..d543dce66349 100644 --- a/programs/keeper-client/Commands.h +++ b/programs/keeper-client/Commands.h @@ -238,4 +238,18 @@ class FourLetterWordCommand : public IKeeperClientCommand String getHelpMessage() const override { return "{} -- Executes four-letter-word command"; } }; +class GetAllChildrenNumberCommand : public IKeeperClientCommand +{ + String getName() const override { return "get_all_children_number"; } + + bool parse(IParser::Pos & pos, std::shared_ptr & node, Expected & expected) const override; + + void execute(const ASTKeeperQuery * query, KeeperClient * client) const override; + + String getHelpMessage() const override + { + return "{} [path] -- Returns the total number of znode nodes for all children"; + } +}; + } diff --git a/programs/keeper-client/KeeperClient.cpp b/programs/keeper-client/KeeperClient.cpp index 1f22a3cd6440..f96975f5ab11 100644 --- a/programs/keeper-client/KeeperClient.cpp +++ b/programs/keeper-client/KeeperClient.cpp @@ -2,6 +2,7 @@ #include "Commands.h" #include #include +#include "Common/VersionNumber.h" #include #include #include @@ -206,6 +207,7 @@ void KeeperClient::initialize(Poco::Util::Application & /* self */) std::make_shared(), std::make_shared(), std::make_shared(), + std::make_shared(), }); String home_path; From ed89451efcf1e55070453e54aeafd59367fe4536 Mon Sep 17 00:00:00 2001 From: guoxiaolong <467887319@qq.com> Date: Wed, 11 Oct 2023 10:41:43 +0800 Subject: [PATCH 02/10] add get_all_children_number command docs in clickhouse-keeper-client.md --- docs/en/operations/utilities/clickhouse-keeper-client.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/operations/utilities/clickhouse-keeper-client.md b/docs/en/operations/utilities/clickhouse-keeper-client.md index 2e84cd260715..d55163b375c4 100644 --- a/docs/en/operations/utilities/clickhouse-keeper-client.md +++ b/docs/en/operations/utilities/clickhouse-keeper-client.md @@ -55,6 +55,7 @@ keeper foo bar - `rmr ` -- Recursively deletes path. Confirmation required - `flwc ` -- Executes four-letter-word command - `help` -- Prints this message +- `get_all_children_number [path]` -- Returns the total number of znode nodes for all children - `get_stat [path]` -- Returns the node's stat (default `.`) - `find_super_nodes [path]` -- Finds nodes with number of children larger than some threshold for the given path (default `.`) - `delete_stale_backups` -- Deletes ClickHouse nodes used for backups that are now inactive From 18a6e624c9f05934e928141840995d7955b88508 Mon Sep 17 00:00:00 2001 From: guoxiaolong <467887319@qq.com> Date: Wed, 11 Oct 2023 17:24:08 +0800 Subject: [PATCH 03/10] fix description --- docs/en/operations/utilities/clickhouse-keeper-client.md | 2 +- programs/keeper-client/Commands.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/utilities/clickhouse-keeper-client.md b/docs/en/operations/utilities/clickhouse-keeper-client.md index d55163b375c4..67aa4babb3f0 100644 --- a/docs/en/operations/utilities/clickhouse-keeper-client.md +++ b/docs/en/operations/utilities/clickhouse-keeper-client.md @@ -55,7 +55,7 @@ keeper foo bar - `rmr ` -- Recursively deletes path. Confirmation required - `flwc ` -- Executes four-letter-word command - `help` -- Prints this message -- `get_all_children_number [path]` -- Returns the total number of znode nodes for all children +- `get_all_children_number [path]` -- Get all numbers of children nodes under a specific path - `get_stat [path]` -- Returns the node's stat (default `.`) - `find_super_nodes [path]` -- Finds nodes with number of children larger than some threshold for the given path (default `.`) - `delete_stale_backups` -- Deletes ClickHouse nodes used for backups that are now inactive diff --git a/programs/keeper-client/Commands.h b/programs/keeper-client/Commands.h index d543dce66349..45a7c85266da 100644 --- a/programs/keeper-client/Commands.h +++ b/programs/keeper-client/Commands.h @@ -248,7 +248,7 @@ class GetAllChildrenNumberCommand : public IKeeperClientCommand String getHelpMessage() const override { - return "{} [path] -- Returns the total number of znode nodes for all children"; + return "{} [path] -- Get all numbers of children nodes under a specific path"; } }; From 31ff669edc6120ea4ebb1b8853b78617afe57c9a Mon Sep 17 00:00:00 2001 From: guoxiaolong <467887319@qq.com> Date: Wed, 11 Oct 2023 17:47:38 +0800 Subject: [PATCH 04/10] change the code to for loop --- programs/keeper-client/Commands.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/programs/keeper-client/Commands.cpp b/programs/keeper-client/Commands.cpp index 68313ae0e54e..c7043baa33c2 100644 --- a/programs/keeper-client/Commands.cpp +++ b/programs/keeper-client/Commands.cpp @@ -324,8 +324,9 @@ void FindBigFamily::execute(const ASTKeeperQuery * query, KeeperClient * client) queue.pop(); auto children = client->zookeeper->getChildren(next_path); - std::transform(children.cbegin(), children.cend(), children.begin(), [&](const String & child) { return next_path / child; }); - + for (auto & child : children) + child = next_path / child; + auto response = client->zookeeper->get(children); for (size_t i = 0; i < response.size(); ++i) @@ -502,7 +503,8 @@ void GetAllChildrenNumberCommand::execute(const ASTKeeperQuery * query, KeeperCl queue.pop(); auto children = client->zookeeper->getChildren(next_path); - std::transform(children.cbegin(), children.cend(), children.begin(), [&](const String & child) { return next_path / child; }); + for (auto & child : children) + child = next_path / child; auto response = client->zookeeper->get(children); From f7e64ff20098955da9edc22168a3861bfc6a1e90 Mon Sep 17 00:00:00 2001 From: guoxiaolong <467887319@qq.com> Date: Wed, 11 Oct 2023 20:45:38 +0800 Subject: [PATCH 05/10] add test_keeper_client --- tests/integration/helpers/keeper_utils.py | 3 +++ tests/integration/test_keeper_client/test.py | 21 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/tests/integration/helpers/keeper_utils.py b/tests/integration/helpers/keeper_utils.py index 3a88688d1107..9135ab72c842 100644 --- a/tests/integration/helpers/keeper_utils.py +++ b/tests/integration/helpers/keeper_utils.py @@ -145,6 +145,9 @@ def find_super_nodes(self, threshold: int, timeout: float = 60.0) -> str: def delete_stale_backups(self, timeout: float = 60.0) -> str: return self.execute_query("delete_stale_backups", timeout) + + def get_all_children_number(self, path: str, timeout: float = 60.0) -> str: + return self.execute_query(f"get_all_children_number {path}", timeout) def reconfig( self, diff --git a/tests/integration/test_keeper_client/test.py b/tests/integration/test_keeper_client/test.py index 77217dd2dcf6..08d0825c118c 100644 --- a/tests/integration/test_keeper_client/test.py +++ b/tests/integration/test_keeper_client/test.py @@ -216,3 +216,24 @@ def test_quoted_argument_parsing(client: KeeperClient): client.execute_query(f"set '{node_path}' \"value4 with some whitespace\" 3") assert client.get(node_path) == "value4 with some whitespace" + +def test_get_all_children_number(client: KeeperClient): + client.touch("/test_get_all_children_number") + client.touch("/test_get_all_children_number/1") + client.touch("/test_get_all_children_number/1/1") + client.touch("/test_get_all_children_number/1/2") + client.touch("/test_get_all_children_number/1/3") + client.touch("/test_get_all_children_number/1/4") + client.touch("/test_get_all_children_number/1/5") + client.touch("/test_get_all_children_number/2") + client.touch("/test_get_all_children_number/2/1") + client.touch("/test_get_all_children_number/2/2") + client.touch("/test_get_all_children_number/2/3") + client.touch("/test_get_all_children_number/2/4") + + response = client.get_all_children_number("/test_get_all_children_number") + assert response == TSV( + [ + ["11"] + ] + ) From 8b47bf5e7ef41d2c4523f27901aa35a2b53cb679 Mon Sep 17 00:00:00 2001 From: guoxiaolong <467887319@qq.com> Date: Thu, 12 Oct 2023 09:58:42 +0800 Subject: [PATCH 06/10] fix CheckSytel --- programs/keeper-client/Commands.cpp | 2 -- tests/integration/helpers/keeper_utils.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/programs/keeper-client/Commands.cpp b/programs/keeper-client/Commands.cpp index c7043baa33c2..543ea8f44498 100644 --- a/programs/keeper-client/Commands.cpp +++ b/programs/keeper-client/Commands.cpp @@ -326,7 +326,6 @@ void FindBigFamily::execute(const ASTKeeperQuery * query, KeeperClient * client) auto children = client->zookeeper->getChildren(next_path); for (auto & child : children) child = next_path / child; - auto response = client->zookeeper->get(children); for (size_t i = 0; i < response.size(); ++i) @@ -505,7 +504,6 @@ void GetAllChildrenNumberCommand::execute(const ASTKeeperQuery * query, KeeperCl auto children = client->zookeeper->getChildren(next_path); for (auto & child : children) child = next_path / child; - auto response = client->zookeeper->get(children); for (size_t i = 0; i < response.size(); ++i) diff --git a/tests/integration/helpers/keeper_utils.py b/tests/integration/helpers/keeper_utils.py index 9135ab72c842..80b221811f7f 100644 --- a/tests/integration/helpers/keeper_utils.py +++ b/tests/integration/helpers/keeper_utils.py @@ -148,7 +148,7 @@ def delete_stale_backups(self, timeout: float = 60.0) -> str: def get_all_children_number(self, path: str, timeout: float = 60.0) -> str: return self.execute_query(f"get_all_children_number {path}", timeout) - + def reconfig( self, joining: tp.Optional[str], From 6e927325e404084f44cd30805cbc5266f07cf8bb Mon Sep 17 00:00:00 2001 From: guoxiaolong <467887319@qq.com> Date: Thu, 12 Oct 2023 20:32:36 +0800 Subject: [PATCH 07/10] fix CheckSytel --- tests/integration/test_keeper_client/test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_keeper_client/test.py b/tests/integration/test_keeper_client/test.py index 08d0825c118c..a00e3d312311 100644 --- a/tests/integration/test_keeper_client/test.py +++ b/tests/integration/test_keeper_client/test.py @@ -217,6 +217,7 @@ def test_quoted_argument_parsing(client: KeeperClient): client.execute_query(f"set '{node_path}' \"value4 with some whitespace\" 3") assert client.get(node_path) == "value4 with some whitespace" + def test_get_all_children_number(client: KeeperClient): client.touch("/test_get_all_children_number") client.touch("/test_get_all_children_number/1") @@ -232,6 +233,7 @@ def test_get_all_children_number(client: KeeperClient): client.touch("/test_get_all_children_number/2/4") response = client.get_all_children_number("/test_get_all_children_number") + assert response == TSV( [ ["11"] From 369ebf3c4f41275fe9cb878f668b848c23672a26 Mon Sep 17 00:00:00 2001 From: guoxiaolong <467887319@qq.com> Date: Thu, 12 Oct 2023 21:15:52 +0800 Subject: [PATCH 08/10] fix CheckSytel --- tests/integration/helpers/keeper_utils.py | 6 +++--- tests/integration/test_keeper_client/test.py | 8 +------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/integration/helpers/keeper_utils.py b/tests/integration/helpers/keeper_utils.py index 80b221811f7f..b287bf5d3f7e 100644 --- a/tests/integration/helpers/keeper_utils.py +++ b/tests/integration/helpers/keeper_utils.py @@ -142,13 +142,13 @@ def find_big_family(self, path: str, n: int = 10, timeout: float = 60.0) -> str: def find_super_nodes(self, threshold: int, timeout: float = 60.0) -> str: return self.execute_query(f"find_super_nodes {threshold}", timeout) + + def get_all_children_number(self, path: str, timeout: float = 60.0) -> str: + return self.execute_query(f"get_all_children_number {path}", timeout) def delete_stale_backups(self, timeout: float = 60.0) -> str: return self.execute_query("delete_stale_backups", timeout) - def get_all_children_number(self, path: str, timeout: float = 60.0) -> str: - return self.execute_query(f"get_all_children_number {path}", timeout) - def reconfig( self, joining: tp.Optional[str], diff --git a/tests/integration/test_keeper_client/test.py b/tests/integration/test_keeper_client/test.py index a00e3d312311..e01c463d94a9 100644 --- a/tests/integration/test_keeper_client/test.py +++ b/tests/integration/test_keeper_client/test.py @@ -232,10 +232,4 @@ def test_get_all_children_number(client: KeeperClient): client.touch("/test_get_all_children_number/2/3") client.touch("/test_get_all_children_number/2/4") - response = client.get_all_children_number("/test_get_all_children_number") - - assert response == TSV( - [ - ["11"] - ] - ) + assert client.get_all_children_number("/test_get_all_children_number") == ["11"] From 7d5c15dfc9ac34ff2ee8f4cd1bd97420a8c09cd7 Mon Sep 17 00:00:00 2001 From: guoxiaolong <467887319@qq.com> Date: Thu, 12 Oct 2023 22:01:37 +0800 Subject: [PATCH 09/10] fix python CheckSytel --- tests/integration/helpers/keeper_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/helpers/keeper_utils.py b/tests/integration/helpers/keeper_utils.py index b287bf5d3f7e..478bb5ebf727 100644 --- a/tests/integration/helpers/keeper_utils.py +++ b/tests/integration/helpers/keeper_utils.py @@ -142,13 +142,13 @@ def find_big_family(self, path: str, n: int = 10, timeout: float = 60.0) -> str: def find_super_nodes(self, threshold: int, timeout: float = 60.0) -> str: return self.execute_query(f"find_super_nodes {threshold}", timeout) - + def get_all_children_number(self, path: str, timeout: float = 60.0) -> str: return self.execute_query(f"get_all_children_number {path}", timeout) def delete_stale_backups(self, timeout: float = 60.0) -> str: return self.execute_query("delete_stale_backups", timeout) - + def reconfig( self, joining: tp.Optional[str], From 98af58d7cf5d6e22999c2ba121ffe03080e507e8 Mon Sep 17 00:00:00 2001 From: guoxiaolong <467887319@qq.com> Date: Mon, 16 Oct 2023 10:02:31 +0800 Subject: [PATCH 10/10] fix test error --- tests/integration/test_keeper_client/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_keeper_client/test.py b/tests/integration/test_keeper_client/test.py index e01c463d94a9..92b5b95dc50b 100644 --- a/tests/integration/test_keeper_client/test.py +++ b/tests/integration/test_keeper_client/test.py @@ -232,4 +232,4 @@ def test_get_all_children_number(client: KeeperClient): client.touch("/test_get_all_children_number/2/3") client.touch("/test_get_all_children_number/2/4") - assert client.get_all_children_number("/test_get_all_children_number") == ["11"] + assert client.get_all_children_number("/test_get_all_children_number") == "11"