farmer|rpc: Introduce get_harvesters_summary RPC endpoint (Chia-Net…

…work#11245)

chia/farmer/farmer.py

@@ -636,13 +636,13 @@ async def generate_login_link(self, launcher_id: bytes32) -> Optional[str]:
 
         return None
 
-    async def get_harvesters(self) -> Dict:
+    async def get_harvesters(self, counts_only: bool = False) -> Dict:
         harvesters: List = []
         for connection in self.server.get_connections(NodeType.HARVESTER):
             self.log.debug(f"get_harvesters host: {connection.peer_host}, node_id: {connection.peer_node_id}")
             receiver = self.plot_sync_receivers.get(connection.peer_node_id)
             if receiver is not None:
-                harvesters.append(receiver.to_dict())
+                harvesters.append(receiver.to_dict(counts_only))
             else:
                 self.log.debug(
                     f"get_harvesters invalid peer: {connection.peer_host}, node_id: {connection.peer_node_id}"

chia/plot_sync/receiver.py

@@ -25,6 +25,7 @@
 from chia.server.ws_connection import ProtocolMessageTypes, WSChiaConnection, make_msg
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.util.ints import int16, uint64
+from chia.util.misc import get_list_or_len
 from chia.util.streamable import _T_Streamable
 
 log = logging.getLogger(__name__)
@@ -287,17 +288,17 @@ async def _sync_done(self, data: PlotSyncDone) -> None:
     async def sync_done(self, data: PlotSyncDone) -> None:
         await self._process(self._sync_done, ProtocolMessageTypes.plot_sync_done, data)
 
-    def to_dict(self) -> Dict[str, Any]:
+    def to_dict(self, counts_only: bool = False) -> Dict[str, Any]:
         result: Dict[str, Any] = {
             "connection": {
                 "node_id": self._connection.peer_node_id,
                 "host": self._connection.peer_host,
                 "port": self._connection.peer_port,
             },
-            "plots": list(self._plots.values()),
-            "failed_to_open_filenames": self._invalid,
-            "no_key_filenames": self._keys_missing,
-            "duplicates": self._duplicates,
+            "plots": get_list_or_len(list(self._plots.values()), counts_only),
+            "failed_to_open_filenames": get_list_or_len(self._invalid, counts_only),
+            "no_key_filenames": get_list_or_len(self._keys_missing, counts_only),
+            "duplicates": get_list_or_len(self._duplicates, counts_only),
         }
         if self._last_sync_time != 0:
             result["last_sync_time"] = self._last_sync_time

chia/rpc/farmer_rpc_api.py

@@ -20,6 +20,7 @@ def get_routes(self) -> Dict[str, Callable]:
             "/get_pool_state": self.get_pool_state,
             "/set_payout_instructions": self.set_payout_instructions,
             "/get_harvesters": self.get_harvesters,
+            "/get_harvesters_summary": self.get_harvesters_summary,
             "/get_pool_login_link": self.get_pool_login_link,
         }
 
@@ -144,7 +145,10 @@ async def set_payout_instructions(self, request: Dict) -> Dict:
         return {}
 
     async def get_harvesters(self, _: Dict):
-        return await self.service.get_harvesters()
+        return await self.service.get_harvesters(False)
+
+    async def get_harvesters_summary(self, _: Dict[str, object]) -> Dict[str, object]:
+        return await self.service.get_harvesters(True)
 
     async def get_pool_login_link(self, request: Dict) -> Dict:
         launcher_id: bytes32 = bytes32(hexstr_to_bytes(request["launcher_id"]))

chia/rpc/farmer_rpc_client.py

@@ -55,6 +55,9 @@ async def set_payout_instructions(self, launcher_id: bytes32, payout_instruction
     async def get_harvesters(self) -> Dict[str, Any]:
         return await self.fetch("get_harvesters", {})
 
+    async def get_harvesters_summary(self) -> Dict[str, object]:
+        return await self.fetch("get_harvesters_summary", {})
+
     async def get_pool_login_link(self, launcher_id: bytes32) -> Optional[str]:
         try:
             return (await self.fetch("get_pool_login_link", {"launcher_id": launcher_id.hex()}))["login_link"]

chia/util/misc.py

@@ -1,3 +1,6 @@
+from typing import Sequence, Union
+
+
 def format_bytes(bytes: int) -> str:
 
     if not isinstance(bytes, int) or bytes < 0:
@@ -68,3 +71,7 @@ def prompt_yes_no(prompt: str = "(y/n) ") -> bool:
             return True
         elif ch == "n":
             return False
+
+
+def get_list_or_len(list_in: Sequence[object], length: bool) -> Union[int, Sequence[object]]:
+    return len(list_in) if length else list_in

tests/core/test_farmer_harvester_rpc.py

@@ -17,6 +17,7 @@
 from chia.util.config import load_config, lock_and_load_config, save_config
 from chia.util.hash import std_hash
 from chia.util.ints import uint8, uint16, uint32, uint64
+from chia.util.misc import get_list_or_len
 from chia.wallet.derive_keys import master_sk_to_wallet_sk, master_sk_to_wallet_sk_unhardened
 from tests.setup_nodes import setup_harvester_farmer, test_constants
 from tests.time_out_assert import time_out_assert, time_out_assert_custom_interval
@@ -102,8 +103,9 @@ async def test_get_routes(harvester_farmer_environment):
     await validate_get_routes(harvester_rpc_client, harvester_rpc_api)
 
 
+@pytest.mark.parametrize("endpoint", ["get_harvesters", "get_harvesters_summary"])
 @pytest.mark.asyncio
-async def test_farmer_get_harvesters(harvester_farmer_environment):
+async def test_farmer_get_harvesters_and_summary(harvester_farmer_environment, endpoint: str):
     (
         farmer_service,
         farmer_rpc_api,
@@ -114,26 +116,38 @@ async def test_farmer_get_harvesters(harvester_farmer_environment):
     ) = harvester_farmer_environment
     harvester = harvester_service._node
 
-    num_plots = 0
+    harvester_plots = []
 
     async def non_zero_plots() -> bool:
         res = await harvester_rpc_client.get_plots()
-        nonlocal num_plots
-        num_plots = len(res["plots"])
-        return num_plots > 0
+        nonlocal harvester_plots
+        harvester_plots = res["plots"]
+        return len(harvester_plots) > 0
 
     await time_out_assert(10, non_zero_plots)
 
     async def test_get_harvesters():
+        nonlocal harvester_plots
         harvester.plot_manager.trigger_refresh()
         await time_out_assert(5, harvester.plot_manager.needs_refresh, value=False)
-        farmer_res = await farmer_rpc_client.get_harvesters()
+        farmer_res = await getattr(farmer_rpc_client, endpoint)()
+
         if len(list(farmer_res["harvesters"])) != 1:
             log.error(f"test_get_harvesters: invalid harvesters {list(farmer_res['harvesters'])}")
             return False
-        if len(list(farmer_res["harvesters"][0]["plots"])) != num_plots:
-            log.error(f"test_get_harvesters: invalid plots {list(farmer_res['harvesters'])}")
-            return False
+
+        harvester_dict = farmer_res["harvesters"][0]
+        counts_only: bool = endpoint == "get_harvesters_summary"
+
+        if not counts_only:
+            harvester_dict["plots"] = sorted(harvester_dict["plots"], key=lambda item: item["filename"])
+            harvester_plots = sorted(harvester_plots, key=lambda item: item["filename"])
+
+        assert harvester_dict["plots"] == get_list_or_len(harvester_plots, counts_only)
+        assert harvester_dict["failed_to_open_filenames"] == get_list_or_len([], counts_only)
+        assert harvester_dict["no_key_filenames"] == get_list_or_len([], counts_only)
+        assert harvester_dict["duplicates"] == get_list_or_len([], counts_only)
+
         return True
 
     await time_out_assert_custom_interval(30, 1, test_get_harvesters)

tests/plot_sync/test_receiver.py

@@ -21,6 +21,7 @@
 from chia.server.ws_connection import NodeType
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.util.ints import uint8, uint32, uint64
+from chia.util.misc import get_list_or_len
 from chia.util.streamable import _T_Streamable
 from tests.plot_sync.util import get_dummy_connection
 
@@ -217,13 +218,15 @@ async def test_reset() -> None:
     assert receiver.connection() == connection_before
 
 
+@pytest.mark.parametrize("counts_only", [True, False])
 @pytest.mark.asyncio
-async def test_to_dict() -> None:
+async def test_to_dict(counts_only: bool) -> None:
     receiver, sync_steps = plot_sync_setup()
-    plot_sync_dict_1 = receiver.to_dict()
-    assert "plots" in plot_sync_dict_1 and len(plot_sync_dict_1["plots"]) == 10
-    assert "failed_to_open_filenames" in plot_sync_dict_1 and len(plot_sync_dict_1["failed_to_open_filenames"]) == 0
-    assert "no_key_filenames" in plot_sync_dict_1 and len(plot_sync_dict_1["no_key_filenames"]) == 0
+    plot_sync_dict_1 = receiver.to_dict(counts_only)
+
+    assert get_list_or_len(plot_sync_dict_1["plots"], not counts_only) == 10
+    assert get_list_or_len(plot_sync_dict_1["failed_to_open_filenames"], not counts_only) == 0
+    assert get_list_or_len(plot_sync_dict_1["no_key_filenames"], not counts_only) == 0
     assert "last_sync_time" not in plot_sync_dict_1
     assert plot_sync_dict_1["connection"] == {
         "node_id": receiver.connection().peer_node_id,
@@ -232,33 +235,21 @@ async def test_to_dict() -> None:
     }
 
     # We should get equal dicts
-    plot_sync_dict_2 = receiver.to_dict()
-    assert plot_sync_dict_1 == plot_sync_dict_2
-
-    dict_2_paths = [x.filename for x in plot_sync_dict_2["plots"]]
-    for plot_info in sync_steps[State.loaded].args[0]:
-        assert plot_info.filename not in dict_2_paths
+    assert plot_sync_dict_1 == receiver.to_dict(counts_only)
+    # But unequal dicts wit the opposite counts_only value
+    assert plot_sync_dict_1 != receiver.to_dict(not counts_only)
 
     # Walk through all states from idle to done and run them with the test data
     for state in State:
         await run_sync_step(receiver, sync_steps[state], state)
 
-    plot_sync_dict_3 = receiver.to_dict()
-    dict_3_paths = [x.filename for x in plot_sync_dict_3["plots"]]
-    for plot_info in sync_steps[State.loaded].args[0]:
-        assert plot_info.filename in dict_3_paths
-
-    for path in sync_steps[State.removed].args[0]:
-        assert path not in plot_sync_dict_3["plots"]
-
-    for path in sync_steps[State.invalid].args[0]:
-        assert path in plot_sync_dict_3["failed_to_open_filenames"]
-
-    for path in sync_steps[State.keys_missing].args[0]:
-        assert path in plot_sync_dict_3["no_key_filenames"]
-
-    for path in sync_steps[State.duplicates].args[0]:
-        assert path in plot_sync_dict_3["duplicates"]
+    plot_sync_dict_3 = receiver.to_dict(counts_only)
+    assert get_list_or_len(sync_steps[State.loaded].args[0], counts_only) == plot_sync_dict_3["plots"]
+    assert (
+        get_list_or_len(sync_steps[State.invalid].args[0], counts_only) == plot_sync_dict_3["failed_to_open_filenames"]
+    )
+    assert get_list_or_len(sync_steps[State.keys_missing].args[0], counts_only) == plot_sync_dict_3["no_key_filenames"]
+    assert get_list_or_len(sync_steps[State.duplicates].args[0], counts_only) == plot_sync_dict_3["duplicates"]
 
     assert plot_sync_dict_3["last_sync_time"] > 0