Make the hard coded 2 reserved cores configurable (Chia-Network#9709)

* Make the hard coded 2 reserved cores configurable

* add missing reserved_cores parameters

* add missing reserved_cores parameters

chia/consensus/blockchain.py

@@ -100,6 +100,7 @@ async def create(
         consensus_constants: ConsensusConstants,
         hint_store: HintStore,
         blockchain_dir: Path,
+        reserved_cores: int,
     ):
         """
         Initializes a blockchain with the BlockRecords from disk, assuming they have all been
@@ -112,7 +113,7 @@ async def create(
         cpu_count = multiprocessing.cpu_count()
         if cpu_count > 61:
             cpu_count = 61  # Windows Server 2016 has an issue https://bugs.python.org/issue26903
-        num_workers = max(cpu_count - 2, 1)
+        num_workers = max(cpu_count - reserved_cores, 1)
         self.pool = ProcessPoolExecutor(max_workers=num_workers)
         log.info(f"Started {num_workers} processes for block validation")
 

chia/full_node/full_node.py

@@ -185,8 +185,9 @@ def sql_trace_callback(req: str):
         self.coin_store = await CoinStore.create(self.db_wrapper)
         self.log.info("Initializing blockchain from disk")
         start_time = time.time()
+        reserved_cores = self.config.get("reserved_cores", 2)
         self.blockchain = await Blockchain.create(
-            self.coin_store, self.block_store, self.constants, self.hint_store, self.db_path.parent
+            self.coin_store, self.block_store, self.constants, self.hint_store, self.db_path.parent, reserved_cores
         )
         self.mempool_manager = MempoolManager(self.coin_store, self.constants)
 

chia/util/initial-config.yaml

@@ -333,6 +333,10 @@ full_node:
   # If node is more than these blocks behind, will do a short batch-sync, if it's less, will do a backtrack sync
   short_sync_blocks_behind_threshold: 20
 
+  # When creating process pools the process count will generally be the CPU count minus
+  # this reserved core count.
+  reserved_cores: 2
+
   # How often to initiate outbound connections to other full nodes.
   peer_connect_interval: 30
   # How long to wait for a peer connection
@@ -459,6 +463,10 @@ wallet:
   wallet_peers_path: wallet/db/wallet_peers.sqlite
   wallet_peers_file_path: wallet/db/wallet_peers.dat
 
+  # When creating process pools the process count will generally be the CPU count minus
+  # this reserved core count.
+  reserved_cores: 2
+
   logging: *logging
   network_overrides: *network_overrides
   selected_network: *selected_network

chia/wallet/wallet_blockchain.py

@@ -88,6 +88,7 @@ async def create(
         new_transaction_block_callback: Callable,  # f(removals: List[Coin], additions: List[Coin], height: uint32)
         reorg_rollback: Callable,
         lock: asyncio.Lock,
+        reserved_cores: int,
     ):
         """
         Initializes a blockchain with the BlockRecords from disk, assuming they have all been
@@ -102,7 +103,7 @@ async def create(
         cpu_count = multiprocessing.cpu_count()
         if cpu_count > 61:
             cpu_count = 61  # Windows Server 2016 has an issue https://bugs.python.org/issue26903
-        num_workers = max(cpu_count - 2, 1)
+        num_workers = max(cpu_count - reserved_cores, 1)
         self.pool = ProcessPoolExecutor(max_workers=num_workers)
         log.info(f"Started {num_workers} processes for block validation")
         self.constants = consensus_constants

chia/wallet/wallet_state_manager.py

@@ -156,6 +156,8 @@ async def create(
         self.interested_store = await WalletInterestedStore.create(self.db_wrapper)
         self.pool_store = await WalletPoolStore.create(self.db_wrapper)
 
+        reserved_cores = self.config.get("reserved_cores", 2)
+
         self.blockchain = await WalletBlockchain.create(
             self.block_store,
             self.coin_store,
@@ -165,6 +167,7 @@ async def create(
             self.new_transaction_block_callback,
             self.reorg_rollback,
             self.lock,
+            reserved_cores,
         )
         self.weight_proof_handler = WeightProofHandler(self.constants, self.blockchain)
 

tests/core/full_node/ram_db.py

@@ -17,5 +17,5 @@ async def create_ram_blockchain(consensus_constants: ConsensusConstants) -> Tupl
     block_store = await BlockStore.create(db_wrapper)
     coin_store = await CoinStore.create(db_wrapper)
     hint_store = await HintStore.create(db_wrapper)
-    blockchain = await Blockchain.create(coin_store, block_store, consensus_constants, hint_store, Path("."))
+    blockchain = await Blockchain.create(coin_store, block_store, consensus_constants, hint_store, Path("."), 2)
     return connection, blockchain

tests/core/full_node/test_block_store.py

@@ -33,7 +33,7 @@ async def test_block_store(self, tmp_dir, db_version):
             coin_store_2 = await CoinStore.create(db_wrapper_2)
             store_2 = await BlockStore.create(db_wrapper_2)
             hint_store = await HintStore.create(db_wrapper_2)
-            bc = await Blockchain.create(coin_store_2, store_2, test_constants, hint_store, tmp_dir)
+            bc = await Blockchain.create(coin_store_2, store_2, test_constants, hint_store, tmp_dir, 2)
 
             store = await BlockStore.create(db_wrapper)
             await BlockStore.create(db_wrapper_2)
@@ -74,7 +74,7 @@ async def test_deadlock(self, tmp_dir, db_version):
             coin_store_2 = await CoinStore.create(wrapper_2)
             store_2 = await BlockStore.create(wrapper_2)
             hint_store = await HintStore.create(wrapper_2)
-            bc = await Blockchain.create(coin_store_2, store_2, test_constants, hint_store, tmp_dir)
+            bc = await Blockchain.create(coin_store_2, store_2, test_constants, hint_store, tmp_dir, 2)
             block_records = []
             for block in blocks:
                 await bc.receive_block(block)
@@ -103,7 +103,7 @@ async def test_rollback(self, tmp_dir):
             coin_store = await CoinStore.create(db_wrapper)
             block_store = await BlockStore.create(db_wrapper)
             hint_store = await HintStore.create(db_wrapper)
-            bc = await Blockchain.create(coin_store, block_store, test_constants, hint_store, tmp_dir)
+            bc = await Blockchain.create(coin_store, block_store, test_constants, hint_store, tmp_dir, 2)
 
             # insert all blocks
             count = 0

tests/core/full_node/test_coin_store.py

@@ -249,7 +249,7 @@ async def test_basic_reorg(self, cache_size: uint32, tmp_dir, db_version):
             coin_store = await CoinStore.create(db_wrapper, cache_size=uint32(cache_size))
             store = await BlockStore.create(db_wrapper)
             hint_store = await HintStore.create(db_wrapper)
-            b: Blockchain = await Blockchain.create(coin_store, store, test_constants, hint_store, tmp_dir)
+            b: Blockchain = await Blockchain.create(coin_store, store, test_constants, hint_store, tmp_dir, 2)
             try:
 
                 records: List[Optional[CoinRecord]] = []
@@ -319,7 +319,7 @@ async def test_get_puzzle_hash(self, cache_size: uint32, tmp_dir, db_version):
             coin_store = await CoinStore.create(db_wrapper, cache_size=uint32(cache_size))
             store = await BlockStore.create(db_wrapper)
             hint_store = await HintStore.create(db_wrapper)
-            b: Blockchain = await Blockchain.create(coin_store, store, test_constants, hint_store, tmp_dir)
+            b: Blockchain = await Blockchain.create(coin_store, store, test_constants, hint_store, tmp_dir, 2)
             for block in blocks:
                 res, err, _, _ = await b.receive_block(block)
                 assert err is None

tests/util/blockchain.py

@@ -30,7 +30,7 @@ async def create_blockchain(constants: ConsensusConstants, db_version: int):
     coin_store = await CoinStore.create(wrapper)
     store = await BlockStore.create(wrapper)
     hint_store = await HintStore.create(wrapper)
-    bc1 = await Blockchain.create(coin_store, store, constants, hint_store, Path("."))
+    bc1 = await Blockchain.create(coin_store, store, constants, hint_store, Path("."), 2)
     assert bc1.get_peak() is None
     return bc1, connection, db_path