From 454009bc31bde7a1c30cc8ff508c18182b5d8a30 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Thu, 20 Jun 2024 13:10:02 +0100 Subject: [PATCH 01/32] swap to threadpool executor and chia_rs validate_clvm_and_signature --- chia/full_node/mempool_manager.py | 93 ++++++++++++++++--------------- 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index 77820bbe36e5..f9e47930e0c0 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -4,12 +4,12 @@ import logging import time from concurrent.futures import Executor -from concurrent.futures.process import ProcessPoolExecutor +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from multiprocessing.context import BaseContext from typing import Awaitable, Callable, Collection, Dict, List, Optional, Set, Tuple, TypeVar -from chia_rs import ELIGIBLE_FOR_DEDUP, ELIGIBLE_FOR_FF, BLSCache, G1Element, supports_fast_forward +from chia_rs import ELIGIBLE_FOR_DEDUP, ELIGIBLE_FOR_FF, BLSCache, G1Element, supports_fast_forward, validate_clvm_and_signature from chiabip158 import PyBIP158 from chia.consensus.block_record import BlockRecordProtocol @@ -48,45 +48,45 @@ # TODO: once the 1.8.0 soft-fork has activated, we don't really need to pass # the constants through here -def validate_clvm_and_signature( - spend_bundle_bytes: bytes, max_cost: int, constants: ConsensusConstants, height: uint32 -) -> Tuple[Optional[Err], bytes, List[Tuple[bytes, bytes]], float]: - """ - Validates CLVM and aggregate signature for a spendbundle. This is meant to be called under a ProcessPoolExecutor - in order to validate the heavy parts of a transaction in a different thread. Returns an optional error, - the NPCResult and a cache of the new pairings validated (if not error) - """ - - start_time = time.monotonic() - additional_data = constants.AGG_SIG_ME_ADDITIONAL_DATA - - try: - bundle: SpendBundle = SpendBundle.from_bytes(spend_bundle_bytes) - program = simple_solution_generator(bundle) - # npc contains names of the coins removed, puzzle_hashes and their spend conditions - result: NPCResult = get_name_puzzle_conditions( - program, max_cost, mempool_mode=True, constants=constants, height=height - ) - - if result.error is not None: - return Err(result.error), b"", [], time.monotonic() - start_time - - pks: List[G1Element] = [] - msgs: List[bytes] = [] - assert result.conds is not None - pks, msgs = pkm_pairs(result.conds, additional_data) - - # Verify aggregated signature - cache = BLSCache(10000) - if not cache.aggregate_verify(pks, msgs, bundle.aggregated_signature): - return Err.BAD_AGGREGATE_SIGNATURE, b"", [], time.monotonic() - start_time - new_cache_entries: List[Tuple[bytes, bytes]] = cache.items() - except ValidationError as e: - return e.code, b"", [], time.monotonic() - start_time - except Exception: - return Err.UNKNOWN, b"", [], time.monotonic() - start_time - - return None, bytes(result), new_cache_entries, time.monotonic() - start_time +# def validate_clvm_and_signature( +# spend_bundle_bytes: bytes, max_cost: int, constants: ConsensusConstants, height: uint32 +# ) -> Tuple[Optional[Err], bytes, List[Tuple[bytes, bytes]], float]: +# """ +# Validates CLVM and aggregate signature for a spendbundle. This is meant to be called under a ProcessPoolExecutor +# in order to validate the heavy parts of a transaction in a different thread. Returns an optional error, +# the NPCResult and a cache of the new pairings validated (if not error) +# """ + +# start_time = time.monotonic() +# additional_data = constants.AGG_SIG_ME_ADDITIONAL_DATA + +# try: +# bundle: SpendBundle = SpendBundle.from_bytes(spend_bundle_bytes) +# program = simple_solution_generator(bundle) +# # npc contains names of the coins removed, puzzle_hashes and their spend conditions +# result: NPCResult = get_name_puzzle_conditions( +# program, max_cost, mempool_mode=True, constants=constants, height=height +# ) + +# if result.error is not None: +# return Err(result.error), b"", [], time.monotonic() - start_time + +# pks: List[G1Element] = [] +# msgs: List[bytes] = [] +# assert result.conds is not None +# pks, msgs = pkm_pairs(result.conds, additional_data) + +# # Verify aggregated signature +# cache = BLSCache(10000) +# if not cache.aggregate_verify(pks, msgs, bundle.aggregated_signature): +# return Err.BAD_AGGREGATE_SIGNATURE, b"", [], time.monotonic() - start_time +# new_cache_entries: List[Tuple[bytes, bytes]] = cache.items() +# except ValidationError as e: +# return e.code, b"", [], time.monotonic() - start_time +# except Exception: +# return Err.UNKNOWN, b"", [], time.monotonic() - start_time + +# return None, bytes(result), new_cache_entries, time.monotonic() - start_time @dataclass @@ -220,7 +220,7 @@ def __init__( if single_threaded: self.pool = InlineExecutor() else: - self.pool = ProcessPoolExecutor( + self.pool = ThreadPoolExecutor( max_workers=2, mp_context=multiprocessing_context, initializer=setproctitle, @@ -324,7 +324,7 @@ async def pre_validate_spendbundle( self._worker_queue_size += 1 try: - err, cached_result_bytes, new_cache_entries, duration = await asyncio.get_running_loop().run_in_executor( + _spend, sbc, new_cache_entries, duration = await asyncio.get_running_loop().run_in_executor( self.pool, validate_clvm_and_signature, new_spend_bytes, @@ -332,15 +332,16 @@ async def pre_validate_spendbundle( self.constants, self.peak.height, ) + except Exception as e: + raise ValidationError(e) finally: self._worker_queue_size -= 1 - if err is not None: - raise ValidationError(err) if bls_cache is not None: bls_cache.update(new_cache_entries) + + ret = NPCResult(None, sbc) - ret: NPCResult = NPCResult.from_bytes(cached_result_bytes) log.log( logging.DEBUG if duration < 2 else logging.WARNING, f"pre_validate_spendbundle took {duration:0.4f} seconds " From 43c9f25f5c4e2f1429ca3cb04450a9685f70fb27 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Thu, 20 Jun 2024 16:48:23 +0100 Subject: [PATCH 02/32] flake8 and black --- chia/full_node/mempool_manager.py | 51 ++----------------------------- 1 file changed, 3 insertions(+), 48 deletions(-) diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index f9e47930e0c0..a2f23a196731 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -9,18 +9,17 @@ from multiprocessing.context import BaseContext from typing import Awaitable, Callable, Collection, Dict, List, Optional, Set, Tuple, TypeVar -from chia_rs import ELIGIBLE_FOR_DEDUP, ELIGIBLE_FOR_FF, BLSCache, G1Element, supports_fast_forward, validate_clvm_and_signature +from chia_rs import ELIGIBLE_FOR_DEDUP, ELIGIBLE_FOR_FF, BLSCache, supports_fast_forward, validate_clvm_and_signature from chiabip158 import PyBIP158 from chia.consensus.block_record import BlockRecordProtocol from chia.consensus.constants import ConsensusConstants from chia.consensus.cost_calculator import NPCResult from chia.full_node.bitcoin_fee_estimator import create_bitcoin_fee_estimator -from chia.full_node.bundle_tools import simple_solution_generator from chia.full_node.fee_estimation import FeeBlockInfo, MempoolInfo, MempoolItemInfo from chia.full_node.fee_estimator_interface import FeeEstimatorInterface from chia.full_node.mempool import MEMPOOL_ITEM_FEE_LIMIT, Mempool, MempoolRemoveInfo, MempoolRemoveReason -from chia.full_node.mempool_check_conditions import get_name_puzzle_conditions, mempool_check_time_locks +from chia.full_node.mempool_check_conditions import mempool_check_time_locks from chia.full_node.pending_tx_cache import ConflictTxCache, PendingTxCache from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.sized_bytes import bytes32 @@ -32,7 +31,6 @@ from chia.types.mempool_item import BundleCoinSpend, MempoolItem from chia.types.spend_bundle import SpendBundle from chia.types.spend_bundle_conditions import SpendBundleConditions -from chia.util.condition_tools import pkm_pairs from chia.util.db_wrapper import SQLITE_INT_MAX from chia.util.errors import Err, ValidationError from chia.util.inline_executor import InlineExecutor @@ -46,49 +44,6 @@ MEMPOOL_MIN_FEE_INCREASE = uint64(10000000) -# TODO: once the 1.8.0 soft-fork has activated, we don't really need to pass -# the constants through here -# def validate_clvm_and_signature( -# spend_bundle_bytes: bytes, max_cost: int, constants: ConsensusConstants, height: uint32 -# ) -> Tuple[Optional[Err], bytes, List[Tuple[bytes, bytes]], float]: -# """ -# Validates CLVM and aggregate signature for a spendbundle. This is meant to be called under a ProcessPoolExecutor -# in order to validate the heavy parts of a transaction in a different thread. Returns an optional error, -# the NPCResult and a cache of the new pairings validated (if not error) -# """ - -# start_time = time.monotonic() -# additional_data = constants.AGG_SIG_ME_ADDITIONAL_DATA - -# try: -# bundle: SpendBundle = SpendBundle.from_bytes(spend_bundle_bytes) -# program = simple_solution_generator(bundle) -# # npc contains names of the coins removed, puzzle_hashes and their spend conditions -# result: NPCResult = get_name_puzzle_conditions( -# program, max_cost, mempool_mode=True, constants=constants, height=height -# ) - -# if result.error is not None: -# return Err(result.error), b"", [], time.monotonic() - start_time - -# pks: List[G1Element] = [] -# msgs: List[bytes] = [] -# assert result.conds is not None -# pks, msgs = pkm_pairs(result.conds, additional_data) - -# # Verify aggregated signature -# cache = BLSCache(10000) -# if not cache.aggregate_verify(pks, msgs, bundle.aggregated_signature): -# return Err.BAD_AGGREGATE_SIGNATURE, b"", [], time.monotonic() - start_time -# new_cache_entries: List[Tuple[bytes, bytes]] = cache.items() -# except ValidationError as e: -# return e.code, b"", [], time.monotonic() - start_time -# except Exception: -# return Err.UNKNOWN, b"", [], time.monotonic() - start_time - -# return None, bytes(result), new_cache_entries, time.monotonic() - start_time - - @dataclass class TimelockConditions: assert_height: uint32 = uint32(0) @@ -339,7 +294,7 @@ async def pre_validate_spendbundle( if bls_cache is not None: bls_cache.update(new_cache_entries) - + ret = NPCResult(None, sbc) log.log( From 0c2c3ae572f85aff6762d5cf2a0dfd7c92999e62 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Thu, 20 Jun 2024 16:57:23 +0100 Subject: [PATCH 03/32] isort --- chia/full_node/mempool_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index a2f23a196731..73517e5f98c0 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -3,8 +3,7 @@ import asyncio import logging import time -from concurrent.futures import Executor -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Executor, ThreadPoolExecutor from dataclasses import dataclass from multiprocessing.context import BaseContext from typing import Awaitable, Callable, Collection, Dict, List, Optional, Set, Tuple, TypeVar From 35eb491b60187daac25640333d18df921f5c7eff Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Mon, 8 Jul 2024 14:48:18 +0100 Subject: [PATCH 04/32] remove mp_context --- chia/full_node/mempool_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index 73517e5f98c0..29d4924b7f03 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -176,7 +176,6 @@ def __init__( else: self.pool = ThreadPoolExecutor( max_workers=2, - mp_context=multiprocessing_context, initializer=setproctitle, initargs=(f"{getproctitle()}_mempool_worker",), ) @@ -278,13 +277,14 @@ async def pre_validate_spendbundle( self._worker_queue_size += 1 try: - _spend, sbc, new_cache_entries, duration = await asyncio.get_running_loop().run_in_executor( + sbc, new_cache_entries, duration = await asyncio.get_running_loop().run_in_executor( self.pool, validate_clvm_and_signature, - new_spend_bytes, + new_spend, self.max_tx_clvm_cost, self.constants, self.peak.height, + bls_cache, ) except Exception as e: raise ValidationError(e) From 03318dddcf7fc85e12418bc513ec5a233c200784 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Mon, 8 Jul 2024 18:10:46 +0100 Subject: [PATCH 05/32] convert TypeError into ValidationError --- chia/full_node/mempool_manager.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index 29d4924b7f03..8cc570b8b61a 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -286,8 +286,10 @@ async def pre_validate_spendbundle( self.peak.height, bls_cache, ) - except Exception as e: - raise ValidationError(e) + except Exception as e: # take returned TypeError and turn into ValidationError class by + error = Err(e.args[0]) + # breakpoint() + raise ValidationError(error) finally: self._worker_queue_size -= 1 From 7aed44ecab760ad893d4acfeb2a9103307fe7437 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Tue, 9 Jul 2024 16:45:18 +0100 Subject: [PATCH 06/32] increase amount for test of exceed cost limit --- chia/_tests/core/mempool/test_mempool_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chia/_tests/core/mempool/test_mempool_manager.py b/chia/_tests/core/mempool/test_mempool_manager.py index 0721fe490bdb..06363b959beb 100644 --- a/chia/_tests/core/mempool/test_mempool_manager.py +++ b/chia/_tests/core/mempool/test_mempool_manager.py @@ -510,7 +510,7 @@ async def test_duplicate_output() -> None: async def test_block_cost_exceeds_max() -> None: mempool_manager = await instantiate_mempool_manager(zero_calls_get_coin_records) conditions = [] - for i in range(2400): + for i in range(3800): conditions.append([ConditionOpcode.CREATE_COIN, IDENTITY_PUZZLE_HASH, i]) sb = spend_bundle_from_conditions(conditions) with pytest.raises(ValidationError, match="BLOCK_COST_EXCEEDS_MAX"): From 91057d31149bd4417ef309dea6d8a237cb9a1726 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Tue, 9 Jul 2024 16:47:18 +0100 Subject: [PATCH 07/32] change hard coded expected costs --- chia/_tests/core/mempool/test_mempool_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chia/_tests/core/mempool/test_mempool_manager.py b/chia/_tests/core/mempool/test_mempool_manager.py index 06363b959beb..6b8df2d16c47 100644 --- a/chia/_tests/core/mempool/test_mempool_manager.py +++ b/chia/_tests/core/mempool/test_mempool_manager.py @@ -581,7 +581,7 @@ async def test_same_sb_twice_with_eligible_coin() -> None: sb = SpendBundle.aggregate([sb1, sb2]) sb_name = sb.name() result = await add_spendbundle(mempool_manager, sb, sb_name) - expected_cost = uint64(10268283) + expected_cost = uint64(6600088) assert result == (expected_cost, MempoolInclusionStatus.SUCCESS, None) assert mempool_manager.get_spendbundle(sb_name) == sb result = await add_spendbundle(mempool_manager, sb, sb_name) @@ -614,7 +614,7 @@ async def test_sb_twice_with_eligible_coin_and_different_spends_order() -> None: assert mempool_manager.get_spendbundle(sb_name) is None assert mempool_manager.get_spendbundle(reordered_sb_name) is None result = await add_spendbundle(mempool_manager, sb, sb_name) - expected_cost = uint64(13091510) + expected_cost = uint64(7800132) assert result == (expected_cost, MempoolInclusionStatus.SUCCESS, None) assert mempool_manager.get_spendbundle(sb_name) == sb assert mempool_manager.get_spendbundle(reordered_sb_name) is None From 72fe562bcb3941f1c22c96a46aaaadb98d90aa3b Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 10 Jul 2024 14:15:49 +0100 Subject: [PATCH 08/32] adjust cost so ttest passes --- chia/_tests/core/mempool/test_mempool_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chia/_tests/core/mempool/test_mempool_manager.py b/chia/_tests/core/mempool/test_mempool_manager.py index 6b8df2d16c47..2550b66cd621 100644 --- a/chia/_tests/core/mempool/test_mempool_manager.py +++ b/chia/_tests/core/mempool/test_mempool_manager.py @@ -1053,7 +1053,7 @@ async def make_and_send_big_cost_sb(coin: Coin) -> None: g1 = sk.get_g1() sig = AugSchemeMPL.sign(sk, IDENTITY_PUZZLE_HASH, g1) aggsig = G2Element() - for _ in range(169): + for _ in range(318): conditions.append([ConditionOpcode.AGG_SIG_UNSAFE, g1, IDENTITY_PUZZLE_HASH]) aggsig += sig conditions.append([ConditionOpcode.CREATE_COIN, IDENTITY_PUZZLE_HASH, coin.amount - 10_000_000]) From 1c5a4b1fc36765ea1efafe57aee51ef2286de8f9 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Tue, 16 Jul 2024 13:25:23 +0100 Subject: [PATCH 09/32] fix fastforward with rust get_npc() call --- .../mempool/test_singleton_fast_forward.py | 2 ++ chia/clvm/spend_sim.py | 1 - chia/full_node/mempool.py | 1 + chia/full_node/mempool_manager.py | 3 +- chia/types/eligible_coin_spends.py | 30 +++++++++++-------- 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/chia/_tests/core/mempool/test_singleton_fast_forward.py b/chia/_tests/core/mempool/test_singleton_fast_forward.py index 8fc4eae2691e..9043d67f69fb 100644 --- a/chia/_tests/core/mempool/test_singleton_fast_forward.py +++ b/chia/_tests/core/mempool/test_singleton_fast_forward.py @@ -461,9 +461,11 @@ async def test_singleton_fast_forward_different_block(is_eligible_for_ff: bool) # version which is the grandchild in this scenario assert status == MempoolInclusionStatus.SUCCESS assert error is None + old_lineage_info = unspent_lineage_info unspent_lineage_info = await sim_client.service.coin_store.get_unspent_lineage_info_for_puzzle_hash( singleton_puzzle_hash ) + assert old_lineage_info != unspent_lineage_info singleton_grandchild, [remaining_coin] = await get_singleton_and_remaining_coins(sim) assert unspent_lineage_info == UnspentLineageInfo( coin_id=singleton_grandchild.name(), diff --git a/chia/clvm/spend_sim.py b/chia/clvm/spend_sim.py index 07d9cfd21c00..ed88e792bdac 100644 --- a/chia/clvm/spend_sim.py +++ b/chia/clvm/spend_sim.py @@ -268,7 +268,6 @@ async def farm_block( get_unspent_lineage_info_for_puzzle_hash=self.coin_store.get_unspent_lineage_info_for_puzzle_hash, item_inclusion_filter=item_inclusion_filter, ) - if result is not None: bundle, additions = result generator_bundle = bundle diff --git a/chia/full_node/mempool.py b/chia/full_node/mempool.py index cf02947eebbe..1bab4f28e4c4 100644 --- a/chia/full_node/mempool.py +++ b/chia/full_node/mempool.py @@ -521,6 +521,7 @@ async def create_bundle_from_mempool_items( unique_additions.extend(spend_data.additions) cost_saving = 0 else: + # item gets modified in the following call await eligible_coin_spends.process_fast_forward_spends( mempool_item=item, get_unspent_lineage_info_for_puzzle_hash=get_unspent_lineage_info_for_puzzle_hash, diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index 8cc570b8b61a..750241de9ccd 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -286,9 +286,8 @@ async def pre_validate_spendbundle( self.peak.height, bls_cache, ) - except Exception as e: # take returned TypeError and turn into ValidationError class by + except Exception as e: # take returned TypeError and turn into ValidationError class by error = Err(e.args[0]) - # breakpoint() raise ValidationError(error) finally: self._worker_queue_size -= 1 diff --git a/chia/types/eligible_coin_spends.py b/chia/types/eligible_coin_spends.py index 5e799cd81d8f..40dd9104ffe2 100644 --- a/chia/types/eligible_coin_spends.py +++ b/chia/types/eligible_coin_spends.py @@ -3,12 +3,13 @@ import dataclasses from typing import Awaitable, Callable, Dict, List, Optional, Tuple -from chia_rs import fast_forward_singleton +from chia_rs import fast_forward_singleton, get_name_puzzle_conditions from chia.consensus.condition_costs import ConditionCost from chia.consensus.constants import ConsensusConstants +from chia.consensus.cost_calculator import NPCResult from chia.full_node.bundle_tools import simple_solution_generator -from chia.full_node.mempool_check_conditions import get_name_puzzle_conditions +from chia.util.errors import Err, ValidationError from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.serialized_program import SerializedProgram from chia.types.blockchain_format.sized_bytes import bytes32 @@ -332,17 +333,20 @@ async def process_fast_forward_spends( coin_spends=new_coin_spends, aggregated_signature=mempool_item.spend_bundle.aggregated_signature ) # We need to run the new spend bundle to make sure it remains valid - generator = simple_solution_generator(new_sb) - new_npc_result = get_name_puzzle_conditions( - generator=generator, - max_cost=mempool_item.conds.cost, - mempool_mode=True, - height=height, - constants=constants, - ) - if new_npc_result.error is not None: - raise ValueError("Mempool item became invalid after singleton fast forward.") - assert new_npc_result.conds is not None + # generator = simple_solution_generator(new_sb) + assert mempool_item.npc_result.conds is not None + try: + new_sbc_result = get_name_puzzle_conditions( + new_sb, + mempool_item.npc_result.conds.cost, + constants, + True, + height, + ) + except TypeError as e: + error = Err(e.args[0]) + raise ValueError(f"Mempool item became invalid after singleton fast forward with error {error}.") + new_npc_result = NPCResult(None, new_sbc_result) # Update bundle_coin_spends using the collected data for coin_id in replaced_coin_ids: mempool_item.bundle_coin_spends.pop(coin_id, None) From 110067ab6aabc1f351a2a7c0c2c6e10ecbc1f524 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 17 Jul 2024 15:43:16 +0100 Subject: [PATCH 10/32] remove unused imports --- chia/types/eligible_coin_spends.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/chia/types/eligible_coin_spends.py b/chia/types/eligible_coin_spends.py index 40dd9104ffe2..bf5ef43aac95 100644 --- a/chia/types/eligible_coin_spends.py +++ b/chia/types/eligible_coin_spends.py @@ -8,8 +8,6 @@ from chia.consensus.condition_costs import ConditionCost from chia.consensus.constants import ConsensusConstants from chia.consensus.cost_calculator import NPCResult -from chia.full_node.bundle_tools import simple_solution_generator -from chia.util.errors import Err, ValidationError from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.serialized_program import SerializedProgram from chia.types.blockchain_format.sized_bytes import bytes32 From a9a7c677d88cc4ca71d119685b5a3eded38b8b40 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Thu, 18 Jul 2024 14:11:09 +0100 Subject: [PATCH 11/32] import Err --- chia/types/eligible_coin_spends.py | 1 + 1 file changed, 1 insertion(+) diff --git a/chia/types/eligible_coin_spends.py b/chia/types/eligible_coin_spends.py index bf5ef43aac95..cd92bfe4e7a2 100644 --- a/chia/types/eligible_coin_spends.py +++ b/chia/types/eligible_coin_spends.py @@ -16,6 +16,7 @@ from chia.types.mempool_item import BundleCoinSpend from chia.types.spend_bundle import SpendBundle from chia.util.ints import uint32, uint64 +from chia.util.errors import Err, ValueError @dataclasses.dataclass(frozen=True) From f506ac63424b96df5e3c17a19d9157f8be6e1b31 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Mon, 29 Jul 2024 14:15:50 +0100 Subject: [PATCH 12/32] swap get_name_puzzle_conditions to get_conditions_from_spendbundle --- chia/types/eligible_coin_spends.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/chia/types/eligible_coin_spends.py b/chia/types/eligible_coin_spends.py index cd92bfe4e7a2..2ec2642ad35f 100644 --- a/chia/types/eligible_coin_spends.py +++ b/chia/types/eligible_coin_spends.py @@ -3,7 +3,7 @@ import dataclasses from typing import Awaitable, Callable, Dict, List, Optional, Tuple -from chia_rs import fast_forward_singleton, get_name_puzzle_conditions +from chia_rs import fast_forward_singleton, get_conditions_from_spendbundle from chia.consensus.condition_costs import ConditionCost from chia.consensus.constants import ConsensusConstants @@ -335,11 +335,10 @@ async def process_fast_forward_spends( # generator = simple_solution_generator(new_sb) assert mempool_item.npc_result.conds is not None try: - new_sbc_result = get_name_puzzle_conditions( + new_sbc_result = get_conditions_from_spendbundle( new_sb, mempool_item.npc_result.conds.cost, constants, - True, height, ) except TypeError as e: From af0f8c0c6aef2a801808b8209e8875e18fd5b96b Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 7 Aug 2024 16:08:09 +0100 Subject: [PATCH 13/32] Fix for latest rust version --- chia/full_node/mempool_manager.py | 3 +-- chia/types/eligible_coin_spends.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index 750241de9ccd..520419256d75 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -284,7 +284,6 @@ async def pre_validate_spendbundle( self.max_tx_clvm_cost, self.constants, self.peak.height, - bls_cache, ) except Exception as e: # take returned TypeError and turn into ValidationError class by error = Err(e.args[0]) @@ -293,7 +292,7 @@ async def pre_validate_spendbundle( self._worker_queue_size -= 1 if bls_cache is not None: - bls_cache.update(new_cache_entries) + bls_cache.update([(e[0], bytes(e[1])) for e in new_cache_entries]) ret = NPCResult(None, sbc) diff --git a/chia/types/eligible_coin_spends.py b/chia/types/eligible_coin_spends.py index 2ec2642ad35f..80e9fa4b2dfc 100644 --- a/chia/types/eligible_coin_spends.py +++ b/chia/types/eligible_coin_spends.py @@ -16,7 +16,7 @@ from chia.types.mempool_item import BundleCoinSpend from chia.types.spend_bundle import SpendBundle from chia.util.ints import uint32, uint64 -from chia.util.errors import Err, ValueError +from chia.util.errors import Err @dataclasses.dataclass(frozen=True) From 91a0f2016076c0647e7687cce923a8237dd28a54 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Fri, 9 Aug 2024 13:42:03 +0100 Subject: [PATCH 14/32] fix fastforwards --- chia/types/eligible_coin_spends.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/chia/types/eligible_coin_spends.py b/chia/types/eligible_coin_spends.py index 80e9fa4b2dfc..46b993884543 100644 --- a/chia/types/eligible_coin_spends.py +++ b/chia/types/eligible_coin_spends.py @@ -7,7 +7,6 @@ from chia.consensus.condition_costs import ConditionCost from chia.consensus.constants import ConsensusConstants -from chia.consensus.cost_calculator import NPCResult from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.serialized_program import SerializedProgram from chia.types.blockchain_format.sized_bytes import bytes32 @@ -333,18 +332,17 @@ async def process_fast_forward_spends( ) # We need to run the new spend bundle to make sure it remains valid # generator = simple_solution_generator(new_sb) - assert mempool_item.npc_result.conds is not None + assert mempool_item.conds is not None try: new_sbc_result = get_conditions_from_spendbundle( new_sb, - mempool_item.npc_result.conds.cost, + mempool_item.conds.cost, constants, height, ) except TypeError as e: error = Err(e.args[0]) raise ValueError(f"Mempool item became invalid after singleton fast forward with error {error}.") - new_npc_result = NPCResult(None, new_sbc_result) # Update bundle_coin_spends using the collected data for coin_id in replaced_coin_ids: mempool_item.bundle_coin_spends.pop(coin_id, None) @@ -356,4 +354,4 @@ async def process_fast_forward_spends( # change. Still, it's good form to update the spend bundle with the # new coin spends mempool_item.spend_bundle = new_sb - mempool_item.conds = new_npc_result.conds + mempool_item.conds = new_sbc_result From 9acc33901c1219102179b658abbf92a4759fbf24 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Fri, 9 Aug 2024 13:47:30 +0100 Subject: [PATCH 15/32] isort --- chia/types/eligible_coin_spends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chia/types/eligible_coin_spends.py b/chia/types/eligible_coin_spends.py index 46b993884543..e943a86cbd33 100644 --- a/chia/types/eligible_coin_spends.py +++ b/chia/types/eligible_coin_spends.py @@ -14,8 +14,8 @@ from chia.types.internal_mempool_item import InternalMempoolItem from chia.types.mempool_item import BundleCoinSpend from chia.types.spend_bundle import SpendBundle -from chia.util.ints import uint32, uint64 from chia.util.errors import Err +from chia.util.ints import uint32, uint64 @dataclasses.dataclass(frozen=True) From d29b1623e4730958c4ff475341338c1c1dd00bc9 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 21 Aug 2024 14:03:48 +0100 Subject: [PATCH 16/32] no longer pass in new_spend_bytes to pre_validate_spendbundle --- .../core/mempool/test_mempool_manager.py | 24 +++++++++---------- chia/clvm/spend_sim.py | 2 +- chia/full_node/full_node.py | 2 +- chia/full_node/mempool_manager.py | 3 --- chia/rpc/full_node_rpc_api.py | 2 +- 5 files changed, 15 insertions(+), 18 deletions(-) diff --git a/chia/_tests/core/mempool/test_mempool_manager.py b/chia/_tests/core/mempool/test_mempool_manager.py index 2550b66cd621..31ac9bf941b8 100644 --- a/chia/_tests/core/mempool/test_mempool_manager.py +++ b/chia/_tests/core/mempool/test_mempool_manager.py @@ -387,7 +387,7 @@ def spend_bundle_from_conditions( async def add_spendbundle( mempool_manager: MempoolManager, sb: SpendBundle, sb_name: bytes32 ) -> Tuple[Optional[uint64], MempoolInclusionStatus, Optional[Err]]: - sbc = await mempool_manager.pre_validate_spendbundle(sb, None, sb_name) + sbc = await mempool_manager.pre_validate_spendbundle(sb, sb_name) ret = await mempool_manager.add_spend_bundle(sb, sbc, sb_name, TEST_HEIGHT) invariant_check_mempool(mempool_manager.mempool) return ret.cost, ret.status, ret.error @@ -461,7 +461,7 @@ async def test_empty_spend_bundle() -> None: mempool_manager = await instantiate_mempool_manager(zero_calls_get_coin_records) sb = SpendBundle([], G2Element()) with pytest.raises(ValidationError, match="INVALID_SPEND_BUNDLE"): - await mempool_manager.pre_validate_spendbundle(sb, None, sb.name()) + await mempool_manager.pre_validate_spendbundle(sb, sb.name()) @pytest.mark.anyio @@ -470,7 +470,7 @@ async def test_negative_addition_amount() -> None: conditions = [[ConditionOpcode.CREATE_COIN, IDENTITY_PUZZLE_HASH, -1]] sb = spend_bundle_from_conditions(conditions) with pytest.raises(ValidationError, match="COIN_AMOUNT_NEGATIVE"): - await mempool_manager.pre_validate_spendbundle(sb, None, sb.name()) + await mempool_manager.pre_validate_spendbundle(sb, sb.name()) @pytest.mark.anyio @@ -481,7 +481,7 @@ async def test_valid_addition_amount() -> None: coin = Coin(IDENTITY_PUZZLE_HASH, IDENTITY_PUZZLE_HASH, max_amount) sb = spend_bundle_from_conditions(conditions, coin) # ensure this does not throw - _ = await mempool_manager.pre_validate_spendbundle(sb, None, sb.name()) + _ = await mempool_manager.pre_validate_spendbundle(sb, sb.name()) @pytest.mark.anyio @@ -491,7 +491,7 @@ async def test_too_big_addition_amount() -> None: conditions = [[ConditionOpcode.CREATE_COIN, IDENTITY_PUZZLE_HASH, max_amount + 1]] sb = spend_bundle_from_conditions(conditions) with pytest.raises(ValidationError, match="COIN_AMOUNT_EXCEEDS_MAXIMUM"): - await mempool_manager.pre_validate_spendbundle(sb, None, sb.name()) + await mempool_manager.pre_validate_spendbundle(sb, sb.name()) @pytest.mark.anyio @@ -503,7 +503,7 @@ async def test_duplicate_output() -> None: ] sb = spend_bundle_from_conditions(conditions) with pytest.raises(ValidationError, match="DUPLICATE_OUTPUT"): - await mempool_manager.pre_validate_spendbundle(sb, None, sb.name()) + await mempool_manager.pre_validate_spendbundle(sb, sb.name()) @pytest.mark.anyio @@ -514,7 +514,7 @@ async def test_block_cost_exceeds_max() -> None: conditions.append([ConditionOpcode.CREATE_COIN, IDENTITY_PUZZLE_HASH, i]) sb = spend_bundle_from_conditions(conditions) with pytest.raises(ValidationError, match="BLOCK_COST_EXCEEDS_MAX"): - await mempool_manager.pre_validate_spendbundle(sb, None, sb.name()) + await mempool_manager.pre_validate_spendbundle(sb, sb.name()) @pytest.mark.anyio @@ -524,7 +524,7 @@ async def test_double_spend_prevalidation() -> None: sb = spend_bundle_from_conditions(conditions) sb_twice: SpendBundle = SpendBundle.aggregate([sb, sb]) with pytest.raises(ValidationError, match="DOUBLE_SPEND"): - await mempool_manager.pre_validate_spendbundle(sb_twice, None, sb_twice.name()) + await mempool_manager.pre_validate_spendbundle(sb_twice, sb_twice.name()) @pytest.mark.anyio @@ -532,11 +532,11 @@ async def test_minting_coin() -> None: mempool_manager = await instantiate_mempool_manager(zero_calls_get_coin_records) conditions = [[ConditionOpcode.CREATE_COIN, IDENTITY_PUZZLE_HASH, TEST_COIN_AMOUNT]] sb = spend_bundle_from_conditions(conditions) - _ = await mempool_manager.pre_validate_spendbundle(sb, None, sb.name()) + _ = await mempool_manager.pre_validate_spendbundle(sb, sb.name()) conditions = [[ConditionOpcode.CREATE_COIN, IDENTITY_PUZZLE_HASH, TEST_COIN_AMOUNT + 1]] sb = spend_bundle_from_conditions(conditions) with pytest.raises(ValidationError, match="MINTING_COIN"): - await mempool_manager.pre_validate_spendbundle(sb, None, sb.name()) + await mempool_manager.pre_validate_spendbundle(sb, sb.name()) @pytest.mark.anyio @@ -544,11 +544,11 @@ async def test_reserve_fee_condition() -> None: mempool_manager = await instantiate_mempool_manager(zero_calls_get_coin_records) conditions = [[ConditionOpcode.RESERVE_FEE, TEST_COIN_AMOUNT]] sb = spend_bundle_from_conditions(conditions) - _ = await mempool_manager.pre_validate_spendbundle(sb, None, sb.name()) + _ = await mempool_manager.pre_validate_spendbundle(sb, sb.name()) conditions = [[ConditionOpcode.RESERVE_FEE, TEST_COIN_AMOUNT + 1]] sb = spend_bundle_from_conditions(conditions) with pytest.raises(ValidationError, match="RESERVE_FEE_CONDITION_FAILED"): - await mempool_manager.pre_validate_spendbundle(sb, None, sb.name()) + await mempool_manager.pre_validate_spendbundle(sb, sb.name()) @pytest.mark.anyio diff --git a/chia/clvm/spend_sim.py b/chia/clvm/spend_sim.py index ed88e792bdac..ea06a14d8dc6 100644 --- a/chia/clvm/spend_sim.py +++ b/chia/clvm/spend_sim.py @@ -336,7 +336,7 @@ def __init__(self, service: SpendSim) -> None: async def push_tx(self, spend_bundle: SpendBundle) -> Tuple[MempoolInclusionStatus, Optional[Err]]: try: spend_bundle_id = spend_bundle.name() - sbc = await self.service.mempool_manager.pre_validate_spendbundle(spend_bundle, None, spend_bundle_id) + sbc = await self.service.mempool_manager.pre_validate_spendbundle(spend_bundle, spend_bundle_id) except ValidationError as e: return MempoolInclusionStatus.FAILED, e.code assert self.service.mempool_manager.peak is not None diff --git a/chia/full_node/full_node.py b/chia/full_node/full_node.py index 0a1eb669685d..6142d78f6572 100644 --- a/chia/full_node/full_node.py +++ b/chia/full_node/full_node.py @@ -2356,7 +2356,7 @@ async def add_transaction( else: try: cost_result = await self.mempool_manager.pre_validate_spendbundle( - transaction, tx_bytes, spend_name, self._bls_cache + transaction, spend_name, self._bls_cache ) except ValidationError as e: self.mempool_manager.remove_seen(spend_name) diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index 520419256d75..0df9a5856a9d 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -259,7 +259,6 @@ def remove_seen(self, bundle_hash: bytes32) -> None: async def pre_validate_spendbundle( self, new_spend: SpendBundle, - new_spend_bytes: Optional[bytes], spend_name: bytes32, bls_cache: Optional[BLSCache] = None, ) -> SpendBundleConditions: @@ -267,8 +266,6 @@ async def pre_validate_spendbundle( Errors are included within the cached_result. This runs in another process so we don't block the main thread """ - if new_spend_bytes is None: - new_spend_bytes = bytes(new_spend) if new_spend.coin_spends == []: raise ValidationError(Err.INVALID_SPEND_BUNDLE, "Empty SpendBundle") diff --git a/chia/rpc/full_node_rpc_api.py b/chia/rpc/full_node_rpc_api.py index 930ca48ebfc8..c6c7e21bdf0c 100644 --- a/chia/rpc/full_node_rpc_api.py +++ b/chia/rpc/full_node_rpc_api.py @@ -868,7 +868,7 @@ async def _validate_fee_estimate_cost(self, request: Dict[str, Any]) -> uint64: spend_bundle: SpendBundle = SpendBundle.from_json_dict(request["spend_bundle"]) spend_name = spend_bundle.name() conds: SpendBundleConditions = await self.service.mempool_manager.pre_validate_spendbundle( - spend_bundle, None, spend_name + spend_bundle, spend_name ) cost = conds.cost elif "cost" in request: From 44cd65aa64b07eda01276c822c1c7a32e096b3b9 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 21 Aug 2024 14:18:22 +0100 Subject: [PATCH 17/32] fix benchmarks for new pre_validate_spendbundle params --- benchmarks/mempool-long-lived.py | 2 +- benchmarks/mempool.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/mempool-long-lived.py b/benchmarks/mempool-long-lived.py index 94bffdec4ff7..564a99812cd0 100644 --- a/benchmarks/mempool-long-lived.py +++ b/benchmarks/mempool-long-lived.py @@ -114,7 +114,7 @@ async def get_coin_record(coin_ids: Collection[bytes32]) -> List[CoinRecord]: coin.name(): CoinRecord(coin, uint32(height // 2), uint32(0), False, uint64(timestamp // 2)) } spend_bundle_id = sb.name() - sbc = await mempool.pre_validate_spendbundle(sb, None, spend_bundle_id) + sbc = await mempool.pre_validate_spendbundle(sb, spend_bundle_id) assert sbc is not None await mempool.add_spend_bundle(sb, sbc, spend_bundle_id, uint32(height)) diff --git a/benchmarks/mempool.py b/benchmarks/mempool.py index c0451ccb06e4..c7dec604b711 100644 --- a/benchmarks/mempool.py +++ b/benchmarks/mempool.py @@ -173,7 +173,7 @@ async def get_unspent_lineage_info_for_puzzle_hash(_: bytes32) -> Optional[Unspe async def add_spend_bundles(spend_bundles: List[SpendBundle]) -> None: for tx in spend_bundles: spend_bundle_id = tx.name() - npc = await mempool.pre_validate_spendbundle(tx, None, spend_bundle_id) + npc = await mempool.pre_validate_spendbundle(tx, spend_bundle_id) assert npc is not None info = await mempool.add_spend_bundle(tx, npc, spend_bundle_id, height) assert info.status == MempoolInclusionStatus.SUCCESS From c8479b1187843c53b95dc21fb60225c0db5ed7bc Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 21 Aug 2024 15:59:52 +0100 Subject: [PATCH 18/32] checks e.args[0] exists before accessing it --- chia/full_node/mempool_manager.py | 7 +++++-- chia/types/eligible_coin_spends.py | 7 ++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index 0df9a5856a9d..af1a2a388e1f 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -283,8 +283,11 @@ async def pre_validate_spendbundle( self.peak.height, ) except Exception as e: # take returned TypeError and turn into ValidationError class by - error = Err(e.args[0]) - raise ValidationError(error) + if len(e.args) > 0: + error = Err(e.args[0]) + raise ValidationError(error) + else: + raise ValidationError finally: self._worker_queue_size -= 1 diff --git a/chia/types/eligible_coin_spends.py b/chia/types/eligible_coin_spends.py index e943a86cbd33..064046e3cf7d 100644 --- a/chia/types/eligible_coin_spends.py +++ b/chia/types/eligible_coin_spends.py @@ -342,7 +342,12 @@ async def process_fast_forward_spends( ) except TypeError as e: error = Err(e.args[0]) - raise ValueError(f"Mempool item became invalid after singleton fast forward with error {error}.") + if len(e.args) > 0: + error = Err(e.args[0]) + raise ValueError(f"Mempool item became invalid after singleton fast forward with error {error}.") + else: + raise ValueError(f"Mempool item became invalid after singleton fast forward with an unspecified error.") + # Update bundle_coin_spends using the collected data for coin_id in replaced_coin_ids: mempool_item.bundle_coin_spends.pop(coin_id, None) From cbb3917f8c7d617205eb0706a547179be5ba4afe Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 21 Aug 2024 16:00:58 +0100 Subject: [PATCH 19/32] fix truncated comment --- chia/full_node/mempool_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index af1a2a388e1f..f00fcbd0247d 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -282,7 +282,7 @@ async def pre_validate_spendbundle( self.constants, self.peak.height, ) - except Exception as e: # take returned TypeError and turn into ValidationError class by + except Exception as e: # take returned TypeError and turn raise it as a ValidationError if len(e.args) > 0: error = Err(e.args[0]) raise ValidationError(error) From c96017aa7a54c3af37167ab13d65cafb390ebf69 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 21 Aug 2024 16:03:57 +0100 Subject: [PATCH 20/32] lint fixes --- chia/types/eligible_coin_spends.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chia/types/eligible_coin_spends.py b/chia/types/eligible_coin_spends.py index 064046e3cf7d..105494b5faab 100644 --- a/chia/types/eligible_coin_spends.py +++ b/chia/types/eligible_coin_spends.py @@ -346,8 +346,8 @@ async def process_fast_forward_spends( error = Err(e.args[0]) raise ValueError(f"Mempool item became invalid after singleton fast forward with error {error}.") else: - raise ValueError(f"Mempool item became invalid after singleton fast forward with an unspecified error.") - + raise ValueError("Mempool item became invalid after singleton fast forward with an unspecified error.") + # Update bundle_coin_spends using the collected data for coin_id in replaced_coin_ids: mempool_item.bundle_coin_spends.pop(coin_id, None) From 62e75425f1e21e8e9881af743c978e66ea5eb5a1 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 21 Aug 2024 16:10:09 +0100 Subject: [PATCH 21/32] return Err.Unknown --- chia/full_node/mempool_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index f00fcbd0247d..4208a96657c9 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -287,7 +287,7 @@ async def pre_validate_spendbundle( error = Err(e.args[0]) raise ValidationError(error) else: - raise ValidationError + raise ValidationError(Err.UNKNOWN) finally: self._worker_queue_size -= 1 From 9eb640f4279e889a8063ac6f135f17744da7e2e6 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 21 Aug 2024 16:28:38 +0100 Subject: [PATCH 22/32] remove multiprocessing_context --- chia/full_node/mempool_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index 4208a96657c9..347e6e7c0248 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -142,7 +142,6 @@ def __init__( self, get_coin_records: Callable[[Collection[bytes32]], Awaitable[List[CoinRecord]]], consensus_constants: ConsensusConstants, - multiprocessing_context: Optional[BaseContext] = None, *, single_threaded: bool = False, max_tx_clvm_cost: Optional[uint64] = None, From 7bbf6a40cebce3a6d81dc4998d5550bb4fe1f246 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 21 Aug 2024 16:31:28 +0100 Subject: [PATCH 23/32] remove oversight --- chia/types/eligible_coin_spends.py | 1 - 1 file changed, 1 deletion(-) diff --git a/chia/types/eligible_coin_spends.py b/chia/types/eligible_coin_spends.py index 105494b5faab..edb281a35a1b 100644 --- a/chia/types/eligible_coin_spends.py +++ b/chia/types/eligible_coin_spends.py @@ -341,7 +341,6 @@ async def process_fast_forward_spends( height, ) except TypeError as e: - error = Err(e.args[0]) if len(e.args) > 0: error = Err(e.args[0]) raise ValueError(f"Mempool item became invalid after singleton fast forward with error {error}.") From a8e771e1770f0f1455b2273076bc57fbcd2308d1 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 21 Aug 2024 16:39:11 +0100 Subject: [PATCH 24/32] remove initialisation of multiprocessing_context in mempool_manager from full_node --- chia/full_node/full_node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/chia/full_node/full_node.py b/chia/full_node/full_node.py index 6142d78f6572..a163bf19196b 100644 --- a/chia/full_node/full_node.py +++ b/chia/full_node/full_node.py @@ -279,7 +279,6 @@ async def manage(self) -> AsyncIterator[None]: self._mempool_manager = MempoolManager( get_coin_records=self.coin_store.get_coin_records, consensus_constants=self.constants, - multiprocessing_context=self.multiprocessing_context, single_threaded=single_threaded, ) From 7f109bb261f578daaa98e9b3b87c67771ff21181 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 21 Aug 2024 16:42:16 +0100 Subject: [PATCH 25/32] remove unused import --- chia/full_node/mempool_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index 347e6e7c0248..6d49674f6c08 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -5,7 +5,6 @@ import time from concurrent.futures import Executor, ThreadPoolExecutor from dataclasses import dataclass -from multiprocessing.context import BaseContext from typing import Awaitable, Callable, Collection, Dict, List, Optional, Set, Tuple, TypeVar from chia_rs import ELIGIBLE_FOR_DEDUP, ELIGIBLE_FOR_FF, BLSCache, supports_fast_forward, validate_clvm_and_signature From 71f38522e7d94a3187f108a3e8b3de01bf8f3c1f Mon Sep 17 00:00:00 2001 From: matt-o-how <48453825+matt-o-how@users.noreply.github.com> Date: Wed, 21 Aug 2024 17:36:54 +0100 Subject: [PATCH 26/32] Use underscore for test Co-authored-by: Amine Khaldi --- chia/_tests/core/mempool/test_mempool_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chia/_tests/core/mempool/test_mempool_manager.py b/chia/_tests/core/mempool/test_mempool_manager.py index 31ac9bf941b8..ec8807b3dd73 100644 --- a/chia/_tests/core/mempool/test_mempool_manager.py +++ b/chia/_tests/core/mempool/test_mempool_manager.py @@ -581,7 +581,7 @@ async def test_same_sb_twice_with_eligible_coin() -> None: sb = SpendBundle.aggregate([sb1, sb2]) sb_name = sb.name() result = await add_spendbundle(mempool_manager, sb, sb_name) - expected_cost = uint64(6600088) + expected_cost = uint64(6_600_088) assert result == (expected_cost, MempoolInclusionStatus.SUCCESS, None) assert mempool_manager.get_spendbundle(sb_name) == sb result = await add_spendbundle(mempool_manager, sb, sb_name) From 7f73e8c3401d05875248119dd2b3ddb184d0af74 Mon Sep 17 00:00:00 2001 From: matt-o-how <48453825+matt-o-how@users.noreply.github.com> Date: Wed, 21 Aug 2024 17:37:11 +0100 Subject: [PATCH 27/32] Use underscore for large test int Co-authored-by: Amine Khaldi --- chia/_tests/core/mempool/test_mempool_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chia/_tests/core/mempool/test_mempool_manager.py b/chia/_tests/core/mempool/test_mempool_manager.py index ec8807b3dd73..894a433a0aae 100644 --- a/chia/_tests/core/mempool/test_mempool_manager.py +++ b/chia/_tests/core/mempool/test_mempool_manager.py @@ -614,7 +614,7 @@ async def test_sb_twice_with_eligible_coin_and_different_spends_order() -> None: assert mempool_manager.get_spendbundle(sb_name) is None assert mempool_manager.get_spendbundle(reordered_sb_name) is None result = await add_spendbundle(mempool_manager, sb, sb_name) - expected_cost = uint64(7800132) + expected_cost = uint64(7_800_132) assert result == (expected_cost, MempoolInclusionStatus.SUCCESS, None) assert mempool_manager.get_spendbundle(sb_name) == sb assert mempool_manager.get_spendbundle(reordered_sb_name) is None From a47454ffae4c88b8dcfbbcc11b8e74d67de8ecb7 Mon Sep 17 00:00:00 2001 From: matt-o-how <48453825+matt-o-how@users.noreply.github.com> Date: Wed, 21 Aug 2024 17:38:58 +0100 Subject: [PATCH 28/32] remove commented out code Co-authored-by: Amine Khaldi --- chia/types/eligible_coin_spends.py | 1 - 1 file changed, 1 deletion(-) diff --git a/chia/types/eligible_coin_spends.py b/chia/types/eligible_coin_spends.py index edb281a35a1b..79c58a7c5fc8 100644 --- a/chia/types/eligible_coin_spends.py +++ b/chia/types/eligible_coin_spends.py @@ -331,7 +331,6 @@ async def process_fast_forward_spends( coin_spends=new_coin_spends, aggregated_signature=mempool_item.spend_bundle.aggregated_signature ) # We need to run the new spend bundle to make sure it remains valid - # generator = simple_solution_generator(new_sb) assert mempool_item.conds is not None try: new_sbc_result = get_conditions_from_spendbundle( From 1298416d96eda9ded6412539a99daaf95e73cc8f Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 21 Aug 2024 17:38:13 +0100 Subject: [PATCH 29/32] rename new_sbc for readability --- chia/types/eligible_coin_spends.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chia/types/eligible_coin_spends.py b/chia/types/eligible_coin_spends.py index 79c58a7c5fc8..e8e8bfe15df5 100644 --- a/chia/types/eligible_coin_spends.py +++ b/chia/types/eligible_coin_spends.py @@ -333,7 +333,7 @@ async def process_fast_forward_spends( # We need to run the new spend bundle to make sure it remains valid assert mempool_item.conds is not None try: - new_sbc_result = get_conditions_from_spendbundle( + new_conditions = get_conditions_from_spendbundle( new_sb, mempool_item.conds.cost, constants, @@ -357,4 +357,4 @@ async def process_fast_forward_spends( # change. Still, it's good form to update the spend bundle with the # new coin spends mempool_item.spend_bundle = new_sb - mempool_item.conds = new_sbc_result + mempool_item.conds = new_conditions From 791bed3e972d6d0209ab6ba5d18e074916ee8933 Mon Sep 17 00:00:00 2001 From: matt-o-how <48453825+matt-o-how@users.noreply.github.com> Date: Thu, 22 Aug 2024 14:13:27 +0100 Subject: [PATCH 30/32] Use more descriptive comment Co-authored-by: Amine Khaldi --- chia/full_node/mempool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chia/full_node/mempool.py b/chia/full_node/mempool.py index 1bab4f28e4c4..8e19a7578250 100644 --- a/chia/full_node/mempool.py +++ b/chia/full_node/mempool.py @@ -521,7 +521,7 @@ async def create_bundle_from_mempool_items( unique_additions.extend(spend_data.additions) cost_saving = 0 else: - # item gets modified in the following call + # Update this item with fast forwarded spends, if it has any. await eligible_coin_spends.process_fast_forward_spends( mempool_item=item, get_unspent_lineage_info_for_puzzle_hash=get_unspent_lineage_info_for_puzzle_hash, From fd5692d64de19177fa72858e77bc38b4ff582e37 Mon Sep 17 00:00:00 2001 From: matt-o-how <48453825+matt-o-how@users.noreply.github.com> Date: Thu, 22 Aug 2024 14:15:53 +0100 Subject: [PATCH 31/32] remove imprecise check from test Co-authored-by: Amine Khaldi --- chia/_tests/core/mempool/test_singleton_fast_forward.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/chia/_tests/core/mempool/test_singleton_fast_forward.py b/chia/_tests/core/mempool/test_singleton_fast_forward.py index 9043d67f69fb..8fc4eae2691e 100644 --- a/chia/_tests/core/mempool/test_singleton_fast_forward.py +++ b/chia/_tests/core/mempool/test_singleton_fast_forward.py @@ -461,11 +461,9 @@ async def test_singleton_fast_forward_different_block(is_eligible_for_ff: bool) # version which is the grandchild in this scenario assert status == MempoolInclusionStatus.SUCCESS assert error is None - old_lineage_info = unspent_lineage_info unspent_lineage_info = await sim_client.service.coin_store.get_unspent_lineage_info_for_puzzle_hash( singleton_puzzle_hash ) - assert old_lineage_info != unspent_lineage_info singleton_grandchild, [remaining_coin] = await get_singleton_and_remaining_coins(sim) assert unspent_lineage_info == UnspentLineageInfo( coin_id=singleton_grandchild.name(), From c53d78bd0843a9a13797c688b6c0bc246a9afe5a Mon Sep 17 00:00:00 2001 From: arvidn Date: Mon, 26 Aug 2024 17:25:59 +0200 Subject: [PATCH 32/32] add temporary work-around for mempool block creation not taking block overhead into account, creating blocks that exceed the max size. --- chia/_tests/core/full_node/test_full_node.py | 4 ++-- chia/full_node/mempool_manager.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/chia/_tests/core/full_node/test_full_node.py b/chia/_tests/core/full_node/test_full_node.py index 2f4cafbf1e0f..8a23b19565a7 100644 --- a/chia/_tests/core/full_node/test_full_node.py +++ b/chia/_tests/core/full_node/test_full_node.py @@ -935,8 +935,8 @@ async def test_new_transaction_and_mempool(self, wallet_nodes, self_hostname, se # these numbers reflect the capacity of the mempool. In these # tests MEMPOOL_BLOCK_BUFFER is 1. The other factors are COST_PER_BYTE # and MAX_BLOCK_COST_CLVM - assert included_tx == 23 - assert not_included_tx == 10 + assert included_tx == 27 + assert not_included_tx == 6 assert seen_bigger_transaction_has_high_fee # Mempool is full diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index 6d49674f6c08..afc48de758aa 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -302,7 +302,9 @@ async def pre_validate_spendbundle( if ret.error is not None: raise ValidationError(Err(ret.error), "pre_validate_spendbundle failed") assert ret.conds is not None - return ret.conds + # TODO: this cost padding is temporary, until we update the mempool's + # block creation logic to take the block overhead into account + return ret.conds.replace(cost=ret.conds.cost + 10000000) async def add_spend_bundle( self,