validate more aspects of conditions earlier (Chia-Network#6654)

* validate more aspects of conditions earlier, in the process clvm execution is offloaded to

* more tests

* use as_int() in SExp rather than int_from_bytes()

* put condition parser tests in its own test class

* capitalize module-scope variable

* abbreviate enum names in parse_condition_args

* use a more realistic cost per byte in test_rom

* merge parse_coin_id and parse_hash, since they are identical

chia/consensus/block_body_validation.py

@@ -330,7 +330,10 @@ async def validate_block_body(
                     curr_block_generator: Optional[BlockGenerator] = await get_block_generator(curr)
                     assert curr_block_generator is not None and curr.transactions_info is not None
                     curr_npc_result = get_name_puzzle_conditions(
-                        curr_block_generator, min(constants.MAX_BLOCK_COST_CLVM, curr.transactions_info.cost), False
+                        curr_block_generator,
+                        min(constants.MAX_BLOCK_COST_CLVM, curr.transactions_info.cost),
+                        cost_per_byte=constants.COST_PER_BYTE,
+                        safe_mode=False,
                     )
                     removals_in_curr, additions_in_curr = tx_removals_and_additions(curr_npc_result.npc_list)
                 else:

chia/consensus/block_creation.py

@@ -126,7 +126,9 @@ def create_foliage(
         # Calculate the cost of transactions
         if block_generator is not None:
             generator_block_heights_list = block_generator.block_height_list()
-            result: NPCResult = get_name_puzzle_conditions(block_generator, constants.MAX_BLOCK_COST_CLVM, True)
+            result: NPCResult = get_name_puzzle_conditions(
+                block_generator, constants.MAX_BLOCK_COST_CLVM, cost_per_byte=constants.COST_PER_BYTE, safe_mode=True
+            )
             cost = calculate_cost_of_program(block_generator.program, result, constants.COST_PER_BYTE)
 
             removal_amount = 0

chia/consensus/blockchain.py

@@ -204,7 +204,10 @@ async def receive_block(
                         return ReceiveBlockResult.INVALID_BLOCK, Err.GENERATOR_REF_HAS_NO_GENERATOR, None
                     assert block_generator is not None and block.transactions_info is not None
                     npc_result = get_name_puzzle_conditions(
-                        block_generator, min(self.constants.MAX_BLOCK_COST_CLVM, block.transactions_info.cost), False
+                        block_generator,
+                        min(self.constants.MAX_BLOCK_COST_CLVM, block.transactions_info.cost),
+                        cost_per_byte=self.constants.COST_PER_BYTE,
+                        safe_mode=False,
                     )
                     removals, tx_additions = tx_removals_and_additions(npc_result.npc_list)
                 else:
@@ -375,7 +378,12 @@ async def get_tx_removals_and_additions(
                 if npc_result is None:
                     block_generator: Optional[BlockGenerator] = await self.get_block_generator(block)
                     assert block_generator is not None
-                    npc_result = get_name_puzzle_conditions(block_generator, self.constants.MAX_BLOCK_COST_CLVM, False)
+                    npc_result = get_name_puzzle_conditions(
+                        block_generator,
+                        self.constants.MAX_BLOCK_COST_CLVM,
+                        cost_per_byte=self.constants.COST_PER_BYTE,
+                        safe_mode=False,
+                    )
                 tx_removals, tx_additions = tx_removals_and_additions(npc_result.npc_list)
                 return tx_removals, tx_additions
             else:
@@ -523,7 +531,10 @@ async def validate_unfinished_block(
             if block_generator is None:
                 return PreValidationResult(uint16(Err.GENERATOR_REF_HAS_NO_GENERATOR.value), None, None)
             npc_result = get_name_puzzle_conditions(
-                block_generator, min(self.constants.MAX_BLOCK_COST_CLVM, block.transactions_info.cost), False
+                block_generator,
+                min(self.constants.MAX_BLOCK_COST_CLVM, block.transactions_info.cost),
+                cost_per_byte=self.constants.COST_PER_BYTE,
+                safe_mode=False,
             )
         error_code, cost_result = await validate_block_body(
             self.constants,

chia/consensus/multiprocess_validation.py

@@ -78,7 +78,10 @@ def batch_pre_validate_blocks(
                     block_generator: BlockGenerator = BlockGenerator.from_bytes(prev_generator_bytes)
                     assert block_generator.program == block.transactions_generator
                     npc_result = get_name_puzzle_conditions(
-                        block_generator, min(constants.MAX_BLOCK_COST_CLVM, block.transactions_info.cost), True
+                        block_generator,
+                        min(constants.MAX_BLOCK_COST_CLVM, block.transactions_info.cost),
+                        cost_per_byte=constants.COST_PER_BYTE,
+                        safe_mode=True,
                     )
                     removals, tx_additions = tx_removals_and_additions(npc_result.npc_list)
 

chia/full_node/mempool_check_conditions.py

@@ -1,7 +1,9 @@
 import time
-from typing import Dict, List, Optional, Set
+from typing import Tuple, Dict, List, Optional, Set
+from clvm import SExp
 
 from chia.consensus.cost_calculator import NPCResult
+from chia.consensus.condition_costs import ConditionCost
 from chia.full_node.generator import create_generator_args, setup_generator_args
 from chia.types.blockchain_format.coin import Coin
 from chia.types.blockchain_format.program import NIL
@@ -10,9 +12,9 @@
 from chia.types.condition_with_args import ConditionWithArgs
 from chia.types.generator_types import BlockGenerator
 from chia.types.name_puzzle_condition import NPC
-from chia.util.clvm import int_from_bytes
+from chia.util.clvm import int_from_bytes, int_to_bytes
 from chia.util.condition_tools import ConditionOpcode, conditions_by_opcode
-from chia.util.errors import Err
+from chia.util.errors import Err, ValidationError
 from chia.util.ints import uint32, uint64, uint16
 from chia.wallet.puzzles.generator_loader import GENERATOR_FOR_SINGLE_COIN_MOD
 from chia.wallet.puzzles.rom_bootstrap_generator import get_generator
@@ -132,40 +134,212 @@ def mempool_assert_my_amount(condition: ConditionWithArgs, unspent: CoinRecord)
     return None
 
 
-def get_name_puzzle_conditions(generator: BlockGenerator, max_cost: int, safe_mode: bool) -> NPCResult:
+def parse_aggsig(args: SExp) -> List[bytes]:
+    pubkey = args.first().atom
+    args = args.rest()
+    message = args.first().atom
+    if len(pubkey) != 48:
+        raise ValidationError(Err.INVALID_CONDITION)
+    if len(message) > 1024:
+        raise ValidationError(Err.INVALID_CONDITION)
+    return [pubkey, message]
+
+
+def parse_create_coin(args: SExp) -> List[bytes]:
+    puzzle_hash = args.first().atom
+    args = args.rest()
+    if len(puzzle_hash) != 32:
+        raise ValidationError(Err.INVALID_CONDITION)
+    amount_int = args.first().as_int()
+    if amount_int >= 2 ** 64:
+        raise ValidationError(Err.COIN_AMOUNT_EXCEEDS_MAXIMUM)
+    if amount_int < 0:
+        raise ValidationError(Err.COIN_AMOUNT_NEGATIVE)
+    # note that this may change the representation of amount. If the original
+    # buffer had redundant leading zeroes, they will be stripped
+    return [puzzle_hash, int_to_bytes(amount_int)]
+
+
+def parse_seconds(args: SExp, error_code: Err) -> Optional[List[bytes]]:
+    seconds_int = args.first().as_int()
+    # this condition is inherently satisified, there is no need to keep it
+    if seconds_int <= 0:
+        return None
+    if seconds_int >= 2 ** 64:
+        raise ValidationError(error_code)
+    # note that this may change the representation of seconds. If the original
+    # buffer had redundant leading zeroes, they will be stripped
+    return [int_to_bytes(seconds_int)]
+
+
+def parse_height(args: SExp, error_code: Err) -> Optional[List[bytes]]:
+    height_int = args.first().as_int()
+    # this condition is inherently satisified, there is no need to keep it
+    if height_int <= 0:
+        return None
+    if height_int >= 2 ** 32:
+        raise ValidationError(error_code)
+    # note that this may change the representation of the height. If the original
+    # buffer had redundant leading zeroes, they will be stripped
+    return [int_to_bytes(height_int)]
+
+
+def parse_fee(args: SExp) -> List[bytes]:
+    fee_int = args.first().as_int()
+    if fee_int >= 2 ** 64 or fee_int < 0:
+        raise ValidationError(Err.RESERVE_FEE_CONDITION_FAILED)
+    # note that this may change the representation of the fee. If the original
+    # buffer had redundant leading zeroes, they will be stripped
+    return [int_to_bytes(fee_int)]
+
+
+def parse_hash(args: SExp, error_code: Err) -> List[bytes]:
+    h = args.first().atom
+    if len(h) != 32:
+        raise ValidationError(error_code)
+    return [h]
+
+
+def parse_amount(args: SExp) -> List[bytes]:
+    amount_int = args.first().as_int()
+    if amount_int < 0:
+        raise ValidationError(Err.ASSERT_MY_AMOUNT_FAILED)
+    if amount_int >= 2 ** 64:
+        raise ValidationError(Err.ASSERT_MY_AMOUNT_FAILED)
+    # note that this may change the representation of amount. If the original
+    # buffer had redundant leading zeroes, they will be stripped
+    return [int_to_bytes(amount_int)]
+
+
+def parse_announcement(args: SExp) -> List[bytes]:
+    msg = args.first().atom
+    if len(msg) > 1024:
+        raise ValidationError(Err.INVALID_CONDITION)
+    return [msg]
+
+
+def parse_condition_args(args: SExp, condition: ConditionOpcode) -> Tuple[int, Optional[List[bytes]]]:
+    """
+    Parse a list with exactly the expected args, given opcode,
+    from an SExp into a list of bytes. If there are fewer or more elements in
+    the list, raise a RuntimeError. If the condition is inherently true (such as
+    a time- or height lock with a negative time or height, the returned list is None
+    """
+    op = ConditionOpcode
+    cc = ConditionCost
+    if condition is op.AGG_SIG_UNSAFE or condition is op.AGG_SIG_ME:
+        return cc.AGG_SIG.value, parse_aggsig(args)
+    elif condition is op.CREATE_COIN:
+        return cc.CREATE_COIN.value, parse_create_coin(args)
+    elif condition is op.ASSERT_SECONDS_ABSOLUTE:
+        return cc.ASSERT_SECONDS_ABSOLUTE.value, parse_seconds(args, Err.ASSERT_SECONDS_ABSOLUTE_FAILED)
+    elif condition is op.ASSERT_SECONDS_RELATIVE:
+        return cc.ASSERT_SECONDS_RELATIVE.value, parse_seconds(args, Err.ASSERT_SECONDS_RELATIVE_FAILED)
+    elif condition is op.ASSERT_HEIGHT_ABSOLUTE:
+        return cc.ASSERT_HEIGHT_ABSOLUTE.value, parse_height(args, Err.ASSERT_HEIGHT_ABSOLUTE_FAILED)
+    elif condition is op.ASSERT_HEIGHT_RELATIVE:
+        return cc.ASSERT_HEIGHT_RELATIVE.value, parse_height(args, Err.ASSERT_HEIGHT_RELATIVE_FAILED)
+    elif condition is op.ASSERT_MY_COIN_ID:
+        return cc.ASSERT_MY_COIN_ID.value, parse_hash(args, Err.ASSERT_MY_COIN_ID_FAILED)
+    elif condition is op.RESERVE_FEE:
+        return cc.RESERVE_FEE.value, parse_fee(args)
+    elif condition is op.CREATE_COIN_ANNOUNCEMENT:
+        return cc.CREATE_COIN_ANNOUNCEMENT.value, parse_announcement(args)
+    elif condition is op.ASSERT_COIN_ANNOUNCEMENT:
+        return cc.ASSERT_COIN_ANNOUNCEMENT.value, parse_hash(args, Err.ASSERT_ANNOUNCE_CONSUMED_FAILED)
+    elif condition is op.CREATE_PUZZLE_ANNOUNCEMENT:
+        return cc.CREATE_PUZZLE_ANNOUNCEMENT.value, parse_announcement(args)
+    elif condition is op.ASSERT_PUZZLE_ANNOUNCEMENT:
+        return cc.ASSERT_PUZZLE_ANNOUNCEMENT.value, parse_hash(args, Err.ASSERT_ANNOUNCE_CONSUMED_FAILED)
+    elif condition is op.ASSERT_MY_PARENT_ID:
+        return cc.ASSERT_MY_PARENT_ID.value, parse_hash(args, Err.ASSERT_MY_PARENT_ID_FAILED)
+    elif condition is op.ASSERT_MY_PUZZLEHASH:
+        return cc.ASSERT_MY_PUZZLEHASH.value, parse_hash(args, Err.ASSERT_MY_PUZZLEHASH_FAILED)
+    elif condition is op.ASSERT_MY_AMOUNT:
+        return cc.ASSERT_MY_AMOUNT.value, parse_amount(args)
+    else:
+        raise ValidationError(Err.INVALID_CONDITION)
+
+
+CONDITION_OPCODES: Set[bytes] = set(item.value for item in ConditionOpcode)
+
+
+def parse_condition(cond: SExp, safe_mode: bool) -> Tuple[int, Optional[ConditionWithArgs]]:
+    condition = cond.first().as_atom()
+    if condition in CONDITION_OPCODES:
+        opcode: ConditionOpcode = ConditionOpcode(condition)
+        cost, args = parse_condition_args(cond.rest(), opcode)
+        cvl = ConditionWithArgs(opcode, args) if args is not None else None
+    elif not safe_mode:
+        opcode = ConditionOpcode.UNKNOWN
+        cvl = ConditionWithArgs(opcode, cond.rest().as_atom_list())
+        cost = 0
+    else:
+        raise ValidationError(Err.INVALID_CONDITION)
+    return cost, cvl
+
+
+def get_name_puzzle_conditions(
+    generator: BlockGenerator, max_cost: int, *, cost_per_byte: int, safe_mode: bool
+) -> NPCResult:
+    """
+    This executes the generator program and returns the coins and their
+    conditions. If the cost of the program (size, CLVM execution and conditions)
+    exceed max_cost, the function fails. In order to accurately take the size
+    of the program into account when calculating cost, cost_per_byte must be
+    specified.
+    safe_mode determines whether the clvm program and conditions are executed in
+    strict mode or not. When in safe/strict mode, unknow operations or conditions
+    are considered failures. This is the mode when accepting transactions into
+    the mempool.
+    """
     try:
         block_program, block_program_args = setup_generator_args(generator)
+        max_cost -= len(bytes(generator.program)) * cost_per_byte
+        if max_cost < 0:
+            return NPCResult(uint16(Err.INVALID_BLOCK_COST.value), [], uint64(0))
         if safe_mode:
-            cost, result = GENERATOR_MOD.run_safe_with_cost(max_cost, block_program, block_program_args)
+            clvm_cost, result = GENERATOR_MOD.run_safe_with_cost(max_cost, block_program, block_program_args)
         else:
-            cost, result = GENERATOR_MOD.run_with_cost(max_cost, block_program, block_program_args)
+            clvm_cost, result = GENERATOR_MOD.run_with_cost(max_cost, block_program, block_program_args)
+
+        max_cost -= clvm_cost
+        if max_cost < 0:
+            return NPCResult(uint16(Err.INVALID_BLOCK_COST.value), [], uint64(0))
         npc_list: List[NPC] = []
-        opcodes: Set[bytes] = set(item.value for item in ConditionOpcode)
 
         for res in result.first().as_iter():
             conditions_list: List[ConditionWithArgs] = []
 
+            if len(res.first().atom) != 32:
+                raise ValidationError(Err.INVALID_CONDITION)
             spent_coin_parent_id: bytes32 = res.first().as_atom()
-            spent_coin_puzzle_hash: bytes32 = res.rest().first().as_atom()
-            spent_coin_amount: uint64 = uint64(res.rest().rest().first().as_int())
+            res = res.rest()
+            if len(res.first().atom) != 32:
+                raise ValidationError(Err.INVALID_CONDITION)
+            spent_coin_puzzle_hash: bytes32 = res.first().as_atom()
+            res = res.rest()
+            spent_coin_amount: uint64 = uint64(res.first().as_int())
+            res = res.rest()
             spent_coin: Coin = Coin(spent_coin_parent_id, spent_coin_puzzle_hash, spent_coin_amount)
 
-            for cond in res.rest().rest().rest().first().as_iter():
-                if cond.first().as_atom() in opcodes:
-                    opcode: ConditionOpcode = ConditionOpcode(cond.first().as_atom())
-                elif not safe_mode:
-                    opcode = ConditionOpcode.UNKNOWN
-                else:
-                    return NPCResult(uint16(Err.GENERATOR_RUNTIME_ERROR.value), [], uint64(0))
-                cvl = ConditionWithArgs(opcode, cond.rest().as_atom_list())
-                conditions_list.append(cvl)
+            for cond in res.first().as_iter():
+                cost, cvl = parse_condition(cond, safe_mode)
+                max_cost -= cost
+                if max_cost < 0:
+                    return NPCResult(uint16(Err.INVALID_BLOCK_COST.value), [], uint64(0))
+                if cvl is not None:
+                    conditions_list.append(cvl)
+
             conditions_dict = conditions_by_opcode(conditions_list)
             if conditions_dict is None:
                 conditions_dict = {}
             npc_list.append(
                 NPC(spent_coin.name(), spent_coin.puzzle_hash, [(a, b) for a, b in conditions_dict.items()])
             )
-        return NPCResult(None, npc_list, uint64(cost))
+        return NPCResult(None, npc_list, uint64(clvm_cost))
+    except ValidationError as e:
+        return NPCResult(uint16(e.code.value), [], uint64(0))
     except Exception:
         return NPCResult(uint16(Err.GENERATOR_RUNTIME_ERROR.value), [], uint64(0))
 

chia/full_node/mempool_manager.py

@@ -38,10 +38,10 @@
 log = logging.getLogger(__name__)
 
 
-def get_npc_multiprocess(spend_bundle_bytes: bytes, max_cost: int) -> bytes:
+def get_npc_multiprocess(spend_bundle_bytes: bytes, max_cost: int, cost_per_byte: int) -> bytes:
     program = simple_solution_generator(SpendBundle.from_bytes(spend_bundle_bytes))
     # npc contains names of the coins removed, puzzle_hashes and their spend conditions
-    return bytes(get_name_puzzle_conditions(program, max_cost, True))
+    return bytes(get_name_puzzle_conditions(program, max_cost, cost_per_byte=cost_per_byte, safe_mode=True))
 
 
 class MempoolManager:
@@ -219,6 +219,7 @@ async def pre_validate_spendbundle(self, new_spend: SpendBundle) -> NPCResult:
             get_npc_multiprocess,
             bytes(new_spend),
             int(self.limit_factor * self.constants.MAX_BLOCK_COST_CLVM),
+            self.constants.COST_PER_BYTE,
         )
         end_time = time.time()
         log.info(f"It took {end_time - start_time} to pre validate transaction")
@@ -248,6 +249,8 @@ async def add_spendbundle(
         log.debug(f"Cost: {cost}")
 
         if cost > int(self.limit_factor * self.constants.MAX_BLOCK_COST_CLVM):
+            # we shouldn't ever end up here, since the cost is limited when we
+            # execute the CLVM program.
             return None, MempoolInclusionStatus.FAILED, Err.BLOCK_COST_EXCEEDS_MAX
 
         if npc_result.error is not None:

chia/types/blockchain_format/foliage.py

@@ -18,7 +18,7 @@ class TransactionsInfo(Streamable):
     generator_refs_root: bytes32  # sha256 of the concatenation of the generator ref list entries
     aggregated_signature: G2Element
     fees: uint64  # This only includes user fees, not block rewards
-    cost: uint64  # This is the total cost of running this block in the CLVM
+    cost: uint64  # This is the total cost of this block, including CLVM cost, cost of program size and conditions
     reward_claims_incorporated: List[Coin]  # These can be in any order
 
 

chia/util/condition_tools.py

@@ -96,10 +96,6 @@ def created_outputs_for_conditions_dict(
 ) -> List[Coin]:
     output_coins = []
     for cvp in conditions_dict.get(ConditionOpcode.CREATE_COIN, []):
-        # TODO: check condition very carefully
-        # (ensure there are the correct number and type of parameters)
-        # maybe write a type-checking framework for conditions
-        # and don't just fail with asserts
         puzzle_hash, amount_bin = cvp.vars[0], cvp.vars[1]
         amount = int_from_bytes(amount_bin)
         coin = Coin(input_coin_name, puzzle_hash, uint64(amount))

chia/wallet/cc_wallet/cc_wallet.py

@@ -246,7 +246,10 @@ async def get_max_send_amount(self, records=None):
             program: BlockGenerator = simple_solution_generator(tx.spend_bundle)
             # npc contains names of the coins removed, puzzle_hashes and their spend conditions
             result: NPCResult = get_name_puzzle_conditions(
-                program, self.wallet_state_manager.constants.MAX_BLOCK_COST_CLVM, True
+                program,
+                self.wallet_state_manager.constants.MAX_BLOCK_COST_CLVM,
+                cost_per_byte=self.wallet_state_manager.constants.COST_PER_BYTE,
+                safe_mode=True,
             )
             cost_result: uint64 = calculate_cost_of_program(
                 program.program, result, self.wallet_state_manager.constants.COST_PER_BYTE