diff --git a/apiary.apib b/apiary.apib index 9c5e7a59b..3a021154f 100644 --- a/apiary.apib +++ b/apiary.apib @@ -1466,7 +1466,7 @@ Returns server information and the list of documented routes in JSON format. "result": { "server_ready": true, "network": "mainnet", - "version": "10.9.0-rc.1", + "version": "10.9.0", "backend_height": 850214, "counterparty_height": 850214, "documentation": "https://counterpartycore.docs.apiary.io/", diff --git a/counterparty-core/counterpartycore/lib/backend/bitcoind.py b/counterparty-core/counterpartycore/lib/backend/bitcoind.py index 329e2927d..39d3fe9fb 100644 --- a/counterparty-core/counterpartycore/lib/backend/bitcoind.py +++ b/counterparty-core/counterpartycore/lib/backend/bitcoind.py @@ -21,6 +21,27 @@ TRANSACTIONS_CACHE_MAX_SIZE = 10000 +# for testing +def should_retry(): + return True + + +def get_json_response(response, retry=0): + try: + return response.json() + except json.decoder.JSONDecodeError as e: # noqa: F841 + if response.status_code == 200: + logger.warning( + f"Received invalid JSON with status 200 from Bitcoin Core: {response.text}. Retrying in 5 seconds..." + ) + time.sleep(5) + if retry < 5: + return get_json_response(response, retry=retry + 1) + raise exceptions.BitcoindRPCError( # noqa: B904 + f"Received invalid JSON from backend with a response of {str(response.status_code)}: {response.text}" + ) from e + + def rpc_call(payload, retry=0): """Calls to bitcoin core and returns the response""" url = config.BACKEND_URL @@ -41,16 +62,8 @@ def rpc_call(payload, retry=0): ) if response is None: # noqa: E711 - if config.TESTNET: - network = "testnet" - elif config.TESTNET4: - network = "testnet4" - elif config.REGTEST: - network = "regtest" - else: - network = "mainnet" raise exceptions.BitcoindRPCError( - f"Cannot communicate with Bitcoin Core at `{util.clean_url_for_log(url)}`. (server is set to run on {network}, is backend?)" + f"Cannot communicate with Bitcoin Core at `{util.clean_url_for_log(url)}`. (server is set to run on {config.NETWORK_NAME}, is backend?)" ) if response.status_code in (401,): raise exceptions.BitcoindRPCError( @@ -75,34 +88,28 @@ def rpc_call(payload, retry=0): raise broken_error # Handle json decode errors - try: - response_json = response.json() - except json.decoder.JSONDecodeError as e: # noqa: F841 - raise exceptions.BitcoindRPCError( # noqa: B904 - f"Received invalid JSON from backend with a response of {str(response.status_code) + ' ' + response.reason}" - ) from e + response_json = get_json_response(response) # Batch query returns a list if isinstance(response_json, list): result = response_json elif "error" not in response_json.keys() or response_json["error"] is None: # noqa: E711 result = response_json["result"] - elif response_json["error"]["code"] == -5: # RPC_INVALID_ADDRESS_OR_KEY - raise exceptions.BitcoindRPCError( - f"{response_json['error']} Is `txindex` enabled in {config.BTC_NAME} Core?" - ) - elif response_json["error"]["code"] in [-28, -8, -2]: + elif "Block height out of range" in response_json["error"]["message"]: + # this error should be managed by the caller + raise exceptions.BlockOutOfRange(response_json["error"]["message"]) + elif response_json["error"]["code"] in [-28, -8, -5, -2]: # "Verifying blocks..." or "Block height out of range" or "The network does not appear to fully agree!"" - logger.debug(f"Backend not ready. Sleeping for ten seconds. ({response_json['error']})") - logger.debug(f"Payload: {payload}") - if retry >= 10: - raise exceptions.BitcoindRPCError( - f"Backend not ready after {retry} retries. ({response_json['error']})" - ) - # If Bitcoin Core takes more than `sys.getrecursionlimit() * 10 = 9970` - # seconds to start, this'll hit the maximum recursion depth limit. - time.sleep(10) - return rpc_call(payload, retry=retry + 1) + warning_message = f"Error calling {payload}: {response_json['error']}. Sleeping for ten seconds and retrying." + if response_json["error"]["code"] == -5: # RPC_INVALID_ADDRESS_OR_KEY + warning_message += f" Is `txindex` enabled in {config.BTC_NAME} Core?" + logger.warning(warning_message) + if should_retry(): + # If Bitcoin Core takes more than `sys.getrecursionlimit() * 10 = 9970` + # seconds to start, this'll hit the maximum recursion depth limit. + time.sleep(10) + return rpc_call(payload, retry=retry + 1) + raise exceptions.BitcoindRPCError(warning_message) else: raise exceptions.BitcoindRPCError(response_json["error"]["message"]) @@ -130,8 +137,8 @@ def is_api_request(): return False -def rpc(method, params): - if is_api_request(): +def rpc(method, params, no_retry=False): + if is_api_request() or no_retry: return safe_rpc(method, params) payload = { @@ -160,6 +167,8 @@ def safe_rpc(method, params): verify=(not config.BACKEND_SSL_NO_VERIFY), timeout=config.REQUESTS_TIMEOUT, ).json() + if "error" in response: + raise exceptions.BitcoindRPCError(response["error"]["message"]) return response["result"] except (requests.exceptions.RequestException, json.decoder.JSONDecodeError, KeyError) as e: raise exceptions.BitcoindRPCError(f"Error calling {method}: {str(e)}") from e @@ -190,8 +199,8 @@ def convert_to_psbt(rawtx): @functools.lru_cache(maxsize=10000) -def getrawtransaction(tx_hash, verbose=False): - return rpc("getrawtransaction", [tx_hash, 1 if verbose else 0]) +def getrawtransaction(tx_hash, verbose=False, no_retry=False): + return rpc("getrawtransaction", [tx_hash, 1 if verbose else 0], no_retry=no_retry) def getrawtransaction_batch(tx_hashes, verbose=False, return_dict=False): @@ -380,7 +389,10 @@ def sendrawtransaction(signedhex: str): Proxy to `sendrawtransaction` RPC call. :param signedhex: The signed transaction hex. """ - return rpc("sendrawtransaction", [signedhex]) + try: + return rpc("sendrawtransaction", [signedhex]) + except Exception as e: + raise exceptions.BitcoindRPCError(f"Error broadcasting transaction: {str(e)}") from e def decoderawtransaction(rawtx: str): @@ -388,7 +400,7 @@ def decoderawtransaction(rawtx: str): Proxy to `decoderawtransaction` RPC call. :param rawtx: The raw transaction hex. (e.g. 0200000000010199c94580cbea44aead18f429be20552e640804dc3b4808e39115197f1312954d000000001600147c6b1112ed7bc76fd03af8b91d02fd6942c5a8d0ffffffff0280f0fa02000000001976a914a11b66a67b3ff69671c8f82254099faf374b800e88ac70da0a27010000001600147c6b1112ed7bc76fd03af8b91d02fd6942c5a8d002000000000000) """ - return deserialize.deserialize_tx(rawtx) + return rpc("decoderawtransaction", [rawtx]) def search_pubkey_in_transactions(pubkeyhash, tx_hashes): diff --git a/counterparty-core/counterpartycore/lib/blocks.py b/counterparty-core/counterpartycore/lib/blocks.py index c3918c715..17896d77d 100644 --- a/counterparty-core/counterpartycore/lib/blocks.py +++ b/counterparty-core/counterpartycore/lib/blocks.py @@ -24,7 +24,6 @@ gas, ledger, log, - mempool, message_type, util, ) @@ -1270,6 +1269,43 @@ def get_next_tx_index(db): return tx_index +def handle_reorg(db): + # search last block with the correct hash + previous_block_index = util.CURRENT_BLOCK_INDEX - 1 + while True: + previous_block_hash = backend.bitcoind.getblockhash(previous_block_index) + + try: + current_block_hash = backend.bitcoind.getblockhash(previous_block_index + 1) + except exceptions.BlockOutOfRange: + # current block is not in the blockchain + logger.debug(f"Current block is not in the blockchain ({previous_block_index + 1}).") + previous_block_index -= 1 + continue + + if previous_block_hash != ledger.get_block_hash(db, previous_block_index): + # hashes don't match + logger.debug(f"Hashes don't match ({previous_block_index}).") + previous_block_index -= 1 + continue + + break + + # rollback to the previous block + current_block_index = previous_block_index + 1 + rollback(db, block_index=current_block_index) + util.CURRENT_BLOCK_INDEX = previous_block_index + + # get the new deserialized current block + current_block = deserialize.deserialize_block( + backend.bitcoind.getblock(current_block_hash), + parse_vouts=True, + block_index=current_block_index, + ) + + return current_block + + def parse_new_block(db, decoded_block, tx_index=None): start_time = time.time() @@ -1290,36 +1326,13 @@ def parse_new_block(db, decoded_block, tx_index=None): else: # get previous block previous_block = ledger.get_block(db, util.CURRENT_BLOCK_INDEX - 1) - # check if reorg is needed if decoded_block["hash_prev"] != previous_block["block_hash"]: - # search last block with the correct hash - previous_block_index = util.CURRENT_BLOCK_INDEX - 1 - while True: - bitcoin_block_hash = backend.bitcoind.getblockhash(previous_block_index) - counterparty_block_hash = ledger.get_block_hash(db, previous_block_index) - if bitcoin_block_hash != counterparty_block_hash: - previous_block_index -= 1 - else: - break - current_block_hash = backend.bitcoind.getblockhash(previous_block_index + 1) - raw_current_block = backend.bitcoind.getblock(current_block_hash) - decoded_block = deserialize.deserialize_block( - raw_current_block, - parse_vouts=True, - block_index=previous_block_index + 1, + logger.warning( + "Blockchain reorganization detected at block %s.", util.CURRENT_BLOCK_INDEX ) - logger.warning("Blockchain reorganization detected at block %s.", previous_block_index) - # rollback to the previous block - rollback(db, block_index=previous_block_index + 1) - previous_block = ledger.get_block(db, previous_block_index) - util.CURRENT_BLOCK_INDEX = previous_block_index + 1 - tx_index = get_next_tx_index(db) - - if "height" not in decoded_block: - decoded_block["block_index"] = util.CURRENT_BLOCK_INDEX - else: - decoded_block["block_index"] = decoded_block["height"] + new_current_block = handle_reorg(db) + return parse_new_block(db, new_current_block) # Sanity checks if decoded_block["block_index"] != config.BLOCK_FIRST: @@ -1489,7 +1502,6 @@ def catch_up(db, check_asset_conservation=True): fetcher = start_rsfetcher() else: assert parsed_block_index == block_height - mempool.clean_mempool(db) parsed_blocks += 1 formatted_duration = util.format_duration(time.time() - start_time) diff --git a/counterparty-core/counterpartycore/lib/bootstrap.py b/counterparty-core/counterpartycore/lib/bootstrap.py index be1cddc73..a0f6de28c 100644 --- a/counterparty-core/counterpartycore/lib/bootstrap.py +++ b/counterparty-core/counterpartycore/lib/bootstrap.py @@ -114,9 +114,11 @@ def clean_data_dir(data_dir): if not os.path.exists(data_dir): os.makedirs(data_dir, mode=0o755) return - files_to_delete = glob.glob(os.path.join(data_dir, "*.db")) - files_to_delete += glob.glob(os.path.join(data_dir, "*.db-wal")) - files_to_delete += glob.glob(os.path.join(data_dir, "*.db-shm")) + network = "" if config.NETWORK_NAME == "mainnet" else f".{config.NETWORK_NAME}" + files_to_delete = [] + for db_name in ["counterparty", "state"]: + for ext in ["db", "db-wal", "db-shm"]: + files_to_delete += glob.glob(os.path.join(data_dir, f"{db_name}{network}.{ext}")) for file in files_to_delete: os.remove(file) diff --git a/counterparty-core/counterpartycore/lib/check.py b/counterparty-core/counterpartycore/lib/check.py index 3b5d5c064..f475f4411 100644 --- a/counterparty-core/counterpartycore/lib/check.py +++ b/counterparty-core/counterpartycore/lib/check.py @@ -695,6 +695,10 @@ "ledger_hash": "4c4d6b660af23bb03a04bbf93ddd0a4b8e615dd7b883ecf827274cabe658bfc2", "txlist_hash": "f6a99d60337c33c1822c048f56e241455cd7e45bb5a9515096f1ac609d50f669", }, + 879058: { + "ledger_hash": "e6bf730a18c148adbd2cce9bd0f361e595e44d53fa98a9d9bdbf4c944f6c233b", + "txlist_hash": "e9946ac128405885f251fbb98a952ed554ea6cc25973261da9f40eabf4f3b429", + }, } CONSENSUS_HASH_VERSION_TESTNET = 7 @@ -875,6 +879,10 @@ "ledger_hash": "33cf0669a0d309d7e6b1bf79494613b69262b58c0ea03c9c221d955eb4c84fe5", "txlist_hash": "33cf0669a0d309d7e6b1bf79494613b69262b58c0ea03c9c221d955eb4c84fe5", }, + 64493: { + "ledger_hash": "af481088fb9303d0f61543fb3646110fcd27d5ebd3ac40e04397000320216699", + "txlist_hash": "4b610518a76f59e8f98922f37434be9bb5567040afebabda6ee69b4f92b80434", + }, } diff --git a/counterparty-core/counterpartycore/lib/config.py b/counterparty-core/counterpartycore/lib/config.py index 18d9684f1..b0e2415bc 100644 --- a/counterparty-core/counterpartycore/lib/config.py +++ b/counterparty-core/counterpartycore/lib/config.py @@ -5,7 +5,7 @@ # Semantic Version -__version__ = "10.9.0-rc.1" # for hatch +__version__ = "10.9.0" # for hatch VERSION_STRING = __version__ version = VERSION_STRING.split("-")[0].split(".") VERSION_MAJOR = int(version[0]) @@ -30,8 +30,8 @@ ] NEED_REPARSE_IF_MINOR_IS_LESS_THAN_TESTNET4 = None -NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN = [(8, 871780)] -NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN_TESTNET = [(8, 3522632)] +NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN = [(8, 871780), (9, 871780)] +NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN_TESTNET = [(8, 3522632), (9, 3522632)] NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN_TESTNET4 = None STATE_DB_NEED_REFRESH_ON_VERSION_UPDATE = ["10.9.0-rc.1", "10.9.0"] diff --git a/counterparty-core/counterpartycore/lib/deserialize.py b/counterparty-core/counterpartycore/lib/deserialize.py index d0a27bcac..d78e57020 100644 --- a/counterparty-core/counterpartycore/lib/deserialize.py +++ b/counterparty-core/counterpartycore/lib/deserialize.py @@ -32,4 +32,5 @@ def deserialize_block(block_hex, parse_vouts=False, block_index=None): } ) current_block_index = block_index or util.CURRENT_BLOCK_INDEX - return deserializer.parse_block(block_hex, current_block_index, parse_vouts) + decoded_block = deserializer.parse_block(block_hex, current_block_index, parse_vouts) + return decoded_block diff --git a/counterparty-core/counterpartycore/lib/exceptions.py b/counterparty-core/counterpartycore/lib/exceptions.py index 67764c7e9..38e2dc1bd 100644 --- a/counterparty-core/counterpartycore/lib/exceptions.py +++ b/counterparty-core/counterpartycore/lib/exceptions.py @@ -145,3 +145,7 @@ class RSFetchError(Exception): class ElectrsError(Exception): pass + + +class BlockOutOfRange(Exception): + pass diff --git a/counterparty-core/counterpartycore/lib/follow.py b/counterparty-core/counterpartycore/lib/follow.py index 461bbe3f4..e050b433e 100644 --- a/counterparty-core/counterpartycore/lib/follow.py +++ b/counterparty-core/counterpartycore/lib/follow.py @@ -138,7 +138,8 @@ def receive_rawblock(self, body): blocks.catch_up(self.db, check_asset_conservation=False) else: blocks.parse_new_block(self.db, decoded_block) - mempool.clean_mempool(self.db) + if not config.NO_MEMPOOL: + mempool.clean_mempool(self.db) if not config.NO_TELEMETRY: TelemetryOneShot().submit() @@ -184,9 +185,9 @@ def receive_sequence(self, body): raw_tx = self.raw_tx_cache.get(item_hash) if raw_tx is None: try: - raw_tx = backend.bitcoind.getrawtransaction(item_hash) + raw_tx = backend.bitcoind.getrawtransaction(item_hash, no_retry=True) except exceptions.BitcoindRPCError: - logger.trace("Transaction not found in bitcoind: %s", item_hash) + logger.warning("Transaction not found in bitcoind: %s", item_hash) return # add transaction to mempool block # logger.trace("Adding transaction to mempool block: %s", item_hash) @@ -204,6 +205,7 @@ def receive_sequence(self, body): logger.trace("Waiting for new transactions in the mempool or a new block...") # transaction removed from mempool for non-block inclusion reasons elif label == "R": + logger.debug("Removing transaction from mempool: %s", item_hash) mempool.clean_transaction_events(self.db, item_hash) def receive_message(self, topic, body, seq): @@ -238,9 +240,8 @@ async def receive_multipart(self, socket, topic_name): self.receive_message(topic, body, seq) except Exception as e: logger.error("Error processing message: %s", e) - import traceback - - print(traceback.format_exc()) # for debugging + # import traceback + # print(traceback.format_exc()) # for debugging capture_exception(e) raise e @@ -297,6 +298,7 @@ async def handle(self): except Exception as e: logger.error("Error in handle loop: %s", e) capture_exception(e) + self.stop() break # Optionally break the loop on other exceptions def start(self): diff --git a/counterparty-core/counterpartycore/lib/gettxinfo.py b/counterparty-core/counterpartycore/lib/gettxinfo.py index 670af4fbb..63cd78e42 100644 --- a/counterparty-core/counterpartycore/lib/gettxinfo.py +++ b/counterparty-core/counterpartycore/lib/gettxinfo.py @@ -127,7 +127,7 @@ def get_vin_info(vin): # have been from a while ago, so this call may not hit the cache. vin_ctx = backend.bitcoind.get_decoded_transaction(vin["hash"]) - is_segwit = len(vin_ctx["vtxinwit"]) > 0 + is_segwit = vin_ctx["segwit"] vout = vin_ctx["vout"][vin["n"]] return vout["value"], vout["script_pub_key"], is_segwit @@ -393,6 +393,7 @@ def get_tx_info_new(db, decoded_tx, block_index, p2sh_is_segwit=False, composing The destinations, if they exists, always comes before the data output; the change, if it exists, always comes after. """ + # Ignore coinbase transactions. if decoded_tx["coinbase"]: raise DecodeError("coinbase transaction") diff --git a/counterparty-core/counterpartycore/test/deserialize_test.py b/counterparty-core/counterpartycore/test/deserialize_test.py index 16ea0ab45..630817adc 100644 --- a/counterparty-core/counterpartycore/test/deserialize_test.py +++ b/counterparty-core/counterpartycore/test/deserialize_test.py @@ -3,8 +3,9 @@ from io import BytesIO import bitcoin as bitcoinlib +import pytest -from counterpartycore.lib import deserialize, util +from counterpartycore.lib import config, deserialize, gettxinfo, util from counterpartycore.lib.util import inverse_hash @@ -13,7 +14,6 @@ def deserialize_bitcoinlib(tx_hex): def deserialize_rust(tx_hex): - # config.NETWORK_NAME = "mainnet" return deserialize.deserialize_tx(tx_hex, parse_vouts=True, block_index=900000) @@ -34,6 +34,21 @@ def create_block_hex(transactions_hex): return block_hex +@pytest.mark.skip +def test_deserialize_mpma(): + config.PREFIX = b"CNTRPRTY" + config.NETWORK_NAME = "mainnet" + config.ADDRESSVERSION = config.ADDRESSVERSION_MAINNET + + hex = "0100000001f9cf03a71930731618f2e0ff897db75d208a587129b96296f3958b0dc146420900000000e5483045022100a72e4be0a0f581e1c438c7048413c65c05793e8328a7acaa1ef081cc8c44909a0220718e772276aaa7adf8392a1d39ab44fc8778f622ee0dea9858cd5894290abb2b014c9a4c6f434e545250525459030003000fc815eeb3172efc23fbd39c41189e83e4e0c8150033dafc6a4dcd8bce30b038305e30e5defad4acd6009081f7ee77f0ef849a213670d4e785c26d71375d40467e543326526fa800000000000000060100000000000000018000000000000000006000752102e6dd23598e1d2428ecf7eb59c27fdfeeb7a27c26906e96dc1f3d5ebba6e54d08ad0075740087ffffffff0100000000000000000e6a0c2bb584c84ba87a60dcab46c100000000" + decoded_tx = deserialize_rust(hex) + assert not decoded_tx["segwit"] + p2sh_encoding_source, data, outputs_value = gettxinfo.get_transaction_source_from_p2sh( + decoded_tx, False + ) + assert p2sh_encoding_source == "18b7eyatTwZ8mvSCXRRxjNjvr3DPwhh6bU" + + def test_deserialize(): hex = "0100000001db3acf37743ac015808f7911a88761530c801819b3b907340aa65dfb6d98ce24030000006a473044022002961f4800cb157f8c0913084db0ee148fa3e1130e0b5e40c3a46a6d4f83ceaf02202c3dd8e631bf24f4c0c5341b3e1382a27f8436d75f3e0a095915995b0bf7dc8e01210395c223fbf96e49e5b9e06a236ca7ef95b10bf18c074bd91a5942fc40360d0b68fdffffff040000000000000000536a4c5058325bd61325dc633fadf05bec9157c23106759cee40954d39d9dbffc17ec5851a2d1feb5d271da422e0e24c7ae8ad29d2eeabf7f9ca3de306bd2bc98e2a39e47731aa000caf400053000c1283000149c8000000000000001976a91462bef4110f98fdcb4aac3c1869dbed9bce8702ed88acc80000000000000017a9144317f779c0a2ccf8f6bc3d440bd9e536a5bff75287fa3e5100000000001976a914bf2646b8ba8b4a143220528bde9c306dac44a01c88ac00000000" decoded_tx = deserialize_rust(hex) @@ -135,3 +150,59 @@ def test_deserialize(): print( f"Time to deserialize {4 * iterations} transactions with bitcoinlib: {end_time - start_time} seconds" ) + + +def mock_get_decoded_transaction(tx_hash): + txs = { + "094246c10d8b95f39662b92971588a205db77d89ffe0f21816733019a703cff9": "0100000001c47705b604b5b375fb43b6a7a632e20a7c10eb11d3202c00bd659e673d4d9396010000006a47304402204bc0847f52965c645e164078cfb5d743eb918c4fddaf4f592056b3470445e2c602202986c27c2f0f3b858b8fee94bf712338bc0ab8ff462edcea285a835143e10532012102e6dd23598e1d2428ecf7eb59c27fdfeeb7a27c26906e96dc1f3d5ebba6e54d08ffffffff02893000000000000017a9148760df63af4701313b244bf5ccd7479914843da18778cb0000000000001976a914533c940b158eae03f5bf71f1195d757c819c2e0c88ac00000000", + "05e7e9f59f155b28311a5e2860388783b839027b6529889de791351fe172752d": "020000000001016d72a3d323f82e76dcbf5fe9448a91ea9e68649e313d9a43822c0b27308a7b080200000017160014f6a785077f78695c12d51078ea7d9c10641f24acffffffff0208420000000000002251202525a906d3d870c6c00a2bfd63824c6597a4eddd8d24392f42ffbb2e6991fc5dcb8d04000000000017a914f54105af74fb10e70e899901b6ac4593ac20eea1870247304402205bd9f7e2ebe915532309548aad4e36f4b4feb856dab74f1b0e4df5292c0dbb4102202ca4d61fca54d08e2fd077c7e11154d2f271cf7102bb355c16dcb480d48dd57001210395c693bfc3a4d00e4380bec0d85871a1d0083618f8f01663199261d011e2a2bb00000000", + "c93934dc5149f771c0a9100302006058c51a13af5146ded1053dae2a219f7852": "020000000001019963e21ab347fbd1527f138a6788ad9d63b589fbab5a15a63ec4dc6f8318ffa34000000000ffffffff02315f00000000000016001457b185fde87fefac8aa2c7c823d4aae4c25aa8539f680000000000001600140b8846404281da37f3c4daa8da3b85d21293b97a024730440220662b27c5aa429153ebbe2ff3844efa7c493226c645573c04d6a4ebf404dc738702200f1c36ba63debb4d38d285980c3ecc8d7b20a70f88605b3358f8d10363d741cf012103c9d887d18d3c3a2bbdaf00c98b50863aa4d1d844e448aa4defe8fc4bdf9036b100000000", + } + decoded_tx = deserialize_rust(txs[tx_hash]) + return decoded_tx + + +@pytest.fixture(scope="function") +def init_mock(monkeypatch): + monkeypatch.setattr( + "counterpartycore.lib.backend.bitcoind.get_decoded_transaction", + mock_get_decoded_transaction, + ) + + +def test_get_vin_info(init_mock): + vout_value, script_pubkey, is_segwit = gettxinfo.get_vin_info( + { + "hash": "094246c10d8b95f39662b92971588a205db77d89ffe0f21816733019a703cff9", + "n": 0, + } + ) + assert vout_value == 12425 + assert script_pubkey == b"\xa9\x14\x87`\xdfc\xafG\x011;$K\xf5\xcc\xd7G\x99\x14\x84=\xa1\x87" + assert not is_segwit + + vout_value, script_pubkey, is_segwit = gettxinfo.get_vin_info( + { + "hash": "05e7e9f59f155b28311a5e2860388783b839027b6529889de791351fe172752d", + "n": 0, + } + ) + assert vout_value == 16904 + assert ( + script_pubkey + == b"Q %%\xa9\x06\xd3\xd8p\xc6\xc0\n+\xfdc\x82Le\x97\xa4\xed\xdd\x8d$9/B\xff\xbb.i\x91\xfc]" + ) + assert is_segwit + + vout_value, script_pubkey, is_segwit = gettxinfo.get_vin_info( + { + "hash": "c93934dc5149f771c0a9100302006058c51a13af5146ded1053dae2a219f7852", + "n": 0, + } + ) + assert vout_value == 24369 + assert ( + script_pubkey + == b"\x00\x14W\xb1\x85\xfd\xe8\x7f\xef\xac\x8a\xa2\xc7\xc8#\xd4\xaa\xe4\xc2Z\xa8S" + ) + assert is_segwit diff --git a/counterparty-core/counterpartycore/test/regtest/apidoc/blueprint-template.md b/counterparty-core/counterpartycore/test/regtest/apidoc/blueprint-template.md index 3777c0b0a..88fb9ded8 100644 --- a/counterparty-core/counterpartycore/test/regtest/apidoc/blueprint-template.md +++ b/counterparty-core/counterpartycore/test/regtest/apidoc/blueprint-template.md @@ -165,7 +165,7 @@ Returns server information and the list of documented routes in JSON format. "result": { "server_ready": true, "network": "mainnet", - "version": "10.9.0-rc.1", + "version": "10.9.0", "backend_height": 850214, "counterparty_height": 850214, "documentation": "https://counterpartycore.docs.apiary.io/", diff --git a/counterparty-core/counterpartycore/test/regtest/regtestnode.py b/counterparty-core/counterpartycore/test/regtest/regtestnode.py index 8086776f2..99d6e7f2a 100644 --- a/counterparty-core/counterpartycore/test/regtest/regtestnode.py +++ b/counterparty-core/counterpartycore/test/regtest/regtestnode.py @@ -361,6 +361,7 @@ def start_bitcoin_node(self): "-acceptnonstdtxn", "-minrelaytxfee=0", "-blockmintxfee=0", + "-mempoolfullrbf", f"-datadir={self.datadir}", _bg=True, _out=sys.stdout, @@ -384,6 +385,7 @@ def start_bitcoin_node_2(self): "-minrelaytxfee=0", "-blockmintxfee=0", "-bind=127.0.0.1:2223=onion", + "-mempoolfullrbf", _bg=True, _out=sys.stdout, ) @@ -662,6 +664,7 @@ def test_reorg(self): assert intermediate_xcp_balance > initial_xcp_balance print("Mine a longest chain on the second node...") + before_reorg_block = int(self.bitcoin_cli_2("getblockcount").strip()) self.bitcoin_cli_2("generatetoaddress", 6, self.addresses[0]) print("Re-connect to the first node...") @@ -679,13 +682,68 @@ def test_reorg(self): self.wait_for_counterparty_server(block=last_block) print("Burn count after reorganization: ", self.get_burn_count(self.addresses[0])) - assert "Blockchain reorganization detected" in self.server_out.getvalue() + + assert ( + f"Blockchain reorganization detected at block {last_block - 1}" + in self.server_out.getvalue() + ) + assert f"Hashes don't match ({before_reorg_block + 1})" in self.server_out.getvalue() assert self.get_burn_count(self.addresses[0]) == 1 final_xcp_balance = self.get_xcp_balance(self.addresses[0]) print("Final XCP balance: ", final_xcp_balance) assert final_xcp_balance == initial_xcp_balance + # Test reorg with same length chain but one different block + print("Disconnect from the first node...") + self.bitcoin_cli_2("disconnectnode", "localhost:18445") + + print("Invalidate the last block on the first node...") + best_block_hash = self.bitcoin_cli("getbestblockhash").strip() + self.bitcoin_cli("invalidateblock", best_block_hash) + + self.mine_blocks(1) + retry = 0 + while ( + f"Current block is not in the blockchain ({last_block + 1})." + not in self.server_out.getvalue() + ): + time.sleep(1) + retry += 1 + print("Waiting for reorg...") + assert retry < 100 + + # Test reorg with a same length chain but three different blocks + print("Invalidate block ", last_block - 3) + previous_block_hash = self.bitcoin_cli("getblockhash", last_block - 3).strip() + self.bitcoin_cli("invalidateblock", previous_block_hash) + self.mine_blocks(3) + retry = 0 + while len(self.server_out.getvalue().split(f"Hashes don't match ({last_block - 3})")) < 3: + time.sleep(1) + retry += 1 + print("Waiting for reorg...") + assert retry < 100 + + self.wait_for_counterparty_server(last_block) + + # other tests expect the second node to be connected if running + print("Re-connect to the first node...") + self.bitcoin_cli_2( + "addnode", + "localhost:18445", + "onetry", + _out=sys.stdout, + _err=sys.stdout, + ) + # fix block count for other tests + self.block_count -= 1 + + print("Wait for the two nodes to sync...") + last_block = self.wait_for_node_to_sync() + + print("Reorg test successful") + def test_electrs(self): self.start_and_wait_second_node() @@ -1078,6 +1136,69 @@ def test_fee_calculation(self): ) assert size * 3 - 3 <= unsigned_tx["btc_fee"] <= size * 3 + 3 + def test_rbf(self): + self.start_and_wait_second_node() + + unsigned_tx = self.compose( + self.addresses[0], + "send", + { + "destination": self.addresses[1], + "quantity": 1, + "asset": "XCP", + "exact_fee": 1, + "verbose": True, + "validate": False, + }, + )["result"] + transaction = Transaction.from_raw(unsigned_tx["rawtransaction"]) + raw_hexs = [] + # create 10 transactions with increasing fees + for _i in range(10): + transaction.outputs[1].amount -= 170 + new_raw_transaction = transaction.to_hex() + signed_tx = json.loads( + self.bitcoin_wallet("signrawtransactionwithwallet", new_raw_transaction).strip() + )["hex"] + raw_hexs.append(signed_tx) + + # check that no transaction is in the mempool + mempool_event_count_before = self.api_call("mempool/events?event_name=TRANSACTION_PARSED")[ + "result_count" + ] + assert mempool_event_count_before == 0 + + # broadcast the transactions to the two nodes + tx_hahses = [] + for i, raw_hex in enumerate(raw_hexs): + tx_hash = self.bitcoin_wallet("sendrawtransaction", raw_hex, 0).strip() + tx_hahses.append(tx_hash) + print(f"Transaction {i} sent: {tx_hash}") + + # check that all transactions are in the mempool + mempool_event_count_after = self.api_call("mempool/events?event_name=TRANSACTION_PARSED")[ + "result_count" + ] + while mempool_event_count_after == 0: + time.sleep(1) + mempool_event_count_after = self.api_call( + "mempool/events?event_name=TRANSACTION_PARSED" + )["result_count"] + time.sleep(10) + + print("Mempool event count: ", mempool_event_count_after) + + # only one event should be in the mempool + assert mempool_event_count_after == 1 + # check that RBFed transactions are removed from the mempool + for tx_hash in tx_hahses[:-1]: + assert f"Removing transaction from mempool: {tx_hash}" in self.server_out.getvalue() + event = self.api_call("mempool/events?event_name=TRANSACTION_PARSED")["result"][0] + # check that the last transaction is the one in the mempool + assert event["tx_hash"] == tx_hahses[-1] + + print("RBF test successful") + class RegtestNodeThread(threading.Thread): def __init__(self, wsgi_server="waitress", burn_in_one_block=True): diff --git a/counterparty-core/counterpartycore/test/regtest/scenarios/scenario_23_detach.py b/counterparty-core/counterpartycore/test/regtest/scenarios/scenario_23_detach.py index aa253b171..f44d1f1ea 100644 --- a/counterparty-core/counterpartycore/test/regtest/scenarios/scenario_23_detach.py +++ b/counterparty-core/counterpartycore/test/regtest/scenarios/scenario_23_detach.py @@ -173,4 +173,52 @@ } ], }, + { + "title": "Attach DETACHA asset to UTXO", + "transaction": "attach", + "source": "$ADDRESS_10", + "params": { + "asset": "DETACHA", + "quantity": 1 * 10**8, + }, + "set_variables": { + "ATTACH2_DETACHA_TX_HASH": "$TX_HASH", + }, + "controls": [], + }, + { + "title": "Move no confirmation", + "transaction": "movetoutxo", + "no_confirmation": True, + "source": "$ATTACH2_DETACHA_TX_HASH:0", + "params": { + "destination": "$ADDRESS_9", + "quantity": 1 * 10**8, + }, + "controls": [ + { + "url": "mempool/events?event_name=UTXO_MOVE", + "result": [ + { + "event": "UTXO_MOVE", + "params": { + "asset": "DETACHA", + "block_index": 9999999, + "destination": "$TX_HASH:0", + "destination_address": "$ADDRESS_9", + "msg_index": 0, + "quantity": 100000000, + "send_type": "move", + "source": "$ATTACH2_DETACHA_TX_HASH:0", + "source_address": "$ADDRESS_10", + "status": "valid", + "tx_hash": "$TX_HASH", + "tx_index": "$TX_INDEX", + }, + "tx_hash": "$TX_HASH", + } + ], + } + ], + }, ] diff --git a/counterparty-core/counterpartycore/test/regtest/testscenarios.py b/counterparty-core/counterpartycore/test/regtest/testscenarios.py index b53a19bb6..dce67245b 100644 --- a/counterparty-core/counterpartycore/test/regtest/testscenarios.py +++ b/counterparty-core/counterpartycore/test/regtest/testscenarios.py @@ -445,6 +445,8 @@ def run_scenarios(serve=False, wsgi_server="gunicorn"): regtest_node_thread.node.test_electrs() print("Testing fee calculation...") regtest_node_thread.node.test_fee_calculation() + print("Testing RBF...") + regtest_node_thread.node.test_rbf() except KeyboardInterrupt: print(regtest_node_thread.node.server_out.getvalue()) pass diff --git a/counterparty-core/counterpartycore/test/rpc_test.py b/counterparty-core/counterpartycore/test/rpc_test.py new file mode 100644 index 000000000..8b7f958c4 --- /dev/null +++ b/counterparty-core/counterpartycore/test/rpc_test.py @@ -0,0 +1,97 @@ +import json + +import pytest + +from counterpartycore.lib import config, exceptions +from counterpartycore.lib.backend import bitcoind + + +class MockResponse: + def __init__(self, status_code, json_data, reason=None): + self.status_code = status_code + self.json_data = json_data + self.reason = reason + + def json(self): + return self.json_data + + +def mock_requests_post(*args, **kwargs): + payload = json.loads(kwargs["data"]) + if payload["method"] == "getblockhash": + return MockResponse(200, {"error": {"message": "Block height out of range", "code": -8}}) + if payload["method"] == "return_none": + return None + if payload["method"] == "return_401": + return MockResponse(401, {}, "Unauthorized") + if payload["method"] == "return_800": + return MockResponse(800, {}, "because I want a 500") + if payload["method"] == "return_batch_list": + return MockResponse(200, ["ok", "ok"]) + if payload["method"] == "return_200": + return MockResponse(200, {"result": "ok"}) + if payload["method"] == "return_code_28": + return MockResponse(200, {"error": {"message": "Error 28", "code": -28}}) + if payload["method"] == "return_code_5": + return MockResponse(200, {"error": {"message": "Error 5", "code": -5}}) + if payload["method"] == "return_code_30": + return MockResponse(200, {"error": {"message": "Error 30", "code": -30}}) + + +@pytest.fixture(scope="function") +def init_mock(monkeypatch): + monkeypatch.setattr("requests.post", mock_requests_post) + monkeypatch.setattr("counterpartycore.lib.backend.bitcoind.should_retry", lambda: False) + config.BACKEND_URL = "http://localhost:14000" + config.BACKEND_SSL_NO_VERIFY = True + config.REQUESTS_TIMEOUT = 5 + + +def test_rpc_call(init_mock): + with pytest.raises(exceptions.BlockOutOfRange): + bitcoind.rpc("getblockhash", [1]) + + with pytest.raises(exceptions.BitcoindRPCError) as exc_info: + bitcoind.rpc("return_none", []) + assert ( + str(exc_info.value) + == "Cannot communicate with Bitcoin Core at `http://localhost:14000`. (server is set to run on testnet, is backend?)" + ) + + with pytest.raises(exceptions.BitcoindRPCError) as exc_info: + bitcoind.rpc("return_401", []) + assert ( + str(exc_info.value) + == "Authorization error connecting to http://localhost:14000: 401 Unauthorized" + ) + + with pytest.raises(exceptions.BitcoindRPCError) as exc_info: + bitcoind.rpc("return_500", []) + assert ( + str(exc_info.value) + == "Cannot communicate with Bitcoin Core at `http://localhost:14000`. (server is set to run on testnet, is backend?)" + ) + + result = bitcoind.rpc("return_batch_list", []) + assert result == ["ok", "ok"] + + result = bitcoind.rpc("return_200", []) + assert result == "ok" + + with pytest.raises(exceptions.BitcoindRPCError) as exc_info: + bitcoind.rpc("return_code_28", []) + assert ( + str(exc_info.value) + == "Error calling {'method': 'return_code_28', 'params': [], 'jsonrpc': '2.0', 'id': 0}: {'message': 'Error 28', 'code': -28}. Sleeping for ten seconds and retrying." + ) + + with pytest.raises(exceptions.BitcoindRPCError) as exc_info: + bitcoind.rpc("return_code_5", []) + assert ( + str(exc_info.value) + == "Error calling {'method': 'return_code_5', 'params': [], 'jsonrpc': '2.0', 'id': 0}: {'message': 'Error 5', 'code': -5}. Sleeping for ten seconds and retrying. Is `txindex` enabled in Bitcoin Core?" + ) + + with pytest.raises(exceptions.BitcoindRPCError) as exc_info: + bitcoind.rpc("return_code_30", []) + assert str(exc_info.value) == "Error 30" diff --git a/counterparty-core/requirements.txt b/counterparty-core/requirements.txt index 477670203..3554ef0da 100644 --- a/counterparty-core/requirements.txt +++ b/counterparty-core/requirements.txt @@ -38,4 +38,4 @@ hypothesis==6.116.0 bitcoin-utils==0.7.1 pyzstd==0.16.2 dredd_hooks==0.2.0 -counterparty-rs==10.9.0-rc.1 +counterparty-rs==10.9.0 diff --git a/counterparty-rs/Cargo.lock b/counterparty-rs/Cargo.lock index 267a4066a..e83c558ae 100644 --- a/counterparty-rs/Cargo.lock +++ b/counterparty-rs/Cargo.lock @@ -394,7 +394,7 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "counterparty-rs" -version = "10.9.0-rc.1" +version = "10.9.0" dependencies = [ "bip32", "bitcoin", diff --git a/counterparty-rs/Cargo.toml b/counterparty-rs/Cargo.toml index 67695d2a8..32faad976 100644 --- a/counterparty-rs/Cargo.toml +++ b/counterparty-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "counterparty-rs" -version = "10.9.0-rc.1" +version = "10.9.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/counterparty-rs/src/indexer/block.rs b/counterparty-rs/src/indexer/block.rs index 7eeb8b970..9a5c42828 100644 --- a/counterparty-rs/src/indexer/block.rs +++ b/counterparty-rs/src/indexer/block.rs @@ -156,6 +156,7 @@ impl IntoPy for Block { fn into_py(self, py: Python<'_>) -> PyObject { let dict = PyDict::new_bound(py); dict.set_item("height", self.height).unwrap(); + dict.set_item("block_index", self.height).unwrap(); dict.set_item("version", self.version).unwrap(); dict.set_item("hash_prev", self.hash_prev).unwrap(); dict.set_item("hash_merkle_root", self.hash_merkle_root) diff --git a/counterparty-wallet/requirements.txt b/counterparty-wallet/requirements.txt index 0bf1e683b..96cc1999f 100644 --- a/counterparty-wallet/requirements.txt +++ b/counterparty-wallet/requirements.txt @@ -5,4 +5,4 @@ colorlog==6.8.0 python-dateutil==2.8.2 requests==2.32.0 termcolor==2.4.0 -counterparty-core==10.9.0-rc.1 +counterparty-core==10.9.0 diff --git a/docker-compose.yml b/docker-compose.yml index 33c42103d..3062f2733 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ x-bitcoind-common: &bitcoind-common restart: unless-stopped x-counterparty-common: &counterparty-common - image: counterparty/counterparty:v10.9.0-rc.1 + image: counterparty/counterparty:v10.9.0 stop_grace_period: 1m volumes: - data:/root/.bitcoin diff --git a/release-notes/release-notes-v10.9.0.md b/release-notes/release-notes-v10.9.0.md index 6e61bcf53..1e3c209ca 100644 --- a/release-notes/release-notes-v10.9.0.md +++ b/release-notes/release-notes-v10.9.0.md @@ -1,4 +1,4 @@ -# Release Notes - Counterparty Core v10.9.0 (2025-01-??) +# Release Notes - Counterparty Core v10.9.0 (2025-01-15) This release represents a major technical milestone in the development of Counterparty Core: Counterparty no longer has AddrIndexRs as an external dependency. Originally, AddrIndexRs was used for transaction construction, and at the end of 2023 it was accidentally turned into a consensus-critical dependency (causing a number of subsequent consensus breaks and reliability issues). As of today, the only external dependency for a Counterparty node is Bitcoin Core itself. @@ -37,11 +37,20 @@ The following transaction construction parameters have been deprecated (but rema - Fix the `dispensers` table in State DB: include dispensers with same the `source` and `asset` but a different `tx_hash` - Fix endpoint to get info from raw transaction when block index is not provided - Fix issue where composed transactions contained `script_pubkey` (lock script) where the `script_sig` (unlock script) should be +- Fix bootstrap when using `--bootstrap-url` flag and don't clean other networks files +- Fix logic for blockchain reorgs of several blocks +- Have the node terminate when the `follow` loop raises an error +- Don't stop the server on "No such mempool or blockchain" error +- Handle correctly RPC call errors from the API +- Don't clean mempool on catchup +- Retry 5 times when getting invalid Json with status 200 from Bitcoin Core +- Don't retry RPC call when parsing mempool transactions + ## Codebase - Remove the AddrIndexRs dependency -- Replacement of `transaction.py` and `transaction_helper/*` with `composer.py` +- Replace `transaction.py` and `transaction_helper/*` with `composer.py` - Use the `bitcoin-utils` library for generating transactions - No longer block the follow process on mempool parsing - Add a timeout when parsing mempool transaction from ZMQ @@ -50,6 +59,9 @@ The following transaction construction parameters have been deprecated (but rema - Trigger State DB refreshes automatically on version bumps - Use only Rust to deserialize blocks and transactions - Add `testnet4` support +- Repeat the RPC call to Bitcoin Core indefinitely until it succeeds +- Raise a specific `BlockOutOfRange` error when querying an unknown block +- Add mainnet checkpoint for block 879058 and testnet4 checkpoint for block 64493 ## API