Merge pull request #2942 from CounterpartyXCP/develop

v10.9.0

apiary.apib

@@ -1466,7 +1466,7 @@ Returns server information and the list of documented routes in JSON format.
         "result": {
             "server_ready": true,
             "network": "mainnet",
-            "version": "10.9.0-rc.1",
+            "version": "10.9.0",
             "backend_height": 850214,
             "counterparty_height": 850214,
             "documentation": "https://counterpartycore.docs.apiary.io/",

counterparty-core/counterpartycore/lib/backend/bitcoind.py

@@ -21,6 +21,27 @@
 TRANSACTIONS_CACHE_MAX_SIZE = 10000
 
 
+# for testing
+def should_retry():
+    return True
+
+
+def get_json_response(response, retry=0):
+    try:
+        return response.json()
+    except json.decoder.JSONDecodeError as e:  # noqa: F841
+        if response.status_code == 200:
+            logger.warning(
+                f"Received invalid JSON with status 200 from Bitcoin Core: {response.text}. Retrying in 5 seconds..."
+            )
+            time.sleep(5)
+            if retry < 5:
+                return get_json_response(response, retry=retry + 1)
+        raise exceptions.BitcoindRPCError(  # noqa: B904
+            f"Received invalid JSON from backend with a response of {str(response.status_code)}: {response.text}"
+        ) from e
+
+
 def rpc_call(payload, retry=0):
     """Calls to bitcoin core and returns the response"""
     url = config.BACKEND_URL
@@ -41,16 +62,8 @@ def rpc_call(payload, retry=0):
             )
 
             if response is None:  # noqa: E711
-                if config.TESTNET:
-                    network = "testnet"
-                elif config.TESTNET4:
-                    network = "testnet4"
-                elif config.REGTEST:
-                    network = "regtest"
-                else:
-                    network = "mainnet"
                 raise exceptions.BitcoindRPCError(
-                    f"Cannot communicate with Bitcoin Core at `{util.clean_url_for_log(url)}`. (server is set to run on {network}, is backend?)"
+                    f"Cannot communicate with Bitcoin Core at `{util.clean_url_for_log(url)}`. (server is set to run on {config.NETWORK_NAME}, is backend?)"
                 )
             if response.status_code in (401,):
                 raise exceptions.BitcoindRPCError(
@@ -75,34 +88,28 @@ def rpc_call(payload, retry=0):
         raise broken_error
 
     # Handle json decode errors
-    try:
-        response_json = response.json()
-    except json.decoder.JSONDecodeError as e:  # noqa: F841
-        raise exceptions.BitcoindRPCError(  # noqa: B904
-            f"Received invalid JSON from backend with a response of {str(response.status_code) + ' ' + response.reason}"
-        ) from e
+    response_json = get_json_response(response)
 
     # Batch query returns a list
     if isinstance(response_json, list):
         result = response_json
     elif "error" not in response_json.keys() or response_json["error"] is None:  # noqa: E711
         result = response_json["result"]
-    elif response_json["error"]["code"] == -5:  # RPC_INVALID_ADDRESS_OR_KEY
-        raise exceptions.BitcoindRPCError(
-            f"{response_json['error']} Is `txindex` enabled in {config.BTC_NAME} Core?"
-        )
-    elif response_json["error"]["code"] in [-28, -8, -2]:
+    elif "Block height out of range" in response_json["error"]["message"]:
+        # this error should be managed by the caller
+        raise exceptions.BlockOutOfRange(response_json["error"]["message"])
+    elif response_json["error"]["code"] in [-28, -8, -5, -2]:
         # "Verifying blocks..." or "Block height out of range" or "The network does not appear to fully agree!""
-        logger.debug(f"Backend not ready. Sleeping for ten seconds. ({response_json['error']})")
-        logger.debug(f"Payload: {payload}")
-        if retry >= 10:
-            raise exceptions.BitcoindRPCError(
-                f"Backend not ready after {retry} retries. ({response_json['error']})"
-            )
-        # If Bitcoin Core takes more than `sys.getrecursionlimit() * 10 = 9970`
-        # seconds to start, this'll hit the maximum recursion depth limit.
-        time.sleep(10)
-        return rpc_call(payload, retry=retry + 1)
+        warning_message = f"Error calling {payload}: {response_json['error']}. Sleeping for ten seconds and retrying."
+        if response_json["error"]["code"] == -5:  # RPC_INVALID_ADDRESS_OR_KEY
+            warning_message += f" Is `txindex` enabled in {config.BTC_NAME} Core?"
+        logger.warning(warning_message)
+        if should_retry():
+            # If Bitcoin Core takes more than `sys.getrecursionlimit() * 10 = 9970`
+            # seconds to start, this'll hit the maximum recursion depth limit.
+            time.sleep(10)
+            return rpc_call(payload, retry=retry + 1)
+        raise exceptions.BitcoindRPCError(warning_message)
     else:
         raise exceptions.BitcoindRPCError(response_json["error"]["message"])
 
@@ -130,8 +137,8 @@ def is_api_request():
     return False
 
 
-def rpc(method, params):
-    if is_api_request():
+def rpc(method, params, no_retry=False):
+    if is_api_request() or no_retry:
         return safe_rpc(method, params)
 
     payload = {
@@ -160,6 +167,8 @@ def safe_rpc(method, params):
             verify=(not config.BACKEND_SSL_NO_VERIFY),
             timeout=config.REQUESTS_TIMEOUT,
         ).json()
+        if "error" in response:
+            raise exceptions.BitcoindRPCError(response["error"]["message"])
         return response["result"]
     except (requests.exceptions.RequestException, json.decoder.JSONDecodeError, KeyError) as e:
         raise exceptions.BitcoindRPCError(f"Error calling {method}: {str(e)}") from e
@@ -190,8 +199,8 @@ def convert_to_psbt(rawtx):
 
 
 @functools.lru_cache(maxsize=10000)
-def getrawtransaction(tx_hash, verbose=False):
-    return rpc("getrawtransaction", [tx_hash, 1 if verbose else 0])
+def getrawtransaction(tx_hash, verbose=False, no_retry=False):
+    return rpc("getrawtransaction", [tx_hash, 1 if verbose else 0], no_retry=no_retry)
 
 
 def getrawtransaction_batch(tx_hashes, verbose=False, return_dict=False):
@@ -380,15 +389,18 @@ def sendrawtransaction(signedhex: str):
     Proxy to `sendrawtransaction` RPC call.
     :param signedhex: The signed transaction hex.
     """
-    return rpc("sendrawtransaction", [signedhex])
+    try:
+        return rpc("sendrawtransaction", [signedhex])
+    except Exception as e:
+        raise exceptions.BitcoindRPCError(f"Error broadcasting transaction: {str(e)}") from e
 
 
 def decoderawtransaction(rawtx: str):
     """
     Proxy to `decoderawtransaction` RPC call.
     :param rawtx: The raw transaction hex. (e.g. 0200000000010199c94580cbea44aead18f429be20552e640804dc3b4808e39115197f1312954d000000001600147c6b1112ed7bc76fd03af8b91d02fd6942c5a8d0ffffffff0280f0fa02000000001976a914a11b66a67b3ff69671c8f82254099faf374b800e88ac70da0a27010000001600147c6b1112ed7bc76fd03af8b91d02fd6942c5a8d002000000000000)
     """
-    return deserialize.deserialize_tx(rawtx)
+    return rpc("decoderawtransaction", [rawtx])
 
 
 def search_pubkey_in_transactions(pubkeyhash, tx_hashes):

counterparty-core/counterpartycore/lib/blocks.py

@@ -24,7 +24,6 @@
     gas,
     ledger,
     log,
-    mempool,
     message_type,
     util,
 )
@@ -1270,6 +1269,43 @@ def get_next_tx_index(db):
     return tx_index
 
 
+def handle_reorg(db):
+    # search last block with the correct hash
+    previous_block_index = util.CURRENT_BLOCK_INDEX - 1
+    while True:
+        previous_block_hash = backend.bitcoind.getblockhash(previous_block_index)
+
+        try:
+            current_block_hash = backend.bitcoind.getblockhash(previous_block_index + 1)
+        except exceptions.BlockOutOfRange:
+            # current block is not in the blockchain
+            logger.debug(f"Current block is not in the blockchain ({previous_block_index + 1}).")
+            previous_block_index -= 1
+            continue
+
+        if previous_block_hash != ledger.get_block_hash(db, previous_block_index):
+            # hashes don't match
+            logger.debug(f"Hashes don't match ({previous_block_index}).")
+            previous_block_index -= 1
+            continue
+
+        break
+
+    # rollback to the previous block
+    current_block_index = previous_block_index + 1
+    rollback(db, block_index=current_block_index)
+    util.CURRENT_BLOCK_INDEX = previous_block_index
+
+    # get the new deserialized current block
+    current_block = deserialize.deserialize_block(
+        backend.bitcoind.getblock(current_block_hash),
+        parse_vouts=True,
+        block_index=current_block_index,
+    )
+
+    return current_block
+
+
 def parse_new_block(db, decoded_block, tx_index=None):
     start_time = time.time()
 
@@ -1290,36 +1326,13 @@ def parse_new_block(db, decoded_block, tx_index=None):
     else:
         # get previous block
         previous_block = ledger.get_block(db, util.CURRENT_BLOCK_INDEX - 1)
-
         # check if reorg is needed
         if decoded_block["hash_prev"] != previous_block["block_hash"]:
-            # search last block with the correct hash
-            previous_block_index = util.CURRENT_BLOCK_INDEX - 1
-            while True:
-                bitcoin_block_hash = backend.bitcoind.getblockhash(previous_block_index)
-                counterparty_block_hash = ledger.get_block_hash(db, previous_block_index)
-                if bitcoin_block_hash != counterparty_block_hash:
-                    previous_block_index -= 1
-                else:
-                    break
-            current_block_hash = backend.bitcoind.getblockhash(previous_block_index + 1)
-            raw_current_block = backend.bitcoind.getblock(current_block_hash)
-            decoded_block = deserialize.deserialize_block(
-                raw_current_block,
-                parse_vouts=True,
-                block_index=previous_block_index + 1,
+            logger.warning(
+                "Blockchain reorganization detected at block %s.", util.CURRENT_BLOCK_INDEX
             )
-            logger.warning("Blockchain reorganization detected at block %s.", previous_block_index)
-            # rollback to the previous block
-            rollback(db, block_index=previous_block_index + 1)
-            previous_block = ledger.get_block(db, previous_block_index)
-            util.CURRENT_BLOCK_INDEX = previous_block_index + 1
-            tx_index = get_next_tx_index(db)
-
-    if "height" not in decoded_block:
-        decoded_block["block_index"] = util.CURRENT_BLOCK_INDEX
-    else:
-        decoded_block["block_index"] = decoded_block["height"]
+            new_current_block = handle_reorg(db)
+            return parse_new_block(db, new_current_block)
 
     # Sanity checks
     if decoded_block["block_index"] != config.BLOCK_FIRST:
@@ -1489,7 +1502,6 @@ def catch_up(db, check_asset_conservation=True):
                 fetcher = start_rsfetcher()
             else:
                 assert parsed_block_index == block_height
-            mempool.clean_mempool(db)
 
             parsed_blocks += 1
             formatted_duration = util.format_duration(time.time() - start_time)

counterparty-core/counterpartycore/lib/bootstrap.py

@@ -114,9 +114,11 @@ def clean_data_dir(data_dir):
     if not os.path.exists(data_dir):
         os.makedirs(data_dir, mode=0o755)
         return
-    files_to_delete = glob.glob(os.path.join(data_dir, "*.db"))
-    files_to_delete += glob.glob(os.path.join(data_dir, "*.db-wal"))
-    files_to_delete += glob.glob(os.path.join(data_dir, "*.db-shm"))
+    network = "" if config.NETWORK_NAME == "mainnet" else f".{config.NETWORK_NAME}"
+    files_to_delete = []
+    for db_name in ["counterparty", "state"]:
+        for ext in ["db", "db-wal", "db-shm"]:
+            files_to_delete += glob.glob(os.path.join(data_dir, f"{db_name}{network}.{ext}"))
     for file in files_to_delete:
         os.remove(file)
 

counterparty-core/counterpartycore/lib/check.py

@@ -695,6 +695,10 @@
         "ledger_hash": "4c4d6b660af23bb03a04bbf93ddd0a4b8e615dd7b883ecf827274cabe658bfc2",
         "txlist_hash": "f6a99d60337c33c1822c048f56e241455cd7e45bb5a9515096f1ac609d50f669",
     },
+    879058: {
+        "ledger_hash": "e6bf730a18c148adbd2cce9bd0f361e595e44d53fa98a9d9bdbf4c944f6c233b",
+        "txlist_hash": "e9946ac128405885f251fbb98a952ed554ea6cc25973261da9f40eabf4f3b429",
+    },
 }
 
 CONSENSUS_HASH_VERSION_TESTNET = 7
@@ -875,6 +879,10 @@
         "ledger_hash": "33cf0669a0d309d7e6b1bf79494613b69262b58c0ea03c9c221d955eb4c84fe5",
         "txlist_hash": "33cf0669a0d309d7e6b1bf79494613b69262b58c0ea03c9c221d955eb4c84fe5",
     },
+    64493: {
+        "ledger_hash": "af481088fb9303d0f61543fb3646110fcd27d5ebd3ac40e04397000320216699",
+        "txlist_hash": "4b610518a76f59e8f98922f37434be9bb5567040afebabda6ee69b4f92b80434",
+    },
 }
 
 

counterparty-core/counterpartycore/lib/config.py

@@ -5,7 +5,7 @@
 
 
 # Semantic Version
-__version__ = "10.9.0-rc.1"  # for hatch
+__version__ = "10.9.0"  # for hatch
 VERSION_STRING = __version__
 version = VERSION_STRING.split("-")[0].split(".")
 VERSION_MAJOR = int(version[0])
@@ -30,8 +30,8 @@
 ]
 NEED_REPARSE_IF_MINOR_IS_LESS_THAN_TESTNET4 = None
 
-NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN = [(8, 871780)]
-NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN_TESTNET = [(8, 3522632)]
+NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN = [(8, 871780), (9, 871780)]
+NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN_TESTNET = [(8, 3522632), (9, 3522632)]
 NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN_TESTNET4 = None
 
 STATE_DB_NEED_REFRESH_ON_VERSION_UPDATE = ["10.9.0-rc.1", "10.9.0"]

counterparty-core/counterpartycore/lib/deserialize.py

@@ -32,4 +32,5 @@ def deserialize_block(block_hex, parse_vouts=False, block_index=None):
         }
     )
     current_block_index = block_index or util.CURRENT_BLOCK_INDEX
-    return deserializer.parse_block(block_hex, current_block_index, parse_vouts)
+    decoded_block = deserializer.parse_block(block_hex, current_block_index, parse_vouts)
+    return decoded_block

counterparty-core/counterpartycore/lib/exceptions.py

@@ -145,3 +145,7 @@ class RSFetchError(Exception):
 
 class ElectrsError(Exception):
     pass
+
+
+class BlockOutOfRange(Exception):
+    pass

counterparty-core/counterpartycore/lib/follow.py

@@ -138,7 +138,8 @@ def receive_rawblock(self, body):
                 blocks.catch_up(self.db, check_asset_conservation=False)
             else:
                 blocks.parse_new_block(self.db, decoded_block)
-            mempool.clean_mempool(self.db)
+            if not config.NO_MEMPOOL:
+                mempool.clean_mempool(self.db)
             if not config.NO_TELEMETRY:
                 TelemetryOneShot().submit()
 
@@ -184,9 +185,9 @@ def receive_sequence(self, body):
                 raw_tx = self.raw_tx_cache.get(item_hash)
                 if raw_tx is None:
                     try:
-                        raw_tx = backend.bitcoind.getrawtransaction(item_hash)
+                        raw_tx = backend.bitcoind.getrawtransaction(item_hash, no_retry=True)
                     except exceptions.BitcoindRPCError:
-                        logger.trace("Transaction not found in bitcoind: %s", item_hash)
+                        logger.warning("Transaction not found in bitcoind: %s", item_hash)
                         return
                 # add transaction to mempool block
                 # logger.trace("Adding transaction to mempool block: %s", item_hash)
@@ -204,6 +205,7 @@ def receive_sequence(self, body):
                     logger.trace("Waiting for new transactions in the mempool or a new block...")
         # transaction removed from mempool for non-block inclusion reasons
         elif label == "R":
+            logger.debug("Removing transaction from mempool: %s", item_hash)
             mempool.clean_transaction_events(self.db, item_hash)
 
     def receive_message(self, topic, body, seq):
@@ -238,9 +240,8 @@ async def receive_multipart(self, socket, topic_name):
             self.receive_message(topic, body, seq)
         except Exception as e:
             logger.error("Error processing message: %s", e)
-            import traceback
-
-            print(traceback.format_exc())  # for debugging
+            # import traceback
+            # print(traceback.format_exc())  # for debugging
             capture_exception(e)
             raise e
 
@@ -297,6 +298,7 @@ async def handle(self):
             except Exception as e:
                 logger.error("Error in handle loop: %s", e)
                 capture_exception(e)
+                self.stop()
                 break  # Optionally break the loop on other exceptions
 
     def start(self):

counterparty-core/counterpartycore/lib/gettxinfo.py

@@ -127,7 +127,7 @@ def get_vin_info(vin):
     # have been from a while ago, so this call may not hit the cache.
     vin_ctx = backend.bitcoind.get_decoded_transaction(vin["hash"])
 
-    is_segwit = len(vin_ctx["vtxinwit"]) > 0
+    is_segwit = vin_ctx["segwit"]
     vout = vin_ctx["vout"][vin["n"]]
 
     return vout["value"], vout["script_pub_key"], is_segwit
@@ -393,6 +393,7 @@ def get_tx_info_new(db, decoded_tx, block_index, p2sh_is_segwit=False, composing
     The destinations, if they exists, always comes before the data output; the
     change, if it exists, always comes after.
     """
+
     # Ignore coinbase transactions.
     if decoded_tx["coinbase"]:
         raise DecodeError("coinbase transaction")