From 1b0eb3a8038384f34d5ca5dd48b8318604b1e30e Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Tue, 4 Jun 2024 09:41:50 +0000 Subject: [PATCH 1/6] Fix mempool log; Try to fix docker compose stop --- .github/workflows/test_compose.yml | 2 +- counterparty-core/counterpartycore/lib/api/api_v1.py | 5 +++-- counterparty-core/counterpartycore/lib/log.py | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_compose.yml b/.github/workflows/test_compose.yml index 13e8f2fa3b..28e516434a 100644 --- a/.github/workflows/test_compose.yml +++ b/.github/workflows/test_compose.yml @@ -2,7 +2,7 @@ name: Docker Compose on: push: - branches: ['develop', 'master'] + branches: ['develop', 'master', 'compose'] jobs: build: diff --git a/counterparty-core/counterpartycore/lib/api/api_v1.py b/counterparty-core/counterpartycore/lib/api/api_v1.py index 803fbbe9f1..fbf49338bc 100644 --- a/counterparty-core/counterpartycore/lib/api/api_v1.py +++ b/counterparty-core/counterpartycore/lib/api/api_v1.py @@ -499,8 +499,9 @@ def __init__(self): def stop(self): logger.info("Stopping API Status Poller...") self.stopping = True - self.join() self.db.close() + self.db = None + self.join() def run(self): logger.debug("Starting API Status Poller...") @@ -514,7 +515,7 @@ def run(self): if ( time.time() - self.last_database_check > 10 * 60 ): # Ten minutes since last check. - if not config.FORCE: + if not config.FORCE and self.db is not None: code = 11 logger.debug("Checking backend state.") check_backend_state() diff --git a/counterparty-core/counterpartycore/lib/log.py b/counterparty-core/counterpartycore/lib/log.py index a186dfb441..71e1bc099d 100644 --- a/counterparty-core/counterpartycore/lib/log.py +++ b/counterparty-core/counterpartycore/lib/log.py @@ -58,10 +58,10 @@ def format(self, record): and util.CURRENT_BLOCK_INDEX is not None and "/counterpartycore/lib/messages/" in record.pathname ): - if util.CURRENT_BLOCK_INDEX != config.MEMPOOL_BLOCK_INDEX: - log_format = f"%(asctime)s - [%(levelname)8s] - Block {util.CURRENT_BLOCK_INDEX} - %(message)s" - else: + if util.PARSING_MEMPOOL: log_format = "%(asctime)s - [%(levelname)8s] - Mempool - %(message)s" + else: + log_format = f"%(asctime)s - [%(levelname)8s] - Block {util.CURRENT_BLOCK_INDEX} - %(message)s" else: log_format = "%(asctime)s - [%(levelname)8s] - %(message)s" attrs = ["bold"] if hasattr(record, "bold") and record.bold else [] From 70baf232eb8aab8926a9d567c63bbad3f95f6fd1 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Tue, 4 Jun 2024 10:57:29 +0000 Subject: [PATCH 2/6] db connection pool for API v1 --- .github/workflows/test_compose.yml | 2 +- .../counterpartycore/lib/api/api_v1.py | 452 +++++++++--------- .../counterpartycore/lib/database.py | 3 +- .../counterpartycore/test/conftest.py | 17 +- 4 files changed, 253 insertions(+), 221 deletions(-) diff --git a/.github/workflows/test_compose.yml b/.github/workflows/test_compose.yml index 28e516434a..13e8f2fa3b 100644 --- a/.github/workflows/test_compose.yml +++ b/.github/workflows/test_compose.yml @@ -2,7 +2,7 @@ name: Docker Compose on: push: - branches: ['develop', 'master', 'compose'] + branches: ['develop', 'master'] jobs: build: diff --git a/counterparty-core/counterpartycore/lib/api/api_v1.py b/counterparty-core/counterpartycore/lib/api/api_v1.py index fbf49338bc..5ac4960464 100644 --- a/counterparty-core/counterpartycore/lib/api/api_v1.py +++ b/counterparty-core/counterpartycore/lib/api/api_v1.py @@ -33,6 +33,7 @@ util, ) from counterpartycore.lib.api import util as api_util +from counterpartycore.lib.database import DBConnectionPool from counterpartycore.lib.messages import ( bet, # noqa: F401 broadcast, # noqa: F401 @@ -543,7 +544,6 @@ class APIServer(threading.Thread): """Handle JSON-RPC API calls.""" def __init__(self, db=None): - self.db = db self.is_ready = False self.server = None self.ctx = None @@ -552,13 +552,12 @@ def __init__(self, db=None): def stop(self): logger.info("Stopping API Server v1...") - self.db.close() + DBConnectionPool().close() self.server.shutdown() self.join() def run(self): logger.info("Starting API Server v1...") - self.db = self.db or database.get_connection(read_only=True) app = flask.Flask(__name__) auth = HTTPBasicAuth() @@ -575,7 +574,8 @@ def get_pw(username): def generate_get_method(table): def get_method(**kwargs): try: - return get_rows(self.db, table=table, **kwargs) + with DBConnectionPool().connection() as db: + return get_rows(db, table=table, **kwargs) except TypeError as e: # TODO: generalise for all API methods raise APIError(str(e)) # noqa: B904 @@ -590,7 +590,8 @@ def get_method(**kwargs): def sql(query, bindings=None): if bindings == None: # noqa: E711 bindings = [] - return db_query(self.db, query, tuple(bindings)) + with DBConnectionPool().connection() as db: + return db_query(db, query, tuple(bindings)) ###################### # WRITE/ACTION API @@ -602,9 +603,10 @@ def create_method(**kwargs): transaction_args, common_args, private_key_wif = ( transaction.split_compose_params(**kwargs) ) - return transaction.compose_transaction( - self.db, name=tx, params=transaction_args, api_v1=True, **common_args - ) + with DBConnectionPool().connection() as db: + return transaction.compose_transaction( + db, name=tx, params=transaction_args, api_v1=True, **common_args + ) except ( TypeError, script.AddressError, @@ -631,8 +633,8 @@ def create_method(**kwargs): def get_messages(block_index): if not isinstance(block_index, int): raise APIError("block_index must be an integer.") - - messages = ledger.get_messages(self.db, block_index=block_index) + with DBConnectionPool().connection() as db: + messages = ledger.get_messages(db, block_index=block_index) return messages @dispatcher.add_method @@ -648,24 +650,26 @@ def get_messages_by_index(message_indexes): for idx in message_indexes: # make sure the data is clean if not isinstance(idx, int): raise APIError("All items in message_indexes are not integers") - - messages = ledger.get_messages(self.db, message_index_in=message_indexes) + with DBConnectionPool().connection() as db: + messages = ledger.get_messages(db, message_index_in=message_indexes) return messages @dispatcher.add_method def get_supply(asset): - if asset == "BTC": - return backend.bitcoind.get_btc_supply(normalize=False) - elif asset == "XCP": - return ledger.xcp_supply(self.db) - else: - asset = ledger.resolve_subasset_longname(self.db, asset) - return ledger.asset_supply(self.db, asset) + with DBConnectionPool().connection() as db: + if asset == "BTC": + return backend.bitcoind.get_btc_supply(normalize=False) + elif asset == "XCP": + return ledger.xcp_supply(db) + else: + asset = ledger.resolve_subasset_longname(db, asset) + return ledger.asset_supply(db, asset) @dispatcher.add_method def get_xcp_supply(): logger.warning("Deprecated method: `get_xcp_supply`") - return ledger.xcp_supply(self.db) + with DBConnectionPool().connection() as db: + return ledger.xcp_supply(db) @dispatcher.add_method def get_asset_info(assets=None, asset=None): @@ -677,70 +681,72 @@ def get_asset_info(assets=None, asset=None): "assets must be a list of asset names, even if it just contains one entry" ) assets_info = [] - for asset in assets: - asset = ledger.resolve_subasset_longname(self.db, asset) # noqa: PLW2901 - - # BTC and XCP. - if asset in [config.BTC, config.XCP]: - if asset == config.BTC: - supply = backend.bitcoind.get_btc_supply(normalize=False) + with DBConnectionPool().connection() as db: + for asset in assets: + asset = ledger.resolve_subasset_longname(db, asset) # noqa: PLW2901 + + # BTC and XCP. + if asset in [config.BTC, config.XCP]: + if asset == config.BTC: + supply = backend.bitcoind.get_btc_supply(normalize=False) + else: + supply = ledger.xcp_supply(db) + + assets_info.append( + { + "asset": asset, + "asset_longname": None, + "owner": None, + "divisible": True, + "locked": False, + "supply": supply, + "description": "", + "issuer": None, + } + ) + continue + + # User‐created asset. + cursor = db.cursor() + issuances = ledger.get_issuances(db, asset=asset, status="valid", first=True) + cursor.close() + if not issuances: + continue # asset not found, most likely else: - supply = ledger.xcp_supply(self.db) - + last_issuance = issuances[-1] + locked = False + for e in issuances: + if e["locked"]: + locked = True assets_info.append( { "asset": asset, - "asset_longname": None, - "owner": None, - "divisible": True, - "locked": False, - "supply": supply, - "description": "", - "issuer": None, + "asset_longname": last_issuance["asset_longname"], + "owner": last_issuance["issuer"], + "divisible": bool(last_issuance["divisible"]), + "locked": locked, + "supply": ledger.asset_supply(db, asset), + "description": last_issuance["description"], + "issuer": last_issuance["issuer"], } ) - continue - - # User‐created asset. - cursor = self.db.cursor() - issuances = ledger.get_issuances(self.db, asset=asset, status="valid", first=True) - cursor.close() - if not issuances: - continue # asset not found, most likely - else: - last_issuance = issuances[-1] - locked = False - for e in issuances: - if e["locked"]: - locked = True - assets_info.append( - { - "asset": asset, - "asset_longname": last_issuance["asset_longname"], - "owner": last_issuance["issuer"], - "divisible": bool(last_issuance["divisible"]), - "locked": locked, - "supply": ledger.asset_supply(self.db, asset), - "description": last_issuance["description"], - "issuer": last_issuance["issuer"], - } - ) return assets_info @dispatcher.add_method def get_block_info(block_index): assert isinstance(block_index, int) - cursor = self.db.cursor() - cursor.execute("""SELECT * FROM blocks WHERE block_index = ?""", (block_index,)) - blocks = list(cursor) # noqa: F811 - if len(blocks) == 1: - block = blocks[0] - elif len(blocks) == 0: - raise exceptions.DatabaseError("No blocks found.") - else: - assert False # noqa: B011 - cursor.close() - return block + with DBConnectionPool().connection() as db: + cursor = db.cursor() + cursor.execute("""SELECT * FROM blocks WHERE block_index = ?""", (block_index,)) + blocks = list(cursor) # noqa: F811 + if len(blocks) == 1: + block = blocks[0] + elif len(blocks) == 0: + raise exceptions.DatabaseError("No blocks found.") + else: + assert False # noqa: B011 + cursor.close() + return block @dispatcher.add_method def fee_per_kb(conf_target=config.ESTIMATE_FEE_CONF_TARGET, mode=config.ESTIMATE_FEE_MODE): @@ -768,61 +774,63 @@ def get_blocks(block_indexes, min_message_index=None): if not isinstance(block_index, int): raise APIError(must_be_non_empty_list_int) - cursor = self.db.cursor() + with DBConnectionPool().connection() as db: + cursor = db.cursor() - block_indexes_placeholder = f"{','.join(['?'] * len(block_indexes))}" - # no sql injection here - cursor.execute( - f"SELECT * FROM blocks WHERE block_index IN ({block_indexes_placeholder}) ORDER BY block_index ASC", # nosec B608 # noqa: S608 - block_indexes, - ) - blocks = cursor.fetchall() # noqa: F811 + block_indexes_placeholder = f"{','.join(['?'] * len(block_indexes))}" + # no sql injection here + cursor.execute( + f"SELECT * FROM blocks WHERE block_index IN ({block_indexes_placeholder}) ORDER BY block_index ASC", # nosec B608 # noqa: S608 + block_indexes, + ) + blocks = cursor.fetchall() # noqa: F811 - messages = collections.deque(ledger.get_messages(self.db, block_index_in=block_indexes)) + messages = collections.deque(ledger.get_messages(db, block_index_in=block_indexes)) - # Discard any messages less than min_message_index - if min_message_index: - while len(messages) and messages[0]["message_index"] < min_message_index: - messages.popleft() + # Discard any messages less than min_message_index + if min_message_index: + while len(messages) and messages[0]["message_index"] < min_message_index: + messages.popleft() - # Packages messages into their appropriate block in the data structure to be returned - for block in blocks: - block["_messages"] = [] - while len(messages) and messages[0]["block_index"] == block["block_index"]: - block["_messages"].append(messages.popleft()) - # NOTE: if len(messages), then we're only returning the messages for the first set of blocks before the reorg + # Packages messages into their appropriate block in the data structure to be returned + for block in blocks: + block["_messages"] = [] + while len(messages) and messages[0]["block_index"] == block["block_index"]: + block["_messages"].append(messages.popleft()) + # NOTE: if len(messages), then we're only returning the messages for the first set of blocks before the reorg - cursor.close() + cursor.close() return blocks @dispatcher.add_method def get_running_info(): latest_block_index = backend.bitcoind.getblockcount() - try: - api_util.check_last_parsed_block(self.db, latest_block_index) - except exceptions.DatabaseError: - caught_up = False - else: - caught_up = True + with DBConnectionPool().connection() as db: + try: + api_util.check_last_parsed_block(db, latest_block_index) + except exceptions.DatabaseError: + caught_up = False + else: + caught_up = True - try: - cursor = self.db.cursor() - blocks = list( - cursor.execute( - """SELECT * FROM blocks WHERE block_index = ?""", - (util.CURRENT_BLOCK_INDEX,), + try: + cursor = db.cursor() + blocks = list( + cursor.execute( + """SELECT * FROM blocks WHERE block_index = ?""", + (util.CURRENT_BLOCK_INDEX,), + ) ) - ) - assert len(blocks) == 1 - last_block = blocks[0] - cursor.close() - except: # noqa: E722 - last_block = None + assert len(blocks) == 1 + last_block = blocks[0] + cursor.close() + except: # noqa: E722 + last_block = None - try: - last_message = ledger.last_message(self.db) - except: # noqa: E722 - last_message = None + try: + last_message = ledger.last_message(db) + except: # noqa: E722 + last_message = None try: indexd_blocks_behind = backend.bitcoind.get_blocks_behind() @@ -859,43 +867,45 @@ def get_running_info(): @dispatcher.add_method def get_element_counts(): - counts = {} - cursor = self.db.cursor() - for element in [ - "transactions", - "blocks", - "debits", - "credits", - "balances", - "sends", - "orders", - "order_matches", - "btcpays", - "issuances", - "broadcasts", - "bets", - "bet_matches", - "dividends", - "burns", - "cancels", - "order_expirations", - "bet_expirations", - "order_match_expirations", - "bet_match_expirations", - "messages", - "destructions", - ]: - # no sql injection here, element is hardcoded - cursor.execute(f"SELECT COUNT(*) AS count FROM {element}") # nosec B608 # noqa: S608 - count_list = cursor.fetchall() - assert len(count_list) == 1 - counts[element] = count_list[0]["count"] - cursor.close() - return counts + with DBConnectionPool().connection() as db: + counts = {} + cursor = db.cursor() + for element in [ + "transactions", + "blocks", + "debits", + "credits", + "balances", + "sends", + "orders", + "order_matches", + "btcpays", + "issuances", + "broadcasts", + "bets", + "bet_matches", + "dividends", + "burns", + "cancels", + "order_expirations", + "bet_expirations", + "order_match_expirations", + "bet_match_expirations", + "messages", + "destructions", + ]: + # no sql injection here, element is hardcoded + cursor.execute(f"SELECT COUNT(*) AS count FROM {element}") # nosec B608 # noqa: S608 + count_list = cursor.fetchall() + assert len(count_list) == 1 + counts[element] = count_list[0]["count"] + cursor.close() + return counts @dispatcher.add_method def get_asset_names(longnames=False): - all_assets = ledger.get_valid_assets(self.db) + with DBConnectionPool().connection() as db: + all_assets = ledger.get_valid_assets(db) if longnames: names = [ {"asset": row["asset"], "asset_longname": row["asset_longname"]} @@ -911,8 +921,9 @@ def get_asset_longnames(): @dispatcher.add_method def get_holder_count(asset): - asset = ledger.resolve_subasset_longname(self.db, asset) - holders = ledger.holders(self.db, asset, True) + with DBConnectionPool().connection() as db: + asset = ledger.resolve_subasset_longname(db, asset) + holders = ledger.holders(db, asset, True) addresses = [] for holder in holders: addresses.append(holder["address"]) @@ -920,8 +931,9 @@ def get_holder_count(asset): @dispatcher.add_method def get_holders(asset): - asset = ledger.resolve_subasset_longname(self.db, asset) - holders = ledger.holders(self.db, asset, True) + with DBConnectionPool().connection() as db: + asset = ledger.resolve_subasset_longname(db, asset) + holders = ledger.holders(db, asset, True) return holders @dispatcher.add_method @@ -963,11 +975,12 @@ def getrawtransaction_batch(txhash_list, verbose=False, skip_missing=False): def get_tx_info(tx_hex, block_index=None): # block_index mandatory for transactions before block 335000 use_txid = util.enabled("correct_segwit_txids", block_index=block_index) - source, destination, btc_amount, fee, data, extra = gettxinfo.get_tx_info( - self.db, - deserialize.deserialize_tx(tx_hex, use_txid=use_txid), - block_index=block_index, - ) + with DBConnectionPool().connection() as db: + source, destination, btc_amount, fee, data, extra = gettxinfo.get_tx_info( + db, + deserialize.deserialize_tx(tx_hex, use_txid=use_txid), + block_index=block_index, + ) return ( source, destination, @@ -983,7 +996,8 @@ def unpack(data_hex): # TODO: Enabled only for `send`. if message_type_id == send.ID: - unpacked = send.unpack(self.db, message, util.CURRENT_BLOCK_INDEX) + with DBConnectionPool().connection() as db: + unpacked = send.unpack(db, message, util.CURRENT_BLOCK_INDEX) elif message_type_id == enhanced_send.ID: unpacked = enhanced_send.unpack(message, util.CURRENT_BLOCK_INDEX) else: @@ -1001,54 +1015,55 @@ def get_dispenser_info(tx_hash=None, tx_index=None): raise APIError("You must provided a tx hash or a tx index") dispensers = [] - if tx_hash is not None: - dispensers = ledger.get_dispenser_info(self.db, tx_hash=tx_hash) - else: - dispensers = ledger.get_dispenser_info(self.db, tx_index=tx_index) - - if len(dispensers) == 1: - dispenser = dispensers[0] - oracle_price = "" - satoshi_price = "" - fiat_price = "" - oracle_price_last_updated = "" - oracle_fiat_label = "" - - if dispenser["oracle_address"] != None: # noqa: E711 - fiat_price = util.satoshirate_to_fiat(dispenser["satoshirate"]) - ( - oracle_price, - oracle_fee, - oracle_fiat_label, - oracle_price_last_updated, - ) = ledger.get_oracle_last_price( - self.db, dispenser["oracle_address"], util.CURRENT_BLOCK_INDEX - ) - - if oracle_price > 0: - satoshi_price = math.ceil((fiat_price / oracle_price) * config.UNIT) - else: - raise APIError("Last oracle price is zero") - - return { - "tx_index": dispenser["tx_index"], - "tx_hash": dispenser["tx_hash"], - "block_index": dispenser["block_index"], - "source": dispenser["source"], - "asset": dispenser["asset"], - "give_quantity": dispenser["give_quantity"], - "escrow_quantity": dispenser["escrow_quantity"], - "mainchainrate": dispenser["satoshirate"], - "fiat_price": fiat_price, - "fiat_unit": oracle_fiat_label, - "oracle_price": oracle_price, - "satoshi_price": satoshi_price, - "status": dispenser["status"], - "give_remaining": dispenser["give_remaining"], - "oracle_address": dispenser["oracle_address"], - "oracle_price_last_updated": oracle_price_last_updated, - "asset_longname": dispenser["asset_longname"], - } + with DBConnectionPool().connection() as db: + if tx_hash is not None: + dispensers = ledger.get_dispenser_info(db, tx_hash=tx_hash) + else: + dispensers = ledger.get_dispenser_info(db, tx_index=tx_index) + + if len(dispensers) == 1: + dispenser = dispensers[0] + oracle_price = "" + satoshi_price = "" + fiat_price = "" + oracle_price_last_updated = "" + oracle_fiat_label = "" + + if dispenser["oracle_address"] != None: # noqa: E711 + fiat_price = util.satoshirate_to_fiat(dispenser["satoshirate"]) + ( + oracle_price, + oracle_fee, + oracle_fiat_label, + oracle_price_last_updated, + ) = ledger.get_oracle_last_price( + db, dispenser["oracle_address"], util.CURRENT_BLOCK_INDEX + ) + + if oracle_price > 0: + satoshi_price = math.ceil((fiat_price / oracle_price) * config.UNIT) + else: + raise APIError("Last oracle price is zero") + + return { + "tx_index": dispenser["tx_index"], + "tx_hash": dispenser["tx_hash"], + "block_index": dispenser["block_index"], + "source": dispenser["source"], + "asset": dispenser["asset"], + "give_quantity": dispenser["give_quantity"], + "escrow_quantity": dispenser["escrow_quantity"], + "mainchainrate": dispenser["satoshirate"], + "fiat_price": fiat_price, + "fiat_unit": oracle_fiat_label, + "oracle_price": oracle_price, + "satoshi_price": satoshi_price, + "status": dispenser["status"], + "give_remaining": dispenser["give_remaining"], + "oracle_address": dispenser["oracle_address"], + "oracle_price_last_updated": oracle_price_last_updated, + "asset_longname": dispenser["asset_longname"], + } return {} @@ -1067,7 +1082,8 @@ def handle_healthz(): with configure_sentry_scope() as scope: scope.set_transaction_name("healthcheck") check_type = request.args.get("type", "light") - return api_util.handle_healthz_route(self.db, check_type) + with DBConnectionPool().connection() as db: + return api_util.handle_healthz_route(db, check_type) @app.route("/", defaults={"args_path": ""}, methods=["GET", "POST", "OPTIONS"]) @app.route("/", methods=["GET", "POST", "OPTIONS"]) @@ -1206,9 +1222,10 @@ def handle_rest(path_args, flask_request): # Compose the transaction. try: - query_data = transaction.compose_transaction( - self.db, name=query_type, params=transaction_args, **common_args - ) + with DBConnectionPool().connection() as db: + query_data = transaction.compose_transaction( + db, name=query_type, params=transaction_args, **common_args + ) except ( script.AddressError, exceptions.ComposeError, @@ -1231,12 +1248,13 @@ def handle_rest(path_args, flask_request): # Run the query. try: - query_data = get_rows( - self.db, - table=query_type, - filters=data_filter, - filterop=operator, - ) + with DBConnectionPool().connection() as db: + query_data = get_rows( + db, + table=query_type, + filters=data_filter, + filterop=operator, + ) except APIError as error: # noqa: F841 return flask.Response("API Error", 400, mimetype="application/json") diff --git a/counterparty-core/counterpartycore/lib/database.py b/counterparty-core/counterpartycore/lib/database.py index f1e43ca83e..57a34e3e03 100644 --- a/counterparty-core/counterpartycore/lib/database.py +++ b/counterparty-core/counterpartycore/lib/database.py @@ -115,7 +115,8 @@ def connection(self): db.close() def close(self): - for db in self.connections: + while len(self.connections) > 0: + db = self.connections.pop() db.close() diff --git a/counterparty-core/counterpartycore/test/conftest.py b/counterparty-core/counterpartycore/test/conftest.py index b5efb60959..8f090fa852 100644 --- a/counterparty-core/counterpartycore/test/conftest.py +++ b/counterparty-core/counterpartycore/test/conftest.py @@ -7,6 +7,7 @@ import json import logging import time +from contextlib import contextmanager from datetime import datetime import apsw @@ -205,14 +206,26 @@ def rawtransactions_db(request): @pytest.fixture(scope="function") -def server_db(request, cp_server, api_server): +def server_db(request, cp_server, api_server, monkeypatch): """Enable database access for unit test vectors.""" db = database.get_connection(read_only=False) - api_server.db = db # inject into api_server cursor = db.cursor() cursor.execute("""BEGIN""") util_test.reset_current_block_index(db) + class DBConnectionPool: + @contextmanager + def connection(self): + try: + yield db + finally: + pass + + def close(self): + db.close() + + database.DBConnectionPool = DBConnectionPool + request.addfinalizer(lambda: cursor.execute("""ROLLBACK""")) request.addfinalizer(lambda: util_test.reset_current_block_index(db)) From e43d7bf929181006202fe00db8588a9fcc7d35e2 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Tue, 4 Jun 2024 12:56:34 +0000 Subject: [PATCH 3/6] fix tests --- .../counterpartycore/lib/api/api_v1.py | 45 ++++++++++--------- .../test/complex_unit_test.py | 19 +++++++- .../counterpartycore/test/conftest.py | 18 ++------ 3 files changed, 44 insertions(+), 38 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/api/api_v1.py b/counterparty-core/counterpartycore/lib/api/api_v1.py index 5ac4960464..fe43279aa7 100644 --- a/counterparty-core/counterpartycore/lib/api/api_v1.py +++ b/counterparty-core/counterpartycore/lib/api/api_v1.py @@ -547,12 +547,13 @@ def __init__(self, db=None): self.is_ready = False self.server = None self.ctx = None + self.connection_pool = DBConnectionPool() threading.Thread.__init__(self) sentry.init() def stop(self): logger.info("Stopping API Server v1...") - DBConnectionPool().close() + self.connection_pool.close() self.server.shutdown() self.join() @@ -574,7 +575,7 @@ def get_pw(username): def generate_get_method(table): def get_method(**kwargs): try: - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: return get_rows(db, table=table, **kwargs) except TypeError as e: # TODO: generalise for all API methods raise APIError(str(e)) # noqa: B904 @@ -590,7 +591,7 @@ def get_method(**kwargs): def sql(query, bindings=None): if bindings == None: # noqa: E711 bindings = [] - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: return db_query(db, query, tuple(bindings)) ###################### @@ -603,7 +604,7 @@ def create_method(**kwargs): transaction_args, common_args, private_key_wif = ( transaction.split_compose_params(**kwargs) ) - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: return transaction.compose_transaction( db, name=tx, params=transaction_args, api_v1=True, **common_args ) @@ -633,7 +634,7 @@ def create_method(**kwargs): def get_messages(block_index): if not isinstance(block_index, int): raise APIError("block_index must be an integer.") - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: messages = ledger.get_messages(db, block_index=block_index) return messages @@ -650,13 +651,13 @@ def get_messages_by_index(message_indexes): for idx in message_indexes: # make sure the data is clean if not isinstance(idx, int): raise APIError("All items in message_indexes are not integers") - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: messages = ledger.get_messages(db, message_index_in=message_indexes) return messages @dispatcher.add_method def get_supply(asset): - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: if asset == "BTC": return backend.bitcoind.get_btc_supply(normalize=False) elif asset == "XCP": @@ -668,7 +669,7 @@ def get_supply(asset): @dispatcher.add_method def get_xcp_supply(): logger.warning("Deprecated method: `get_xcp_supply`") - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: return ledger.xcp_supply(db) @dispatcher.add_method @@ -681,7 +682,7 @@ def get_asset_info(assets=None, asset=None): "assets must be a list of asset names, even if it just contains one entry" ) assets_info = [] - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: for asset in assets: asset = ledger.resolve_subasset_longname(db, asset) # noqa: PLW2901 @@ -735,7 +736,7 @@ def get_asset_info(assets=None, asset=None): @dispatcher.add_method def get_block_info(block_index): assert isinstance(block_index, int) - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: cursor = db.cursor() cursor.execute("""SELECT * FROM blocks WHERE block_index = ?""", (block_index,)) blocks = list(cursor) # noqa: F811 @@ -774,7 +775,7 @@ def get_blocks(block_indexes, min_message_index=None): if not isinstance(block_index, int): raise APIError(must_be_non_empty_list_int) - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: cursor = db.cursor() block_indexes_placeholder = f"{','.join(['?'] * len(block_indexes))}" @@ -805,7 +806,7 @@ def get_blocks(block_indexes, min_message_index=None): @dispatcher.add_method def get_running_info(): latest_block_index = backend.bitcoind.getblockcount() - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: try: api_util.check_last_parsed_block(db, latest_block_index) except exceptions.DatabaseError: @@ -867,7 +868,7 @@ def get_running_info(): @dispatcher.add_method def get_element_counts(): - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: counts = {} cursor = db.cursor() for element in [ @@ -904,7 +905,7 @@ def get_element_counts(): @dispatcher.add_method def get_asset_names(longnames=False): - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: all_assets = ledger.get_valid_assets(db) if longnames: names = [ @@ -921,7 +922,7 @@ def get_asset_longnames(): @dispatcher.add_method def get_holder_count(asset): - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: asset = ledger.resolve_subasset_longname(db, asset) holders = ledger.holders(db, asset, True) addresses = [] @@ -931,7 +932,7 @@ def get_holder_count(asset): @dispatcher.add_method def get_holders(asset): - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: asset = ledger.resolve_subasset_longname(db, asset) holders = ledger.holders(db, asset, True) return holders @@ -975,7 +976,7 @@ def getrawtransaction_batch(txhash_list, verbose=False, skip_missing=False): def get_tx_info(tx_hex, block_index=None): # block_index mandatory for transactions before block 335000 use_txid = util.enabled("correct_segwit_txids", block_index=block_index) - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: source, destination, btc_amount, fee, data, extra = gettxinfo.get_tx_info( db, deserialize.deserialize_tx(tx_hex, use_txid=use_txid), @@ -996,7 +997,7 @@ def unpack(data_hex): # TODO: Enabled only for `send`. if message_type_id == send.ID: - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: unpacked = send.unpack(db, message, util.CURRENT_BLOCK_INDEX) elif message_type_id == enhanced_send.ID: unpacked = enhanced_send.unpack(message, util.CURRENT_BLOCK_INDEX) @@ -1015,7 +1016,7 @@ def get_dispenser_info(tx_hash=None, tx_index=None): raise APIError("You must provided a tx hash or a tx index") dispensers = [] - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: if tx_hash is not None: dispensers = ledger.get_dispenser_info(db, tx_hash=tx_hash) else: @@ -1082,7 +1083,7 @@ def handle_healthz(): with configure_sentry_scope() as scope: scope.set_transaction_name("healthcheck") check_type = request.args.get("type", "light") - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: return api_util.handle_healthz_route(db, check_type) @app.route("/", defaults={"args_path": ""}, methods=["GET", "POST", "OPTIONS"]) @@ -1222,7 +1223,7 @@ def handle_rest(path_args, flask_request): # Compose the transaction. try: - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: query_data = transaction.compose_transaction( db, name=query_type, params=transaction_args, **common_args ) @@ -1248,7 +1249,7 @@ def handle_rest(path_args, flask_request): # Run the query. try: - with DBConnectionPool().connection() as db: + with self.connection_pool.connection() as db: query_data = get_rows( db, table=query_type, diff --git a/counterparty-core/counterpartycore/test/complex_unit_test.py b/counterparty-core/counterpartycore/test/complex_unit_test.py index cb590e5754..538b708474 100644 --- a/counterparty-core/counterpartycore/test/complex_unit_test.py +++ b/counterparty-core/counterpartycore/test/complex_unit_test.py @@ -1,5 +1,6 @@ import json import tempfile +from contextlib import contextmanager import pytest from apsw import ConstraintError @@ -21,7 +22,8 @@ @pytest.mark.usefixtures("server_db") @pytest.mark.usefixtures("api_server") -def test_alice_bob(server_db): +@pytest.mark.usefixtures("cp_server") +def test_alice_bob(server_db, cp_server, api_server): alice = ADDR[0] bob = "miJqNkHhC5xsB61gsiSWXeTLnEGSQnWbXB" @@ -65,6 +67,17 @@ def test_alice_bob(server_db): assert alice_balance2 == alice_balance - v assert bob_balance2 == bob_balance + v + class DBConnectionPoolMock(metaclass=util.SingletonMeta): + @contextmanager + def connection(self): + try: + yield server_db + finally: + pass + + api_server_connection_pool = api_server.connection_pool + api_server.connection_pool = DBConnectionPoolMock() + # check API result result = util.api( "get_balances", @@ -76,6 +89,10 @@ def test_alice_bob(server_db): }, ) + api_server.connection_pool = api_server_connection_pool + + print(result) + assert result[0]["quantity"] == alice_balance2 # -- do another TX diff --git a/counterparty-core/counterpartycore/test/conftest.py b/counterparty-core/counterpartycore/test/conftest.py index 8f090fa852..91432a9df0 100644 --- a/counterparty-core/counterpartycore/test/conftest.py +++ b/counterparty-core/counterpartycore/test/conftest.py @@ -7,7 +7,6 @@ import json import logging import time -from contextlib import contextmanager from datetime import datetime import apsw @@ -206,26 +205,13 @@ def rawtransactions_db(request): @pytest.fixture(scope="function") -def server_db(request, cp_server, api_server, monkeypatch): +def server_db(request, cp_server, api_server): """Enable database access for unit test vectors.""" db = database.get_connection(read_only=False) cursor = db.cursor() cursor.execute("""BEGIN""") util_test.reset_current_block_index(db) - class DBConnectionPool: - @contextmanager - def connection(self): - try: - yield db - finally: - pass - - def close(self): - db.close() - - database.DBConnectionPool = DBConnectionPool - request.addfinalizer(lambda: cursor.execute("""ROLLBACK""")) request.addfinalizer(lambda: util_test.reset_current_block_index(db)) @@ -348,6 +334,8 @@ def cp_server(request): request.addfinalizer(lambda: util_test.remove_database_files(dbfile)) + return db + class MockUTXOSet(object): """ From 436317d493b282fed897b0b00d5106d7879bbc84 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Tue, 4 Jun 2024 15:39:37 +0000 Subject: [PATCH 4/6] Add indexes for some quantity fields --- counterparty-core/counterpartycore/lib/blocks.py | 3 +++ .../counterpartycore/lib/messages/dispenser.py | 2 ++ .../counterpartycore/lib/messages/order.py | 2 ++ release-notes/release-notes-v10.2.0.md | 14 ++++++++++++-- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/blocks.py b/counterparty-core/counterpartycore/lib/blocks.py index e19bab91dd..9913001083 100644 --- a/counterparty-core/counterpartycore/lib/blocks.py +++ b/counterparty-core/counterpartycore/lib/blocks.py @@ -483,6 +483,7 @@ def initialise(db): ["block_index"], ["event"], ["action"], + ["quantity"], ], ) @@ -512,6 +513,7 @@ def initialise(db): ["block_index"], ["event"], ["calling_function"], + ["quantity"], ], ) @@ -539,6 +541,7 @@ def initialise(db): ["address"], ["asset"], ["block_index"], + ["quantity"], ], ) diff --git a/counterparty-core/counterpartycore/lib/messages/dispenser.py b/counterparty-core/counterpartycore/lib/messages/dispenser.py index 1cd5747f49..cacf9d24b6 100644 --- a/counterparty-core/counterpartycore/lib/messages/dispenser.py +++ b/counterparty-core/counterpartycore/lib/messages/dispenser.py @@ -103,6 +103,7 @@ def initialise(db): ["source", "asset", "origin", "status"], ["last_status_tx_hash"], ["close_block_index", "status"], + ["give_quantity"], ], ) @@ -139,6 +140,7 @@ def initialise(db): ["asset"], ["source"], ["destination"], + ["dispense_quantity"], ], ) diff --git a/counterparty-core/counterpartycore/lib/messages/order.py b/counterparty-core/counterpartycore/lib/messages/order.py index 10abaa8a48..b780468a34 100644 --- a/counterparty-core/counterpartycore/lib/messages/order.py +++ b/counterparty-core/counterpartycore/lib/messages/order.py @@ -85,6 +85,8 @@ def initialise(db): ["get_asset", "give_asset"], ["status"], ["source", "give_asset"], + ["get_quantity"], + ["give_quantity"], ], ) diff --git a/release-notes/release-notes-v10.2.0.md b/release-notes/release-notes-v10.2.0.md index ea01109a13..4ed74c22d9 100644 --- a/release-notes/release-notes-v10.2.0.md +++ b/release-notes/release-notes-v10.2.0.md @@ -25,7 +25,17 @@ Note: A reparse from block 819250 is automatically launched during the update. * Optimizes `get_pending_dispensers()` by adding the `last_status_tx_source` and `close_block_index` fields in the `dispensers` table. * Optimizes `is_dispensable()` by caching a list of all dispenser addresses. * Add `transaction_count` field in `blocks` table. -* Added the following indexes: `credits.calling_function`, `debits.action`, `transactions.source` +* Added the following indexes: + - `credits.calling_function` + - `debits.action` + - `transactions.source` + - `credit.quantity` + - `debit.quanity` + - `balance.quantity` + - `dispenser.give_quantity` + - `order.get_quantity` + - `order.give_quantity` + - `dispense.dispense_quantity` * Remove checking of impossible edge case in `list_tx()` function. * Add `EVENT` log level. * Update Pyo3 to the latest version. @@ -66,7 +76,7 @@ Note: A reparse from block 819250 is automatically launched during the update. * The `asset`, `assets`, `give_asset`, and `get_asset` parameters are no longer case-sensitive. * `/v2/assets` accepts now the paramater `named=true|false` to return only named or numeric assets. * Publish events on ZMQ Pub/Sub channel (see Documentation). -* Database connection pooling. +* Database connection pooling for API v1 and v2. * If `verbose=true`, enrich results containing `block_index` with `block_time`. * Added an `action` filter for the `*/credits` and `*/debits` routes. * Added an `event_name` filter for the `*/events` routes. From d38c51f74ca89a3fe87003b57a9d0280607694af Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 5 Jun 2024 07:29:42 +0000 Subject: [PATCH 5/6] update fixtures --- .../test/fixtures/scenarios/multisig_1_of_2.sql | 16 ++++++++++++++++ .../test/fixtures/scenarios/multisig_1_of_3.sql | 16 ++++++++++++++++ .../test/fixtures/scenarios/multisig_2_of_2.sql | 16 ++++++++++++++++ .../test/fixtures/scenarios/multisig_2_of_3.sql | 16 ++++++++++++++++ .../test/fixtures/scenarios/multisig_3_of_3.sql | 16 ++++++++++++++++ .../scenarios/parseblock_unittest_fixture.sql | 16 ++++++++++++++++ .../test/fixtures/scenarios/simplesig.sql | 16 ++++++++++++++++ .../test/fixtures/scenarios/unittest_fixture.sql | 16 ++++++++++++++++ 8 files changed, 128 insertions(+) diff --git a/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_2.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_2.sql index 8320b3b638..ca49560225 100644 --- a/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_2.sql +++ b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_2.sql @@ -276,6 +276,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -348,6 +350,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -412,6 +416,8 @@ CREATE INDEX credits_calling_function_idx ON credits (calling_function) ; CREATE INDEX credits_event_idx ON credits (event) ; +CREATE INDEX credits_quantity_idx ON credits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -464,6 +470,8 @@ CREATE INDEX debits_block_index_idx ON debits (block_index) ; CREATE INDEX debits_event_idx ON debits (event) ; +CREATE INDEX debits_quantity_idx ON debits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -1008,8 +1016,12 @@ CREATE INDEX orders_expire_index_idx ON orders (expire_index) ; CREATE INDEX orders_get_asset_give_asset_idx ON orders (get_asset, give_asset) ; +CREATE INDEX orders_get_quantity_idx ON orders (get_quantity) + ; CREATE INDEX orders_give_asset_idx ON orders (give_asset) ; +CREATE INDEX orders_give_quantity_idx ON orders (give_quantity) + ; CREATE INDEX orders_source_give_asset_idx ON orders (source, give_asset) ; CREATE INDEX orders_status_idx ON orders (status) @@ -1817,6 +1829,8 @@ CREATE INDEX dispensers_block_index_idx ON dispensers (block_index) ; CREATE INDEX dispensers_close_block_index_status_idx ON dispensers (close_block_index, status) ; +CREATE INDEX dispensers_give_quantity_idx ON dispensers (give_quantity) + ; CREATE INDEX dispensers_give_remaining_idx ON dispensers (give_remaining) ; CREATE INDEX dispensers_last_status_tx_hash_idx ON dispensers (last_status_tx_hash) @@ -1869,6 +1883,8 @@ CREATE INDEX dispenses_block_index_idx ON dispenses (block_index) ; CREATE INDEX dispenses_destination_idx ON dispenses (destination) ; +CREATE INDEX dispenses_dispense_quantity_idx ON dispenses (dispense_quantity) + ; CREATE INDEX dispenses_dispenser_tx_hash_idx ON dispenses (dispenser_tx_hash) ; CREATE INDEX dispenses_source_idx ON dispenses (source) diff --git a/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_3.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_3.sql index 0330f4374f..397860448d 100644 --- a/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_3.sql +++ b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_3.sql @@ -276,6 +276,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -348,6 +350,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -412,6 +416,8 @@ CREATE INDEX credits_calling_function_idx ON credits (calling_function) ; CREATE INDEX credits_event_idx ON credits (event) ; +CREATE INDEX credits_quantity_idx ON credits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -464,6 +470,8 @@ CREATE INDEX debits_block_index_idx ON debits (block_index) ; CREATE INDEX debits_event_idx ON debits (event) ; +CREATE INDEX debits_quantity_idx ON debits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -1008,8 +1016,12 @@ CREATE INDEX orders_expire_index_idx ON orders (expire_index) ; CREATE INDEX orders_get_asset_give_asset_idx ON orders (get_asset, give_asset) ; +CREATE INDEX orders_get_quantity_idx ON orders (get_quantity) + ; CREATE INDEX orders_give_asset_idx ON orders (give_asset) ; +CREATE INDEX orders_give_quantity_idx ON orders (give_quantity) + ; CREATE INDEX orders_source_give_asset_idx ON orders (source, give_asset) ; CREATE INDEX orders_status_idx ON orders (status) @@ -1817,6 +1829,8 @@ CREATE INDEX dispensers_block_index_idx ON dispensers (block_index) ; CREATE INDEX dispensers_close_block_index_status_idx ON dispensers (close_block_index, status) ; +CREATE INDEX dispensers_give_quantity_idx ON dispensers (give_quantity) + ; CREATE INDEX dispensers_give_remaining_idx ON dispensers (give_remaining) ; CREATE INDEX dispensers_last_status_tx_hash_idx ON dispensers (last_status_tx_hash) @@ -1869,6 +1883,8 @@ CREATE INDEX dispenses_block_index_idx ON dispenses (block_index) ; CREATE INDEX dispenses_destination_idx ON dispenses (destination) ; +CREATE INDEX dispenses_dispense_quantity_idx ON dispenses (dispense_quantity) + ; CREATE INDEX dispenses_dispenser_tx_hash_idx ON dispenses (dispenser_tx_hash) ; CREATE INDEX dispenses_source_idx ON dispenses (source) diff --git a/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_2.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_2.sql index baf9d2a209..38a41bfe4c 100644 --- a/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_2.sql +++ b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_2.sql @@ -276,6 +276,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -348,6 +350,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -412,6 +416,8 @@ CREATE INDEX credits_calling_function_idx ON credits (calling_function) ; CREATE INDEX credits_event_idx ON credits (event) ; +CREATE INDEX credits_quantity_idx ON credits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -464,6 +470,8 @@ CREATE INDEX debits_block_index_idx ON debits (block_index) ; CREATE INDEX debits_event_idx ON debits (event) ; +CREATE INDEX debits_quantity_idx ON debits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -1008,8 +1016,12 @@ CREATE INDEX orders_expire_index_idx ON orders (expire_index) ; CREATE INDEX orders_get_asset_give_asset_idx ON orders (get_asset, give_asset) ; +CREATE INDEX orders_get_quantity_idx ON orders (get_quantity) + ; CREATE INDEX orders_give_asset_idx ON orders (give_asset) ; +CREATE INDEX orders_give_quantity_idx ON orders (give_quantity) + ; CREATE INDEX orders_source_give_asset_idx ON orders (source, give_asset) ; CREATE INDEX orders_status_idx ON orders (status) @@ -1817,6 +1829,8 @@ CREATE INDEX dispensers_block_index_idx ON dispensers (block_index) ; CREATE INDEX dispensers_close_block_index_status_idx ON dispensers (close_block_index, status) ; +CREATE INDEX dispensers_give_quantity_idx ON dispensers (give_quantity) + ; CREATE INDEX dispensers_give_remaining_idx ON dispensers (give_remaining) ; CREATE INDEX dispensers_last_status_tx_hash_idx ON dispensers (last_status_tx_hash) @@ -1869,6 +1883,8 @@ CREATE INDEX dispenses_block_index_idx ON dispenses (block_index) ; CREATE INDEX dispenses_destination_idx ON dispenses (destination) ; +CREATE INDEX dispenses_dispense_quantity_idx ON dispenses (dispense_quantity) + ; CREATE INDEX dispenses_dispenser_tx_hash_idx ON dispenses (dispenser_tx_hash) ; CREATE INDEX dispenses_source_idx ON dispenses (source) diff --git a/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_3.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_3.sql index f303076122..4e1d471424 100644 --- a/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_3.sql +++ b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_3.sql @@ -276,6 +276,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -348,6 +350,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -412,6 +416,8 @@ CREATE INDEX credits_calling_function_idx ON credits (calling_function) ; CREATE INDEX credits_event_idx ON credits (event) ; +CREATE INDEX credits_quantity_idx ON credits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -464,6 +470,8 @@ CREATE INDEX debits_block_index_idx ON debits (block_index) ; CREATE INDEX debits_event_idx ON debits (event) ; +CREATE INDEX debits_quantity_idx ON debits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -1008,8 +1016,12 @@ CREATE INDEX orders_expire_index_idx ON orders (expire_index) ; CREATE INDEX orders_get_asset_give_asset_idx ON orders (get_asset, give_asset) ; +CREATE INDEX orders_get_quantity_idx ON orders (get_quantity) + ; CREATE INDEX orders_give_asset_idx ON orders (give_asset) ; +CREATE INDEX orders_give_quantity_idx ON orders (give_quantity) + ; CREATE INDEX orders_source_give_asset_idx ON orders (source, give_asset) ; CREATE INDEX orders_status_idx ON orders (status) @@ -1817,6 +1829,8 @@ CREATE INDEX dispensers_block_index_idx ON dispensers (block_index) ; CREATE INDEX dispensers_close_block_index_status_idx ON dispensers (close_block_index, status) ; +CREATE INDEX dispensers_give_quantity_idx ON dispensers (give_quantity) + ; CREATE INDEX dispensers_give_remaining_idx ON dispensers (give_remaining) ; CREATE INDEX dispensers_last_status_tx_hash_idx ON dispensers (last_status_tx_hash) @@ -1869,6 +1883,8 @@ CREATE INDEX dispenses_block_index_idx ON dispenses (block_index) ; CREATE INDEX dispenses_destination_idx ON dispenses (destination) ; +CREATE INDEX dispenses_dispense_quantity_idx ON dispenses (dispense_quantity) + ; CREATE INDEX dispenses_dispenser_tx_hash_idx ON dispenses (dispenser_tx_hash) ; CREATE INDEX dispenses_source_idx ON dispenses (source) diff --git a/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_3_of_3.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_3_of_3.sql index 17f0467caa..b02dd63f16 100644 --- a/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_3_of_3.sql +++ b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_3_of_3.sql @@ -276,6 +276,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -348,6 +350,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -412,6 +416,8 @@ CREATE INDEX credits_calling_function_idx ON credits (calling_function) ; CREATE INDEX credits_event_idx ON credits (event) ; +CREATE INDEX credits_quantity_idx ON credits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -464,6 +470,8 @@ CREATE INDEX debits_block_index_idx ON debits (block_index) ; CREATE INDEX debits_event_idx ON debits (event) ; +CREATE INDEX debits_quantity_idx ON debits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -1008,8 +1016,12 @@ CREATE INDEX orders_expire_index_idx ON orders (expire_index) ; CREATE INDEX orders_get_asset_give_asset_idx ON orders (get_asset, give_asset) ; +CREATE INDEX orders_get_quantity_idx ON orders (get_quantity) + ; CREATE INDEX orders_give_asset_idx ON orders (give_asset) ; +CREATE INDEX orders_give_quantity_idx ON orders (give_quantity) + ; CREATE INDEX orders_source_give_asset_idx ON orders (source, give_asset) ; CREATE INDEX orders_status_idx ON orders (status) @@ -1817,6 +1829,8 @@ CREATE INDEX dispensers_block_index_idx ON dispensers (block_index) ; CREATE INDEX dispensers_close_block_index_status_idx ON dispensers (close_block_index, status) ; +CREATE INDEX dispensers_give_quantity_idx ON dispensers (give_quantity) + ; CREATE INDEX dispensers_give_remaining_idx ON dispensers (give_remaining) ; CREATE INDEX dispensers_last_status_tx_hash_idx ON dispensers (last_status_tx_hash) @@ -1869,6 +1883,8 @@ CREATE INDEX dispenses_block_index_idx ON dispenses (block_index) ; CREATE INDEX dispenses_destination_idx ON dispenses (destination) ; +CREATE INDEX dispenses_dispense_quantity_idx ON dispenses (dispense_quantity) + ; CREATE INDEX dispenses_dispenser_tx_hash_idx ON dispenses (dispenser_tx_hash) ; CREATE INDEX dispenses_source_idx ON dispenses (source) diff --git a/counterparty-core/counterpartycore/test/fixtures/scenarios/parseblock_unittest_fixture.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/parseblock_unittest_fixture.sql index e185a15344..18a46d6e72 100644 --- a/counterparty-core/counterpartycore/test/fixtures/scenarios/parseblock_unittest_fixture.sql +++ b/counterparty-core/counterpartycore/test/fixtures/scenarios/parseblock_unittest_fixture.sql @@ -728,6 +728,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -824,6 +826,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -896,6 +900,8 @@ CREATE INDEX credits_calling_function_idx ON credits (calling_function) ; CREATE INDEX credits_event_idx ON credits (event) ; +CREATE INDEX credits_quantity_idx ON credits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -964,6 +970,8 @@ CREATE INDEX debits_block_index_idx ON debits (block_index) ; CREATE INDEX debits_event_idx ON debits (event) ; +CREATE INDEX debits_quantity_idx ON debits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -2394,8 +2402,12 @@ CREATE INDEX orders_expire_index_idx ON orders (expire_index) ; CREATE INDEX orders_get_asset_give_asset_idx ON orders (get_asset, give_asset) ; +CREATE INDEX orders_get_quantity_idx ON orders (get_quantity) + ; CREATE INDEX orders_give_asset_idx ON orders (give_asset) ; +CREATE INDEX orders_give_quantity_idx ON orders (give_quantity) + ; CREATE INDEX orders_source_give_asset_idx ON orders (source, give_asset) ; CREATE INDEX orders_status_idx ON orders (status) @@ -3226,6 +3238,8 @@ CREATE INDEX dispensers_block_index_idx ON dispensers (block_index) ; CREATE INDEX dispensers_close_block_index_status_idx ON dispensers (close_block_index, status) ; +CREATE INDEX dispensers_give_quantity_idx ON dispensers (give_quantity) + ; CREATE INDEX dispensers_give_remaining_idx ON dispensers (give_remaining) ; CREATE INDEX dispensers_last_status_tx_hash_idx ON dispensers (last_status_tx_hash) @@ -3278,6 +3292,8 @@ CREATE INDEX dispenses_block_index_idx ON dispenses (block_index) ; CREATE INDEX dispenses_destination_idx ON dispenses (destination) ; +CREATE INDEX dispenses_dispense_quantity_idx ON dispenses (dispense_quantity) + ; CREATE INDEX dispenses_dispenser_tx_hash_idx ON dispenses (dispenser_tx_hash) ; CREATE INDEX dispenses_source_idx ON dispenses (source) diff --git a/counterparty-core/counterpartycore/test/fixtures/scenarios/simplesig.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/simplesig.sql index 068f52771b..e322b4d14a 100644 --- a/counterparty-core/counterpartycore/test/fixtures/scenarios/simplesig.sql +++ b/counterparty-core/counterpartycore/test/fixtures/scenarios/simplesig.sql @@ -276,6 +276,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -348,6 +350,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -412,6 +416,8 @@ CREATE INDEX credits_calling_function_idx ON credits (calling_function) ; CREATE INDEX credits_event_idx ON credits (event) ; +CREATE INDEX credits_quantity_idx ON credits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -464,6 +470,8 @@ CREATE INDEX debits_block_index_idx ON debits (block_index) ; CREATE INDEX debits_event_idx ON debits (event) ; +CREATE INDEX debits_quantity_idx ON debits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -1008,8 +1016,12 @@ CREATE INDEX orders_expire_index_idx ON orders (expire_index) ; CREATE INDEX orders_get_asset_give_asset_idx ON orders (get_asset, give_asset) ; +CREATE INDEX orders_get_quantity_idx ON orders (get_quantity) + ; CREATE INDEX orders_give_asset_idx ON orders (give_asset) ; +CREATE INDEX orders_give_quantity_idx ON orders (give_quantity) + ; CREATE INDEX orders_source_give_asset_idx ON orders (source, give_asset) ; CREATE INDEX orders_status_idx ON orders (status) @@ -1817,6 +1829,8 @@ CREATE INDEX dispensers_block_index_idx ON dispensers (block_index) ; CREATE INDEX dispensers_close_block_index_status_idx ON dispensers (close_block_index, status) ; +CREATE INDEX dispensers_give_quantity_idx ON dispensers (give_quantity) + ; CREATE INDEX dispensers_give_remaining_idx ON dispensers (give_remaining) ; CREATE INDEX dispensers_last_status_tx_hash_idx ON dispensers (last_status_tx_hash) @@ -1869,6 +1883,8 @@ CREATE INDEX dispenses_block_index_idx ON dispenses (block_index) ; CREATE INDEX dispenses_destination_idx ON dispenses (destination) ; +CREATE INDEX dispenses_dispense_quantity_idx ON dispenses (dispense_quantity) + ; CREATE INDEX dispenses_dispenser_tx_hash_idx ON dispenses (dispenser_tx_hash) ; CREATE INDEX dispenses_source_idx ON dispenses (source) diff --git a/counterparty-core/counterpartycore/test/fixtures/scenarios/unittest_fixture.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/unittest_fixture.sql index 9fce2249b0..09ea3b632b 100644 --- a/counterparty-core/counterpartycore/test/fixtures/scenarios/unittest_fixture.sql +++ b/counterparty-core/counterpartycore/test/fixtures/scenarios/unittest_fixture.sql @@ -727,6 +727,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -823,6 +825,8 @@ CREATE INDEX balances_asset_idx ON balances (asset) ; CREATE INDEX balances_block_index_idx ON balances (block_index) ; +CREATE INDEX balances_quantity_idx ON balances (quantity) + ; CREATE TRIGGER block_update_balances BEFORE UPDATE ON balances BEGIN SELECT RAISE(FAIL, "UPDATES NOT ALLOWED"); @@ -895,6 +899,8 @@ CREATE INDEX credits_calling_function_idx ON credits (calling_function) ; CREATE INDEX credits_event_idx ON credits (event) ; +CREATE INDEX credits_quantity_idx ON credits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -963,6 +969,8 @@ CREATE INDEX debits_block_index_idx ON debits (block_index) ; CREATE INDEX debits_event_idx ON debits (event) ; +CREATE INDEX debits_quantity_idx ON debits (quantity) + ; COMMIT TRANSACTION; PRAGMA page_size=4096; @@ -2392,8 +2400,12 @@ CREATE INDEX orders_expire_index_idx ON orders (expire_index) ; CREATE INDEX orders_get_asset_give_asset_idx ON orders (get_asset, give_asset) ; +CREATE INDEX orders_get_quantity_idx ON orders (get_quantity) + ; CREATE INDEX orders_give_asset_idx ON orders (give_asset) ; +CREATE INDEX orders_give_quantity_idx ON orders (give_quantity) + ; CREATE INDEX orders_source_give_asset_idx ON orders (source, give_asset) ; CREATE INDEX orders_status_idx ON orders (status) @@ -3224,6 +3236,8 @@ CREATE INDEX dispensers_block_index_idx ON dispensers (block_index) ; CREATE INDEX dispensers_close_block_index_status_idx ON dispensers (close_block_index, status) ; +CREATE INDEX dispensers_give_quantity_idx ON dispensers (give_quantity) + ; CREATE INDEX dispensers_give_remaining_idx ON dispensers (give_remaining) ; CREATE INDEX dispensers_last_status_tx_hash_idx ON dispensers (last_status_tx_hash) @@ -3276,6 +3290,8 @@ CREATE INDEX dispenses_block_index_idx ON dispenses (block_index) ; CREATE INDEX dispenses_destination_idx ON dispenses (destination) ; +CREATE INDEX dispenses_dispense_quantity_idx ON dispenses (dispense_quantity) + ; CREATE INDEX dispenses_dispenser_tx_hash_idx ON dispenses (dispenser_tx_hash) ; CREATE INDEX dispenses_source_idx ON dispenses (source) From 8fb50111af1b2a1fd299190de7628e3d4cef3da2 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 5 Jun 2024 07:36:11 +0000 Subject: [PATCH 6/6] Fix typo --- counterparty-core/counterpartycore/lib/api/util.py | 1 + counterparty-core/counterpartycore/lib/script.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/counterparty-core/counterpartycore/lib/api/util.py b/counterparty-core/counterpartycore/lib/api/util.py index 2f722f28d3..e8942cb049 100644 --- a/counterparty-core/counterpartycore/lib/api/util.py +++ b/counterparty-core/counterpartycore/lib/api/util.py @@ -56,6 +56,7 @@ def healthz(db, check_type: str = "heavy"): healthz_light(db) healthz_heavy(db) except Exception as e: + # logger.exception(e) logger.error(f"Health check failed: {e}") return False return True diff --git a/counterparty-core/counterpartycore/lib/script.py b/counterparty-core/counterpartycore/lib/script.py index 735b16b55f..4796dd36eb 100644 --- a/counterparty-core/counterpartycore/lib/script.py +++ b/counterparty-core/counterpartycore/lib/script.py @@ -534,6 +534,6 @@ def ensure_script_pub_key_for_inputs(coins): txid = coin["txid"] for vout in txs[txid]["vout"]: if vout["n"] == coin["vout"]: - coin["script_pub_key"] = vout["script_pub_key"]["hex"] + coin["script_pub_key"] = vout["scriptPubKey"]["hex"] return coins