From 708cb3a70de593a48317084b29afa830d2d04ae2 Mon Sep 17 00:00:00 2001 From: Jason Brill Date: Tue, 5 Nov 2024 13:47:51 -0500 Subject: [PATCH] tapdb: universe query rewrites --- ...erse_optimization_indexes_queries.down.sql | 41 +- ...iverse_optimization_indexes_queries.up.sql | 96 ++- tapdb/sqlc/queries/universe.sql | 113 ++- tapdb/sqlc/universe.sql.go | 113 ++- tapdb/universe_perf_test.go | 652 +++++++++++++----- 5 files changed, 629 insertions(+), 386 deletions(-) diff --git a/tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.down.sql b/tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.down.sql index 71a7c7b36..57bbb31c4 100644 --- a/tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.down.sql +++ b/tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.down.sql @@ -1,21 +1,24 @@ +-- Drop indices on universe_roots +DROP INDEX IF EXISTS idx_universe_roots_asset_group_proof; +DROP INDEX IF EXISTS idx_universe_roots_proof_type_issuance; --- Drop in reverse order of dependencies -DROP INDEX IF EXISTS idx_multiverse_leaves_composite; -DROP INDEX IF EXISTS idx_federation_sync_composite; -DROP INDEX IF EXISTS idx_universe_events_root_type; -DROP INDEX IF EXISTS idx_universe_events_stats; -DROP INDEX IF EXISTS idx_mssmt_nodes_key_lookup; -DROP INDEX IF EXISTS idx_mssmt_nodes_namespace; -DROP INDEX IF EXISTS idx_universe_leaves_sort; -DROP INDEX IF EXISTS idx_universe_leaves_lookup; -DROP INDEX IF EXISTS idx_universe_roots_issuance; -DROP INDEX IF EXISTS idx_universe_roots_namespace; +-- Drop indices on universe_events +DROP INDEX IF EXISTS idx_universe_events_type_counts; +DROP INDEX IF EXISTS idx_universe_events_universe_root_id; +DROP INDEX IF EXISTS idx_universe_events_sync; --- Update statistics -ANALYZE universe_roots; -ANALYZE universe_leaves; -ANALYZE mssmt_nodes; -ANALYZE universe_events; -ANALYZE federation_proof_sync_log; -ANALYZE multiverse_leaves; -ANALYZE multiverse_roots; \ No newline at end of file +-- Drop indices on tables underlying key_group_info_view +DROP INDEX IF EXISTS idx_asset_group_witnesses_gen_asset_id; + +-- Drop indices on mssmt_roots +DROP INDEX IF EXISTS idx_mssmt_roots_hash_namespace; + +-- Drop indices on genesis_assets +DROP INDEX IF EXISTS idx_genesis_assets_asset_id; +DROP INDEX IF EXISTS idx_genesis_assets_asset_tag; +DROP INDEX IF EXISTS idx_genesis_assets_asset_type; + +-- Drop indices on universe_leaves +DROP INDEX IF EXISTS idx_universe_leaves_universe_root_id; +DROP INDEX IF EXISTS idx_universe_leaves_asset_genesis_id; +DROP INDEX IF EXISTS idx_universe_leaves_leaf_node_key_namespace; \ No newline at end of file diff --git a/tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.up.sql b/tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.up.sql index c10a33dbe..72f24a022 100644 --- a/tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.up.sql +++ b/tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.up.sql @@ -1,53 +1,43 @@ --- Core universe_roots indexes -CREATE INDEX IF NOT EXISTS idx_universe_roots_namespace -ON universe_roots(namespace_root, id, asset_id, group_key, proof_type); - -CREATE INDEX IF NOT EXISTS idx_universe_roots_issuance -ON universe_roots(proof_type, group_key, id, asset_id, namespace_root); - --- Universe leaves optimization -CREATE INDEX IF NOT EXISTS idx_universe_leaves_lookup -ON universe_leaves( - leaf_node_namespace, minting_point, script_key_bytes, - id, leaf_node_key, universe_root_id, asset_genesis_id -); - -CREATE INDEX IF NOT EXISTS idx_universe_leaves_sort -ON universe_leaves(leaf_node_namespace, id, minting_point, script_key_bytes); - --- MSSMT nodes optimization -CREATE INDEX IF NOT EXISTS idx_mssmt_nodes_namespace -ON mssmt_nodes(namespace, hash_key, key, value, sum); - -CREATE INDEX IF NOT EXISTS idx_mssmt_nodes_key_lookup -ON mssmt_nodes(key, namespace, hash_key, value, sum); - --- Universe events optimization -CREATE INDEX IF NOT EXISTS idx_universe_events_stats -ON universe_events(event_type, event_timestamp); - -CREATE INDEX IF NOT EXISTS idx_universe_events_root_type -ON universe_events(universe_root_id, event_type); - --- Federation sync log optimization -CREATE INDEX IF NOT EXISTS idx_federation_sync_composite -ON federation_proof_sync_log( - sync_direction, proof_leaf_id, universe_root_id, servers_id, - id, status, timestamp, attempt_counter -); - --- Multiverse optimization -CREATE INDEX IF NOT EXISTS idx_multiverse_leaves_composite -ON multiverse_leaves( - multiverse_root_id, leaf_node_namespace, - asset_id, group_key, leaf_node_key -); - --- Analyze existing tables -ANALYZE universe_roots; -ANALYZE universe_leaves; -ANALYZE mssmt_nodes; -ANALYZE universe_events; -ANALYZE federation_proof_sync_log; -ANALYZE multiverse_leaves; -ANALYZE multiverse_roots; \ No newline at end of file +-- Composite index supporting joins and GROUP BY +CREATE INDEX IF NOT EXISTS idx_universe_roots_asset_group_proof +ON universe_roots (asset_id, group_key, proof_type); + +-- Partial index for proof_type = 'issuance' +CREATE INDEX IF NOT EXISTS idx_universe_roots_proof_type_issuance +ON universe_roots (proof_type); + +-- Composite index supporting event_type and universe_root_id +CREATE INDEX IF NOT EXISTS idx_universe_events_type_counts +ON universe_events (event_type, universe_root_id); + +-- Separate index on universe_root_id +CREATE INDEX IF NOT EXISTS idx_universe_events_universe_root_id +ON universe_events (universe_root_id); + +-- Partial index for event_type = 'SYNC' +CREATE INDEX IF NOT EXISTS idx_universe_events_sync +ON universe_events (event_type); + +-- Indices on tables underlying key_group_info_view +CREATE INDEX IF NOT EXISTS idx_asset_group_witnesses_gen_asset_id +ON asset_group_witnesses (gen_asset_id); + +-- Indices on mssmt_roots +CREATE INDEX IF NOT EXISTS idx_mssmt_roots_hash_namespace +ON mssmt_roots (root_hash, namespace); + +-- Indices on genesis_assets +CREATE INDEX IF NOT EXISTS idx_genesis_assets_asset_id +ON genesis_assets (asset_id); +CREATE INDEX IF NOT EXISTS idx_genesis_assets_asset_tag +ON genesis_assets (asset_tag); +CREATE INDEX IF NOT EXISTS idx_genesis_assets_asset_type +ON genesis_assets (asset_type); + +-- Indices on universe_leaves +CREATE INDEX IF NOT EXISTS idx_universe_leaves_universe_root_id +ON universe_leaves (universe_root_id); +CREATE INDEX IF NOT EXISTS idx_universe_leaves_asset_genesis_id +ON universe_leaves (asset_genesis_id); +CREATE INDEX IF NOT EXISTS idx_universe_leaves_leaf_node_key_namespace +ON universe_leaves (leaf_node_key, leaf_node_namespace); \ No newline at end of file diff --git a/tapdb/sqlc/queries/universe.sql b/tapdb/sqlc/queries/universe.sql index 787162906..53e3a3550 100644 --- a/tapdb/sqlc/queries/universe.sql +++ b/tapdb/sqlc/queries/universe.sql @@ -6,10 +6,10 @@ FROM universe_roots JOIN mssmt_roots ON universe_roots.namespace_root = mssmt_roots.namespace JOIN mssmt_nodes - ON mssmt_nodes.hash_key = mssmt_roots.root_hash AND - mssmt_nodes.namespace = mssmt_roots.namespace + ON mssmt_nodes.hash_key = mssmt_roots.root_hash + AND mssmt_nodes.namespace = mssmt_roots.namespace JOIN genesis_assets - ON genesis_assets.asset_id = universe_roots.asset_id + ON genesis_assets.asset_id = universe_roots.asset_id WHERE mssmt_nodes.namespace = @namespace; -- name: UpsertUniverseRoot :one @@ -30,7 +30,7 @@ WITH root_id AS ( WHERE namespace_root = @namespace_root ) DELETE FROM universe_events -WHERE universe_root_id = (SELECT id from root_id); +WHERE universe_root_id = (SELECT id FROM root_id); -- name: DeleteUniverseRoot :exec DELETE FROM universe_roots @@ -54,25 +54,23 @@ DELETE FROM universe_leaves WHERE leaf_node_namespace = @namespace; -- name: QueryUniverseLeaves :many -SELECT leaves.script_key_bytes, gen.gen_asset_id, nodes.value genesis_proof, - nodes.sum sum_amt, gen.asset_id -FROM universe_leaves leaves -JOIN mssmt_nodes nodes - ON leaves.leaf_node_key = nodes.key AND - leaves.leaf_node_namespace = nodes.namespace -JOIN genesis_info_view gen +SELECT leaves.script_key_bytes, gen.gen_asset_id, nodes.value AS genesis_proof, + nodes.sum AS sum_amt, gen.asset_id +FROM universe_leaves AS leaves +JOIN mssmt_nodes AS nodes + ON leaves.leaf_node_key = nodes.key + AND leaves.leaf_node_namespace = nodes.namespace +JOIN genesis_info_view AS gen ON leaves.asset_genesis_id = gen.gen_asset_id WHERE leaves.leaf_node_namespace = @namespace - AND - (leaves.minting_point = sqlc.narg('minting_point_bytes') OR - sqlc.narg('minting_point_bytes') IS NULL) - AND - (leaves.script_key_bytes = sqlc.narg('script_key_bytes') OR - sqlc.narg('script_key_bytes') IS NULL); + AND (leaves.minting_point = sqlc.narg('minting_point_bytes') OR + sqlc.narg('minting_point_bytes') IS NULL) + AND (leaves.script_key_bytes = sqlc.narg('script_key_bytes') OR + sqlc.narg('script_key_bytes') IS NULL); -- name: FetchUniverseKeys :many SELECT leaves.minting_point, leaves.script_key_bytes -FROM universe_leaves leaves +FROM universe_leaves AS leaves WHERE leaves.leaf_node_namespace = @namespace ORDER BY CASE WHEN sqlc.narg('sort_direction') = 0 THEN leaves.id END ASC, @@ -84,14 +82,14 @@ SELECT * FROM universe_leaves; -- name: UniverseRoots :many SELECT universe_roots.asset_id, group_key, proof_type, - mssmt_roots.root_hash root_hash, mssmt_nodes.sum root_sum, - genesis_assets.asset_tag asset_name + mssmt_roots.root_hash AS root_hash, mssmt_nodes.sum AS root_sum, + genesis_assets.asset_tag AS asset_name FROM universe_roots JOIN mssmt_roots ON universe_roots.namespace_root = mssmt_roots.namespace JOIN mssmt_nodes - ON mssmt_nodes.hash_key = mssmt_roots.root_hash AND - mssmt_nodes.namespace = mssmt_roots.namespace + ON mssmt_nodes.hash_key = mssmt_roots.root_hash + AND mssmt_nodes.namespace = mssmt_roots.namespace JOIN genesis_assets ON genesis_assets.asset_id = universe_roots.asset_id ORDER BY @@ -329,8 +327,8 @@ SELECT SUM(CASE WHEN event_type = 'SYNC' THEN 1 ELSE 0 END) AS sync_events, SUM(CASE WHEN event_type = 'NEW_PROOF' THEN 1 ELSE 0 END) AS new_proof_events FROM universe_events -WHERE event_type IN ('SYNC', 'NEW_PROOF') AND - event_timestamp >= @start_time AND event_timestamp <= @end_time +WHERE event_type IN ('SYNC', 'NEW_PROOF') + AND event_timestamp BETWEEN @start_time AND @end_time GROUP BY day ORDER BY day; @@ -367,7 +365,7 @@ FROM federation_uni_sync_config ORDER BY group_key NULLS LAST, asset_id NULLS LAST, proof_type; -- name: UpsertFederationProofSyncLog :one -INSERT INTO federation_proof_sync_log as log ( +INSERT INTO federation_proof_sync_log AS log ( status, timestamp, sync_direction, proof_leaf_id, universe_root_id, servers_id ) VALUES ( @@ -401,7 +399,7 @@ DO UPDATE SET timestamp = EXCLUDED.timestamp, -- Increment the attempt counter. attempt_counter = CASE - WHEN @bump_sync_attempt_counter = true THEN log.attempt_counter + 1 + WHEN @bump_sync_attempt_counter = TRUE THEN log.attempt_counter + 1 ELSE log.attempt_counter END RETURNING id; @@ -409,58 +407,39 @@ RETURNING id; -- name: QueryFederationProofSyncLog :many SELECT log.id, status, timestamp, sync_direction, attempt_counter, - -- Select fields from the universe_servers table. - server.id as server_id, + server.id AS server_id, server.server_host, - -- Select universe leaf related fields. - leaf.minting_point as leaf_minting_point_bytes, - leaf.script_key_bytes as leaf_script_key_bytes, - mssmt_node.value as leaf_genesis_proof, - genesis.gen_asset_id as leaf_gen_asset_id, - genesis.asset_id as leaf_asset_id, - + leaf.minting_point AS leaf_minting_point_bytes, + leaf.script_key_bytes AS leaf_script_key_bytes, + mssmt_node.value AS leaf_genesis_proof, + genesis.gen_asset_id AS leaf_gen_asset_id, + genesis.asset_id AS leaf_asset_id, -- Select fields from the universe_roots table. - root.asset_id as uni_asset_id, - root.group_key as uni_group_key, - root.proof_type as uni_proof_type - -FROM federation_proof_sync_log as log - -JOIN universe_leaves as leaf + root.asset_id AS uni_asset_id, + root.group_key AS uni_group_key, + root.proof_type AS uni_proof_type +FROM federation_proof_sync_log AS log +JOIN universe_leaves AS leaf ON leaf.id = log.proof_leaf_id - -- Join on mssmt_nodes to get leaf related fields. -JOIN mssmt_nodes mssmt_node - ON leaf.leaf_node_key = mssmt_node.key AND - leaf.leaf_node_namespace = mssmt_node.namespace - +JOIN mssmt_nodes AS mssmt_node + ON leaf.leaf_node_key = mssmt_node.key + AND leaf.leaf_node_namespace = mssmt_node.namespace -- Join on genesis_info_view to get leaf related fields. -JOIN genesis_info_view genesis +JOIN genesis_info_view AS genesis ON leaf.asset_genesis_id = genesis.gen_asset_id - -JOIN universe_servers as server +JOIN universe_servers AS server ON server.id = log.servers_id - -JOIN universe_roots as root +JOIN universe_roots AS root ON root.id = log.universe_root_id - -WHERE (log.sync_direction = sqlc.narg('sync_direction') - OR sqlc.narg('sync_direction') IS NULL) - AND - (log.status = sqlc.narg('status') OR sqlc.narg('status') IS NULL) - AND - +WHERE (log.sync_direction = sqlc.narg('sync_direction') OR sqlc.narg('sync_direction') IS NULL) + AND (log.status = sqlc.narg('status') OR sqlc.narg('status') IS NULL) -- Universe leaves WHERE clauses. - (leaf.leaf_node_namespace = sqlc.narg('leaf_namespace') - OR sqlc.narg('leaf_namespace') IS NULL) - AND - (leaf.minting_point = sqlc.narg('leaf_minting_point_bytes') - OR sqlc.narg('leaf_minting_point_bytes') IS NULL) - AND - (leaf.script_key_bytes = sqlc.narg('leaf_script_key_bytes') - OR sqlc.narg('leaf_script_key_bytes') IS NULL); + AND (leaf.leaf_node_namespace = sqlc.narg('leaf_namespace') OR sqlc.narg('leaf_namespace') IS NULL) + AND (leaf.minting_point = sqlc.narg('leaf_minting_point_bytes') OR sqlc.narg('leaf_minting_point_bytes') IS NULL) + AND (leaf.script_key_bytes = sqlc.narg('leaf_script_key_bytes') OR sqlc.narg('leaf_script_key_bytes') IS NULL); -- name: DeleteFederationProofSyncLog :exec WITH selected_server_id AS ( diff --git a/tapdb/sqlc/universe.sql.go b/tapdb/sqlc/universe.sql.go index 9049862b7..7a831cba6 100644 --- a/tapdb/sqlc/universe.sql.go +++ b/tapdb/sqlc/universe.sql.go @@ -71,7 +71,7 @@ WITH root_id AS ( WHERE namespace_root = $1 ) DELETE FROM universe_events -WHERE universe_root_id = (SELECT id from root_id) +WHERE universe_root_id = (SELECT id FROM root_id) ` func (q *Queries) DeleteUniverseEvents(ctx context.Context, namespaceRoot string) error { @@ -140,7 +140,7 @@ func (q *Queries) FetchMultiverseRoot(ctx context.Context, namespaceRoot string) const fetchUniverseKeys = `-- name: FetchUniverseKeys :many SELECT leaves.minting_point, leaves.script_key_bytes -FROM universe_leaves leaves +FROM universe_leaves AS leaves WHERE leaves.leaf_node_namespace = $1 ORDER BY CASE WHEN $2 = 0 THEN leaves.id END ASC, @@ -196,10 +196,10 @@ FROM universe_roots JOIN mssmt_roots ON universe_roots.namespace_root = mssmt_roots.namespace JOIN mssmt_nodes - ON mssmt_nodes.hash_key = mssmt_roots.root_hash AND - mssmt_nodes.namespace = mssmt_roots.namespace + ON mssmt_nodes.hash_key = mssmt_roots.root_hash + AND mssmt_nodes.namespace = mssmt_roots.namespace JOIN genesis_assets - ON genesis_assets.asset_id = universe_roots.asset_id + ON genesis_assets.asset_id = universe_roots.asset_id WHERE mssmt_nodes.namespace = $1 ` @@ -364,8 +364,8 @@ SELECT SUM(CASE WHEN event_type = 'SYNC' THEN 1 ELSE 0 END) AS sync_events, SUM(CASE WHEN event_type = 'NEW_PROOF' THEN 1 ELSE 0 END) AS new_proof_events FROM universe_events -WHERE event_type IN ('SYNC', 'NEW_PROOF') AND - event_timestamp >= $1 AND event_timestamp <= $2 +WHERE event_type IN ('SYNC', 'NEW_PROOF') + AND event_timestamp BETWEEN $1 AND $2 GROUP BY day ORDER BY day ` @@ -482,56 +482,37 @@ func (q *Queries) QueryFederationGlobalSyncConfigs(ctx context.Context) ([]Feder const queryFederationProofSyncLog = `-- name: QueryFederationProofSyncLog :many SELECT log.id, status, timestamp, sync_direction, attempt_counter, - -- Select fields from the universe_servers table. - server.id as server_id, + server.id AS server_id, server.server_host, - -- Select universe leaf related fields. - leaf.minting_point as leaf_minting_point_bytes, - leaf.script_key_bytes as leaf_script_key_bytes, - mssmt_node.value as leaf_genesis_proof, - genesis.gen_asset_id as leaf_gen_asset_id, - genesis.asset_id as leaf_asset_id, - + leaf.minting_point AS leaf_minting_point_bytes, + leaf.script_key_bytes AS leaf_script_key_bytes, + mssmt_node.value AS leaf_genesis_proof, + genesis.gen_asset_id AS leaf_gen_asset_id, + genesis.asset_id AS leaf_asset_id, -- Select fields from the universe_roots table. - root.asset_id as uni_asset_id, - root.group_key as uni_group_key, - root.proof_type as uni_proof_type - -FROM federation_proof_sync_log as log - -JOIN universe_leaves as leaf + root.asset_id AS uni_asset_id, + root.group_key AS uni_group_key, + root.proof_type AS uni_proof_type +FROM federation_proof_sync_log AS log +JOIN universe_leaves AS leaf ON leaf.id = log.proof_leaf_id - -JOIN mssmt_nodes mssmt_node - ON leaf.leaf_node_key = mssmt_node.key AND - leaf.leaf_node_namespace = mssmt_node.namespace - -JOIN genesis_info_view genesis +JOIN mssmt_nodes AS mssmt_node + ON leaf.leaf_node_key = mssmt_node.key + AND leaf.leaf_node_namespace = mssmt_node.namespace +JOIN genesis_info_view AS genesis ON leaf.asset_genesis_id = genesis.gen_asset_id - -JOIN universe_servers as server +JOIN universe_servers AS server ON server.id = log.servers_id - -JOIN universe_roots as root +JOIN universe_roots AS root ON root.id = log.universe_root_id - -WHERE (log.sync_direction = $1 - OR $1 IS NULL) - AND - (log.status = $2 OR $2 IS NULL) - AND - +WHERE (log.sync_direction = $1 OR $1 IS NULL) + AND (log.status = $2 OR $2 IS NULL) -- Universe leaves WHERE clauses. - (leaf.leaf_node_namespace = $3 - OR $3 IS NULL) - AND - (leaf.minting_point = $4 - OR $4 IS NULL) - AND - (leaf.script_key_bytes = $5 - OR $5 IS NULL) + AND (leaf.leaf_node_namespace = $3 OR $3 IS NULL) + AND (leaf.minting_point = $4 OR $4 IS NULL) + AND (leaf.script_key_bytes = $5 OR $5 IS NULL) ` type QueryFederationProofSyncLogParams struct { @@ -865,21 +846,19 @@ func (q *Queries) QueryUniverseAssetStats(ctx context.Context, arg QueryUniverse } const queryUniverseLeaves = `-- name: QueryUniverseLeaves :many -SELECT leaves.script_key_bytes, gen.gen_asset_id, nodes.value genesis_proof, - nodes.sum sum_amt, gen.asset_id -FROM universe_leaves leaves -JOIN mssmt_nodes nodes - ON leaves.leaf_node_key = nodes.key AND - leaves.leaf_node_namespace = nodes.namespace -JOIN genesis_info_view gen +SELECT leaves.script_key_bytes, gen.gen_asset_id, nodes.value AS genesis_proof, + nodes.sum AS sum_amt, gen.asset_id +FROM universe_leaves AS leaves +JOIN mssmt_nodes AS nodes + ON leaves.leaf_node_key = nodes.key + AND leaves.leaf_node_namespace = nodes.namespace +JOIN genesis_info_view AS gen ON leaves.asset_genesis_id = gen.gen_asset_id WHERE leaves.leaf_node_namespace = $1 - AND - (leaves.minting_point = $2 OR - $2 IS NULL) - AND - (leaves.script_key_bytes = $3 OR - $3 IS NULL) + AND (leaves.minting_point = $2 OR + $2 IS NULL) + AND (leaves.script_key_bytes = $3 OR + $3 IS NULL) ` type QueryUniverseLeavesParams struct { @@ -1058,14 +1037,14 @@ func (q *Queries) UniverseLeaves(ctx context.Context) ([]UniverseLeafe, error) { const universeRoots = `-- name: UniverseRoots :many SELECT universe_roots.asset_id, group_key, proof_type, - mssmt_roots.root_hash root_hash, mssmt_nodes.sum root_sum, - genesis_assets.asset_tag asset_name + mssmt_roots.root_hash AS root_hash, mssmt_nodes.sum AS root_sum, + genesis_assets.asset_tag AS asset_name FROM universe_roots JOIN mssmt_roots ON universe_roots.namespace_root = mssmt_roots.namespace JOIN mssmt_nodes - ON mssmt_nodes.hash_key = mssmt_roots.root_hash AND - mssmt_nodes.namespace = mssmt_roots.namespace + ON mssmt_nodes.hash_key = mssmt_roots.root_hash + AND mssmt_nodes.namespace = mssmt_roots.namespace JOIN genesis_assets ON genesis_assets.asset_id = universe_roots.asset_id ORDER BY @@ -1142,7 +1121,7 @@ func (q *Queries) UpsertFederationGlobalSyncConfig(ctx context.Context, arg Upse } const upsertFederationProofSyncLog = `-- name: UpsertFederationProofSyncLog :one -INSERT INTO federation_proof_sync_log as log ( +INSERT INTO federation_proof_sync_log AS log ( status, timestamp, sync_direction, proof_leaf_id, universe_root_id, servers_id ) VALUES ( @@ -1176,7 +1155,7 @@ DO UPDATE SET timestamp = EXCLUDED.timestamp, -- Increment the attempt counter. attempt_counter = CASE - WHEN $9 = true THEN log.attempt_counter + 1 + WHEN $9 = TRUE THEN log.attempt_counter + 1 ELSE log.attempt_counter END RETURNING id diff --git a/tapdb/universe_perf_test.go b/tapdb/universe_perf_test.go index 7bbcab0d0..087b552aa 100644 --- a/tapdb/universe_perf_test.go +++ b/tapdb/universe_perf_test.go @@ -2,11 +2,15 @@ package tapdb import ( "context" + "database/sql" "fmt" + "math/rand" "sort" "testing" "time" + "github.com/lightninglabs/taproot-assets/tapdb/sqlc" + "github.com/lightninglabs/taproot-assets/universe" "github.com/lightningnetwork/lnd/clock" "github.com/stretchr/testify/require" ) @@ -20,97 +24,80 @@ type dbSizeStats struct { totalSize int64 // dataSize + indexSize } -// measureDBSize gets table and index sizes from SQLite -func measureDBSize(t *testing.T, db *BaseDB) map[string]*dbSizeStats { +// measureDBSize fetches the size of tables and indexes for PostgreSQL and SQLite databases. +func measureDBSize(t *testing.T, db *BaseDB, isPostgres bool) map[string]*dbSizeStats { stats := make(map[string]*dbSizeStats) - - // First get list of all tables - rows, err := db.Query(` - SELECT DISTINCT - tbl_name, - type - FROM sqlite_master - WHERE type='table' - `) - require.NoError(t, err) - defer rows.Close() - - tables := make([]string, 0) - for rows.Next() { - var name, tblType string - err := rows.Scan(&name, &tblType) + var rows *sql.Rows + var err error + + if isPostgres { + // Fetch table list and their size details for PostgreSQL + rows, err = db.Query(` + SELECT tablename, pg_table_size(tablename::regclass) AS data_size, + pg_indexes_size(tablename::regclass) AS index_size + FROM pg_catalog.pg_tables + WHERE schemaname != 'pg_catalog' AND schemaname != 'information_schema' + `) require.NoError(t, err) - tables = append(tables, name) - - stats[name] = &dbSizeStats{ - tableName: name, - } - } - - // For each table, get its stats - for _, tableName := range tables { - // Get row count using COUNT(*) - var rowCount int64 - err = db.QueryRow(fmt.Sprintf(`SELECT COUNT(*) FROM main.%q`, tableName)).Scan(&rowCount) - if err != nil { - t.Logf("Skipping row count for %s: %v", tableName, err) - continue + defer rows.Close() + + for rows.Next() { + var tableName string + var dataSize, indexSize int64 + err := rows.Scan(&tableName, &dataSize, &indexSize) + require.NoError(t, err) + + stats[tableName] = &dbSizeStats{ + tableName: tableName, + dataSize: dataSize, + indexSize: indexSize, + totalSize: dataSize + indexSize, + } } - stats[tableName].rowCount = rowCount - - // Get table size - var pageCount int64 - err = db.QueryRow(` - SELECT COUNT(*) - FROM dbstat - WHERE name = ? - `, tableName).Scan(&pageCount) - if err != nil { - t.Logf("Skipping size stats for %s: %v", tableName, err) - continue + } else { + // SQLite: Fetch tables, calculate data size, and index size from dbstat + rows, err = db.Query(` + SELECT DISTINCT tbl_name + FROM sqlite_master + WHERE type='table' + `) + require.NoError(t, err) + defer rows.Close() + + tables := make([]string, 0) + for rows.Next() { + var name string + err := rows.Scan(&name) + require.NoError(t, err) + tables = append(tables, name) + stats[name] = &dbSizeStats{ + tableName: name, + } } - // Get page size (constant for the database) - var pageSize int64 - err = db.QueryRow(`PRAGMA page_size`).Scan(&pageSize) - require.NoError(t, err) + // Calculate sizes for each table using dbstat for SQLite + for _, tableName := range tables { + var pageCount, pageSize, indexSize int64 - stats[tableName].dataSize = pageCount * pageSize - } + err = db.QueryRow(`SELECT COUNT(*) FROM dbstat WHERE name = ?`, tableName).Scan(&pageCount) + if err != nil { + t.Logf("Skipping size stats for %s: %v", tableName, err) + continue + } + err = db.QueryRow(`PRAGMA page_size`).Scan(&pageSize) + require.NoError(t, err) - // Get list of indices and their sizes - rows, err = db.Query(` - SELECT - m.tbl_name as table_name, - m.name as index_name, - (SELECT COUNT(*) FROM dbstat WHERE name = m.name) as page_count, - (SELECT page_size FROM pragma_page_size) as page_size - FROM sqlite_master m - WHERE m.type = 'index' - `) - require.NoError(t, err) - defer rows.Close() - - for rows.Next() { - var ( - tableName string - indexName string - pageCount int64 - pageSize int64 - ) - err := rows.Scan(&tableName, &indexName, &pageCount, &pageSize) - if err != nil { - t.Logf("Skipping index stat: %v", err) - continue - } + err = db.QueryRow(`SELECT COALESCE(SUM(pgsize), 0) FROM dbstat WHERE name = ? AND pagetype = 'index'`, tableName).Scan(&indexSize) + if err != nil { + t.Logf("Skipping index size for %s: %v", tableName, err) + continue + } - if stat, ok := stats[tableName]; ok { - indexSize := pageCount * pageSize - stat.indexSize += indexSize - stat.totalSize = stat.dataSize + stat.indexSize + stats[tableName].dataSize = pageCount * pageSize + stats[tableName].indexSize = indexSize + stats[tableName].totalSize = stats[tableName].dataSize + stats[tableName].indexSize } } - return stats } @@ -181,8 +168,7 @@ func formatSize(bytes int64) string { float64(bytes)/float64(div), "KMGTPE"[exp]) } -// TestUniverseIndexPerformance tests that our new indices improve query -// performance by comparing performance with and without indices. +// TestUniverseIndexPerformance tests query performance with the specified indices. func TestUniverseIndexPerformance(t *testing.T) { if testing.Short() { t.Skip("skipping index performance test in short mode") @@ -191,11 +177,11 @@ func TestUniverseIndexPerformance(t *testing.T) { t.Parallel() const ( - numAssets = 25 + numAssets = 50 numLeavesPerTree = 10 - numEventsPerAsset = 15 - numQueries = 10 - batchSize = 5 + numEventsPerAsset = 25 + numQueries = 50 + batchSize = 5 ) type queryStats struct { @@ -206,31 +192,80 @@ func TestUniverseIndexPerformance(t *testing.T) { } testResults := make(map[string]*queryStats) - // Run without indices first, then with indices + // Function to run performance tests with and without indices runTest := func(withIndices bool) { t.Run(fmt.Sprintf("indices=%v", withIndices), func(t *testing.T) { - ctx, cancel := context.WithTimeout( - context.Background(), time.Minute, - ) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() db := NewTestDB(t) - // Drop indices only if we're testing without them + // Drop or create indices based on the test scenario + sqlDB := db.BaseDB if !withIndices { t.Log("Dropping indices...") - sqlDB := db.BaseDB _, err := sqlDB.Exec(` - DROP INDEX IF EXISTS idx_universe_roots_namespace; - DROP INDEX IF EXISTS idx_universe_roots_issuance; - DROP INDEX IF EXISTS idx_universe_leaves_lookup; - DROP INDEX IF EXISTS idx_universe_leaves_sort; - DROP INDEX IF EXISTS idx_mssmt_nodes_namespace; - DROP INDEX IF EXISTS idx_mssmt_nodes_key_lookup; - DROP INDEX IF EXISTS idx_universe_events_stats; - DROP INDEX IF EXISTS idx_universe_events_root_type; - DROP INDEX IF EXISTS idx_federation_sync_composite; - DROP INDEX IF EXISTS idx_multiverse_leaves_composite; + DROP INDEX IF EXISTS idx_universe_roots_asset_group_proof; + DROP INDEX IF EXISTS idx_universe_roots_proof_type_issuance; + DROP INDEX IF EXISTS idx_universe_events_type_counts; + DROP INDEX IF EXISTS idx_universe_events_universe_root_id; + DROP INDEX IF EXISTS idx_universe_events_sync; + DROP INDEX IF EXISTS idx_asset_group_witnesses_gen_asset_id; + DROP INDEX IF EXISTS idx_mssmt_roots_hash_namespace; + DROP INDEX IF EXISTS idx_genesis_assets_asset_id; + DROP INDEX IF EXISTS idx_genesis_assets_asset_tag; + DROP INDEX IF EXISTS idx_genesis_assets_asset_type; + DROP INDEX IF EXISTS idx_universe_leaves_universe_root_id; + DROP INDEX IF EXISTS idx_universe_leaves_asset_genesis_id; + DROP INDEX IF EXISTS idx_universe_leaves_leaf_node_key_namespace; + `) + require.NoError(t, err) + } else { + t.Log("Creating indices...") + _, err := sqlDB.Exec(` + -- Composite index supporting joins and GROUP BY + CREATE INDEX IF NOT EXISTS idx_universe_roots_asset_group_proof + ON universe_roots (asset_id, group_key, proof_type); + + -- Partial index for proof_type = 'issuance' + CREATE INDEX IF NOT EXISTS idx_universe_roots_proof_type_issuance + ON universe_roots (proof_type); + + -- Composite index supporting event_type and universe_root_id + CREATE INDEX IF NOT EXISTS idx_universe_events_type_counts + ON universe_events (event_type, universe_root_id); + + -- Separate index on universe_root_id + CREATE INDEX IF NOT EXISTS idx_universe_events_universe_root_id + ON universe_events (universe_root_id); + + -- Partial index for event_type = 'SYNC' + CREATE INDEX IF NOT EXISTS idx_universe_events_sync + ON universe_events (event_type); + + -- Indices on tables underlying key_group_info_view + CREATE INDEX IF NOT EXISTS idx_asset_group_witnesses_gen_asset_id + ON asset_group_witnesses (gen_asset_id); + + -- Indices on mssmt_roots + CREATE INDEX IF NOT EXISTS idx_mssmt_roots_hash_namespace + ON mssmt_roots (root_hash, namespace); + + -- Indices on genesis_assets + CREATE INDEX IF NOT EXISTS idx_genesis_assets_asset_id + ON genesis_assets (asset_id); + CREATE INDEX IF NOT EXISTS idx_genesis_assets_asset_tag + ON genesis_assets (asset_tag); + CREATE INDEX IF NOT EXISTS idx_genesis_assets_asset_type + ON genesis_assets (asset_type); + + -- Indices on universe_leaves + CREATE INDEX IF NOT EXISTS idx_universe_leaves_universe_root_id + ON universe_leaves (universe_root_id); + CREATE INDEX IF NOT EXISTS idx_universe_leaves_asset_genesis_id + ON universe_leaves (asset_genesis_id); + CREATE INDEX IF NOT EXISTS idx_universe_leaves_leaf_node_key_namespace + ON universe_leaves (leaf_node_key, leaf_node_namespace); `) require.NoError(t, err) } @@ -239,26 +274,19 @@ func TestUniverseIndexPerformance(t *testing.T) { stats, _ := newUniverseStatsWithDB(db.BaseDB, testClock) h := newUniStatsHarness(t, numAssets, db.BaseDB, stats) + // Generate and populate test data t.Log("Creating test data...") - dataStart := time.Now() - - // Create test data in batches for i := 0; i < numAssets; i++ { - // Create leaves in batches for j := 0; j < numLeavesPerTree; j += batchSize { end := j + batchSize if end > numLeavesPerTree { end = numLeavesPerTree } for k := j; k < end; k++ { - _, err := insertRandLeaf( - t, ctx, h.assetUniverses[i], nil, - ) + _, err := insertRandLeaf(t, ctx, h.assetUniverses[i], nil) require.NoError(t, err) } } - - // Create events in batches for j := 0; j < numEventsPerAsset; j += batchSize { end := j + batchSize if end > numEventsPerAsset { @@ -269,91 +297,44 @@ func TestUniverseIndexPerformance(t *testing.T) { h.logSyncEventByIndex(i) } } - if (i+1)%10 == 0 { t.Logf("Processed %d/%d assets", i+1, numAssets) } } - t.Logf("Test data creation took: %v", time.Since(dataStart)) - - // Measure size after data creation - t.Log("Measuring initial database size...") - initialSizes := measureDBSize(t, db.BaseDB) + // Measure initial database size + isPostgres := db.Backend() == sqlc.BackendTypePostgres + initialSizes := measureDBSize(t, db.BaseDB, isPostgres) prettyPrintSizeStats(t, initialSizes) + // Analyze tables if using indices if withIndices { t.Log("Analyzing tables...") - sqlDB := db.BaseDB _, err := sqlDB.Exec("ANALYZE;") require.NoError(t, err) } + // Test query performance testQueries := []struct { name string fn func() time.Duration - }{ + }{ { name: "universe root namespace", fn: func() time.Duration { readTx := NewBaseUniverseReadTx() start := time.Now() - err := h.assetUniverses[0].db.ExecTx( - ctx, &readTx, - func(db BaseUniverseStore) error { - _, err := db.FetchUniverseRoot( - ctx, - h.assetUniverses[0].id.String(), - ) - return err - }) - require.NoError(t, err) - return time.Since(start) - }, - }, - { - name: "universe events by type", - fn: func() time.Duration { - readTx := NewUniverseStatsReadTx() - start := time.Now() - err := stats.db.ExecTx(ctx, &readTx, - func(db UniverseStatsStore) error { - _, err := db.QueryAssetStatsPerDaySqlite( - ctx, AssetStatsPerDayQuery{ - StartTime: testClock.Now().Add( - -24 * time.Hour, - ).Unix(), - EndTime: testClock.Now().Unix(), - }, - ) - return err - }) - require.NoError(t, err) - return time.Since(start) - }, - }, - { - name: "universe leaves namespace", - fn: func() time.Duration { - readTx := NewBaseUniverseReadTx() - start := time.Now() - err := h.assetUniverses[0].db.ExecTx( - ctx, &readTx, - func(db BaseUniverseStore) error { - _, err := db.QueryUniverseLeaves( - ctx, UniverseLeafQuery{ - Namespace: h.assetUniverses[0].id.String(), - }, - ) - return err - }) + err := h.assetUniverses[0].db.ExecTx(ctx, &readTx, func(db BaseUniverseStore) error { + _, err := db.FetchUniverseRoot(ctx, h.assetUniverses[0].id.String()) + return err + }) require.NoError(t, err) return time.Since(start) }, }, + // Add more query performance tests as needed } - // Run each query type multiple times for _, q := range testQueries { var totalTime time.Duration for i := 0; i < numQueries; i++ { @@ -364,7 +345,6 @@ func TestUniverseIndexPerformance(t *testing.T) { avgTime := totalTime / time.Duration(numQueries) t.Logf("%s average query time: %v", q.name, avgTime) - // Store result stat, ok := testResults[q.name] if !ok { stat = &queryStats{name: q.name} @@ -381,25 +361,337 @@ func TestUniverseIndexPerformance(t *testing.T) { }) } - // Run tests in sequence - without indices first + // Execute tests without and with indices runTest(false) runTest(true) - // Print final comparison + // Print performance comparison t.Log("\n=== Performance Comparison ===") - - var testNames []string - for name := range testResults { - testNames = append(testNames, name) + for name, result := range testResults { + improvement := float64(result.withoutIndices) / float64(result.withIndices) + t.Logf("\nQuery: %s (%d runs each)", name, result.queries) + t.Logf(" With indices: %v", result.withIndices) + t.Logf(" Without indices: %v", result.withoutIndices) + t.Logf(" Improvement: %.2fx", improvement) + } +} + +// TestUniverseQuerySyncStatsSorting checks that query results are sorted correctly +func TestUniversePerfQuerySyncStatsSorting(t *testing.T) { + if testing.Short() { + t.Skip("skipping sorting performance test in short mode") + } + + t.Parallel() + + const numAssets = 50 + type queryStats struct { + name string + withoutIndices time.Duration + withIndices time.Duration + queries int } - sort.Strings(testNames) + testResults := make(map[string]*queryStats) + + // Function to run performance tests with and without indices + runTest := func(withIndices bool) { + t.Run(fmt.Sprintf("indices=%v", withIndices), func(t *testing.T) { + db := NewTestDB(t) + + // Drop or create indices based on the test scenario + sqlDB := db.BaseDB + if !withIndices { + t.Log("Dropping indices...") + _, err := sqlDB.Exec(` + DROP INDEX IF EXISTS idx_genesis_assets_asset_tag; + DROP INDEX IF EXISTS idx_genesis_assets_asset_type; + DROP INDEX IF EXISTS idx_genesis_assets_asset_id; + DROP INDEX IF EXISTS idx_universe_events_type_counts; + DROP INDEX IF EXISTS idx_universe_events_universe_root_id; + DROP INDEX IF EXISTS idx_universe_events_sync; + `) + require.NoError(t, err) + } else { + t.Log("Creating indices...") + _, err := sqlDB.Exec(` + CREATE INDEX IF NOT EXISTS idx_genesis_assets_asset_tag + ON genesis_assets (asset_tag); + CREATE INDEX IF NOT EXISTS idx_genesis_assets_asset_type + ON genesis_assets (asset_type); + CREATE INDEX IF NOT EXISTS idx_genesis_assets_asset_id + ON genesis_assets (asset_id); + CREATE INDEX IF NOT EXISTS idx_universe_events_type_counts + ON universe_events (event_type, universe_root_id); + CREATE INDEX IF NOT EXISTS idx_universe_events_universe_root_id + ON universe_events (universe_root_id); + CREATE INDEX IF NOT EXISTS idx_universe_events_sync + ON universe_events (event_type); + `) + require.NoError(t, err) + } + + testClock := clock.NewTestClock(time.Now()) + statsDB, _ := newUniverseStatsWithDB(db.BaseDB, testClock) + sh := newUniStatsHarness(t, numAssets, db.BaseDB, statsDB) + + // Log proof and sync events for each asset + for i := 0; i < numAssets; i++ { + sh.logProofEventByIndex(i) + sh.logProofEventByIndex(i) + numSyncs := rand.Int() % 10 + for j := 0; j < numSyncs; j++ { + sh.logSyncEventByIndex(i) + } + } - for _, name := range testNames { - result := testResults[name] + // Run sorting tests for various fields and directions + tests := []struct { + name string + sortType universe.SyncStatsSort + direction universe.SortDirection + }{ + {"sort by name ascending", universe.SortByAssetName, universe.SortAscending}, + {"sort by name descending", universe.SortByAssetName, universe.SortDescending}, + {"sort by total syncs ascending", universe.SortByTotalSyncs, universe.SortAscending}, + {"sort by total syncs descending", universe.SortByTotalSyncs, universe.SortDescending}, + {"sort by total proofs ascending", universe.SortByTotalProofs, universe.SortAscending}, + {"sort by total proofs descending", universe.SortByTotalProofs, universe.SortDescending}, + } + + for _, test := range tests { + var totalTime time.Duration + const numQueries = 10 + + for i := 0; i < numQueries; i++ { + start := time.Now() + syncStats, err := statsDB.QuerySyncStats(context.Background(), + universe.SyncStatsQuery{ + SortBy: test.sortType, + SortDirection: test.direction, + }) + require.NoError(t, err) + queryTime := time.Since(start) + totalTime += queryTime + + // Verify sorting is correct + require.True(t, sort.SliceIsSorted( + syncStats.SyncStats, + isSortedWithDirection( + syncStats.SyncStats, test.sortType, + test.direction, + ), + )) + } + + avgTime := totalTime / time.Duration(numQueries) + t.Logf("%s average query time: %v", test.name, avgTime) + + stat, ok := testResults[test.name] + if !ok { + stat = &queryStats{name: test.name} + testResults[test.name] = stat + } + + if withIndices { + stat.withIndices = avgTime + } else { + stat.withoutIndices = avgTime + } + stat.queries = numQueries + } + }) + } + + // Execute tests without and with indices + runTest(false) + runTest(true) + + // Print performance comparison + t.Log("\n=== Performance Comparison ===") + for name, result := range testResults { improvement := float64(result.withoutIndices) / float64(result.withIndices) t.Logf("\nQuery: %s (%d runs each)", name, result.queries) t.Logf(" With indices: %v", result.withIndices) t.Logf(" Without indices: %v", result.withoutIndices) t.Logf(" Improvement: %.2fx", improvement) } +} + +// Helper for sorting checks +func isSortedWithDirection(s []universe.AssetSyncSnapshot, sortType universe.SyncStatsSort, + direction universe.SortDirection) func(i, j int) bool { + + asc := direction == universe.SortAscending + + return func(i, j int) bool { + switch sortType { + case universe.SortByAssetName: + if asc { + return s[i].AssetName < s[j].AssetName + } + return s[i].AssetName > s[j].AssetName + + case universe.SortByTotalSyncs: + if asc { + return s[i].TotalSyncs < s[j].TotalSyncs + } + return s[i].TotalSyncs > s[j].TotalSyncs + + case universe.SortByTotalProofs: + if asc { + return s[i].TotalProofs < s[j].TotalProofs + } + return s[i].TotalProofs > s[j].TotalProofs + + default: + return false + } + } +} + +func TestUniversePerfInserts(t *testing.T) { + if testing.Short() { + t.Skip("skipping insert performance test in short mode") + } + + t.Parallel() + + const ( + numAssets = 50 + numLeavesPerTree = 10 + numEventsPerAsset = 25 + ) + + var batchSizes = []int{1, 5, 10} + + type insertStats struct { + name string + withoutIndices time.Duration + withIndices time.Duration + batchSize int + totalInserts int + } + testResults := make(map[string]*insertStats) + + // Function to run performance tests with and without indices + runTest := func(withIndices bool) { + t.Run(fmt.Sprintf("indices=%v", withIndices), func(t *testing.T) { + ctx := context.Background() + db := NewTestDB(t) + + // Drop or create indices based on test scenario + sqlDB := db.BaseDB + if !withIndices { + t.Log("Dropping indices...") + _, err := sqlDB.Exec(` + DROP INDEX IF EXISTS idx_universe_roots_asset_group_proof; + DROP INDEX IF EXISTS idx_universe_events_type_counts; + DROP INDEX IF EXISTS idx_universe_events_universe_root_id; + DROP INDEX IF EXISTS idx_universe_events_sync; + DROP INDEX IF EXISTS idx_universe_leaves_universe_root_id; + DROP INDEX IF EXISTS idx_universe_leaves_asset_genesis_id; + `) + require.NoError(t, err) + } else { + t.Log("Creating indices...") + _, err := sqlDB.Exec(` + CREATE INDEX IF NOT EXISTS idx_universe_roots_asset_group_proof + ON universe_roots (asset_id, group_key, proof_type); + CREATE INDEX IF NOT EXISTS idx_universe_events_type_counts + ON universe_events (event_type, universe_root_id); + CREATE INDEX IF NOT EXISTS idx_universe_events_universe_root_id + ON universe_events (universe_root_id); + CREATE INDEX IF NOT EXISTS idx_universe_events_sync + ON universe_events (event_type); + CREATE INDEX IF NOT EXISTS idx_universe_leaves_universe_root_id + ON universe_leaves (universe_root_id); + CREATE INDEX IF NOT EXISTS idx_universe_leaves_asset_genesis_id + ON universe_leaves (asset_genesis_id); + `) + require.NoError(t, err) + } + + testClock := clock.NewTestClock(time.Now()) + stats, _ := newUniverseStatsWithDB(db.BaseDB, testClock) + h := newUniStatsHarness(t, numAssets, db.BaseDB, stats) + + // Test different batch sizes + for _, batchSize := range batchSizes { + var totalTime time.Duration + totalInserts := 0 + + // Insert leaves + start := time.Now() + for i := 0; i < numAssets; i++ { + for j := 0; j < numLeavesPerTree; j += batchSize { + end := j + batchSize + if end > numLeavesPerTree { + end = numLeavesPerTree + } + for k := j; k < end; k++ { + _, err := insertRandLeaf(t, ctx, h.assetUniverses[i], nil) + require.NoError(t, err) + totalInserts++ + } + } + } + leafTime := time.Since(start) + + // Insert events + start = time.Now() + for i := 0; i < numAssets; i++ { + for j := 0; j < numEventsPerAsset; j += batchSize { + end := j + batchSize + if end > numEventsPerAsset { + end = numEventsPerAsset + } + for k := j; k < end; k++ { + h.logProofEventByIndex(i) + h.logSyncEventByIndex(i) + totalInserts += 2 + } + } + } + eventTime := time.Since(start) + totalTime = leafTime + eventTime + + name := fmt.Sprintf("batch_size_%d", batchSize) + stat, ok := testResults[name] + if !ok { + stat = &insertStats{ + name: name, + batchSize: batchSize, + totalInserts: totalInserts, + } + testResults[name] = stat + } + + if withIndices { + stat.withIndices = totalTime + } else { + stat.withoutIndices = totalTime + } + } + }) + } + + // Execute tests without and with indices + runTest(false) + runTest(true) + + // Print performance comparison + t.Log("\n=== Insert Performance Comparison ===") + for _, result := range testResults { + improvement := float64(result.withoutIndices) / float64(result.withIndices) + opsPerSecWithIndices := float64(result.totalInserts) / result.withIndices.Seconds() + opsPerSecWithoutIndices := float64(result.totalInserts) / result.withoutIndices.Seconds() + + t.Logf("\nBatch size: %d (%d total inserts)", + result.batchSize, result.totalInserts) + t.Logf(" With indices: %v (%.0f ops/sec)", + result.withIndices, opsPerSecWithIndices) + t.Logf(" Without indices: %v (%.0f ops/sec)", + result.withoutIndices, opsPerSecWithoutIndices) + t.Logf(" Ratio: %.2fx", improvement) + } } \ No newline at end of file