diff --git a/make/testing_flags.mk b/make/testing_flags.mk index f657d89e4..7115ad610 100644 --- a/make/testing_flags.mk +++ b/make/testing_flags.mk @@ -84,6 +84,11 @@ else TEST_FLAGS += -test.timeout=20m endif +# Run universe tests with increased scale for performance testing. +ifneq ($(universe-scale),) +TEST_FLAGS += -test.universe-scale +endif + # UNIT_TARGTED is undefined iff a specific package and/or unit test case is # not being targeted. UNIT_TARGETED ?= no diff --git a/tapdb/universe_perf_test.go b/tapdb/universe_perf_test.go index 087b552aa..62c6860a0 100644 --- a/tapdb/universe_perf_test.go +++ b/tapdb/universe_perf_test.go @@ -3,6 +3,7 @@ package tapdb import ( "context" "database/sql" + "flag" "fmt" "math/rand" "sort" @@ -15,6 +16,14 @@ import ( "github.com/stretchr/testify/require" ) +var ( + numAssets = 50 + numLeavesPerTree = 10 + numEventsPerAsset = 25 + numQueries = 50 + batchSize = 5 +) + type dbSizeStats struct { tableName string @@ -170,19 +179,20 @@ func formatSize(bytes int64) string { // TestUniverseIndexPerformance tests query performance with the specified indices. func TestUniverseIndexPerformance(t *testing.T) { - if testing.Short() { - t.Skip("skipping index performance test in short mode") - } + if testing.Short() { + t.Skip("skipping index performance test in short mode") + } - t.Parallel() + t.Parallel() - const ( - numAssets = 50 - numLeavesPerTree = 10 - numEventsPerAsset = 25 - numQueries = 50 - batchSize = 5 - ) + // Check for universe-scale test flag + if flag.Lookup("test.universe-scale") != nil { + t.Log("Running with increased universe scale parameters") + numAssets = 500 + numLeavesPerTree = 100 + numEventsPerAsset = 250 + numQueries = 500 + } type queryStats struct { name string @@ -205,67 +215,30 @@ func TestUniverseIndexPerformance(t *testing.T) { if !withIndices { t.Log("Dropping indices...") _, err := sqlDB.Exec(` - DROP INDEX IF EXISTS idx_universe_roots_asset_group_proof; - DROP INDEX IF EXISTS idx_universe_roots_proof_type_issuance; - DROP INDEX IF EXISTS idx_universe_events_type_counts; - DROP INDEX IF EXISTS idx_universe_events_universe_root_id; - DROP INDEX IF EXISTS idx_universe_events_sync; - DROP INDEX IF EXISTS idx_asset_group_witnesses_gen_asset_id; - DROP INDEX IF EXISTS idx_mssmt_roots_hash_namespace; - DROP INDEX IF EXISTS idx_genesis_assets_asset_id; - DROP INDEX IF EXISTS idx_genesis_assets_asset_tag; - DROP INDEX IF EXISTS idx_genesis_assets_asset_type; - DROP INDEX IF EXISTS idx_universe_leaves_universe_root_id; - DROP INDEX IF EXISTS idx_universe_leaves_asset_genesis_id; - DROP INDEX IF EXISTS idx_universe_leaves_leaf_node_key_namespace; + DROP INDEX IF EXISTS idx_universe_roots_asset_group; + DROP INDEX IF EXISTS idx_universe_events_type_root; + DROP INDEX IF EXISTS idx_mssmt_roots_namespace; + DROP INDEX IF EXISTS idx_genesis_assets_id; + DROP INDEX IF EXISTS idx_universe_leaves_composite; `) require.NoError(t, err) } else { t.Log("Creating indices...") _, err := sqlDB.Exec(` - -- Composite index supporting joins and GROUP BY - CREATE INDEX IF NOT EXISTS idx_universe_roots_asset_group_proof - ON universe_roots (asset_id, group_key, proof_type); - - -- Partial index for proof_type = 'issuance' - CREATE INDEX IF NOT EXISTS idx_universe_roots_proof_type_issuance - ON universe_roots (proof_type); + CREATE INDEX IF NOT EXISTS idx_universe_roots_asset_group + ON universe_roots (asset_id, group_key); - -- Composite index supporting event_type and universe_root_id - CREATE INDEX IF NOT EXISTS idx_universe_events_type_counts + CREATE INDEX IF NOT EXISTS idx_universe_events_type_root ON universe_events (event_type, universe_root_id); - -- Separate index on universe_root_id - CREATE INDEX IF NOT EXISTS idx_universe_events_universe_root_id - ON universe_events (universe_root_id); - - -- Partial index for event_type = 'SYNC' - CREATE INDEX IF NOT EXISTS idx_universe_events_sync - ON universe_events (event_type); - - -- Indices on tables underlying key_group_info_view - CREATE INDEX IF NOT EXISTS idx_asset_group_witnesses_gen_asset_id - ON asset_group_witnesses (gen_asset_id); - - -- Indices on mssmt_roots - CREATE INDEX IF NOT EXISTS idx_mssmt_roots_hash_namespace - ON mssmt_roots (root_hash, namespace); + CREATE INDEX IF NOT EXISTS idx_mssmt_roots_namespace + ON mssmt_roots (namespace, root_hash); - -- Indices on genesis_assets - CREATE INDEX IF NOT EXISTS idx_genesis_assets_asset_id + CREATE INDEX IF NOT EXISTS idx_genesis_assets_id ON genesis_assets (asset_id); - CREATE INDEX IF NOT EXISTS idx_genesis_assets_asset_tag - ON genesis_assets (asset_tag); - CREATE INDEX IF NOT EXISTS idx_genesis_assets_asset_type - ON genesis_assets (asset_type); - -- Indices on universe_leaves - CREATE INDEX IF NOT EXISTS idx_universe_leaves_universe_root_id - ON universe_leaves (universe_root_id); - CREATE INDEX IF NOT EXISTS idx_universe_leaves_asset_genesis_id - ON universe_leaves (asset_genesis_id); - CREATE INDEX IF NOT EXISTS idx_universe_leaves_leaf_node_key_namespace - ON universe_leaves (leaf_node_key, leaf_node_namespace); + CREATE INDEX IF NOT EXISTS idx_universe_leaves_composite + ON universe_leaves (universe_root_id, asset_genesis_id); `) require.NoError(t, err) } @@ -376,15 +349,21 @@ func TestUniverseIndexPerformance(t *testing.T) { } } -// TestUniverseQuerySyncStatsSorting checks that query results are sorted correctly +// TestUniversePerfQuerySyncStatsSorting checks that query results are sorted correctly func TestUniversePerfQuerySyncStatsSorting(t *testing.T) { - if testing.Short() { - t.Skip("skipping sorting performance test in short mode") - } - - t.Parallel() - - const numAssets = 50 + if testing.Short() { + t.Skip("skipping sorting performance test in short mode") + } + + t.Parallel() + + numAssets := 50 + + // Check for universe-scale test flag + if flag.Lookup("test.universe-scale") != nil { + t.Log("Running with increased universe scale parameters") + numAssets = 500 + } type queryStats struct { name string withoutIndices time.Duration @@ -549,20 +528,24 @@ func isSortedWithDirection(s []universe.AssetSyncSnapshot, sortType universe.Syn } } +// TestUniversePerfInserts tests the performance of database inserts func TestUniversePerfInserts(t *testing.T) { - if testing.Short() { - t.Skip("skipping insert performance test in short mode") - } + if testing.Short() { + t.Skip("skipping insert performance test in short mode") + } t.Parallel() - const ( - numAssets = 50 - numLeavesPerTree = 10 - numEventsPerAsset = 25 - ) + // Check for universe-scale test flag and adjust parameters + if flag.Lookup("test.universe-scale") != nil { + t.Log("Running with increased universe scale parameters") + numAssets = 500 + numLeavesPerTree = 100 + numEventsPerAsset = 250 + numQueries = 500 + } - var batchSizes = []int{1, 5, 10} + var batchSizes = []int{1, batchSize, batchSize * 2} type insertStats struct { name string @@ -584,29 +567,30 @@ func TestUniversePerfInserts(t *testing.T) { if !withIndices { t.Log("Dropping indices...") _, err := sqlDB.Exec(` - DROP INDEX IF EXISTS idx_universe_roots_asset_group_proof; - DROP INDEX IF EXISTS idx_universe_events_type_counts; - DROP INDEX IF EXISTS idx_universe_events_universe_root_id; - DROP INDEX IF EXISTS idx_universe_events_sync; - DROP INDEX IF EXISTS idx_universe_leaves_universe_root_id; - DROP INDEX IF EXISTS idx_universe_leaves_asset_genesis_id; + DROP INDEX IF EXISTS idx_universe_roots_asset_group; + DROP INDEX IF EXISTS idx_universe_events_type_root; + DROP INDEX IF EXISTS idx_mssmt_roots_namespace; + DROP INDEX IF EXISTS idx_genesis_assets_id; + DROP INDEX IF EXISTS idx_universe_leaves_composite; `) require.NoError(t, err) } else { t.Log("Creating indices...") _, err := sqlDB.Exec(` - CREATE INDEX IF NOT EXISTS idx_universe_roots_asset_group_proof - ON universe_roots (asset_id, group_key, proof_type); - CREATE INDEX IF NOT EXISTS idx_universe_events_type_counts + CREATE INDEX IF NOT EXISTS idx_universe_roots_asset_group + ON universe_roots (asset_id, group_key); + + CREATE INDEX IF NOT EXISTS idx_universe_events_type_root ON universe_events (event_type, universe_root_id); - CREATE INDEX IF NOT EXISTS idx_universe_events_universe_root_id - ON universe_events (universe_root_id); - CREATE INDEX IF NOT EXISTS idx_universe_events_sync - ON universe_events (event_type); - CREATE INDEX IF NOT EXISTS idx_universe_leaves_universe_root_id - ON universe_leaves (universe_root_id); - CREATE INDEX IF NOT EXISTS idx_universe_leaves_asset_genesis_id - ON universe_leaves (asset_genesis_id); + + CREATE INDEX IF NOT EXISTS idx_mssmt_roots_namespace + ON mssmt_roots (namespace, root_hash); + + CREATE INDEX IF NOT EXISTS idx_genesis_assets_id + ON genesis_assets (asset_id); + + CREATE INDEX IF NOT EXISTS idx_universe_leaves_composite + ON universe_leaves (universe_root_id, asset_genesis_id); `) require.NoError(t, err) }