Skip to content

Commit

Permalink
tapdb: testing flags for universe scale
Browse files Browse the repository at this point in the history
  • Loading branch information
@jbrill
jbrill committed Nov 8, 2024
1 parent d52c3fc commit a607625
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 95 deletions.
5 changes: 5 additions & 0 deletions make/testing_flags.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,11 @@ else
TEST_FLAGS += -test.timeout=20m
endif

# Run universe tests with increased scale for performance testing.
ifneq ($(universe-scale),)
TEST_FLAGS += -test.universe-scale
endif

# UNIT_TARGTED is undefined iff a specific package and/or unit test case is
# not being targeted.
UNIT_TARGETED ?= no
Expand Down
174 changes: 79 additions & 95 deletions tapdb/universe_perf_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package tapdb
import (
"context"
"database/sql"
"flag"
"fmt"
"math/rand"
"sort"
Expand All @@ -15,6 +16,14 @@ import (
"github.com/stretchr/testify/require"
)

var (
numAssets = 50
numLeavesPerTree = 10
numEventsPerAsset = 25
numQueries = 50
batchSize = 5
)


type dbSizeStats struct {
tableName string
Expand Down Expand Up @@ -170,19 +179,20 @@ func formatSize(bytes int64) string {

// TestUniverseIndexPerformance tests query performance with the specified indices.
func TestUniverseIndexPerformance(t *testing.T) {
if testing.Short() {
t.Skip("skipping index performance test in short mode")
}
if testing.Short() {
t.Skip("skipping index performance test in short mode")
}

t.Parallel()
t.Parallel()

const (
numAssets = 50
numLeavesPerTree = 10
numEventsPerAsset = 25
numQueries = 50
batchSize = 5
)
// Check for universe-scale test flag
if flag.Lookup("test.universe-scale") != nil {
t.Log("Running with increased universe scale parameters")
numAssets = 500
numLeavesPerTree = 100
numEventsPerAsset = 250
numQueries = 500
}

type queryStats struct {
name string
Expand All @@ -205,67 +215,30 @@ func TestUniverseIndexPerformance(t *testing.T) {
if !withIndices {
t.Log("Dropping indices...")
_, err := sqlDB.Exec(`
DROP INDEX IF EXISTS idx_universe_roots_asset_group_proof;
DROP INDEX IF EXISTS idx_universe_roots_proof_type_issuance;
DROP INDEX IF EXISTS idx_universe_events_type_counts;
DROP INDEX IF EXISTS idx_universe_events_universe_root_id;
DROP INDEX IF EXISTS idx_universe_events_sync;
DROP INDEX IF EXISTS idx_asset_group_witnesses_gen_asset_id;
DROP INDEX IF EXISTS idx_mssmt_roots_hash_namespace;
DROP INDEX IF EXISTS idx_genesis_assets_asset_id;
DROP INDEX IF EXISTS idx_genesis_assets_asset_tag;
DROP INDEX IF EXISTS idx_genesis_assets_asset_type;
DROP INDEX IF EXISTS idx_universe_leaves_universe_root_id;
DROP INDEX IF EXISTS idx_universe_leaves_asset_genesis_id;
DROP INDEX IF EXISTS idx_universe_leaves_leaf_node_key_namespace;
DROP INDEX IF EXISTS idx_universe_roots_asset_group;
DROP INDEX IF EXISTS idx_universe_events_type_root;
DROP INDEX IF EXISTS idx_mssmt_roots_namespace;
DROP INDEX IF EXISTS idx_genesis_assets_id;
DROP INDEX IF EXISTS idx_universe_leaves_composite;
`)
require.NoError(t, err)
} else {
t.Log("Creating indices...")
_, err := sqlDB.Exec(`
-- Composite index supporting joins and GROUP BY
CREATE INDEX IF NOT EXISTS idx_universe_roots_asset_group_proof
ON universe_roots (asset_id, group_key, proof_type);
-- Partial index for proof_type = 'issuance'
CREATE INDEX IF NOT EXISTS idx_universe_roots_proof_type_issuance
ON universe_roots (proof_type);
CREATE INDEX IF NOT EXISTS idx_universe_roots_asset_group
ON universe_roots (asset_id, group_key);
-- Composite index supporting event_type and universe_root_id
CREATE INDEX IF NOT EXISTS idx_universe_events_type_counts
CREATE INDEX IF NOT EXISTS idx_universe_events_type_root
ON universe_events (event_type, universe_root_id);
-- Separate index on universe_root_id
CREATE INDEX IF NOT EXISTS idx_universe_events_universe_root_id
ON universe_events (universe_root_id);
-- Partial index for event_type = 'SYNC'
CREATE INDEX IF NOT EXISTS idx_universe_events_sync
ON universe_events (event_type);
-- Indices on tables underlying key_group_info_view
CREATE INDEX IF NOT EXISTS idx_asset_group_witnesses_gen_asset_id
ON asset_group_witnesses (gen_asset_id);
-- Indices on mssmt_roots
CREATE INDEX IF NOT EXISTS idx_mssmt_roots_hash_namespace
ON mssmt_roots (root_hash, namespace);
CREATE INDEX IF NOT EXISTS idx_mssmt_roots_namespace
ON mssmt_roots (namespace, root_hash);
-- Indices on genesis_assets
CREATE INDEX IF NOT EXISTS idx_genesis_assets_asset_id
CREATE INDEX IF NOT EXISTS idx_genesis_assets_id
ON genesis_assets (asset_id);
CREATE INDEX IF NOT EXISTS idx_genesis_assets_asset_tag
ON genesis_assets (asset_tag);
CREATE INDEX IF NOT EXISTS idx_genesis_assets_asset_type
ON genesis_assets (asset_type);
-- Indices on universe_leaves
CREATE INDEX IF NOT EXISTS idx_universe_leaves_universe_root_id
ON universe_leaves (universe_root_id);
CREATE INDEX IF NOT EXISTS idx_universe_leaves_asset_genesis_id
ON universe_leaves (asset_genesis_id);
CREATE INDEX IF NOT EXISTS idx_universe_leaves_leaf_node_key_namespace
ON universe_leaves (leaf_node_key, leaf_node_namespace);
CREATE INDEX IF NOT EXISTS idx_universe_leaves_composite
ON universe_leaves (universe_root_id, asset_genesis_id);
`)
require.NoError(t, err)
}
Expand Down Expand Up @@ -376,15 +349,21 @@ func TestUniverseIndexPerformance(t *testing.T) {
}
}

// TestUniverseQuerySyncStatsSorting checks that query results are sorted correctly
// TestUniversePerfQuerySyncStatsSorting checks that query results are sorted correctly
func TestUniversePerfQuerySyncStatsSorting(t *testing.T) {
if testing.Short() {
t.Skip("skipping sorting performance test in short mode")
}

t.Parallel()

const numAssets = 50
if testing.Short() {
t.Skip("skipping sorting performance test in short mode")
}

t.Parallel()

numAssets := 50

// Check for universe-scale test flag
if flag.Lookup("test.universe-scale") != nil {
t.Log("Running with increased universe scale parameters")
numAssets = 500
}
type queryStats struct {
name string
withoutIndices time.Duration
Expand Down Expand Up @@ -549,20 +528,24 @@ func isSortedWithDirection(s []universe.AssetSyncSnapshot, sortType universe.Syn
}
}

// TestUniversePerfInserts tests the performance of database inserts
func TestUniversePerfInserts(t *testing.T) {
if testing.Short() {
t.Skip("skipping insert performance test in short mode")
}
if testing.Short() {
t.Skip("skipping insert performance test in short mode")
}

t.Parallel()

const (
numAssets = 50
numLeavesPerTree = 10
numEventsPerAsset = 25
)
// Check for universe-scale test flag and adjust parameters
if flag.Lookup("test.universe-scale") != nil {
t.Log("Running with increased universe scale parameters")
numAssets = 500
numLeavesPerTree = 100
numEventsPerAsset = 250
numQueries = 500
}

var batchSizes = []int{1, 5, 10}
var batchSizes = []int{1, batchSize, batchSize * 2}

type insertStats struct {
name string
Expand All @@ -584,29 +567,30 @@ func TestUniversePerfInserts(t *testing.T) {
if !withIndices {
t.Log("Dropping indices...")
_, err := sqlDB.Exec(`
DROP INDEX IF EXISTS idx_universe_roots_asset_group_proof;
DROP INDEX IF EXISTS idx_universe_events_type_counts;
DROP INDEX IF EXISTS idx_universe_events_universe_root_id;
DROP INDEX IF EXISTS idx_universe_events_sync;
DROP INDEX IF EXISTS idx_universe_leaves_universe_root_id;
DROP INDEX IF EXISTS idx_universe_leaves_asset_genesis_id;
DROP INDEX IF EXISTS idx_universe_roots_asset_group;
DROP INDEX IF EXISTS idx_universe_events_type_root;
DROP INDEX IF EXISTS idx_mssmt_roots_namespace;
DROP INDEX IF EXISTS idx_genesis_assets_id;
DROP INDEX IF EXISTS idx_universe_leaves_composite;
`)
require.NoError(t, err)
} else {
t.Log("Creating indices...")
_, err := sqlDB.Exec(`
CREATE INDEX IF NOT EXISTS idx_universe_roots_asset_group_proof
ON universe_roots (asset_id, group_key, proof_type);
CREATE INDEX IF NOT EXISTS idx_universe_events_type_counts
CREATE INDEX IF NOT EXISTS idx_universe_roots_asset_group
ON universe_roots (asset_id, group_key);
CREATE INDEX IF NOT EXISTS idx_universe_events_type_root
ON universe_events (event_type, universe_root_id);
CREATE INDEX IF NOT EXISTS idx_universe_events_universe_root_id
ON universe_events (universe_root_id);
CREATE INDEX IF NOT EXISTS idx_universe_events_sync
ON universe_events (event_type);
CREATE INDEX IF NOT EXISTS idx_universe_leaves_universe_root_id
ON universe_leaves (universe_root_id);
CREATE INDEX IF NOT EXISTS idx_universe_leaves_asset_genesis_id
ON universe_leaves (asset_genesis_id);
CREATE INDEX IF NOT EXISTS idx_mssmt_roots_namespace
ON mssmt_roots (namespace, root_hash);
CREATE INDEX IF NOT EXISTS idx_genesis_assets_id
ON genesis_assets (asset_id);
CREATE INDEX IF NOT EXISTS idx_universe_leaves_composite
ON universe_leaves (universe_root_id, asset_genesis_id);
`)
require.NoError(t, err)
}
Expand Down

0 comments on commit a607625

Please sign in to comment.