Skip to content

Commit

Permalink
Add initial db tests making use of dockertest, docker spinup of postg…
Browse files Browse the repository at this point in the history
…res, testify suite builder, and a few tests around db utils. Remove a number of unused functions around database ops
  • Loading branch information
pharr117 committed Feb 16, 2024
1 parent 9a695aa commit 17bc2e2
Show file tree
Hide file tree
Showing 7 changed files with 244 additions and 138 deletions.
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ repos:
# - id: go-cyclo
# args: [-over=15]
- id: validate-toml
- id: no-go-testing
# - id: no-go-testing
# - id: go-critic
# - id: go-unit-tests
- id: go-build
Expand Down
55 changes: 0 additions & 55 deletions backend_test.go

This file was deleted.

43 changes: 0 additions & 43 deletions cmd/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package cmd
import (
"fmt"
"io"
"log"
"os"
"strings"
"sync"
Expand Down Expand Up @@ -364,48 +363,6 @@ func index(cmd *cobra.Command, args []string) {
wg.Wait()
}

func GetBlockEventsStartIndexHeight(db *gorm.DB, chainID uint) int64 {
block, err := dbTypes.GetHighestEventIndexedBlock(db, chainID)
if err != nil && err.Error() != "record not found" {
log.Fatalf("Cannot retrieve highest indexed block event. Err: %v", err)
}

return block.Height
}

// GetIndexerStartingHeight will determine which block to start at
// if start block is set to -1, it will start at the highest block indexed
// otherwise, it will start at the first missing block between the start and end height
func (idxr *Indexer) GetIndexerStartingHeight(chainID uint) int64 {
// If the start height is set to -1, resume from the highest block already indexed
if idxr.cfg.Base.StartBlock == -1 {
latestBlock, err := rpc.GetLatestBlockHeight(idxr.cl)
if err != nil {
log.Fatalf("Error getting blockchain latest height. Err: %v", err)
}

fmt.Println("Found latest block", latestBlock)
highestIndexedBlock := dbTypes.GetHighestIndexedBlock(idxr.db, chainID)
if highestIndexedBlock.Height < latestBlock {
return highestIndexedBlock.Height + 1
}
}

// if we are re-indexing, just start at the configured start block
if idxr.cfg.Base.ReIndex {
return idxr.cfg.Base.StartBlock
}

maxStart := idxr.cfg.Base.EndBlock
if maxStart == -1 {
heighestBlock := dbTypes.GetHighestIndexedBlock(idxr.db, chainID)
maxStart = heighestBlock.Height
}

// Otherwise, start at the first block after the configured start block that we have not yet indexed.
return dbTypes.GetFirstMissingBlockInRange(idxr.db, idxr.cfg.Base.StartBlock, maxStart, chainID)
}

type dbData struct {
txDBWrappers []dbTypes.TxDBWrapper
block models.Block
Expand Down
37 changes: 0 additions & 37 deletions db/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package db
import (
"errors"
"fmt"
"strings"

"github.com/DefiantLabs/cosmos-indexer/config"
"github.com/DefiantLabs/cosmos-indexer/db/models"
Expand Down Expand Up @@ -121,36 +120,6 @@ func MigrateInterfaces(db *gorm.DB, interfaces []any) error {
return db.AutoMigrate(interfaces...)
}

func GetFailedBlocks(db *gorm.DB, chainID uint) []models.FailedBlock {
var failedBlocks []models.FailedBlock
db.Table("failed_blocks").Where("chain_id = ?::int", chainID).Order("height asc").Scan(&failedBlocks)
return failedBlocks
}

func GetFirstMissingBlockInRange(db *gorm.DB, start, end int64, chainID uint) int64 {
// Find the highest block we have indexed so far
currMax := GetHighestIndexedBlock(db, chainID)

// If this is after the start date, fine the first missing block between the desired start, and the highest we have indexed +1
if currMax.Height > start {
end = currMax.Height + 1
}

var firstMissingBlock int64
err := db.Raw(`SELECT s.i AS missing_blocks
FROM generate_series($1::int,$2::int) s(i)
WHERE NOT EXISTS (SELECT 1 FROM blocks WHERE height = s.i AND chain_id = $3::int AND tx_indexed = true AND time_stamp != '0001-01-01T00:00:00.000Z')
ORDER BY s.i ASC LIMIT 1;`, start, end, chainID).Row().Scan(&firstMissingBlock)
if err != nil {
if !strings.Contains(err.Error(), "no rows in result set") {
config.Log.Fatalf("Unable to find start block. Err: %v", err)
}
firstMissingBlock = start
}

return firstMissingBlock
}

func GetDBChainID(db *gorm.DB, chain models.Chain) (uint, error) {
if err := db.Where("chain_id = ?", chain.ChainID).FirstOrCreate(&chain).Error; err != nil {
config.Log.Error("Error getting/creating chain DB object.", err)
Expand Down Expand Up @@ -194,12 +163,6 @@ func GetHighestEventIndexedBlock(db *gorm.DB, chainID uint) (models.Block, error
return block, err
}

func BlockEventsAlreadyIndexed(blockHeight int64, chainID uint, db *gorm.DB) (bool, error) {
var exists bool
err := db.Raw(`SELECT count(*) > 0 FROM blocks WHERE height = ?::int AND chain_id = ?::int AND block_events_indexed = true AND time_stamp != '0001-01-01T00:00:00.000Z';`, blockHeight, chainID).Row().Scan(&exists)
return exists, err
}

func UpsertFailedBlock(db *gorm.DB, blockHeight int64, chainID string, chainName string) error {
return db.Transaction(func(dbTransaction *gorm.DB) error {
failedBlock := models.FailedBlock{Height: blockHeight, Chain: models.Chain{ChainID: chainID, Name: chainName}}
Expand Down
163 changes: 163 additions & 0 deletions db/db_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
package db

import (
"log"
"testing"
"time"

"github.com/DefiantLabs/cosmos-indexer/db/models"
"github.com/ory/dockertest/v3"
"github.com/stretchr/testify/suite"
"gorm.io/gorm"
)

// TODO: Optimize tests to use a single database instance, clean database after each test, and teardown database after all tests are done

type DBTestSuite struct {
suite.Suite
db *gorm.DB
clean func()
}

func (suite *DBTestSuite) SetupTest() {
clean, db, err := SetupTestDatabase()
suite.Require().NoError(err)

suite.db = db
suite.clean = clean
}

func (suite *DBTestSuite) TearDownTest() {
if suite.clean != nil {
suite.clean()
}

suite.db = nil
suite.clean = nil
}

func (suite *DBTestSuite) TestMigrateModels() {
err := MigrateModels(suite.db)
suite.Require().NoError(err)
}

func (suite *DBTestSuite) TestGetDBChainID() {
err := MigrateModels(suite.db)
suite.Require().NoError(err)

initChain := models.Chain{
ChainID: "testchain-1",
}

err = suite.db.Create(&initChain).Error
suite.Require().NoError(err)

chainID, err := GetDBChainID(suite.db, initChain)
suite.Require().NoError(err)
suite.Assert().NotZero(chainID)
}

func SetupTestDatabase() (func(), *gorm.DB, error) {
// TODO: allow environment overrides to skip creating mock database
pool, err := dockertest.NewPool("")
if err != nil {
return nil, nil, err
}

err = pool.Client.Ping()
if err != nil {
return nil, nil, err
}

resource, err := pool.Run("postgres", "15-alpine", []string{"POSTGRES_USER=test", "POSTGRES_PASSWORD=test", "POSTGRES_DB=test"})
if err != nil {
return nil, nil, err
}

var db *gorm.DB
if err := pool.Retry(func() error {
var err error
db, err = PostgresDbConnect(resource.GetBoundIP("5432/tcp"), resource.GetPort("5432/tcp"), "test", "test", "test", "debug")
if err != nil {
return err
}
return nil
}); err != nil {
return nil, nil, err
}

clean := func() {
if err := pool.Purge(resource); err != nil {
log.Fatalf("Could not purge resource: %s", err)
}
}

return clean, db, nil
}

func createMockBlock(mockDb *gorm.DB, chain models.Chain, address models.Address, height int64, txIndexed bool, eventIndexed bool) (models.Block, error) {
block := models.Block{
Chain: chain,
Height: height,
TimeStamp: time.Now(),
TxIndexed: txIndexed,
BlockEventsIndexed: eventIndexed,
ProposerConsAddress: address,
}

err := mockDb.Create(&block).Error
return block, err
}

func (suite *DBTestSuite) TestGetHighestBlockFunctions() {
err := MigrateModels(suite.db)
suite.Require().NoError(err)

initChain := models.Chain{
ChainID: "testchain-1",
}

err = suite.db.Create(&initChain).Error
suite.Require().NoError(err)

initConsAddress := models.Address{
Address: "testchainaddress",
}

err = suite.db.Create(&initConsAddress).Error
suite.Require().NoError(err)

block1, err := createMockBlock(suite.db, initChain, initConsAddress, 1, true, true)
suite.Require().NoError(err)

txBlock := GetHighestIndexedBlock(suite.db, initChain.ID)
eventBlock, err := GetHighestEventIndexedBlock(suite.db, initChain.ID)
suite.Require().NoError(err)

suite.Assert().Equal(block1.Height, txBlock.Height)
suite.Assert().Equal(block1.Height, eventBlock.Height)

_, err = createMockBlock(suite.db, initChain, initConsAddress, 2, false, false)
suite.Require().NoError(err)

txBlock = GetHighestIndexedBlock(suite.db, initChain.ID)
eventBlock, err = GetHighestEventIndexedBlock(suite.db, initChain.ID)
suite.Require().NoError(err)

suite.Assert().Equal(block1.Height, txBlock.Height)
suite.Assert().Equal(block1.Height, eventBlock.Height)

block3, err := createMockBlock(suite.db, initChain, initConsAddress, 3, true, true)
suite.Require().NoError(err)

txBlock = GetHighestIndexedBlock(suite.db, initChain.ID)
eventBlock, err = GetHighestEventIndexedBlock(suite.db, initChain.ID)
suite.Require().NoError(err)

suite.Assert().Equal(block3.Height, txBlock.Height)
suite.Assert().Equal(block3.Height, eventBlock.Height)
}

func TestDBSuite(t *testing.T) {
suite.Run(t, new(DBTestSuite))
}
Loading

0 comments on commit 17bc2e2

Please sign in to comment.