Skip to content

Commit

Permalink
Merge pull request #70 from DefiantLabs/feat/database-testing
Browse files Browse the repository at this point in the history
Add initial db tests making use of dockertest, docker spinup of postg…
  • Loading branch information
pharr117 authored Feb 16, 2024
2 parents 9a695aa + 17bc2e2 commit 271dac7
Show file tree
Hide file tree
Showing 7 changed files with 244 additions and 138 deletions.
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ repos:
# - id: go-cyclo
# args: [-over=15]
- id: validate-toml
- id: no-go-testing
# - id: no-go-testing
# - id: go-critic
# - id: go-unit-tests
- id: go-build
Expand Down
55 changes: 0 additions & 55 deletions backend_test.go

This file was deleted.

43 changes: 0 additions & 43 deletions cmd/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package cmd
import (
"fmt"
"io"
"log"
"os"
"strings"
"sync"
Expand Down Expand Up @@ -364,48 +363,6 @@ func index(cmd *cobra.Command, args []string) {
wg.Wait()
}

func GetBlockEventsStartIndexHeight(db *gorm.DB, chainID uint) int64 {
block, err := dbTypes.GetHighestEventIndexedBlock(db, chainID)
if err != nil && err.Error() != "record not found" {
log.Fatalf("Cannot retrieve highest indexed block event. Err: %v", err)
}

return block.Height
}

// GetIndexerStartingHeight will determine which block to start at
// if start block is set to -1, it will start at the highest block indexed
// otherwise, it will start at the first missing block between the start and end height
func (idxr *Indexer) GetIndexerStartingHeight(chainID uint) int64 {
// If the start height is set to -1, resume from the highest block already indexed
if idxr.cfg.Base.StartBlock == -1 {
latestBlock, err := rpc.GetLatestBlockHeight(idxr.cl)
if err != nil {
log.Fatalf("Error getting blockchain latest height. Err: %v", err)
}

fmt.Println("Found latest block", latestBlock)
highestIndexedBlock := dbTypes.GetHighestIndexedBlock(idxr.db, chainID)
if highestIndexedBlock.Height < latestBlock {
return highestIndexedBlock.Height + 1
}
}

// if we are re-indexing, just start at the configured start block
if idxr.cfg.Base.ReIndex {
return idxr.cfg.Base.StartBlock
}

maxStart := idxr.cfg.Base.EndBlock
if maxStart == -1 {
heighestBlock := dbTypes.GetHighestIndexedBlock(idxr.db, chainID)
maxStart = heighestBlock.Height
}

// Otherwise, start at the first block after the configured start block that we have not yet indexed.
return dbTypes.GetFirstMissingBlockInRange(idxr.db, idxr.cfg.Base.StartBlock, maxStart, chainID)
}

type dbData struct {
txDBWrappers []dbTypes.TxDBWrapper
block models.Block
Expand Down
37 changes: 0 additions & 37 deletions db/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package db
import (
"errors"
"fmt"
"strings"

"github.com/DefiantLabs/cosmos-indexer/config"
"github.com/DefiantLabs/cosmos-indexer/db/models"
Expand Down Expand Up @@ -121,36 +120,6 @@ func MigrateInterfaces(db *gorm.DB, interfaces []any) error {
return db.AutoMigrate(interfaces...)
}

func GetFailedBlocks(db *gorm.DB, chainID uint) []models.FailedBlock {
var failedBlocks []models.FailedBlock
db.Table("failed_blocks").Where("chain_id = ?::int", chainID).Order("height asc").Scan(&failedBlocks)
return failedBlocks
}

func GetFirstMissingBlockInRange(db *gorm.DB, start, end int64, chainID uint) int64 {
// Find the highest block we have indexed so far
currMax := GetHighestIndexedBlock(db, chainID)

// If this is after the start date, fine the first missing block between the desired start, and the highest we have indexed +1
if currMax.Height > start {
end = currMax.Height + 1
}

var firstMissingBlock int64
err := db.Raw(`SELECT s.i AS missing_blocks
FROM generate_series($1::int,$2::int) s(i)
WHERE NOT EXISTS (SELECT 1 FROM blocks WHERE height = s.i AND chain_id = $3::int AND tx_indexed = true AND time_stamp != '0001-01-01T00:00:00.000Z')
ORDER BY s.i ASC LIMIT 1;`, start, end, chainID).Row().Scan(&firstMissingBlock)
if err != nil {
if !strings.Contains(err.Error(), "no rows in result set") {
config.Log.Fatalf("Unable to find start block. Err: %v", err)
}
firstMissingBlock = start
}

return firstMissingBlock
}

func GetDBChainID(db *gorm.DB, chain models.Chain) (uint, error) {
if err := db.Where("chain_id = ?", chain.ChainID).FirstOrCreate(&chain).Error; err != nil {
config.Log.Error("Error getting/creating chain DB object.", err)
Expand Down Expand Up @@ -194,12 +163,6 @@ func GetHighestEventIndexedBlock(db *gorm.DB, chainID uint) (models.Block, error
return block, err
}

func BlockEventsAlreadyIndexed(blockHeight int64, chainID uint, db *gorm.DB) (bool, error) {
var exists bool
err := db.Raw(`SELECT count(*) > 0 FROM blocks WHERE height = ?::int AND chain_id = ?::int AND block_events_indexed = true AND time_stamp != '0001-01-01T00:00:00.000Z';`, blockHeight, chainID).Row().Scan(&exists)
return exists, err
}

func UpsertFailedBlock(db *gorm.DB, blockHeight int64, chainID string, chainName string) error {
return db.Transaction(func(dbTransaction *gorm.DB) error {
failedBlock := models.FailedBlock{Height: blockHeight, Chain: models.Chain{ChainID: chainID, Name: chainName}}
Expand Down
163 changes: 163 additions & 0 deletions db/db_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
package db

import (
"log"
"testing"
"time"

"github.com/DefiantLabs/cosmos-indexer/db/models"
"github.com/ory/dockertest/v3"
"github.com/stretchr/testify/suite"
"gorm.io/gorm"
)

// TODO: Optimize tests to use a single database instance, clean database after each test, and teardown database after all tests are done

type DBTestSuite struct {
suite.Suite
db *gorm.DB
clean func()
}

func (suite *DBTestSuite) SetupTest() {
clean, db, err := SetupTestDatabase()
suite.Require().NoError(err)

suite.db = db
suite.clean = clean
}

func (suite *DBTestSuite) TearDownTest() {
if suite.clean != nil {
suite.clean()
}

suite.db = nil
suite.clean = nil
}

func (suite *DBTestSuite) TestMigrateModels() {
err := MigrateModels(suite.db)
suite.Require().NoError(err)
}

func (suite *DBTestSuite) TestGetDBChainID() {
err := MigrateModels(suite.db)
suite.Require().NoError(err)

initChain := models.Chain{
ChainID: "testchain-1",
}

err = suite.db.Create(&initChain).Error
suite.Require().NoError(err)

chainID, err := GetDBChainID(suite.db, initChain)
suite.Require().NoError(err)
suite.Assert().NotZero(chainID)
}

func SetupTestDatabase() (func(), *gorm.DB, error) {
// TODO: allow environment overrides to skip creating mock database
pool, err := dockertest.NewPool("")
if err != nil {
return nil, nil, err
}

err = pool.Client.Ping()
if err != nil {
return nil, nil, err
}

resource, err := pool.Run("postgres", "15-alpine", []string{"POSTGRES_USER=test", "POSTGRES_PASSWORD=test", "POSTGRES_DB=test"})
if err != nil {
return nil, nil, err
}

var db *gorm.DB
if err := pool.Retry(func() error {
var err error
db, err = PostgresDbConnect(resource.GetBoundIP("5432/tcp"), resource.GetPort("5432/tcp"), "test", "test", "test", "debug")
if err != nil {
return err
}
return nil
}); err != nil {
return nil, nil, err
}

clean := func() {
if err := pool.Purge(resource); err != nil {
log.Fatalf("Could not purge resource: %s", err)
}
}

return clean, db, nil
}

func createMockBlock(mockDb *gorm.DB, chain models.Chain, address models.Address, height int64, txIndexed bool, eventIndexed bool) (models.Block, error) {
block := models.Block{
Chain: chain,
Height: height,
TimeStamp: time.Now(),
TxIndexed: txIndexed,
BlockEventsIndexed: eventIndexed,
ProposerConsAddress: address,
}

err := mockDb.Create(&block).Error
return block, err
}

func (suite *DBTestSuite) TestGetHighestBlockFunctions() {
err := MigrateModels(suite.db)
suite.Require().NoError(err)

initChain := models.Chain{
ChainID: "testchain-1",
}

err = suite.db.Create(&initChain).Error
suite.Require().NoError(err)

initConsAddress := models.Address{
Address: "testchainaddress",
}

err = suite.db.Create(&initConsAddress).Error
suite.Require().NoError(err)

block1, err := createMockBlock(suite.db, initChain, initConsAddress, 1, true, true)
suite.Require().NoError(err)

txBlock := GetHighestIndexedBlock(suite.db, initChain.ID)
eventBlock, err := GetHighestEventIndexedBlock(suite.db, initChain.ID)
suite.Require().NoError(err)

suite.Assert().Equal(block1.Height, txBlock.Height)
suite.Assert().Equal(block1.Height, eventBlock.Height)

_, err = createMockBlock(suite.db, initChain, initConsAddress, 2, false, false)
suite.Require().NoError(err)

txBlock = GetHighestIndexedBlock(suite.db, initChain.ID)
eventBlock, err = GetHighestEventIndexedBlock(suite.db, initChain.ID)
suite.Require().NoError(err)

suite.Assert().Equal(block1.Height, txBlock.Height)
suite.Assert().Equal(block1.Height, eventBlock.Height)

block3, err := createMockBlock(suite.db, initChain, initConsAddress, 3, true, true)
suite.Require().NoError(err)

txBlock = GetHighestIndexedBlock(suite.db, initChain.ID)
eventBlock, err = GetHighestEventIndexedBlock(suite.db, initChain.ID)
suite.Require().NoError(err)

suite.Assert().Equal(block3.Height, txBlock.Height)
suite.Assert().Equal(block3.Height, eventBlock.Height)
}

func TestDBSuite(t *testing.T) {
suite.Run(t, new(DBTestSuite))
}
Loading

0 comments on commit 271dac7

Please sign in to comment.