Fix race conditions detected by --race (#205)

* Fix race conditions detected by --race

* comments

Makefile

@@ -54,6 +54,7 @@ endif
 ldflags += $(LDFLAGS)
 ldflags := $(strip $(ldflags))
 
+# BUILD_FLAGS := -tags "$(build_tags)" -ldflags '$(ldflags)' -race
 BUILD_FLAGS := -tags "$(build_tags)" -ldflags '$(ldflags)'
 
 #### Command List ####

loadtest/contracts/deploy_ten_contracts.sh

@@ -143,6 +143,8 @@ marsproposalid4=$(python3 parser.py proposal_id $marspair4)
 printf "12345678\n" | $seidbin tx gov deposit $marsproposalid4 10000000usei -y --from=$keyname --chain-id=$chainid --fees=10000000usei --gas=500000 --broadcast-mode=block
 printf "12345678\n" | $seidbin tx gov vote $marsproposalid4 yes -y --from=$keyname --chain-id=$chainid --fees=10000000usei --gas=500000 --broadcast-mode=block
 
+sleep 90
+
 printf "12345678\n" | $seidbin tx staking unbond $valaddr 1000000000usei --from=$keyname --chain-id=$chainid -b block -y
 
 echo $marsaddr

scripts/old_initialize_local.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+echo -n OS Password:
+read -s password
+echo
+echo -n Key Name:
+read keyname
+echo
+echo -n Number of Test Accounts:
+read numtestaccount
+echo
+
+docker stop jaeger
+docker rm jaeger
+docker run -d --name jaeger \
+  -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
+  -p 5775:5775/udp \
+  -p 6831:6831/udp \
+  -p 6832:6832/udp \
+  -p 5778:5778 \
+  -p 16686:16686 \
+  -p 14250:14250 \
+  -p 14268:14268 \
+  -p 14269:14269 \
+  -p 9411:9411 \
+  jaegertracing/all-in-one:1.33
+
+echo "Building..."
+make install
+echo $password | sudo -S rm -r ~/.sei/
+echo $password | sudo -S rm -r ~/test_accounts/
+~/go/bin/seid tendermint unsafe-reset-all
+~/go/bin/seid init demo --chain-id sei-chain
+yes | ~/go/bin/seid keys add $keyname
+yes | ~/go/bin/seid keys add faucet
+printf '12345678\n' | ~/go/bin/seid add-genesis-account $(~/go/bin/seid keys show $keyname -a) 100000000000000000000usei,100000000000000000000uusdc,100000000000000000000uatom
+printf '12345678\n' | ~/go/bin/seid add-genesis-account $(~/go/bin/seid keys show faucet -a) 100000000000000000000usei,100000000000000000000uusdc,100000000000000000000uatom
+python3 ./loadtest/scripts/populate_genesis_accounts.py $numtestaccount loc
+printf '12345678\n' | ~/go/bin/seid gentx $keyname 70000000000000000000usei --chain-id sei-chain
+~/go/bin/seid collect-gentxs
+cat ~/.sei/config/genesis.json | jq '.app_state["crisis"]["constant_fee"]["denom"]="usei"' > ~/.sei/config/tmp_genesis.json && mv ~/.sei/config/tmp_genesis.json ~/.sei/config/genesis.json
+cat ~/.sei/config/genesis.json | jq '.app_state["gov"]["deposit_params"]["min_deposit"][0]["denom"]="usei"' > ~/.sei/config/tmp_genesis.json && mv ~/.sei/config/tmp_genesis.json ~/.sei/config/genesis.json
+cat ~/.sei/config/genesis.json | jq '.app_state["mint"]["params"]["mint_denom"]="usei"' > ~/.sei/config/tmp_genesis.json && mv ~/.sei/config/tmp_genesis.json ~/.sei/config/genesis.json
+cat ~/.sei/config/genesis.json | jq '.app_state["staking"]["params"]["bond_denom"]="usei"' > ~/.sei/config/tmp_genesis.json && mv ~/.sei/config/tmp_genesis.json ~/.sei/config/genesis.json
+cat ~/.sei/config/genesis.json | jq '.app_state["gov"]["deposit_params"]["max_deposit_period"]="300s"' > ~/.sei/config/tmp_genesis.json && mv ~/.sei/config/tmp_genesis.json ~/.sei/config/genesis.json
+cat ~/.sei/config/genesis.json | jq '.app_state["gov"]["voting_params"]["voting_period"]="5s"' > ~/.sei/config/tmp_genesis.json && mv ~/.sei/config/tmp_genesis.json ~/.sei/config/genesis.json
+cat ~/.sei/config/genesis.json | jq '.consensus_params["block"]["time_iota_ms"]="50"' > ~/.sei/config/tmp_genesis.json && mv ~/.sei/config/tmp_genesis.json ~/.sei/config/genesis.json
+
+# set block time to 2s
+if [ ! -z "$1" ]; then
+  CONFIG_PATH="$1"
+else
+  CONFIG_PATH="$HOME/.sei/config/config.toml"
+fi
+
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+  sed -i 's/timeout_prevote =.*/timeout_prevote = "2000ms"/g' $CONFIG_PATH
+  sed -i 's/timeout_precommit =.*/timeout_precommit = "2000ms"/g' $CONFIG_PATH
+  sed -i 's/timeout_commit =.*/timeout_commit = "2000ms"/g' $CONFIG_PATH
+  sed -i 's/skip_timeout_commit =.*/skip_timeout_commit = false/g' $CONFIG_PATH
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+  sed -i '' 's/timeout_prevote =.*/timeout_prevote = "2000ms"/g' $CONFIG_PATH
+  sed -i '' 's/timeout_precommit =.*/timeout_precommit = "2000ms"/g' $CONFIG_PATH
+  sed -i '' 's/timeout_commit =.*/timeout_commit = "2000ms"/g' $CONFIG_PATH
+  sed -i '' 's/skip_timeout_commit =.*/skip_timeout_commit = false/g' $CONFIG_PATH
+else
+  printf "Platform not supported, please ensure that the following values are set in your config.toml:\n"
+  printf "###         Consensus Configuration Options         ###\n"
+  printf "\t timeout_prevote = \"2000ms\"\n"
+  printf "\t timeout_precommit = \"2000ms\"\n"
+  printf "\t timeout_commit = \"2000ms\"\n"
+  printf "\t skip_timeout_commit = false\n"
+  exit 1
+fi
+
+# start the chain with log tracing
+GORACE="log_path=/tmp/race/seid_race" ~/go/bin/seid start --trace

sync/gas.go

@@ -0,0 +1,30 @@
+package sync
+
+import (
+	"sync"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+type GasWrapper struct {
+	sdk.GasMeter
+	mu *sync.Mutex
+}
+
+func NewGasWrapper(wrapped sdk.GasMeter) sdk.GasMeter {
+	return GasWrapper{GasMeter: wrapped, mu: &sync.Mutex{}}
+}
+
+func (g GasWrapper) ConsumeGas(amount sdk.Gas, descriptor string) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
+	g.GasMeter.ConsumeGas(amount, descriptor)
+}
+
+func (g GasWrapper) RefundGas(amount sdk.Gas, descriptor string) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
+	g.GasMeter.RefundGas(amount, descriptor)
+}

utils/panic.go

@@ -17,7 +17,7 @@ func PanicHandler(recoverCallback func(any)) func() {
 }
 
 func MetricsPanicCallback(err any, ctx sdk.Context, key string) {
-	ctx.Logger().Error(fmt.Sprintf("panic occurred during order matching for: %s", key))
+	ctx.Logger().Error(fmt.Sprintf("panic %s occurred during order matching for: %s", err, key))
 	telemetry.IncrCounterWithLabels(
 		[]string{key},
 		1,

x/dex/cache/cache.go

@@ -2,6 +2,7 @@ package dex
 
 import (
 	"fmt"
+	"sync"
 	"time"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
@@ -19,21 +20,29 @@ type memStateItem interface {
 type memStateItems[T memStateItem] struct {
 	internal []T
 	copier   func(T) T
+
+	mu *sync.Mutex
 }
 
 func NewItems[T memStateItem](copier func(T) T) memStateItems[T] {
-	return memStateItems[T]{internal: []T{}, copier: copier}
+	return memStateItems[T]{internal: []T{}, copier: copier, mu: &sync.Mutex{}}
 }
 
 func (i *memStateItems[T]) Get() []T {
+	i.mu.Lock()
+	defer i.mu.Unlock()
 	return i.internal
 }
 
 func (i *memStateItems[T]) Add(newItem T) {
+	i.mu.Lock()
+	defer i.mu.Unlock()
 	i.internal = append(i.internal, newItem)
 }
 
 func (i *memStateItems[T]) FilterByAccount(account string) {
+	i.mu.Lock()
+	defer i.mu.Unlock()
 	newItems := []T{}
 	for _, item := range i.internal {
 		if item.GetAccount() == account {
@@ -45,6 +54,8 @@ func (i *memStateItems[T]) FilterByAccount(account string) {
 }
 
 func (i *memStateItems[T]) Copy() *memStateItems[T] {
+	i.mu.Lock()
+	defer i.mu.Unlock()
 	copy := NewItems(i.copier)
 	for _, item := range i.internal {
 		copy.Add(i.copier(item))

x/dex/cache/cancel.go

@@ -19,6 +19,9 @@ func (o *BlockCancellations) Copy() *BlockCancellations {
 }
 
 func (o *BlockCancellations) FilterByIds(idsToRemove []uint64) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+
 	newItems := []*types.Cancellation{}
 	badIDSet := datastructures.NewSyncSet(idsToRemove)
 	for _, cancel := range o.internal {
@@ -30,6 +33,9 @@ func (o *BlockCancellations) FilterByIds(idsToRemove []uint64) {
 }
 
 func (o *BlockCancellations) GetIdsToCancel() []uint64 {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+
 	res := []uint64{}
 	for _, cancel := range o.internal {
 		res = append(res, cancel.Id)

x/dex/cache/liquidation.go

@@ -24,6 +24,9 @@ func (lrs *LiquidationRequests) Copy() *LiquidationRequests {
 }
 
 func (lrs *LiquidationRequests) IsAccountLiquidating(accountToLiquidate string) bool {
+	lrs.mu.Lock()
+	defer lrs.mu.Unlock()
+
 	for _, lr := range lrs.internal {
 		if lr.AccountToLiquidate == accountToLiquidate {
 			return true

x/dex/cache/order.go

@@ -22,6 +22,8 @@ func (o *BlockOrders) Copy() *BlockOrders {
 }
 
 func (o *BlockOrders) MarkFailedToPlaceByAccounts(accounts []string) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
 	badAccountSet := datastructures.NewSyncSet(accounts)
 	newOrders := []*types.Order{}
 	for _, order := range o.internal {
@@ -35,6 +37,8 @@ func (o *BlockOrders) MarkFailedToPlaceByAccounts(accounts []string) {
 }
 
 func (o *BlockOrders) MarkFailedToPlace(failedOrders []wasm.UnsuccessfulOrder) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
 	failedOrdersMap := map[uint64]wasm.UnsuccessfulOrder{}
 	for _, failedOrder := range failedOrders {
 		failedOrdersMap[failedOrder.ID] = failedOrder
@@ -51,6 +55,9 @@ func (o *BlockOrders) MarkFailedToPlace(failedOrders []wasm.UnsuccessfulOrder) {
 }
 
 func (o *BlockOrders) GetSortedMarketOrders(direction types.PositionDirection, includeLiquidationOrders bool) []*types.Order {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+
 	res := o.getOrdersByCriteria(types.OrderType_MARKET, direction)
 	if includeLiquidationOrders {
 		res = append(res, o.getOrdersByCriteria(types.OrderType_LIQUIDATION, direction)...)
@@ -76,6 +83,8 @@ func (o *BlockOrders) GetSortedMarketOrders(direction types.PositionDirection, i
 }
 
 func (o *BlockOrders) GetLimitOrders(direction types.PositionDirection) []*types.Order {
+	o.mu.Lock()
+	defer o.mu.Unlock()
 	return o.getOrdersByCriteria(types.OrderType_LIMIT, direction)
 }