[CORE-813] - Add pricefeed config tomls back into the repository (#863)

This restores configurability to the pricefeed daemon while we wait to refactor configs for the daemon.

protocol/app/app.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"github.com/dydxprotocol/v4-chain/protocol/daemons/configs"
 	"io"
 	"math/big"
 	"net/http"
@@ -627,7 +628,7 @@ func New(
 
 		// Non-validating full-nodes have no need to run the price daemon.
 		if !appFlags.NonValidatingFullNode && daemonFlags.Price.Enabled {
-			exchangeQueryConfig := constants.StaticExchangeQueryConfig
+			exchangeQueryConfig := configs.ReadExchangeQueryConfigFile(homePath)
 			// Start pricefeed client for sending prices for the pricefeed server to consume. These prices
 			// are retrieved via third-party APIs like Binance and then are encoded in-memory and
 			// periodically sent via gRPC to a shared socket with the server.

protocol/cmd/dydxprotocold/cmd/init.go

@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"os"
+
+	"github.com/cosmos/cosmos-sdk/client"
+	"github.com/dydxprotocol/v4-chain/protocol/daemons/configs"
+	"github.com/spf13/cobra"
+)
+
+// AddInitCmdPostRunE adds a PostRunE to the `init` subcommand.
+func AddInitCmdPostRunE(rootCmd *cobra.Command) {
+	// Fetch init subcommand.
+	initCmd, _, err := rootCmd.Find([]string{"init"})
+	if err != nil {
+		os.Exit(1)
+	}
+
+	// Add PostRun to configure required setups after `init`.
+	initCmd.PostRunE = func(cmd *cobra.Command, args []string) error {
+		// Get home directory.
+		clientCtx := client.GetClientContextFromCmd(cmd)
+
+		// Add default pricefeed exchange config toml file if it does not exist.
+		configs.WriteDefaultPricefeedExchangeToml(clientCtx.HomeDir)
+		return nil
+	}
+}

protocol/cmd/dydxprotocold/main.go

@@ -16,6 +16,7 @@ func main() {
 	rootCmd := cmd.NewRootCmd(option)
 
 	cmd.AddTendermintSubcommands(rootCmd)
+	cmd.AddInitCmdPostRunE(rootCmd)
 
 	if err := svrcmd.Execute(rootCmd, app.AppDaemonName, app.DefaultNodeHome); err != nil {
 		os.Exit(1)

protocol/daemons/configs/default_pricefeed_exchange_config.go

@@ -0,0 +1,124 @@
+package configs
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"text/template"
+
+	tmos "github.com/cometbft/cometbft/libs/os"
+	daemonconstants "github.com/dydxprotocol/v4-chain/protocol/daemons/constants"
+	"github.com/dydxprotocol/v4-chain/protocol/daemons/pricefeed/client/constants"
+	"github.com/dydxprotocol/v4-chain/protocol/daemons/pricefeed/client/types"
+	"github.com/pelletier/go-toml"
+)
+
+// Note: any changes to the comments/variables/mapstructure must be reflected in the appropriate
+// struct in daemons/pricefeed/client/static_exchange_startup_config.go.
+const (
+	defaultTomlTemplate = `# This is a TOML config file.
+	# StaticExchangeStartupConfig represents the mapping of exchanges to the parameters for
+	# querying from them.
+	#
+	# ExchangeId - Unique string identifying an exchange.
+	#
+	# IntervalMs - Delays between sending API requests to get exchange market prices - cannot be 0.
+	#
+	# TimeoutMs - Max time to wait on an API call to an exchange - cannot be 0.
+	#
+	# MaxQueries - Max api calls to get market prices for an exchange to make in a task-loop -
+	# cannot be 0. For multi-market API exchanges, the behavior will default to 1.{{ range $exchangeId, $element := .}}
+	[[exchanges]]
+	ExchangeId = "{{$element.ExchangeId}}"
+	IntervalMs = {{$element.IntervalMs}}
+	TimeoutMs = {{$element.TimeoutMs}}
+	MaxQueries = {{$element.MaxQueries}}{{end}}
+`
+)
+
+// GenerateDefaultExchangeTomlString creates the toml file string containing the default configs
+// for querying each exchange.
+func GenerateDefaultExchangeTomlString() bytes.Buffer {
+	// Create the template for turning each `parsableExchangeStartupConfig` into a toml map config in
+	// a stringified toml file.
+	template, err := template.New("").Parse(defaultTomlTemplate)
+	// Panic if failure occurs when parsing the template.
+	if err != nil {
+		panic(err)
+	}
+
+	// Encode toml string into `defaultExchangeToml` and return if successful. Otherwise, panic.
+	var defaultExchangeToml bytes.Buffer
+	err = template.Execute(&defaultExchangeToml, constants.StaticExchangeQueryConfig)
+	if err != nil {
+		panic(err)
+	}
+	return defaultExchangeToml
+}
+
+// WriteDefaultPricefeedExchangeToml reads in the toml string for the pricefeed client and
+// writes said string to the config folder as a toml file if the config file does not exist.
+func WriteDefaultPricefeedExchangeToml(homeDir string) {
+	// Write file into config folder if file does not exist.
+	configFilePath := getConfigFilePath(homeDir)
+	if !tmos.FileExists(configFilePath) {
+		buffer := GenerateDefaultExchangeTomlString()
+		tmos.MustWriteFile(configFilePath, buffer.Bytes(), 0644)
+	}
+}
+
+// ReadExchangeQueryConfigFile gets a mapping of `exchangeIds` to `ExchangeQueryConfigs`
+// where `ExchangeQueryConfig` for querying exchanges for market prices comes from parsing a TOML
+// file in the config directory.
+// NOTE: if the config file is not found for the price-daemon, return the static exchange query
+// config.
+func ReadExchangeQueryConfigFile(homeDir string) map[types.ExchangeId]*types.ExchangeQueryConfig {
+	// Read file for exchange query configurations.
+	tomlFile, err := os.ReadFile(getConfigFilePath(homeDir))
+	if err != nil {
+		panic(err)
+	}
+
+	// Unmarshal `tomlFile` into `exchanges` for `exchangeStartupConfigMap`.
+	exchanges := map[string][]types.ExchangeQueryConfig{}
+	if err = toml.Unmarshal(tomlFile, &exchanges); err != nil {
+		panic(err)
+	}
+
+	// Populate configs for exchanges.
+	exchangeStartupConfigMap := make(map[types.ExchangeId]*types.ExchangeQueryConfig, len(exchanges))
+	for _, exchange := range exchanges["exchanges"] {
+		// Zero is an invalid configuration value for all parameters. This could also point to the
+		// configuration file being setup wrong with one or more exchange parameters unset.
+		if exchange.IntervalMs == 0 ||
+			exchange.TimeoutMs == 0 ||
+			exchange.MaxQueries == 0 {
+			panic(
+				fmt.Errorf(
+					"One or more query config values are unset or are set to zero for exchange with id: '%v'",
+					exchange.ExchangeId,
+				),
+			)
+		}
+
+		// Insert Key-Value pair into `exchangeStartupConfigMap`.
+		exchangeStartupConfigMap[exchange.ExchangeId] = &types.ExchangeQueryConfig{
+			ExchangeId: exchange.ExchangeId,
+			IntervalMs: exchange.IntervalMs,
+			TimeoutMs:  exchange.TimeoutMs,
+			MaxQueries: exchange.MaxQueries,
+		}
+	}
+
+	return exchangeStartupConfigMap
+}
+
+// getConfigFilePath returns the path to the pricefeed exchange config file.
+func getConfigFilePath(homeDir string) string {
+	return filepath.Join(
+		homeDir,
+		"config",
+		daemonconstants.PricefeedExchangeConfigFileName,
+	)
+}

protocol/daemons/configs/default_pricefeed_exchange_config_test.go

@@ -0,0 +1,248 @@
+package configs_test
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/dydxprotocol/v4-chain/protocol/daemons/configs"
+	"github.com/dydxprotocol/v4-chain/protocol/daemons/constants"
+	pfconstants "github.com/dydxprotocol/v4-chain/protocol/daemons/pricefeed/client/constants"
+	"github.com/dydxprotocol/v4-chain/protocol/daemons/pricefeed/client/constants/exchange_common"
+	"github.com/dydxprotocol/v4-chain/protocol/daemons/pricefeed/client/types"
+
+	tmos "github.com/cometbft/cometbft/libs/os"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	binanceId = exchange_common.EXCHANGE_ID_BINANCE
+	filePath  = fmt.Sprintf("config/%v", constants.PricefeedExchangeConfigFileName)
+)
+
+const (
+	tomlString = `# This is a TOML config file.
+	# StaticExchangeStartupConfig represents the mapping of exchanges to the parameters for
+	# querying from them.
+	#
+	# ExchangeId - Unique string identifying an exchange.
+	#
+	# IntervalMs - Delays between sending API requests to get exchange market prices - cannot be 0.
+	#
+	# TimeoutMs - Max time to wait on an API call to an exchange - cannot be 0.
+	#
+	# MaxQueries - Max api calls to get market prices for an exchange to make in a task-loop -
+	# cannot be 0. For multi-market API exchanges, the behavior will default to 1.
+	[[exchanges]]
+	ExchangeId = "Binance"
+	IntervalMs = 2500
+	TimeoutMs = 3000
+	MaxQueries = 1
+	[[exchanges]]
+	ExchangeId = "BinanceUS"
+	IntervalMs = 2500
+	TimeoutMs = 3000
+	MaxQueries = 1
+	[[exchanges]]
+	ExchangeId = "Bitfinex"
+	IntervalMs = 2500
+	TimeoutMs = 3000
+	MaxQueries = 1
+	[[exchanges]]
+	ExchangeId = "Bitstamp"
+	IntervalMs = 2000
+	TimeoutMs = 3000
+	MaxQueries = 1
+	[[exchanges]]
+	ExchangeId = "Bybit"
+	IntervalMs = 2000
+	TimeoutMs = 3000
+	MaxQueries = 1
+	[[exchanges]]
+	ExchangeId = "CoinbasePro"
+	IntervalMs = 2000
+	TimeoutMs = 3000
+	MaxQueries = 3
+	[[exchanges]]
+	ExchangeId = "CryptoCom"
+	IntervalMs = 2000
+	TimeoutMs = 3000
+	MaxQueries = 1
+	[[exchanges]]
+	ExchangeId = "Gate"
+	IntervalMs = 2000
+	TimeoutMs = 3000
+	MaxQueries = 1
+	[[exchanges]]
+	ExchangeId = "Huobi"
+	IntervalMs = 2000
+	TimeoutMs = 3000
+	MaxQueries = 1
+	[[exchanges]]
+	ExchangeId = "Kraken"
+	IntervalMs = 2000
+	TimeoutMs = 3000
+	MaxQueries = 1
+	[[exchanges]]
+	ExchangeId = "Kucoin"
+	IntervalMs = 2000
+	TimeoutMs = 3000
+	MaxQueries = 1
+	[[exchanges]]
+	ExchangeId = "Mexc"
+	IntervalMs = 2000
+	TimeoutMs = 3000
+	MaxQueries = 1
+	[[exchanges]]
+	ExchangeId = "Okx"
+	IntervalMs = 2000
+	TimeoutMs = 3000
+	MaxQueries = 1
+	[[exchanges]]
+	ExchangeId = "TestFixedPriceExchange"
+	IntervalMs = 2000
+	TimeoutMs = 3000
+	MaxQueries = 3
+	[[exchanges]]
+	ExchangeId = "TestVolatileExchange"
+	IntervalMs = 2000
+	TimeoutMs = 3000
+	MaxQueries = 3
+`
+)
+
+func TestGenerateDefaultExchangeTomlString(t *testing.T) {
+	defaultConfigStringBuffer := configs.GenerateDefaultExchangeTomlString()
+	require.Equal(
+		t,
+		tomlString,
+		defaultConfigStringBuffer.String(),
+	)
+}
+
+func TestWriteDefaultPricefeedExchangeToml(t *testing.T) {
+	err := os.Mkdir("config", 0700)
+	require.NoError(t, err)
+	configs.WriteDefaultPricefeedExchangeToml("")
+
+	buffer, err := os.ReadFile(filePath)
+	require.NoError(t, err)
+
+	require.Equal(t, tomlString, string(buffer[:]))
+	os.RemoveAll("config")
+}
+
+func TestWriteDefaultPricefeedExchangeToml_FileExists(t *testing.T) {
+	helloWorld := "Hello World"
+
+	err := os.Mkdir("config", 0700)
+	require.NoError(t, err)
+
+	tmos.MustWriteFile(filePath, bytes.NewBuffer([]byte(helloWorld)).Bytes(), 0644)
+	configs.WriteDefaultPricefeedExchangeToml("")
+
+	buffer, err := os.ReadFile(filePath)
+	require.NoError(t, err)
+
+	require.Equal(t, helloWorld, string(buffer[:]))
+	os.RemoveAll("config")
+}
+
+func TestReadExchangeStartupConfigFile(t *testing.T) {
+	pwd, _ := os.Getwd()
+
+	tests := map[string]struct {
+		// parameters
+		exchangeConfigSourcePath string
+		doNotWriteFile           bool
+
+		// expectations
+		expectedExchangeId         types.ExchangeId
+		expectedIntervalMsExchange uint32
+		expectedTimeoutMsExchange  uint32
+		expectedMaxQueries         uint32
+		expectedPanic              error
+	}{
+		"valid": {
+			exchangeConfigSourcePath:   "test_data/valid_test.toml",
+			expectedExchangeId:         binanceId,
+			expectedIntervalMsExchange: pfconstants.StaticExchangeQueryConfig[binanceId].IntervalMs,
+			expectedTimeoutMsExchange:  pfconstants.StaticExchangeQueryConfig[binanceId].TimeoutMs,
+			expectedMaxQueries:         pfconstants.StaticExchangeQueryConfig[binanceId].MaxQueries,
+		},
+		"config file cannot be found": {
+			exchangeConfigSourcePath: "test_data/notexisting_test.toml",
+			doNotWriteFile:           true,
+			expectedPanic: fmt.Errorf(
+				"open %s%s: no such file or directory",
+				pwd+"/config/",
+				constants.PricefeedExchangeConfigFileName,
+			),
+		},
+		"config file cannot be unmarshalled": {
+			exchangeConfigSourcePath: "test_data/broken_test.toml",
+			expectedPanic:            errors.New("(1, 12): was expecting token [[, but got unclosed table array key instead"),
+		},
+		"config file has malformed values": {
+			exchangeConfigSourcePath: "test_data/missingvals_test.toml",
+			expectedPanic: errors.New(
+				"One or more query config values are unset or are set to zero for exchange with id: 'BinanceUS'",
+			),
+		},
+		"config file has incorrect values": {
+			exchangeConfigSourcePath: "test_data/wrongvaltype_test.toml",
+			expectedPanic: errors.New(
+				"(3, 1): Can't convert a(string) to uint32",
+			),
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			if !tc.doNotWriteFile {
+				err := os.Mkdir("config", 0700)
+				require.NoError(t, err)
+
+				file, err := os.Open(tc.exchangeConfigSourcePath)
+				require.NoError(t, err)
+
+				config, err := os.Create(filepath.Join("config", constants.PricefeedExchangeConfigFileName))
+				require.NoError(t, err)
+				_, err = config.ReadFrom(file)
+				require.NoError(t, err)
+			}
+
+			if tc.expectedPanic != nil {
+				require.PanicsWithError(
+					t,
+					tc.expectedPanic.Error(),
+					func() { configs.ReadExchangeQueryConfigFile(pwd) },
+				)
+
+				os.RemoveAll("config")
+				return
+			}
+
+			exchangeStartupConfigMap := configs.ReadExchangeQueryConfigFile(pwd)
+
+			require.Equal(
+				t,
+				&types.ExchangeQueryConfig{
+					ExchangeId: tc.expectedExchangeId,
+					IntervalMs: tc.expectedIntervalMsExchange,
+					TimeoutMs:  tc.expectedTimeoutMsExchange,
+					MaxQueries: tc.expectedMaxQueries,
+				},
+				exchangeStartupConfigMap[tc.expectedExchangeId],
+			)
+
+			os.RemoveAll("config")
+		})
+	}
+
+	// In case tests fail and the path was never removed.
+	os.RemoveAll("config")
+}

protocol/daemons/configs/test_data/broken_test.toml

@@ -0,0 +1,6 @@
+[[exchanges]
+ExchangeId = "BinanceUS"
+IntervalMs = 4_250
+TimeoutMs = 3_000
+MaxQueries = 1
+MaxBufferSize = 10

protocol/daemons/configs/test_data/missingvals_test.toml

@@ -0,0 +1,4 @@
+[[exchanges]]
+ExchangeId = "BinanceUS"
+IntervalMs = 4_250
+TimeoutMs = 3_000

protocol/daemons/configs/test_data/valid_test.toml

@@ -0,0 +1,5 @@
+[[exchanges]]
+ExchangeId = "Binance"
+IntervalMs = 2_500
+TimeoutMs = 3_000
+MaxQueries = 1

protocol/daemons/configs/test_data/wrongvaltype_test.toml

@@ -0,0 +1,6 @@
+[[exchanges]]
+ExchangeId = "BinanceUS"
+IntervalMs = "a"
+TimeoutMs = 3_000
+MaxQueries = 1
+MaxBufferSize = 10

protocol/daemons/constants/pricefeed.go

@@ -3,4 +3,7 @@ package constants
 const (
 	// DefaultPrice is the default value for `Price` field in `UpdateMarketPricesRequest`.
 	DefaultPrice = 0
+
+	// PricefeedExchangeConfigFileName names the config file containing the exchange startup config.
+	PricefeedExchangeConfigFileName = "pricefeed_exchange_config.toml"
 )

protocol/go.mod

@@ -49,6 +49,7 @@ require (
 	github.com/deckarep/golang-set/v2 v2.3.0
 	github.com/ethereum/go-ethereum v1.12.0
 	github.com/ory/dockertest/v3 v3.10.0
+	github.com/pelletier/go-toml v1.9.5
 	github.com/rs/zerolog v1.29.1
 	github.com/shopspring/decimal v1.3.1
 	google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529

protocol/go.sum

@@ -1175,6 +1175,8 @@ github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0Mw
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
+github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
 github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=

protocol/testing/containertest/Dockerfile

@@ -7,4 +7,9 @@ COPY ./testing/delaymsg_config /dydxprotocol/delaymsg_config
 
 RUN /dydxprotocol/containertest.sh
 
+COPY ./testing/containertest/config/pricefeed_exchange_config.toml /dydxprotocol/chain/.alice/config/pricefeed_exchange_config.toml
+COPY ./testing/containertest/config/pricefeed_exchange_config.toml /dydxprotocol/chain/.bob/config/pricefeed_exchange_config.toml
+COPY ./testing/containertest/config/pricefeed_exchange_config.toml /dydxprotocol/chain/.carl/config/pricefeed_exchange_config.toml
+COPY ./testing/containertest/config/pricefeed_exchange_config.toml /dydxprotocol/chain/.dave/config/pricefeed_exchange_config.toml
+
 ENTRYPOINT ["dydxprotocold"]

protocol/testing/containertest/config/pricefeed_exchange_config.toml

@@ -0,0 +1,5 @@
+[[exchanges]]
+ExchangeId = "TestExchange"
+IntervalMs = 1000
+TimeoutMs = 1000
+MaxQueries = 33

protocol/x/prices/client/cli/prices_cli_test.go

@@ -5,6 +5,8 @@ package cli_test
 import (
 	"fmt"
 	appflags "github.com/dydxprotocol/v4-chain/protocol/app/flags"
+	"github.com/dydxprotocol/v4-chain/protocol/daemons/configs"
+	"path/filepath"
 	"time"
 
 	"testing"
@@ -95,6 +97,9 @@ func (s *PricesIntegrationTestSuite) SetupTest() {
 			appOptions.Set(daemonflags.FlagPriceDaemonEnabled, true)
 			appOptions.Set(daemonflags.FlagPriceDaemonLoopDelayMs, 1_000)
 
+			homeDir := filepath.Join(testval.Dir, "simd")
+			configs.WriteDefaultPricefeedExchangeToml(homeDir) // must manually create config file.
+
 			// Make sure the daemon is using the correct GRPC address.
 			appOptions.Set(appflags.GrpcAddress, testval.AppConfig.GRPC.Address)
 		},