feat: providertest (skip-mev#791)

providers/apis/dydx/multi_market_map_fetcher.go

@@ -24,7 +24,7 @@ var (
 	}
 )
 
-// NewDYDXResearchMarketMapFetcher returns a MultiMarketMapFetcher composed of dydx mainnet + research
+// DefaultDYDXResearchMarketMapFetcher returns a MultiMarketMapFetcher composed of dydx mainnet + research
 // apiDataHandlers.
 func DefaultDYDXResearchMarketMapFetcher(
 	rh apihandlers.RequestHandler,

providers/providertest/README.md

@@ -0,0 +1,67 @@
+# Provider testing
+
+## Example
+
+The following example can be used as a base for testing providers.
+
+```go
+package providertest_test
+
+import (
+	"context"
+	"testing"
+
+	"go.uber.org/zap"
+
+	"github.com/stretchr/testify/require"
+
+	connecttypes "github.com/skip-mev/connect/v2/pkg/types"
+	"github.com/skip-mev/connect/v2/providers/providertest"
+	mmtypes "github.com/skip-mev/connect/v2/x/marketmap/types"
+)
+
+var (
+	usdtusd = mmtypes.Market{
+		Ticker: mmtypes.Ticker{
+			CurrencyPair: connecttypes.CurrencyPair{
+				Base:  "USDT",
+				Quote: "USD",
+			},
+			Decimals:         8,
+			MinProviderCount: 1,
+			Enabled:          true,
+		},
+		ProviderConfigs: []mmtypes.ProviderConfig{
+			{
+				Name:           "okx_ws",
+				OffChainTicker: "USDC-USDT",
+				Invert:         true,
+			},
+		},
+	}
+
+	mm = mmtypes.MarketMap{
+		Markets: map[string]mmtypes.Market{
+			usdtusd.Ticker.String(): usdtusd,
+		},
+	}
+)
+
+func TestProvider(t *testing.T) {
+	// take in a market map and filter it to output N market maps with only a single provider
+	marketsPerProvider := providertest.FilterMarketMapToProviders(mm)
+
+	// run this check for each provider (here only okx_ws)
+	for provider, marketMap := range marketsPerProvider {
+		ctx := context.Background()
+		p, err := providertest.NewTestingOracle(ctx, provider)
+		require.NoError(t, err)
+
+		results, err := p.RunMarketMap(ctx, marketMap, providertest.DefaultProviderTestConfig())
+		require.NoError(t, err)
+
+		p.Logger.Info("results", zap.Any("results", results))
+	}
+}
+
+```

providers/providertest/provider.go

@@ -0,0 +1,184 @@
+package providertest
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go.uber.org/zap"
+
+	"github.com/skip-mev/connect/v2/oracle"
+	oraclemetrics "github.com/skip-mev/connect/v2/oracle/metrics"
+	oracletypes "github.com/skip-mev/connect/v2/oracle/types"
+	"github.com/skip-mev/connect/v2/pkg/log"
+	oraclemath "github.com/skip-mev/connect/v2/pkg/math/oracle"
+	oraclefactory "github.com/skip-mev/connect/v2/providers/factories/oracle"
+	mmtypes "github.com/skip-mev/connect/v2/x/marketmap/types"
+)
+
+type TestingOracle struct {
+	Oracle *oracle.OracleImpl
+	Logger *zap.Logger
+}
+
+func (o *TestingOracle) Start(ctx context.Context) error {
+	return o.Oracle.Start(ctx)
+}
+
+func (o *TestingOracle) Stop() {
+	o.Oracle.Stop()
+}
+
+func (o *TestingOracle) GetPrices() oracletypes.Prices {
+	return o.Oracle.GetPrices()
+}
+
+func (o *TestingOracle) UpdateMarketMap(mm mmtypes.MarketMap) error {
+	return o.Oracle.UpdateMarketMap(mm)
+}
+
+func NewTestingOracle(ctx context.Context, providerNames ...string) (TestingOracle, error) {
+	logCfg := log.NewDefaultConfig()
+	logCfg.StdOutLogLevel = "debug"
+	logCfg.FileOutLogLevel = "debug"
+	logCfg.LogSamplePeriod = 250 * time.Millisecond
+	logger := log.NewLogger(logCfg)
+
+	agg, err := oraclemath.NewIndexPriceAggregator(logger, mmtypes.MarketMap{}, oraclemetrics.NewNopMetrics())
+	if err != nil {
+		return TestingOracle{}, fmt.Errorf("failed to create oracle index price aggregator: %w", err)
+	}
+
+	cfg, err := OracleConfigForProvider(providerNames...)
+	if err != nil {
+		return TestingOracle{}, fmt.Errorf("failed to create oracle config: %w", err)
+	}
+
+	orc, err := oracle.New(
+		cfg,
+		agg,
+		oracle.WithLogger(logger),
+		oracle.WithPriceAPIQueryHandlerFactory(oraclefactory.APIQueryHandlerFactory),
+		oracle.WithPriceWebSocketQueryHandlerFactory(oraclefactory.WebSocketQueryHandlerFactory),
+		oracle.WithMarketMapperFactory(oraclefactory.MarketMapProviderFactory),
+	)
+	if err != nil {
+		return TestingOracle{}, fmt.Errorf("failed to create oracle: %w", err)
+	}
+
+	o := orc.(*oracle.OracleImpl)
+	err = o.Init(ctx)
+	if err != nil {
+		return TestingOracle{}, err
+	}
+
+	return TestingOracle{
+		Oracle: o,
+		Logger: logger,
+	}, nil
+}
+
+// Config is used to configure tests when calling RunMarket or RunMarketMap.
+type Config struct {
+	// TestDuration is the total duration of testing time.
+	TestDuration time.Duration
+	// PolInterval is the interval at which the provider will be queried for prices.
+	PollInterval time.Duration
+	// BurnInInterval is the amount of time to allow the provider to run before querying it.
+	BurnInInterval time.Duration
+}
+
+func (c *Config) Validate() error {
+	if c.TestDuration == 0 {
+		return fmt.Errorf("test duration cannot be 0")
+	}
+
+	if c.PollInterval == 0 {
+		return fmt.Errorf("poll interval cannot be 0")
+	}
+
+	if c.TestDuration/c.PollInterval < 1 {
+		return fmt.Errorf("ratio of test duration to poll interval must be GTE 1")
+	}
+
+	return nil
+}
+
+// DefaultProviderTestConfig tests by:
+// - allow the providers to run for 5 seconds
+// - test for a total of 1 minute
+// - poll each 5 seconds for prices.
+func DefaultProviderTestConfig() Config {
+	return Config{
+		TestDuration:   1 * time.Minute,
+		PollInterval:   5 * time.Second,
+		BurnInInterval: 5 * time.Second,
+	}
+}
+
+// PriceResults is a type alias for an array of PriceResult.
+type PriceResults []PriceResult
+
+// PriceResult is a snapshot of Prices results at a given time point when testing.
+type PriceResult struct {
+	Prices oracletypes.Prices
+	Time   time.Time
+}
+
+func (o *TestingOracle) RunMarketMap(ctx context.Context, mm mmtypes.MarketMap, cfg Config) (PriceResults, error) {
+	if err := cfg.Validate(); err != nil {
+		return nil, fmt.Errorf("invalid config: %w", err)
+	}
+
+	err := o.UpdateMarketMap(mm)
+	if err != nil {
+		return nil, fmt.Errorf("failed to update oracle market map: %w", err)
+	}
+
+	expectedNumPrices := len(mm.Markets)
+	if expectedNumPrices == 0 {
+		return nil, fmt.Errorf("cannot test with empty market map")
+	}
+
+	go o.Start(ctx)
+	time.Sleep(cfg.BurnInInterval)
+
+	priceResults := make(PriceResults, 0, cfg.TestDuration/cfg.PollInterval)
+
+	ticker := time.NewTicker(cfg.PollInterval)
+	defer ticker.Stop()
+
+	timer := time.NewTicker(cfg.TestDuration)
+	defer timer.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			prices := o.GetPrices()
+			if len(prices) != expectedNumPrices {
+				return nil, fmt.Errorf("expected %d prices, got %d", expectedNumPrices, len(prices))
+			}
+			o.Logger.Info("provider prices", zap.Any("prices", prices))
+			priceResults = append(priceResults, PriceResult{
+				Prices: prices,
+				Time:   time.Now(),
+			})
+
+		case <-timer.C:
+			o.Stop()
+
+			// cleanup
+			return priceResults, nil
+		}
+	}
+}
+
+func (o *TestingOracle) RunMarket(ctx context.Context, market mmtypes.Market, cfg Config) (PriceResults, error) {
+	mm := mmtypes.MarketMap{
+		Markets: map[string]mmtypes.Market{
+			market.Ticker.String(): market,
+		},
+	}
+
+	return o.RunMarketMap(ctx, mm, cfg)
+}

providers/providertest/util.go

@@ -0,0 +1,75 @@
+package providertest
+
+import (
+	"fmt"
+
+	cmdconfig "github.com/skip-mev/connect/v2/cmd/connect/config"
+	"github.com/skip-mev/connect/v2/cmd/constants"
+	"github.com/skip-mev/connect/v2/oracle/config"
+	mmtypes "github.com/skip-mev/connect/v2/x/marketmap/types"
+)
+
+func FilterMarketMapToProviders(mm mmtypes.MarketMap) map[string]mmtypes.MarketMap {
+	m := make(map[string]mmtypes.MarketMap)
+
+	for _, market := range mm.Markets {
+		// check each provider config
+		for _, pc := range market.ProviderConfigs {
+			// create a market from the given provider config
+			isolatedMarket := mmtypes.Market{
+				Ticker: market.Ticker,
+				ProviderConfigs: []mmtypes.ProviderConfig{
+					pc,
+				},
+			}
+
+			// always enable and set minprovider count to 1 so that it can be run isolated
+			isolatedMarket.Ticker.Enabled = true
+			isolatedMarket.Ticker.MinProviderCount = 1
+
+			// init mm if necessary
+			if _, found := m[pc.Name]; !found {
+				m[pc.Name] = mmtypes.MarketMap{
+					Markets: map[string]mmtypes.Market{
+						isolatedMarket.Ticker.String(): isolatedMarket,
+					},
+				}
+				// otherwise insert
+			} else {
+				m[pc.Name].Markets[isolatedMarket.Ticker.String()] = isolatedMarket
+			}
+		}
+	}
+
+	return m
+}
+
+func OracleConfigForProvider(providerNames ...string) (config.OracleConfig, error) {
+	cfg := config.OracleConfig{
+		UpdateInterval: cmdconfig.DefaultUpdateInterval,
+		MaxPriceAge:    cmdconfig.DefaultMaxPriceAge,
+		Metrics: config.MetricsConfig{
+			Enabled: false,
+			Telemetry: config.TelemetryConfig{
+				Disabled: true,
+			},
+		},
+		Providers: make(map[string]config.ProviderConfig),
+		Host:      cmdconfig.DefaultHost,
+		Port:      cmdconfig.DefaultPort,
+	}
+
+	for _, provider := range append(constants.Providers, constants.AlternativeMarketMapProviders...) {
+		for _, providerName := range providerNames {
+			if provider.Name == providerName {
+				cfg.Providers[provider.Name] = provider
+			}
+		}
+	}
+
+	if err := cfg.ValidateBasic(); err != nil {
+		return cfg, fmt.Errorf("default oracle config is invalid: %w", err)
+	}
+
+	return cfg, nil
+}