Skip to content

Commit

Permalink
Squashed commit of the following: (#1109)
Browse files Browse the repository at this point in the history 
commit 4061b7330cb8c046c83e8596eb97e9f0666f5524
Author: Eric <[email protected]>
Date:   Tue Mar 12 14:23:48 2024 -0700

    Fix test failure

commit d59fdc8
Author: Eric <[email protected]>
Date:   Mon Mar 11 13:03:53 2024 -0700

    Update sample pregenesis

commit 18028fd
Merge: b23d461 340801a
Author: Eric <[email protected]>
Date:   Mon Mar 11 12:57:44 2024 -0700

    Merge remote-tracking branch 'upstream' into eric/sidecar-integration

commit b23d461
Author: Eric <[email protected]>
Date:   Mon Mar 11 12:57:40 2024 -0700

    Fix config paths in Dockerfile

commit 9c202cd
Author: Eric <[email protected]>
Date:   Fri Mar 8 10:01:21 2024 -0800

    Remove min exchanges check

commit 63b4a0c
Author: Eric <[email protected]>
Date:   Wed Mar 6 18:57:45 2024 -0800

    Write multiple index prices for slinky to cache

commit 11df103
Author: Eric <[email protected]>
Date:   Wed Mar 6 18:38:48 2024 -0800

    Push prices directly to indexPriceCache

commit 51d448d
Merge: 54a8f95 a5118b7
Author: Eric <[email protected]>
Date:   Tue Mar 5 22:51:20 2024 -0800

    Merge remote-tracking branch 'upstream' into eric/sidecar-integration

commit 54a8f95
Author: Eric <[email protected]>
Date:   Tue Mar 5 22:47:54 2024 -0800

    Remove dupe flag

commit b4bed6a
Author: Eric <[email protected]>
Date:   Tue Mar 5 22:46:06 2024 -0800

    Move flags and provide cli overrides

commit b685cfc
Author: Eric <[email protected]>
Date:   Tue Mar 5 22:06:02 2024 -0800

    Change log to error level

commit 10f86ab
Author: Eric <[email protected]>
Date:   Tue Mar 5 16:51:34 2024 -0800

    Add slinky configs to repo

commit 88675f5
Author: Eric <[email protected]>
Date:   Tue Feb 27 12:46:33 2024 -0800

    Register daemon with health monitor

commit 8438d54
Author: Eric <[email protected]>
Date:   Tue Feb 27 12:32:36 2024 -0800

    PR updates

commit 3046d69
Author: Eric <[email protected]>
Date:   Mon Feb 26 15:05:52 2024 -0800

    Make unflaky

commit 7799d29
Author: Eric <[email protected]>
Date:   Mon Feb 26 15:01:18 2024 -0800

    Lints, more tests

commit e0a8c37
Author: Eric <[email protected]>
Date:   Mon Feb 26 14:35:11 2024 -0800

    Add tests

commit d8e5e90
Author: Eric <[email protected]>
Date:   Sat Feb 24 21:01:35 2024 -0800

    Add interfaces, generate mocks

commit 3a8ebec
Author: Eric <[email protected]>
Date:   Sat Feb 24 10:18:33 2024 -0800

    Lint

commit 5cdd35f
Author: Eric <[email protected]>
Date:   Sat Feb 24 10:09:56 2024 -0800

    Initial commit of sidecar integration
  • Loading branch information
@Eric-Warehime
Eric-Warehime authored Mar 13, 2024
1 parent d92b5b2 commit 3c05233
Show file tree
Hide file tree
Showing 28 changed files with 4,638 additions and 125 deletions.
15 changes: 12 additions & 3 deletions protocol/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# NB: This is a digest for a multi-arch manifest list, you will want to get this by running
# `docker buildx imagetools inspect golang:1.21-alpine`
ARG GOLANG_1_21_ALPINE_DIGEST="926f7f7e1ab8509b4e91d5ec6d5916ebb45155b0c8920291ba9f361d65385806"
ARG GOLANG_1_22_ALPINE_DIGEST="8e96e6cff6a388c2f70f5f662b64120941fcd7d4b89d62fec87520323a316bd9"

# This Dockerfile is a stateless build of the `dydxprotocold` binary as a Docker container.
# It does not include any configuration, state, or genesis information.
Expand All @@ -9,7 +9,7 @@ ARG GOLANG_1_21_ALPINE_DIGEST="926f7f7e1ab8509b4e91d5ec6d5916ebb45155b0c8920291b
# Builder
# --------------------------------------------------------

FROM golang@sha256:${GOLANG_1_21_ALPINE_DIGEST} as builder
FROM golang@sha256:${GOLANG_1_22_ALPINE_DIGEST} as builder
ARG VERSION
ARG COMMIT

Expand Down Expand Up @@ -41,15 +41,24 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
-o /dydxprotocol/build/ \
./...

# Build the oracle binary
WORKDIR /
RUN git clone https://github.com/skip-mev/slinky.git
WORKDIR /slinky
RUN make build

# --------------------------------------------------------
# Runner
# --------------------------------------------------------

FROM golang@sha256:${GOLANG_1_21_ALPINE_DIGEST}
FROM golang@sha256:${GOLANG_1_22_ALPINE_DIGEST}

RUN apk add --no-cache bash

COPY --from=builder /dydxprotocol/build/dydxprotocold /bin/dydxprotocold
COPY --from=builder /dydxprotocol/daemons/slinky/config/oracle.json /etc/oracle.json
COPY --from=builder /dydxprotocol/daemons/slinky/config/market.json /etc/market.json
COPY --from=builder /slinky/build/oracle /bin/slinky

ENV HOME /dydxprotocol
WORKDIR $HOME
Expand Down
86 changes: 67 additions & 19 deletions protocol/app/app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ import (
bridgedaemontypes "github.com/dydxprotocol/v4-chain/protocol/daemons/server/types/bridge"
liquidationtypes "github.com/dydxprotocol/v4-chain/protocol/daemons/server/types/liquidations"
pricefeedtypes "github.com/dydxprotocol/v4-chain/protocol/daemons/server/types/pricefeed"
slinkyclient "github.com/dydxprotocol/v4-chain/protocol/daemons/slinky/client"
daemontypes "github.com/dydxprotocol/v4-chain/protocol/daemons/types"

// Modules
Expand Down Expand Up @@ -202,6 +203,11 @@ import (
"github.com/dydxprotocol/v4-chain/protocol/indexer/indexer_manager"
"github.com/dydxprotocol/v4-chain/protocol/indexer/msgsender"

// Slinky
oracleconfig "github.com/skip-mev/slinky/oracle/config"
oracleclient "github.com/skip-mev/slinky/service/clients/oracle"
servicemetrics "github.com/skip-mev/slinky/service/metrics"

// Grpc Streaming
streaming "github.com/dydxprotocol/v4-chain/protocol/streaming/grpc"
streamingtypes "github.com/dydxprotocol/v4-chain/protocol/streaming/grpc/types"
Expand Down Expand Up @@ -322,6 +328,9 @@ type App struct {
BridgeClient *bridgeclient.Client

DaemonHealthMonitor *daemonservertypes.HealthMonitor

// Slinky
SlinkyClient *slinkyclient.Client
}

// assertAppPreconditions assert invariants required for an application to start.
Expand Down Expand Up @@ -432,6 +441,9 @@ func New(
if app.Server != nil {
app.Server.Stop()
}
if app.SlinkyClient != nil {
app.SlinkyClient.Stop()
}
return nil
},
)
Expand Down Expand Up @@ -766,25 +778,39 @@ func New(
}()
}

// Non-validating full-nodes have no need to run the price daemon.
if !appFlags.NonValidatingFullNode && daemonFlags.Price.Enabled {
exchangeQueryConfig := configs.ReadExchangeQueryConfigFile(homePath)
// Start pricefeed client for sending prices for the pricefeed server to consume. These prices
// are retrieved via third-party APIs like Binance and then are encoded in-memory and
// periodically sent via gRPC to a shared socket with the server.
app.PriceFeedClient = pricefeedclient.StartNewClient(
// The client will use `context.Background` so that it can have a different context from
// the main application.
context.Background(),
daemonFlags,
appFlags,
logger,
&daemontypes.GrpcClientImpl{},
exchangeQueryConfig,
constants.StaticExchangeDetails,
&pricefeedclient.SubTaskRunnerImpl{},
)
app.RegisterDaemonWithHealthMonitor(app.PriceFeedClient, maxDaemonUnhealthyDuration)
// Non-validating full-nodes have no need to run the oracle.
if !appFlags.NonValidatingFullNode {
if daemonFlags.Price.Enabled {
exchangeQueryConfig := configs.ReadExchangeQueryConfigFile(homePath)
// Start pricefeed client for sending prices for the pricefeed server to consume. These prices
// are retrieved via third-party APIs like Binance and then are encoded in-memory and
// periodically sent via gRPC to a shared socket with the server.
app.PriceFeedClient = pricefeedclient.StartNewClient(
// The client will use `context.Background` so that it can have a different context from
// the main application.
context.Background(),
daemonFlags,
appFlags,
logger,
&daemontypes.GrpcClientImpl{},
exchangeQueryConfig,
constants.StaticExchangeDetails,
&pricefeedclient.SubTaskRunnerImpl{},
)
app.RegisterDaemonWithHealthMonitor(app.PriceFeedClient, maxDaemonUnhealthyDuration)
}
if daemonFlags.Slinky.Enabled {
app.SlinkyClient = slinkyclient.StartNewClient(
context.Background(),
app.initSlinkySidecarClient(appOpts),
indexPriceCache,
&daemontypes.GrpcClientImpl{},
daemonFlags,
appFlags,
logger,
)
app.RegisterDaemonWithHealthMonitor(app.SlinkyClient, maxDaemonUnhealthyDuration)
}
}

// Start Bridge Daemon.
Expand Down Expand Up @@ -1420,6 +1446,28 @@ func New(
return app
}

func (app *App) initSlinkySidecarClient(appOpts servertypes.AppOptions) oracleclient.OracleClient {
// Slinky setup
cfg, err := oracleconfig.ReadConfigFromAppOpts(appOpts)
if err != nil {
panic(err)
}
oracleMetrics, err := servicemetrics.NewMetricsFromConfig(cfg, app.ChainID())
if err != nil {
panic(err)
}
// Create the oracle service.
slinkyClient, err := oracleclient.NewClientFromConfig(
cfg,
app.Logger().With("client", "oracle"),
oracleMetrics,
)
if err != nil {
panic(err)
}
return slinkyClient
}

// RegisterDaemonWithHealthMonitor registers a daemon service with the update monitor, which will commence monitoring
// the health of the daemon. If the daemon does not register, the method will panic.
func (app *App) RegisterDaemonWithHealthMonitor(
Expand Down
1 change: 1 addition & 0 deletions protocol/app/app_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ func TestAppIsFullyInitialized(t *testing.T) {
"PriceFeedClient",
"LiquidationsClient",
"BridgeClient",
"SlinkyClient",

// Any default constructed type can be considered initialized if the default is what is
// expected. getUninitializedStructFields relies on fields being the non-default and
Expand Down
4 changes: 3 additions & 1 deletion protocol/cmd/dydxprotocold/cmd/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
serverconfig "github.com/cosmos/cosmos-sdk/server/config"
assettypes "github.com/dydxprotocol/v4-chain/protocol/x/assets/types"
clobtypes "github.com/dydxprotocol/v4-chain/protocol/x/clob/types"
oracleconfig "github.com/skip-mev/slinky/oracle/config"
)

const (
Expand All @@ -25,6 +26,7 @@ const (
// DydxAppConfig specifies dYdX app specific config.
type DydxAppConfig struct {
serverconfig.Config
Oracle oracleconfig.AppConfig `mapstructure:"oracle"`
}

// TODO(DEC-1718): Audit tendermint and app config parameters for mainnet.
Expand Down Expand Up @@ -65,7 +67,7 @@ func initAppConfig() (string, *DydxAppConfig) {
// GRPC.
appConfig.GRPC.Address = "0.0.0.0:9090"

appTemplate := serverconfig.DefaultConfigTemplate
appTemplate := serverconfig.DefaultConfigTemplate + oracleconfig.DefaultConfigTemplate

return appTemplate, &appConfig
}
Expand Down
76 changes: 76 additions & 0 deletions protocol/daemons/flags/flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ package flags

import (
servertypes "github.com/cosmos/cosmos-sdk/server/types"
oracleconfig "github.com/skip-mev/slinky/oracle/config"
"github.com/spf13/cast"
"github.com/spf13/cobra"
"time"
)

// List of CLI flags for Server and Client.
Expand All @@ -23,6 +25,13 @@ const (
FlagLiquidationDaemonEnabled = "liquidation-daemon-enabled"
FlagLiquidationDaemonLoopDelayMs = "liquidation-daemon-loop-delay-ms"
FlagLiquidationDaemonQueryPageLimit = "liquidation-daemon-query-page-limit"

// Oracle flags
FlagOracleEnabled = "oracle.enabled"
FlagOracleAddress = "oracle.oracle_address"
FlagOracleClientTimeout = "oracle.client_timeout"
FlagOracleMetricsEnabled = "oracle.metrics_enabled"
FlagOraclePrometheusServerAddress = "oracle.prometheus_server_address"
)

// Shared flags contains configuration flags shared by all daemons.
Expand Down Expand Up @@ -63,12 +72,17 @@ type PriceFlags struct {
LoopDelayMs uint32
}

type SlinkyFlags struct {
oracleconfig.AppConfig
}

// DaemonFlags contains the collected configuration flags for all daemons.
type DaemonFlags struct {
Shared SharedFlags
Bridge BridgeFlags
Liquidation LiquidationFlags
Price PriceFlags
Slinky SlinkyFlags
}

var defaultDaemonFlags *DaemonFlags
Expand Down Expand Up @@ -96,6 +110,14 @@ func GetDefaultDaemonFlags() DaemonFlags {
Enabled: true,
LoopDelayMs: 3_000,
},
Slinky: SlinkyFlags{
AppConfig: oracleconfig.AppConfig{
OracleAddress: "localhost:8080",
ClientTimeout: time.Second * 2,
MetricsEnabled: false,
PrometheusServerAddress: "",
},
},
}
}
return *defaultDaemonFlags
Expand Down Expand Up @@ -173,6 +195,33 @@ func AddDaemonFlagsToCmd(
df.Price.LoopDelayMs,
"Delay in milliseconds between sending price updates to the application.",
)

// Slinky Daemon.
cmd.Flags().Bool(
FlagOracleEnabled,
df.Slinky.AppConfig.Enabled,
"Enable the slinky oracle.",
)
cmd.Flags().String(
FlagOracleAddress,
df.Slinky.AppConfig.OracleAddress,
"Address of the oracle sidecar.",
)
cmd.Flags().Duration(
FlagOracleClientTimeout,
df.Slinky.AppConfig.ClientTimeout,
"Time out of the oracle sidecar client.",
)
cmd.Flags().Bool(
FlagOracleMetricsEnabled,
df.Slinky.AppConfig.MetricsEnabled,
"Enable the oracle metrics reporting for Slinky.",
)
cmd.Flags().String(
FlagOraclePrometheusServerAddress,
df.Slinky.AppConfig.PrometheusServerAddress,
"The address of the exposed prometheus address for Slinky metrics.",
)
}

// GetDaemonFlagValuesFromOptions gets all daemon flag values from the `AppOptions` struct.
Expand Down Expand Up @@ -245,5 +294,32 @@ func GetDaemonFlagValuesFromOptions(
}
}

// Slinky Daemon.
if option := appOpts.Get(FlagOracleEnabled); option != nil {
if v, err := cast.ToBoolE(option); err == nil {
result.Slinky.AppConfig.Enabled = v
}
}
if option := appOpts.Get(FlagOracleAddress); option != nil {
if v, err := cast.ToStringE(option); err == nil {
result.Slinky.AppConfig.OracleAddress = v
}
}
if option := appOpts.Get(FlagOracleClientTimeout); option != nil {
if v, err := cast.ToDurationE(option); err == nil {
result.Slinky.AppConfig.ClientTimeout = v
}
}
if option := appOpts.Get(FlagOracleMetricsEnabled); option != nil {
if v, err := cast.ToBoolE(option); err == nil {
result.Slinky.AppConfig.MetricsEnabled = v
}
}
if option := appOpts.Get(FlagOraclePrometheusServerAddress); option != nil {
if v, err := cast.ToStringE(option); err == nil {
result.Slinky.AppConfig.PrometheusServerAddress = v
}
}

return result
}
37 changes: 16 additions & 21 deletions protocol/daemons/server/types/pricefeed/market_to_exchange_prices.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,6 @@ func (mte *MarketToExchangePrices) UpdatePrices(
// a price is valid iff
// 1) the last update time is within a predefined threshold away from the given
// read time.
// 2) the number of prices that meet 1) are greater than the minimum number of
// exchanges specified in the given input.
func (mte *MarketToExchangePrices) GetValidMedianPrices(
marketParams []types.MarketParam,
readTime time.Time,
Expand Down Expand Up @@ -101,26 +99,23 @@ func (mte *MarketToExchangePrices) GetValidMedianPrices(
},
)

// The number of valid prices must be >= min number of exchanges.
if len(validPrices) >= int(marketParam.MinExchanges) {
// Calculate the median. Returns an error if the input is empty.
median, err := lib.Median(validPrices)
if err != nil {
telemetry.IncrCounterWithLabels(
[]string{
metrics.PricefeedServer,
metrics.NoValidMedianPrice,
metrics.Count,
},
1,
[]gometrics.Label{
pricefeedmetrics.GetLabelForMarketId(marketId),
},
)
continue
}
marketIdToMedianPrice[marketId] = median
// Calculate the median. Returns an error if the input is empty.
median, err := lib.Median(validPrices)
if err != nil {
telemetry.IncrCounterWithLabels(
[]string{
metrics.PricefeedServer,
metrics.NoValidMedianPrice,
metrics.Count,
},
1,
[]gometrics.Label{
pricefeedmetrics.GetLabelForMarketId(marketId),
},
)
continue
}
marketIdToMedianPrice[marketId] = median
}

return marketIdToMedianPrice
Expand Down
Loading

0 comments on commit 3c05233

Please sign in to comment.