diff --git a/.changelog/4539.feature.md b/.changelog/4539.feature.md new file mode 100644 index 00000000000..d9e95641966 --- /dev/null +++ b/.changelog/4539.feature.md @@ -0,0 +1,6 @@ +Add archive mode support + +Node started in archive mode only serves existing consensus and runtime +states. The node has all unneeded consensus and P2P functionality disabled so +it wont participate in the network. Archive mode can be set using the +`consensus.tendermint.mode` setting. diff --git a/go/consensus/api/api.go b/go/consensus/api/api.go index 7a5876667e7..38c4bf4a53b 100644 --- a/go/consensus/api/api.go +++ b/go/consensus/api/api.go @@ -4,6 +4,7 @@ package api import ( "context" + "fmt" "strings" "time" @@ -34,6 +35,61 @@ const ( HeightLatest int64 = 0 ) +// Mode is the consensus node mode. +type Mode string + +const ( + // ModeFull is the name of the full node consensus mode. + ModeFull Mode = "full" + // ModeSeed is the name of the seed-only node consensus mode. + ModeSeed Mode = "seed" + // ModeArchive is the name of the archive node consensus mode. + ModeArchive Mode = "archive" +) + +// MarshalText encodes a Mode into text form. +func (m Mode) MarshalText() ([]byte, error) { + switch m { + case ModeFull: + return []byte(ModeFull.String()), nil + case ModeSeed: + return []byte(ModeSeed.String()), nil + case ModeArchive: + return []byte(ModeArchive.String()), nil + default: + return nil, fmt.Errorf("invalid mode: %s", string(m)) + } +} + +// UnmarshalText decodes a text marshaled consensus mode. +func (m *Mode) UnmarshalText(text []byte) error { + switch string(text) { + case ModeFull.String(): + *m = ModeFull + case ModeSeed.String(): + *m = ModeSeed + case ModeArchive.String(): + *m = ModeArchive + default: + return fmt.Errorf("invalid consensus mode: %s", string(text)) + } + return nil +} + +// String returns a string representation of the mode. +func (m Mode) String() string { + switch m { + case ModeFull: + return string(ModeFull) + case ModeSeed: + return string(ModeSeed) + case ModeArchive: + return string(ModeArchive) + default: + return fmt.Sprintf("[unknown consensus mode: %s]", string(m)) + } +} + var ( // ErrNoCommittedBlocks is the error returned when there are no committed // blocks and as such no state can be queried. diff --git a/go/consensus/api/api_test.go b/go/consensus/api/api_test.go new file mode 100644 index 00000000000..5f27549779e --- /dev/null +++ b/go/consensus/api/api_test.go @@ -0,0 +1,37 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConsensusMode(t *testing.T) { + require := require.New(t) + + // Test valid Modes. + for _, k := range []Mode{ + ModeArchive, + ModeFull, + ModeSeed, + } { + enc, err := k.MarshalText() + require.NoError(err, "MarshalText") + + var s Mode + err = s.UnmarshalText(enc) + require.NoError(err, "UnmarshalText") + + require.Equal(k, s, "consensus mode should round-trip") + } + + // Test invalid Mode. + sr := Mode("abc") + require.Equal("[unknown consensus mode: abc]", sr.String()) + enc, err := sr.MarshalText() + require.Nil(enc, "MarshalText on invalid consensus mode should be nil") + require.Error(err, "MarshalText on invalid consensus mode should error") + + err = sr.UnmarshalText([]byte("invalid consensus mode")) + require.Error(err, "UnmarshalText on invalid consensus mode should error") +} diff --git a/go/consensus/api/submission.go b/go/consensus/api/submission.go index 5c9168741fb..ab8e3d14f25 100644 --- a/go/consensus/api/submission.go +++ b/go/consensus/api/submission.go @@ -44,6 +44,12 @@ func (pd *staticPriceDiscovery) GasPrice(ctx context.Context) (*quantity.Quantit return pd.price.Clone(), nil } +type noOpPriceDiscovery struct{} + +func (pd *noOpPriceDiscovery) GasPrice(ctx context.Context) (*quantity.Quantity, error) { + return nil, transaction.ErrMethodNotSupported +} + // SubmissionManager is a transaction submission manager interface. type SubmissionManager interface { // SignAndSubmitTx populates the nonce and fee fields in the transaction, signs the transaction @@ -170,3 +176,11 @@ func NewSubmissionManager(backend ClientBackend, priceDiscovery PriceDiscovery, func SignAndSubmitTx(ctx context.Context, backend Backend, signer signature.Signer, tx *transaction.Transaction) error { return backend.SubmissionManager().SignAndSubmitTx(ctx, signer, tx) } + +// NoOpSubmissionManager implements a submission manager that doesn't support submitting transactions. +type NoOpSubmissionManager struct{} + +// Implements SubmissionManager. +func (m *NoOpSubmissionManager) SignAndSubmitTx(ctx context.Context, signer signature.Signer, tx *transaction.Transaction) error { + return transaction.ErrMethodNotSupported +} diff --git a/go/consensus/api/transaction/transaction.go b/go/consensus/api/transaction/transaction.go index 46e48f76100..1eb0109b8ae 100644 --- a/go/consensus/api/transaction/transaction.go +++ b/go/consensus/api/transaction/transaction.go @@ -23,6 +23,9 @@ var ( // ErrInvalidNonce is the error returned when a nonce is invalid. ErrInvalidNonce = errors.New(moduleName, 1, "transaction: invalid nonce") + // ErrMethodNotSupported is the error returned if transaction method is not supported. + ErrMethodNotSupported = errors.New(moduleName, 5, "transaction: method not supported") + // SignatureContext is the context used for signing transactions. SignatureContext = signature.NewContext("oasis-core/consensus: tx", signature.WithChainSeparation()) diff --git a/go/consensus/tendermint/api/api.go b/go/consensus/tendermint/api/api.go index 8a9a175a1b8..05a01ac68be 100644 --- a/go/consensus/tendermint/api/api.go +++ b/go/consensus/tendermint/api/api.go @@ -199,7 +199,7 @@ type Backend interface { // WatchTendermintBlocks returns a stream of Tendermint blocks as they are // returned via the `EventDataNewBlock` query. - WatchTendermintBlocks() (<-chan *tmtypes.Block, *pubsub.Subscription) + WatchTendermintBlocks() (<-chan *tmtypes.Block, *pubsub.Subscription, error) // GetLastRetainedVersion returns the earliest retained version the ABCI // state. @@ -308,8 +308,7 @@ type ServiceClient interface { // BaseServiceClient is a default ServiceClient implementation that provides noop implementations of // all the delivery methods. Implementations should override them as needed. -type BaseServiceClient struct { -} +type BaseServiceClient struct{} // Implements ServiceClient. func (bsc *BaseServiceClient) DeliverBlock(ctx context.Context, height int64) error { diff --git a/go/consensus/tendermint/full/archive.go b/go/consensus/tendermint/full/archive.go new file mode 100644 index 00000000000..9b094c9382f --- /dev/null +++ b/go/consensus/tendermint/full/archive.go @@ -0,0 +1,230 @@ +package full + +import ( + "context" + "fmt" + "path/filepath" + "sync" + "time" + + "github.com/spf13/viper" + abcicli "github.com/tendermint/tendermint/abci/client" + tmconfig "github.com/tendermint/tendermint/config" + tmsync "github.com/tendermint/tendermint/libs/sync" + tmnode "github.com/tendermint/tendermint/node" + tmproxy "github.com/tendermint/tendermint/proxy" + tmcore "github.com/tendermint/tendermint/rpc/core" + tmrpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types" + "github.com/tendermint/tendermint/store" + + "github.com/oasisprotocol/oasis-core/go/common/identity" + "github.com/oasisprotocol/oasis-core/go/common/logging" + cmservice "github.com/oasisprotocol/oasis-core/go/common/service" + consensusAPI "github.com/oasisprotocol/oasis-core/go/consensus/api" + "github.com/oasisprotocol/oasis-core/go/consensus/api/transaction" + "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/abci" + "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/api" + tmcommon "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/common" + "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/db" + genesisAPI "github.com/oasisprotocol/oasis-core/go/genesis/api" + cmbackground "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/background" +) + +var _ api.Backend = (*archiveService)(nil) + +type archiveService struct { + sync.Mutex + commonNode + + abciClient abcicli.Client + + isStarted bool + + startedCh chan struct{} + quitCh chan struct{} + + stopOnce sync.Once +} + +func (srv *archiveService) started() bool { + srv.Lock() + defer srv.Unlock() + + return srv.isStarted +} + +// Start starts the service. +func (srv *archiveService) Start() error { + if srv.started() { + return fmt.Errorf("tendermint: service already started") + } + + if err := srv.commonNode.Start(); err != nil { + return err + } + + if err := srv.abciClient.Start(); err != nil { + return err + } + + // Make sure the quit channel is closed when the node shuts down. + go func() { + select { + case <-srv.quitCh: + case <-srv.mux.Quit(): + select { + case <-srv.quitCh: + default: + close(srv.quitCh) + } + } + }() + + srv.Lock() + srv.isStarted = true + srv.Unlock() + close(srv.startedCh) + + return nil +} + +// Stop halts the service. +func (srv *archiveService) Stop() { + if !srv.started() { + return + } + srv.stopOnce.Do(func() { + if err := srv.abciClient.Stop(); err != nil { + srv.Logger.Error("error on stopping abci client", "err", err) + } + srv.commonNode.Stop() + }) +} + +// Quit returns a channel that will be closed when the service terminates. +func (srv *archiveService) Quit() <-chan struct{} { + return srv.quitCh +} + +// Implements Backend. +func (srv *archiveService) Synced() <-chan struct{} { + // Archive node is always considered synced. + ch := make(chan struct{}) + close(ch) + return ch +} + +// Implements Backend. +func (srv *archiveService) EstimateGas(ctx context.Context, req *consensusAPI.EstimateGasRequest) (transaction.Gas, error) { + return 0, consensusAPI.ErrUnsupported +} + +// Implements Backend. +func (srv *archiveService) GetSignerNonce(ctx context.Context, req *consensusAPI.GetSignerNonceRequest) (uint64, error) { + return 0, consensusAPI.ErrUnsupported +} + +// New creates a new archive-only consensus service. +func NewArchive( + ctx context.Context, + dataDir string, + identity *identity.Identity, + genesisProvider genesisAPI.Provider, +) (consensusAPI.Backend, error) { + var err error + + srv := &archiveService{ + commonNode: commonNode{ + BaseBackgroundService: *cmservice.NewBaseBackgroundService("tendermint"), + ctx: ctx, + rpcCtx: &tmrpctypes.Context{}, + identity: identity, + dataDir: dataDir, + svcMgr: cmbackground.NewServiceManager(logging.GetLogger("tendermint/servicemanager")), + startedCh: make(chan struct{}), + }, + startedCh: make(chan struct{}), + quitCh: make(chan struct{}), + } + // Common node needs access to parent struct for initializing consensus services. + srv.commonNode.parentNode = srv + + doc, err := genesisProvider.GetGenesisDocument() + if err != nil { + return nil, fmt.Errorf("tendermint/archive: failed to get genesis document: %w", err) + } + srv.genesis = doc + + appConfig := &abci.ApplicationConfig{ + DataDir: filepath.Join(srv.dataDir, tmcommon.StateDir), + StorageBackend: db.GetBackendName(), + Pruning: abci.PruneConfig{ + Strategy: abci.PruneNone, + }, + DisableCheckpointer: true, + CheckpointerCheckInterval: 100 * time.Hour, // Disabled. + HaltEpochHeight: srv.genesis.HaltEpoch, + OwnTxSigner: srv.identity.NodeSigner.Public(), + InitialHeight: uint64(srv.genesis.Height), + // ReadOnly should actually be preferable for archive but there is a badger issue with read-only: + // https://discuss.dgraph.io/t/read-only-log-truncate-required-to-run-db/16444/2 + ReadOnlyStorage: false, + } + srv.mux, err = abci.NewApplicationServer(srv.ctx, nil, appConfig) + if err != nil { + return nil, fmt.Errorf("tendermint/archive: failed to create application server: %w", err) + } + + // Setup needed tendermint services. + logger := tmcommon.NewLogAdapter(!viper.GetBool(tmcommon.CfgLogDebug)) + srv.abciClient = abcicli.NewLocalClient(new(tmsync.Mutex), srv.mux.Mux()) + + dbProvider, err := db.GetProvider() + if err != nil { + return nil, err + } + tmConfig := tmconfig.DefaultConfig() + _ = viper.Unmarshal(&tmConfig) + tmConfig.SetRoot(filepath.Join(srv.dataDir, tmcommon.StateDir)) + + // NOTE: DBContext uses a full tendermint config but the only thing that is actually used + // is the data dir field. + srv.blockStoreDB, err = dbProvider(&tmnode.DBContext{ID: "blockstore", Config: tmConfig}) + if err != nil { + return nil, err + } + + // NOTE: DBContext uses a full tendermint config but the only thing that is actually used + // is the data dir field. + srv.stateStore, err = dbProvider(&tmnode.DBContext{ID: "state", Config: tmConfig}) + if err != nil { + return nil, err + } + + tmGenDoc, err := api.GetTendermintGenesisDocument(genesisProvider) + if err != nil { + return nil, err + } + + // Setup minimal tendermint environment needed to support consensus queries. + tmcore.SetEnvironment(&tmcore.Environment{ + ProxyAppQuery: tmproxy.NewAppConnQuery(srv.abciClient), + ProxyAppMempool: nil, + StateDB: srv.stateStore, + BlockStore: store.NewBlockStore(srv.blockStoreDB), + EvidencePool: nil, + ConsensusState: nil, + GenDoc: tmGenDoc, + Logger: logger, + Config: *tmConfig.RPC, + EventBus: nil, + P2PPeers: nil, + P2PTransport: nil, + PubKey: nil, + TxIndexer: nil, + ConsensusReactor: nil, + Mempool: nil, + }) + + return srv, srv.initialize() +} diff --git a/go/consensus/tendermint/full/common.go b/go/consensus/tendermint/full/common.go new file mode 100644 index 00000000000..c4e377789ac --- /dev/null +++ b/go/consensus/tendermint/full/common.go @@ -0,0 +1,735 @@ +package full + +import ( + "context" + "fmt" + "sync" + + "github.com/spf13/viper" + tmcore "github.com/tendermint/tendermint/rpc/core" + tmcoretypes "github.com/tendermint/tendermint/rpc/core/types" + tmrpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types" + tmstate "github.com/tendermint/tendermint/state" + "github.com/tendermint/tendermint/store" + tmtypes "github.com/tendermint/tendermint/types" + tmdb "github.com/tendermint/tm-db" + + beaconAPI "github.com/oasisprotocol/oasis-core/go/beacon/api" + "github.com/oasisprotocol/oasis-core/go/common/crypto/signature" + "github.com/oasisprotocol/oasis-core/go/common/identity" + "github.com/oasisprotocol/oasis-core/go/common/node" + "github.com/oasisprotocol/oasis-core/go/common/pubsub" + cmservice "github.com/oasisprotocol/oasis-core/go/common/service" + "github.com/oasisprotocol/oasis-core/go/common/version" + consensusAPI "github.com/oasisprotocol/oasis-core/go/consensus/api" + "github.com/oasisprotocol/oasis-core/go/consensus/api/transaction" + "github.com/oasisprotocol/oasis-core/go/consensus/api/transaction/results" + "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/abci" + "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/api" + "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/apps/supplementarysanity" + tmbeacon "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/beacon" + tmcommon "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/common" + "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/crypto" + tmepochtime "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/epochtime" + tmkeymanager "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/keymanager" + tmregistry "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/registry" + tmroothash "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/roothash" + tmscheduler "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/scheduler" + tmstaking "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/staking" + epochtimeAPI "github.com/oasisprotocol/oasis-core/go/epochtime/api" + genesisAPI "github.com/oasisprotocol/oasis-core/go/genesis/api" + keymanagerAPI "github.com/oasisprotocol/oasis-core/go/keymanager/api" + cmbackground "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/background" + cmmetrics "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" + "github.com/oasisprotocol/oasis-core/go/registry" + registryAPI "github.com/oasisprotocol/oasis-core/go/registry/api" + roothashAPI "github.com/oasisprotocol/oasis-core/go/roothash/api" + schedulerAPI "github.com/oasisprotocol/oasis-core/go/scheduler/api" + stakingAPI "github.com/oasisprotocol/oasis-core/go/staking/api" +) + +// commonNode implements the common tendermint node functionality shared between + +// full and archive nodes. +type commonNode struct { + sync.Mutex + cmservice.BaseBackgroundService + + svcMgr *cmbackground.ServiceManager + + serviceClients []api.ServiceClient + serviceClientsWg sync.WaitGroup + + ctx context.Context + rpcCtx *tmrpctypes.Context + + dataDir string + identity *identity.Identity + + genesis *genesisAPI.Document + + mux *abci.ApplicationServer + + beacon beaconAPI.Backend + epochtime epochtimeAPI.Backend + keymanager keymanagerAPI.Backend + registry registryAPI.Backend + roothash roothashAPI.Backend + scheduler schedulerAPI.Backend + staking stakingAPI.Backend + + blockStoreDB tmdb.DB + stateStore tmdb.DB + + // Guarded by the lock. + isStarted, isInitialized bool + + startedCh chan struct{} + + parentNode api.Backend +} + +func (n *commonNode) initialized() bool { + n.Lock() + defer n.Unlock() + + return n.isInitialized +} + +func (n *commonNode) started() bool { + n.Lock() + defer n.Unlock() + + return n.isStarted +} + +func (n *commonNode) ensureStarted(ctx context.Context) error { + // Make sure that the Tendermint service is setup and started. + select { + case <-n.startedCh: + case <-n.ctx.Done(): + return n.ctx.Err() + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} + +func (n *commonNode) Start() error { + n.Lock() + defer n.Unlock() + + if n.isStarted { + return fmt.Errorf("tendermint/common_node: already started") + } + + if !n.isInitialized { + return fmt.Errorf("tendermint/common_node: not initialized") + } + + if err := n.mux.Start(); err != nil { + return err + } + + n.isStarted = true + close(n.startedCh) + + return nil +} + +func (n *commonNode) Stop() { + n.Lock() + defer n.Unlock() + + if !n.isStarted || !n.isInitialized { + return + } + + n.svcMgr.Stop() + n.mux.Stop() + if err := n.blockStoreDB.Close(); err != nil { + n.Logger.Error("error on stopping block store", "err", err) + } + if err := n.stateStore.Close(); err != nil { + n.Logger.Error("error on stopping state store", "err", err) + } +} + +func (n *commonNode) initialize() error { + n.Lock() + defer n.Unlock() + + if n.isInitialized { + return nil + } + + // Apply the genesis public key blacklist. + for _, v := range n.genesis.Consensus.Parameters.PublicKeyBlacklist { + if err := v.Blacklist(); err != nil { + n.Logger.Error("initialize: failed to blacklist key", + "err", err, + "pk", v, + ) + return err + } + } + + // Initialize the beacon/epochtime backend. + var ( + err error + + scEpochTime tmepochtime.ServiceClient + scBeacon tmbeacon.ServiceClient + ) + + scEpochTime, err = tmepochtime.New(n.ctx, n.parentNode, n.genesis.EpochTime.Parameters.Interval) + if err != nil { + n.Logger.Error("initEpochtime: failed to initialize epochtime backend", + "err", err, + ) + return err + } + n.epochtime = scEpochTime + n.serviceClients = append(n.serviceClients, scEpochTime) + if err := n.mux.SetEpochtime(n.epochtime); err != nil { + return err + } + + if scBeacon, err = tmbeacon.New(n.ctx, n.parentNode); err != nil { + n.Logger.Error("initialize: failed to initialize beapoch backend", + "err", err, + ) + return err + } + n.beacon = scBeacon + n.serviceClients = append(n.serviceClients, scBeacon) + + // Initialize the rest of backends. + var scKeyManager tmkeymanager.ServiceClient + if scKeyManager, err = tmkeymanager.New(n.ctx, n.parentNode); err != nil { + n.Logger.Error("initialize: failed to initialize keymanager backend", + "err", err, + ) + return err + } + n.keymanager = scKeyManager + n.serviceClients = append(n.serviceClients, scKeyManager) + + var scRegistry tmregistry.ServiceClient + if scRegistry, err = tmregistry.New(n.ctx, n.parentNode); err != nil { + n.Logger.Error("initialize: failed to initialize registry backend", + "err", err, + ) + return err + } + n.registry = scRegistry + if cmmetrics.Enabled() { + n.svcMgr.RegisterCleanupOnly(registry.NewMetricsUpdater(n.ctx, n.registry), "registry metrics updater") + } + n.serviceClients = append(n.serviceClients, scRegistry) + n.svcMgr.RegisterCleanupOnly(n.registry, "registry backend") + + var scStaking tmstaking.ServiceClient + if scStaking, err = tmstaking.New(n.ctx, n.parentNode); err != nil { + n.Logger.Error("staking: failed to initialize staking backend", + "err", err, + ) + return err + } + n.staking = scStaking + n.serviceClients = append(n.serviceClients, scStaking) + n.svcMgr.RegisterCleanupOnly(n.staking, "staking backend") + + var scScheduler tmscheduler.ServiceClient + if scScheduler, err = tmscheduler.New(n.ctx, n.parentNode); err != nil { + n.Logger.Error("scheduler: failed to initialize scheduler backend", + "err", err, + ) + return err + } + n.scheduler = scScheduler + n.serviceClients = append(n.serviceClients, scScheduler) + n.svcMgr.RegisterCleanupOnly(n.scheduler, "scheduler backend") + + var scRootHash tmroothash.ServiceClient + if scRootHash, err = tmroothash.New(n.ctx, n.dataDir, n.parentNode); err != nil { + n.Logger.Error("roothash: failed to initialize roothash backend", + "err", err, + ) + return err + } + n.roothash = scRootHash + n.serviceClients = append(n.serviceClients, scRootHash) + n.svcMgr.RegisterCleanupOnly(n.roothash, "roothash backend") + + // Enable supplementary sanity checks when enabled. + if viper.GetBool(CfgSupplementarySanityEnabled) { + ssa := supplementarysanity.New(viper.GetUint64(CfgSupplementarySanityInterval)) + if err = n.RegisterApplication(ssa); err != nil { + return fmt.Errorf("failed to register supplementary sanity check app: %w", err) + } + } + + n.isInitialized = true + + return nil +} + +// Implements service.BackgroundService. +func (n *commonNode) Cleanup() { + n.serviceClientsWg.Wait() + n.svcMgr.Cleanup() +} + +func (t *commonNode) ConsensusKey() signature.PublicKey { + return t.identity.ConsensusSigner.Public() +} + +func (n *commonNode) SupportedFeatures() consensusAPI.FeatureMask { + return consensusAPI.FeatureServices | consensusAPI.FeatureFullNode +} + +func (n *commonNode) GetAddresses() ([]node.ConsensusAddress, error) { + u, err := tmcommon.GetExternalAddress() + if err != nil { + return nil, err + } + + var addr node.ConsensusAddress + if err = addr.Address.UnmarshalText([]byte(u.Host)); err != nil { + return nil, fmt.Errorf("tendermint: failed to parse external address host: %w", err) + } + addr.ID = n.identity.P2PSigner.Public() + + return []node.ConsensusAddress{addr}, nil +} + +func (n *commonNode) StateToGenesis(ctx context.Context, blockHeight int64) (*genesisAPI.Document, error) { + blk, err := n.GetTendermintBlock(ctx, blockHeight) + if err != nil { + return nil, err + } + if blk == nil { + return nil, consensusAPI.ErrNoCommittedBlocks + } + blockHeight = blk.Header.Height + + // Get initial genesis doc. + genesisDoc, err := n.GetGenesisDocument(ctx) + if err != nil { + return nil, err + } + + // Call StateToGenesis on all backends and merge the results together. + epochtimeGenesis, err := n.epochtime.StateToGenesis(ctx, blockHeight) + if err != nil { + return nil, err + } + + beaconGenesis, err := n.Beacon().StateToGenesis(ctx, blockHeight) + if err != nil { + return nil, err + } + + registryGenesis, err := n.Registry().StateToGenesis(ctx, blockHeight) + if err != nil { + return nil, err + } + + roothashGenesis, err := n.RootHash().StateToGenesis(ctx, blockHeight) + if err != nil { + return nil, err + } + + stakingGenesis, err := n.Staking().StateToGenesis(ctx, blockHeight) + if err != nil { + return nil, err + } + + keymanagerGenesis, err := n.KeyManager().StateToGenesis(ctx, blockHeight) + if err != nil { + return nil, err + } + + schedulerGenesis, err := n.Scheduler().StateToGenesis(ctx, blockHeight) + if err != nil { + return nil, err + } + + return &genesisAPI.Document{ + Height: blockHeight, + ChainID: genesisDoc.ChainID, + HaltEpoch: genesisDoc.HaltEpoch, + Time: blk.Header.Time, + EpochTime: *epochtimeGenesis, + Beacon: *beaconGenesis, + Registry: *registryGenesis, + RootHash: *roothashGenesis, + Staking: *stakingGenesis, + KeyManager: *keymanagerGenesis, + Scheduler: *schedulerGenesis, + Consensus: genesisDoc.Consensus, + }, nil +} + +func (n *commonNode) GetGenesisDocument(ctx context.Context) (*genesisAPI.Document, error) { + return n.genesis, nil +} + +func (n *commonNode) GetChainContext(ctx context.Context) (string, error) { + return n.genesis.ChainContext(), nil +} + +func (n *commonNode) EpochTime() epochtimeAPI.Backend { + return n.epochtime +} + +func (n *commonNode) Beacon() beaconAPI.Backend { + return n.beacon +} + +func (n *commonNode) KeyManager() keymanagerAPI.Backend { + return n.keymanager +} + +func (n *commonNode) Registry() registryAPI.Backend { + return n.registry +} + +func (n *commonNode) RootHash() roothashAPI.Backend { + return n.roothash +} + +func (n *commonNode) Staking() stakingAPI.Backend { + return n.staking +} + +func (n *commonNode) Scheduler() schedulerAPI.Backend { + return n.scheduler +} + +func (n *commonNode) RegisterApplication(app api.Application) error { + return n.mux.Register(app) +} + +func (n *commonNode) SetTransactionAuthHandler(handler api.TransactionAuthHandler) error { + return n.mux.SetTransactionAuthHandler(handler) +} + +func (n *commonNode) TransactionAuthHandler() consensusAPI.TransactionAuthHandler { + return n.mux.TransactionAuthHandler() +} + +func (n *commonNode) EstimateGas(ctx context.Context, req *consensusAPI.EstimateGasRequest) (transaction.Gas, error) { + return n.mux.EstimateGas(req.Signer, req.Transaction) +} + +func (n *commonNode) RegisterHaltHook(hook consensusAPI.HaltHook) { + if !n.initialized() { + return + } + + n.mux.RegisterHaltHook(hook) +} + +func (n *commonNode) heightToTendermintHeight(height int64) (int64, error) { + var tmHeight int64 + if height == consensusAPI.HeightLatest { + // Do not let Tendermint determine the latest height (e.g., by passing nil) as that + // completely ignores ABCI processing so it can return a block for which local state does + // not yet exist. Use our mux notion of latest height instead. + tmHeight = n.mux.State().BlockHeight() + if tmHeight == 0 { + // No committed blocks yet. + return 0, consensusAPI.ErrNoCommittedBlocks + } + } else { + tmHeight = height + } + + return tmHeight, nil +} + +func (n *commonNode) GetSignerNonce(ctx context.Context, req *consensusAPI.GetSignerNonceRequest) (uint64, error) { + return n.mux.TransactionAuthHandler().GetSignerNonce(ctx, req) +} + +// These method need to be provided. +func (n *commonNode) GetTendermintBlock(ctx context.Context, height int64) (*tmtypes.Block, error) { + if err := n.ensureStarted(ctx); err != nil { + return nil, err + } + + tmHeight, err := n.heightToTendermintHeight(height) + switch err { + case nil: + // Continues bellow. + case consensusAPI.ErrNoCommittedBlocks: + // No committed blocks yet. + return nil, nil + default: + return nil, err + } + result, err := tmcore.Block(n.rpcCtx, &tmHeight) + if err != nil { + return nil, fmt.Errorf("tendermint: block query failed: %w", err) + } + return result.Block, nil +} + +func (n *commonNode) GetBlockResults(height int64) (*tmcoretypes.ResultBlockResults, error) { + if err := n.ensureStarted(n.ctx); err != nil { + return nil, err + } + + tmHeight, err := n.heightToTendermintHeight(height) + if err != nil { + return nil, err + } + result, err := tmcore.BlockResults(n.rpcCtx, &tmHeight) + if err != nil { + return nil, fmt.Errorf("tendermint: block results query failed: %w", err) + } + + return result, nil +} + +func (n *commonNode) GetLastRetainedVersion(ctx context.Context) (int64, error) { + if err := n.ensureStarted(ctx); err != nil { + return -1, err + } + state := store.LoadBlockStoreState(n.blockStoreDB) + return state.Base, nil +} + +// Following use the provided methods. +func (n *commonNode) GetBlock(ctx context.Context, height int64) (*consensusAPI.Block, error) { + blk, err := n.GetTendermintBlock(ctx, height) + if err != nil { + return nil, err + } + if blk == nil { + return nil, consensusAPI.ErrNoCommittedBlocks + } + + return api.NewBlock(blk), nil +} + +func (n *commonNode) GetEpoch(ctx context.Context, height int64) (epochtimeAPI.EpochTime, error) { + if n.epochtime == nil { + return epochtimeAPI.EpochInvalid, consensusAPI.ErrUnsupported + } + return n.epochtime.GetEpoch(ctx, height) +} + +func (n *commonNode) WaitEpoch(ctx context.Context, epoch epochtimeAPI.EpochTime) error { + if n.epochtime == nil { + return consensusAPI.ErrUnsupported + } + + ch, sub := n.epochtime.WatchEpochs() + defer sub.Close() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case e, ok := <-ch: + if !ok { + return context.Canceled + } + if e >= epoch { + return nil + } + } + } +} + +func (n *commonNode) GetTransactions(ctx context.Context, height int64) ([][]byte, error) { + blk, err := n.GetTendermintBlock(ctx, height) + if err != nil { + return nil, err + } + if blk == nil { + return nil, consensusAPI.ErrNoCommittedBlocks + } + + txs := make([][]byte, 0, len(blk.Data.Txs)) + for _, v := range blk.Data.Txs { + txs = append(txs, v[:]) + } + return txs, nil +} + +func (n *commonNode) GetTransactionsWithResults(ctx context.Context, height int64) (*consensusAPI.TransactionsWithResults, error) { + var txsWithResults consensusAPI.TransactionsWithResults + + blk, err := n.GetTendermintBlock(ctx, height) + if err != nil { + return nil, err + } + if blk == nil { + return nil, consensusAPI.ErrNoCommittedBlocks + } + for _, tx := range blk.Data.Txs { + txsWithResults.Transactions = append(txsWithResults.Transactions, tx[:]) + } + + res, err := n.GetBlockResults(blk.Height) + if err != nil { + return nil, err + } + for txIdx, rs := range res.TxsResults { + // Transaction result. + result := &results.Result{ + Error: results.Error{ + Module: rs.GetCodespace(), + Code: rs.GetCode(), + Message: rs.GetLog(), + }, + } + + // Transaction staking events. + stakingEvents, err := tmstaking.EventsFromTendermint( + txsWithResults.Transactions[txIdx], + blk.Height, + rs.Events, + ) + if err != nil { + return nil, err + } + for _, e := range stakingEvents { + result.Events = append(result.Events, &results.Event{Staking: e}) + } + + // Transaction registry events. + registryEvents, _, err := tmregistry.EventsFromTendermint( + txsWithResults.Transactions[txIdx], + blk.Height, + rs.Events, + ) + if err != nil { + return nil, err + } + for _, e := range registryEvents { + result.Events = append(result.Events, &results.Event{Registry: e}) + } + + // Transaction roothash events. + roothashEvents, err := tmroothash.EventsFromTendermint( + txsWithResults.Transactions[txIdx], + blk.Height, + rs.Events, + ) + if err != nil { + return nil, err + } + for _, e := range roothashEvents { + result.Events = append(result.Events, &results.Event{RootHash: e}) + } + + txsWithResults.Results = append(txsWithResults.Results, result) + } + return &txsWithResults, nil +} + +func (n *commonNode) GetStatus(ctx context.Context) (*consensusAPI.Status, error) { + status := &consensusAPI.Status{ + ConsensusVersion: version.ConsensusProtocol.String(), + Backend: api.BackendName, + Features: n.SupportedFeatures(), + } + + status.GenesisHeight = n.genesis.Height + if n.started() { + // Only attempt to fetch blocks in case the consensus service has started as otherwise + // requests will block. + genBlk, err := n.GetBlock(ctx, n.genesis.Height) + switch err { + case nil: + status.GenesisHash = genBlk.Hash + default: + // We may not be able to fetch the genesis block in case it has been pruned. + } + + lastRetainedHeight, err := n.GetLastRetainedVersion(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get last retained height: %w", err) + } + // Some pruning configurations return 0 instead of a valid block height. Clamp those to the genesis height. + if lastRetainedHeight < n.genesis.Height { + lastRetainedHeight = n.genesis.Height + } + status.LastRetainedHeight = lastRetainedHeight + lastRetainedBlock, err := n.GetBlock(ctx, lastRetainedHeight) + switch err { + case nil: + status.LastRetainedHash = lastRetainedBlock.Hash + default: + // Before we commit the first block, we can't load it from GetBlock. Don't give its hash in this case. + } + + // Latest block. + latestBlk, err := n.GetBlock(ctx, consensusAPI.HeightLatest) + switch err { + case nil: + status.LatestHeight = latestBlk.Height + status.LatestHash = latestBlk.Hash + status.LatestTime = latestBlk.Time + status.LatestStateRoot = latestBlk.StateRoot + case consensusAPI.ErrNoCommittedBlocks: + // No committed blocks yet. + default: + return nil, fmt.Errorf("failed to fetch current block: %w", err) + } + + // Check if the local node is in the validator set for the latest (uncommitted) block. + valSetHeight := status.LatestHeight + 1 + if valSetHeight < status.GenesisHeight { + valSetHeight = status.GenesisHeight + } + vals, err := tmstate.LoadValidators(n.stateStore, valSetHeight) + if err != nil { + // Failed to load validator set. + status.IsValidator = false + } else { + consensusPk := n.identity.ConsensusSigner.Public() + consensusAddr := []byte(crypto.PublicKeyToTendermint(&consensusPk).Address()) + status.IsValidator = vals.HasAddress(consensusAddr) + } + } + + return status, nil +} + +// Unimplemented methods. + +func (n *commonNode) WatchTendermintBlocks() (<-chan *tmtypes.Block, *pubsub.Subscription, error) { + return nil, nil, consensusAPI.ErrUnsupported +} + +// Implements Backend. +func (n *commonNode) SubmitEvidence(ctx context.Context, evidence *consensusAPI.Evidence) error { + return consensusAPI.ErrUnsupported +} + +// Implements Backend. +func (n *commonNode) SubmitTx(ctx context.Context, tx *transaction.SignedTransaction) error { + return consensusAPI.ErrUnsupported +} + +// Implements Backend. +func (n *commonNode) GetUnconfirmedTransactions(ctx context.Context) ([][]byte, error) { + return nil, consensusAPI.ErrUnsupported +} + +// Implements Backend. +func (n *commonNode) WatchBlocks(ctx context.Context) (<-chan *consensusAPI.Block, pubsub.ClosableSubscription, error) { + return nil, nil, consensusAPI.ErrUnsupported +} + +// Implements Backend. +func (n *commonNode) SubmissionManager() consensusAPI.SubmissionManager { + return &consensusAPI.NoOpSubmissionManager{} +} diff --git a/go/consensus/tendermint/full/full.go b/go/consensus/tendermint/full/full.go index a00ab46008a..1b17300e043 100644 --- a/go/consensus/tendermint/full/full.go +++ b/go/consensus/tendermint/full/full.go @@ -25,54 +25,33 @@ import ( tmproto "github.com/tendermint/tendermint/proto/tendermint/types" tmproxy "github.com/tendermint/tendermint/proxy" tmcli "github.com/tendermint/tendermint/rpc/client/local" - tmrpctypes "github.com/tendermint/tendermint/rpc/core/types" - tmstate "github.com/tendermint/tendermint/state" + tmrpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types" tmstatesync "github.com/tendermint/tendermint/statesync" tmtypes "github.com/tendermint/tendermint/types" tmdb "github.com/tendermint/tm-db" - beaconAPI "github.com/oasisprotocol/oasis-core/go/beacon/api" "github.com/oasisprotocol/oasis-core/go/common/cbor" "github.com/oasisprotocol/oasis-core/go/common/crypto/hash" - "github.com/oasisprotocol/oasis-core/go/common/crypto/signature" "github.com/oasisprotocol/oasis-core/go/common/errors" "github.com/oasisprotocol/oasis-core/go/common/identity" "github.com/oasisprotocol/oasis-core/go/common/logging" "github.com/oasisprotocol/oasis-core/go/common/node" "github.com/oasisprotocol/oasis-core/go/common/pubsub" cmservice "github.com/oasisprotocol/oasis-core/go/common/service" - "github.com/oasisprotocol/oasis-core/go/common/version" consensusAPI "github.com/oasisprotocol/oasis-core/go/consensus/api" "github.com/oasisprotocol/oasis-core/go/consensus/api/transaction" - "github.com/oasisprotocol/oasis-core/go/consensus/api/transaction/results" "github.com/oasisprotocol/oasis-core/go/consensus/metrics" "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/abci" "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/api" - "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/apps/supplementarysanity" - tmbeacon "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/beacon" tmcommon "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/common" "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/crypto" "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/db" - tmepochtime "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/epochtime" - tmepochtimemock "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/epochtime_mock" - tmkeymanager "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/keymanager" "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/light" - tmregistry "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/registry" - tmroothash "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/roothash" - tmscheduler "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/scheduler" - tmstaking "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/staking" epochtimeAPI "github.com/oasisprotocol/oasis-core/go/epochtime/api" genesisAPI "github.com/oasisprotocol/oasis-core/go/genesis/api" - keymanagerAPI "github.com/oasisprotocol/oasis-core/go/keymanager/api" cmbackground "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/background" cmflags "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/flags" cmmetrics "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" - "github.com/oasisprotocol/oasis-core/go/registry" - registryAPI "github.com/oasisprotocol/oasis-core/go/registry/api" - "github.com/oasisprotocol/oasis-core/go/roothash" - roothashAPI "github.com/oasisprotocol/oasis-core/go/roothash/api" - schedulerAPI "github.com/oasisprotocol/oasis-core/go/scheduler/api" - stakingAPI "github.com/oasisprotocol/oasis-core/go/staking/api" upgradeAPI "github.com/oasisprotocol/oasis-core/go/upgrade/api" ) @@ -153,39 +132,21 @@ var ( // fullService implements a full Tendermint node. type fullService struct { // nolint: maligned sync.Mutex - cmservice.BaseBackgroundService + commonNode - ctx context.Context - svcMgr *cmbackground.ServiceManager upgrader upgradeAPI.Backend - mux *abci.ApplicationServer node *tmnode.Node client *tmcli.Local blockNotifier *pubsub.Broker failMonitor *failMonitor - stateDb tmdb.DB - - beacon beaconAPI.Backend - epochtime epochtimeAPI.Backend - keymanager keymanagerAPI.Backend - registry registryAPI.Backend - roothash roothashAPI.Backend - staking stakingAPI.Backend - scheduler schedulerAPI.Backend submissionMgr consensusAPI.SubmissionManager - serviceClients []api.ServiceClient - serviceClientsWg sync.WaitGroup - - genesis *genesisAPI.Document - genesisProvider genesisAPI.Provider - identity *identity.Identity - dataDir string - isInitialized, isStarted bool - startedCh chan struct{} - syncedCh chan struct{} - quitCh chan struct{} + genesisProvider genesisAPI.Provider + isStarted bool + startedCh chan struct{} + syncedCh chan struct{} + quitCh chan struct{} startFn func() error stopOnce sync.Once @@ -193,13 +154,6 @@ type fullService struct { // nolint: maligned nextSubscriberID uint64 } -func (t *fullService) initialized() bool { - t.Lock() - defer t.Unlock() - - return t.isInitialized -} - func (t *fullService) started() bool { t.Lock() defer t.Unlock() @@ -215,7 +169,7 @@ func (t *fullService) Start() error { switch t.initialized() { case true: - if err := t.mux.Start(); err != nil { + if err := t.commonNode.Start(); err != nil { return err } if err := t.startFn(); err != nil { @@ -269,12 +223,6 @@ func (t *fullService) Quit() <-chan struct{} { return t.quitCh } -// Implements service.BackgroundService. -func (t *fullService) Cleanup() { - t.serviceClientsWg.Wait() - t.svcMgr.Cleanup() -} - // Implements service.BackgroundService. func (t *fullService) Stop() { if !t.initialized() || !t.started() { @@ -286,9 +234,7 @@ func (t *fullService) Stop() { if err := t.node.Stop(); err != nil { t.Logger.Error("Error on stopping node", err) } - - t.svcMgr.Stop() - t.mux.Stop() + t.commonNode.Stop() }) } @@ -296,135 +242,10 @@ func (t *fullService) Started() <-chan struct{} { return t.startedCh } -func (t *fullService) SupportedFeatures() consensusAPI.FeatureMask { - return consensusAPI.FeatureServices | consensusAPI.FeatureFullNode -} - func (t *fullService) Synced() <-chan struct{} { return t.syncedCh } -func (t *fullService) GetAddresses() ([]node.ConsensusAddress, error) { - u, err := tmcommon.GetExternalAddress() - if err != nil { - return nil, err - } - - var addr node.ConsensusAddress - if err = addr.Address.UnmarshalText([]byte(u.Host)); err != nil { - return nil, fmt.Errorf("tendermint: failed to parse external address host: %w", err) - } - addr.ID = t.identity.P2PSigner.Public() - - return []node.ConsensusAddress{addr}, nil -} - -func (t *fullService) StateToGenesis(ctx context.Context, blockHeight int64) (*genesisAPI.Document, error) { - blk, err := t.GetTendermintBlock(ctx, blockHeight) - if err != nil { - t.Logger.Error("failed to get tendermint block", - "err", err, - "block_height", blockHeight, - ) - return nil, err - } - if blk == nil { - return nil, consensusAPI.ErrNoCommittedBlocks - } - blockHeight = blk.Header.Height - - // Get initial genesis doc. - genesisDoc, err := t.GetGenesisDocument(ctx) - if err != nil { - t.Logger.Error("failed getting genesis document", - "err", err, - ) - return nil, err - } - - // Call StateToGenesis on all backends and merge the results together. - epochtimeGenesis, err := t.epochtime.StateToGenesis(ctx, blockHeight) - if err != nil { - t.Logger.Error("epochtime StateToGenesis failure", - "err", err, - "block_height", blockHeight, - ) - return nil, err - } - - registryGenesis, err := t.registry.StateToGenesis(ctx, blockHeight) - if err != nil { - t.Logger.Error("registry StateToGenesis failure", - "err", err, - "block_height", blockHeight, - ) - return nil, err - } - - roothashGenesis, err := t.roothash.StateToGenesis(ctx, blockHeight) - if err != nil { - t.Logger.Error("roothash StateToGenesis failure", - "err", err, - "block_height", blockHeight, - ) - return nil, err - } - - stakingGenesis, err := t.staking.StateToGenesis(ctx, blockHeight) - if err != nil { - t.Logger.Error("staking StateToGenesis failure", - "err", err, - "block_height", blockHeight, - ) - return nil, err - } - - keymanagerGenesis, err := t.keymanager.StateToGenesis(ctx, blockHeight) - if err != nil { - t.Logger.Error("keymanager StateToGenesis failure", - "err", err, - "block_height", blockHeight, - ) - return nil, err - } - - schedulerGenesis, err := t.scheduler.StateToGenesis(ctx, blockHeight) - if err != nil { - t.Logger.Error("scheduler StateToGenesis failure", - "err", err, - "block_height", blockHeight, - ) - return nil, err - } - - return &genesisAPI.Document{ - Height: blockHeight, - ChainID: genesisDoc.ChainID, - HaltEpoch: genesisDoc.HaltEpoch, - Time: blk.Header.Time, - EpochTime: *epochtimeGenesis, - Registry: *registryGenesis, - RootHash: *roothashGenesis, - Staking: *stakingGenesis, - KeyManager: *keymanagerGenesis, - Scheduler: *schedulerGenesis, - Beacon: genesisDoc.Beacon, - Consensus: genesisDoc.Consensus, - }, nil -} - -func (t *fullService) GetGenesisDocument(ctx context.Context) (*genesisAPI.Document, error) { - return t.genesis, nil -} - -func (t *fullService) RegisterHaltHook(hook consensusAPI.HaltHook) { - if !t.initialized() { - return - } - - t.mux.RegisterHaltHook(hook) -} - func (t *fullService) SubmitTx(ctx context.Context, tx *transaction.SignedTransaction) error { // Subscribe to the transaction being included in a block. data := cbor.Marshal(tx) @@ -532,10 +353,6 @@ func (t *fullService) SubmitEvidence(ctx context.Context, evidence *consensusAPI return nil } -func (t *fullService) EstimateGas(ctx context.Context, req *consensusAPI.EstimateGasRequest) (transaction.Gas, error) { - return t.mux.EstimateGas(req.Signer, req.Transaction) -} - func (t *fullService) subscribe(subscriber string, query tmpubsub.Query) (tmtypes.Subscription, error) { // Note: The tendermint documentation claims using SubscribeUnbuffered can // freeze the server, however, the buffered Subscribe can drop events, and @@ -584,183 +401,10 @@ func (t *fullService) unsubscribe(subscriber string, query tmpubsub.Query) error return fmt.Errorf("tendermint: unsubscribe called with no backing service") } -func (t *fullService) RegisterApplication(app api.Application) error { - return t.mux.Register(app) -} - -func (t *fullService) SetTransactionAuthHandler(handler api.TransactionAuthHandler) error { - return t.mux.SetTransactionAuthHandler(handler) -} - -func (t *fullService) TransactionAuthHandler() consensusAPI.TransactionAuthHandler { - return t.mux.TransactionAuthHandler() -} - func (t *fullService) SubmissionManager() consensusAPI.SubmissionManager { return t.submissionMgr } -func (t *fullService) EpochTime() epochtimeAPI.Backend { - return t.epochtime -} - -func (t *fullService) Beacon() beaconAPI.Backend { - return t.beacon -} - -func (t *fullService) KeyManager() keymanagerAPI.Backend { - return t.keymanager -} - -func (t *fullService) Registry() registryAPI.Backend { - return t.registry -} - -func (t *fullService) RootHash() roothashAPI.Backend { - return t.roothash -} - -func (t *fullService) Staking() stakingAPI.Backend { - return t.staking -} - -func (t *fullService) Scheduler() schedulerAPI.Backend { - return t.scheduler -} - -func (t *fullService) GetEpoch(ctx context.Context, height int64) (epochtimeAPI.EpochTime, error) { - if t.epochtime == nil { - return epochtimeAPI.EpochInvalid, consensusAPI.ErrUnsupported - } - return t.epochtime.GetEpoch(ctx, height) -} - -func (t *fullService) WaitEpoch(ctx context.Context, epoch epochtimeAPI.EpochTime) error { - if t.epochtime == nil { - return consensusAPI.ErrUnsupported - } - - ch, sub := t.epochtime.WatchEpochs() - defer sub.Close() - - for { - select { - case <-ctx.Done(): - return ctx.Err() - case e, ok := <-ch: - if !ok { - return context.Canceled - } - if e >= epoch { - return nil - } - } - } -} - -func (t *fullService) GetBlock(ctx context.Context, height int64) (*consensusAPI.Block, error) { - blk, err := t.GetTendermintBlock(ctx, height) - if err != nil { - return nil, err - } - if blk == nil { - return nil, consensusAPI.ErrNoCommittedBlocks - } - - return api.NewBlock(blk), nil -} - -func (t *fullService) GetSignerNonce(ctx context.Context, req *consensusAPI.GetSignerNonceRequest) (uint64, error) { - return t.mux.TransactionAuthHandler().GetSignerNonce(ctx, req) -} - -func (t *fullService) GetTransactions(ctx context.Context, height int64) ([][]byte, error) { - blk, err := t.GetTendermintBlock(ctx, height) - if err != nil { - return nil, err - } - if blk == nil { - return nil, consensusAPI.ErrNoCommittedBlocks - } - - txs := make([][]byte, 0, len(blk.Data.Txs)) - for _, v := range blk.Data.Txs { - txs = append(txs, v[:]) - } - return txs, nil -} - -func (t *fullService) GetTransactionsWithResults(ctx context.Context, height int64) (*consensusAPI.TransactionsWithResults, error) { - var txsWithResults consensusAPI.TransactionsWithResults - - blk, err := t.GetTendermintBlock(ctx, height) - if err != nil { - return nil, err - } - if blk == nil { - return nil, consensusAPI.ErrNoCommittedBlocks - } - for _, tx := range blk.Data.Txs { - txsWithResults.Transactions = append(txsWithResults.Transactions, tx[:]) - } - - res, err := t.GetBlockResults(blk.Height) - if err != nil { - return nil, err - } - for txIdx, rs := range res.TxsResults { - // Transaction result. - result := &results.Result{ - Error: results.Error{ - Module: rs.GetCodespace(), - Code: rs.GetCode(), - Message: rs.GetLog(), - }, - } - - // Transaction staking events. - stakingEvents, err := tmstaking.EventsFromTendermint( - txsWithResults.Transactions[txIdx], - blk.Height, - rs.Events, - ) - if err != nil { - return nil, err - } - for _, e := range stakingEvents { - result.Events = append(result.Events, &results.Event{Staking: e}) - } - - // Transaction registry events. - registryEvents, _, err := tmregistry.EventsFromTendermint( - txsWithResults.Transactions[txIdx], - blk.Height, - rs.Events, - ) - if err != nil { - return nil, err - } - for _, e := range registryEvents { - result.Events = append(result.Events, &results.Event{Registry: e}) - } - - // Transaction roothash events. - roothashEvents, err := tmroothash.EventsFromTendermint( - txsWithResults.Transactions[txIdx], - blk.Height, - rs.Events, - ) - if err != nil { - return nil, err - } - for _, e := range roothashEvents { - result.Events = append(result.Events, &results.Event{RootHash: e}) - } - txsWithResults.Results = append(txsWithResults.Results, result) - } - return &txsWithResults, nil -} - func (t *fullService) GetUnconfirmedTransactions(ctx context.Context) ([][]byte, error) { mempoolTxs := t.node.Mempool().ReapMaxTxs(-1) txs := make([][]byte, 0, len(mempoolTxs)) @@ -770,84 +414,11 @@ func (t *fullService) GetUnconfirmedTransactions(ctx context.Context) ([][]byte, return txs, nil } -func (t *fullService) GetStatus(ctx context.Context) (*consensusAPI.Status, error) { - status := &consensusAPI.Status{ - ConsensusVersion: version.ConsensusProtocol.String(), - Backend: api.BackendName, - Features: t.SupportedFeatures(), - } - - status.GenesisHeight = t.genesis.Height - if t.started() { - // Only attempt to fetch blocks in case the consensus service has started as otherwise - // requests will block. - genBlk, err := t.GetBlock(ctx, t.genesis.Height) - switch err { - case nil: - status.GenesisHash = genBlk.Hash - default: - // We may not be able to fetch the genesis block in case it has been pruned. - } - - lastRetainedHeight, err := t.GetLastRetainedVersion(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get last retained height: %w", err) - } - // Some pruning configurations return 0 instead of a valid block height. Clamp those to the genesis height. - if lastRetainedHeight < t.genesis.Height { - lastRetainedHeight = t.genesis.Height - } - status.LastRetainedHeight = lastRetainedHeight - lastRetainedBlock, err := t.GetBlock(ctx, lastRetainedHeight) - switch err { - case nil: - status.LastRetainedHash = lastRetainedBlock.Hash - default: - // Before we commit the first block, we can't load it from GetBlock. Don't give its hash in this case. - } - - // Latest block. - latestBlk, err := t.GetBlock(ctx, consensusAPI.HeightLatest) - switch err { - case nil: - status.LatestHeight = latestBlk.Height - status.LatestHash = latestBlk.Hash - status.LatestTime = latestBlk.Time - status.LatestStateRoot = latestBlk.StateRoot - case consensusAPI.ErrNoCommittedBlocks: - // No committed blocks yet. - default: - return nil, fmt.Errorf("failed to fetch current block: %w", err) - } - - // List of consensus peers. - tmpeers := t.node.Switch().Peers().List() - peers := make([]string, 0, len(tmpeers)) - for _, tmpeer := range tmpeers { - p := string(tmpeer.ID()) + "@" + tmpeer.RemoteAddr().String() - peers = append(peers, p) - } - status.NodePeers = peers - - // Check if the local node is in the validator set for the latest (uncommitted) block. - valSetHeight := status.LatestHeight + 1 - if valSetHeight < status.GenesisHeight { - valSetHeight = status.GenesisHeight - } - vals, err := tmstate.LoadValidators(t.stateDb, valSetHeight) - if err != nil { - return nil, fmt.Errorf("failed to load validator set: %w", err) - } - consensusPk := t.identity.ConsensusSigner.Public() - consensusAddr := []byte(crypto.PublicKeyToTendermint(&consensusPk).Address()) - status.IsValidator = vals.HasAddress(consensusAddr) - } - - return status, nil -} - func (t *fullService) WatchBlocks(ctx context.Context) (<-chan *consensusAPI.Block, pubsub.ClosableSubscription, error) { - ch, sub := t.WatchTendermintBlocks() + ch, sub, err := t.WatchTendermintBlocks() + if err != nil { + return nil, nil, err + } mapCh := make(chan *consensusAPI.Block) go func() { defer close(mapCh) @@ -869,6 +440,26 @@ func (t *fullService) WatchBlocks(ctx context.Context) (<-chan *consensusAPI.Blo return mapCh, sub, nil } +func (t *fullService) GetStatus(ctx context.Context) (*consensusAPI.Status, error) { + status, err := t.commonNode.GetStatus(ctx) + if err != nil { + return nil, err + } + + if t.started() { + // List of consensus peers. + tmpeers := t.node.Switch().Peers().List() + peers := make([]string, 0, len(tmpeers)) + for _, tmpeer := range tmpeers { + p := string(tmpeer.ID()) + "@" + tmpeer.RemoteAddr().String() + peers = append(peers, p) + } + status.NodePeers = peers + } + + return status, nil +} + func (t *fullService) ensureStarted(ctx context.Context) error { // Make sure that the Tendermint service has started so that we // have the client interface available. @@ -883,213 +474,16 @@ func (t *fullService) ensureStarted(ctx context.Context) error { return nil } -func (t *fullService) initialize() error { - t.Lock() - defer t.Unlock() - - if t.isInitialized { - return nil - } - - if err := t.lazyInit(); err != nil { - return err - } - - // Apply the genesis public key blacklist. - for _, v := range t.genesis.Consensus.Parameters.PublicKeyBlacklist { - if err := v.Blacklist(); err != nil { - t.Logger.Error("initialize: failed to blacklist key", - "err", err, - "pk", v, - ) - return err - } - } - - if err := t.initEpochtime(); err != nil { - return err - } - if err := t.mux.SetEpochtime(t.epochtime); err != nil { - return err - } - - // Initialize the rest of backends. - var err error - var scBeacon tmbeacon.ServiceClient - if scBeacon, err = tmbeacon.New(t.ctx, t); err != nil { - t.Logger.Error("initialize: failed to initialize beacon backend", - "err", err, - ) - return err - } - t.beacon = scBeacon - t.serviceClients = append(t.serviceClients, scBeacon) - - var scKeyManager tmkeymanager.ServiceClient - if scKeyManager, err = tmkeymanager.New(t.ctx, t); err != nil { - t.Logger.Error("initialize: failed to initialize keymanager backend", - "err", err, - ) - return err - } - t.keymanager = scKeyManager - t.serviceClients = append(t.serviceClients, scKeyManager) - - var scRegistry tmregistry.ServiceClient - if scRegistry, err = tmregistry.New(t.ctx, t); err != nil { - t.Logger.Error("initialize: failed to initialize registry backend", - "err", err, - ) - return err - } - t.registry = scRegistry - if cmmetrics.Enabled() { - t.svcMgr.RegisterCleanupOnly(registry.NewMetricsUpdater(t.ctx, t.registry), "registry metrics updater") - } - t.serviceClients = append(t.serviceClients, scRegistry) - t.svcMgr.RegisterCleanupOnly(t.registry, "registry backend") - - var scStaking tmstaking.ServiceClient - if scStaking, err = tmstaking.New(t.ctx, t); err != nil { - t.Logger.Error("staking: failed to initialize staking backend", - "err", err, - ) - return err - } - t.staking = scStaking - t.serviceClients = append(t.serviceClients, scStaking) - t.svcMgr.RegisterCleanupOnly(t.staking, "staking backend") - - var scScheduler tmscheduler.ServiceClient - if scScheduler, err = tmscheduler.New(t.ctx, t); err != nil { - t.Logger.Error("scheduler: failed to initialize scheduler backend", - "err", err, - ) - return err - } - t.scheduler = scScheduler - t.serviceClients = append(t.serviceClients, scScheduler) - t.svcMgr.RegisterCleanupOnly(t.scheduler, "scheduler backend") - - var scRootHash tmroothash.ServiceClient - if scRootHash, err = tmroothash.New(t.ctx, t.dataDir, t); err != nil { - t.Logger.Error("roothash: failed to initialize roothash backend", - "err", err, - ) - return err - } - t.roothash = roothash.NewMetricsWrapper(scRootHash) - t.serviceClients = append(t.serviceClients, scRootHash) - t.svcMgr.RegisterCleanupOnly(t.roothash, "roothash backend") - - // Enable supplementary sanity checks when enabled. - if viper.GetBool(CfgSupplementarySanityEnabled) { - ssa := supplementarysanity.New(viper.GetUint64(CfgSupplementarySanityInterval)) - if err = t.RegisterApplication(ssa); err != nil { - return fmt.Errorf("failed to register supplementary sanity check app: %w", err) - } - } - - return nil -} - -func (t *fullService) GetLastRetainedVersion(ctx context.Context) (int64, error) { - return t.mux.State().LastRetainedVersion() -} - -func (t *fullService) GetTendermintBlock(ctx context.Context, height int64) (*tmtypes.Block, error) { - if err := t.ensureStarted(ctx); err != nil { - return nil, err - } - - var tmHeight int64 - if height == consensusAPI.HeightLatest { - // Do not let Tendermint determine the latest height (e.g., by passing nil here) as that - // completely ignores ABCI processing so it can return a block for which local state does - // not yet exist. Use our mux notion of latest height instead. - tmHeight = t.mux.State().BlockHeight() - if tmHeight == 0 { - // No committed blocks yet. - return nil, nil - } - } else { - tmHeight = height - } - result, err := t.client.Block(&tmHeight) - if err != nil { - return nil, fmt.Errorf("tendermint: block query failed: %w", err) - } - return result.Block, nil -} - -func (t *fullService) GetBlockResults(height int64) (*tmrpctypes.ResultBlockResults, error) { - if t.client == nil { - panic("client not available yet") - } - - // As in GetTendermintBlock above, get the latest tendermint block height - // from our mux. - var tmHeight int64 - if height == consensusAPI.HeightLatest { - tmHeight = t.mux.State().BlockHeight() - if tmHeight == 0 { - // No committed blocks yet. - return nil, consensusAPI.ErrNoCommittedBlocks - } - } else { - tmHeight = height - } - - result, err := t.client.BlockResults(&tmHeight) - if err != nil { - return nil, fmt.Errorf("tendermint: block results query failed: %w", err) - } - - return result, nil -} - -func (t *fullService) WatchTendermintBlocks() (<-chan *tmtypes.Block, *pubsub.Subscription) { +func (t *fullService) WatchTendermintBlocks() (<-chan *tmtypes.Block, *pubsub.Subscription, error) { typedCh := make(chan *tmtypes.Block) sub := t.blockNotifier.Subscribe() sub.Unwrap(typedCh) - return typedCh, sub -} - -func (t *fullService) ConsensusKey() signature.PublicKey { - return t.identity.ConsensusSigner.Public() -} - -func (t *fullService) initEpochtime() error { - var err error - if t.genesis.EpochTime.Parameters.DebugMockBackend { - var scEpochTime tmepochtimemock.ServiceClient - scEpochTime, err = tmepochtimemock.New(t.ctx, t) - if err != nil { - t.Logger.Error("initEpochtime: failed to initialize mock epochtime backend", - "err", err, - ) - return err - } - t.epochtime = scEpochTime - t.serviceClients = append(t.serviceClients, scEpochTime) - } else { - var scEpochTime tmepochtime.ServiceClient - scEpochTime, err = tmepochtime.New(t.ctx, t, t.genesis.EpochTime.Parameters.Interval) - if err != nil { - t.Logger.Error("initEpochtime: failed to initialize epochtime backend", - "err", err, - ) - return err - } - t.epochtime = scEpochTime - t.serviceClients = append(t.serviceClients, scEpochTime) - } - return nil + return typedCh, sub, nil } func (t *fullService) lazyInit() error { - if t.isInitialized { + if t.initialized() { return nil } @@ -1235,7 +629,9 @@ func (t *fullService) lazyInit() error { switch dbCtx.ID { case "state": // Tendermint state database. - t.stateDb = db + t.stateStore = db + case "blockstore": + t.blockStoreDB = db default: } @@ -1310,7 +706,7 @@ func (t *fullService) lazyInit() error { if err != nil { return fmt.Errorf("tendermint: failed to create node: %w", err) } - if t.stateDb == nil { + if t.stateStore == nil { // Sanity check for the above wrapDbProvider hack in case the DB provider changes. return fmt.Errorf("tendermint: internal error: state database not set") } @@ -1355,8 +751,6 @@ func (t *fullService) lazyInit() error { return nil } - t.isInitialized = true - return nil } @@ -1442,7 +836,13 @@ func (t *fullService) blockNotifierWorker() { // metrics updates oasis_consensus metrics by checking last accepted block info. func (t *fullService) metrics() { - ch, sub := t.WatchTendermintBlocks() + ch, sub, err := t.WatchTendermintBlocks() + if err != nil { + t.Logger.Error("failed to watch tendermint blocks", + "err", err, + ) + return + } defer sub.Close() // Tendermint uses specific public key encoding. @@ -1502,18 +902,22 @@ func New( } t := &fullService{ - BaseBackgroundService: *cmservice.NewBaseBackgroundService("tendermint"), - svcMgr: cmbackground.NewServiceManager(logging.GetLogger("tendermint/servicemanager")), - upgrader: upgrader, - blockNotifier: pubsub.NewBroker(false), - identity: identity, - genesis: genesisDoc, - genesisProvider: genesisProvider, - ctx: ctx, - dataDir: dataDir, - startedCh: make(chan struct{}), - syncedCh: make(chan struct{}), - quitCh: make(chan struct{}), + commonNode: commonNode{ + BaseBackgroundService: *cmservice.NewBaseBackgroundService("tendermint"), + ctx: ctx, + identity: identity, + rpcCtx: &tmrpctypes.Context{}, + genesis: genesisDoc, + dataDir: dataDir, + svcMgr: cmbackground.NewServiceManager(logging.GetLogger("tendermint/servicemanager")), + startedCh: make(chan struct{}), + }, + upgrader: upgrader, + blockNotifier: pubsub.NewBroker(false), + genesisProvider: genesisProvider, + startedCh: make(chan struct{}), + syncedCh: make(chan struct{}), + quitCh: make(chan struct{}), } t.Logger.Info("starting a full consensus node") @@ -1525,6 +929,10 @@ func New( } t.submissionMgr = consensusAPI.NewSubmissionManager(t, pd, viper.GetUint64(tmcommon.CfgSubmissionMaxFee)) + if err := t.lazyInit(); err != nil { + return nil, fmt.Errorf("lazy init: %w", err) + } + return t, t.initialize() } diff --git a/go/consensus/tendermint/full/light.go b/go/consensus/tendermint/full/light.go index d095cd4724b..e4026df406f 100644 --- a/go/consensus/tendermint/full/light.go +++ b/go/consensus/tendermint/full/light.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + tmcore "github.com/tendermint/tendermint/rpc/core" tmstate "github.com/tendermint/tendermint/state" tmtypes "github.com/tendermint/tendermint/types" @@ -14,26 +15,31 @@ import ( ) // Implements LightClientBackend. -func (t *fullService) GetLightBlock(ctx context.Context, height int64) (*consensusAPI.LightBlock, error) { - if err := t.ensureStarted(ctx); err != nil { +func (n *commonNode) GetLightBlock(ctx context.Context, height int64) (*consensusAPI.LightBlock, error) { + if err := n.ensureStarted(ctx); err != nil { return nil, err } - commit, err := t.client.Commit(&height) + tmHeight, err := n.heightToTendermintHeight(height) if err != nil { - return nil, fmt.Errorf("%w: tendermint: header query failed: %s", consensusAPI.ErrVersionNotFound, err.Error()) - } - - if commit.Header == nil { - return nil, fmt.Errorf("tendermint: header is nil") + return nil, err } // Don't use the client as that imposes stupid pagination. Access the state database directly. - vals, err := tmstate.LoadValidators(t.stateDb, height) + vals, err := tmstate.LoadValidators(n.stateStore, tmHeight) if err != nil { return nil, consensusAPI.ErrVersionNotFound } + commit, err := tmcore.Commit(n.rpcCtx, &height) + if err != nil { + return nil, fmt.Errorf("%w: tendermint: header query failed: %s", consensusAPI.ErrVersionNotFound, err.Error()) + } + + if commit.Header == nil { + return nil, fmt.Errorf("tendermint: header is nil") + } + lb := tmtypes.LightBlock{ SignedHeader: &commit.SignedHeader, ValidatorSet: vals, @@ -54,33 +60,45 @@ func (t *fullService) GetLightBlock(ctx context.Context, height int64) (*consens } // Implements LightClientBackend. -func (t *fullService) GetParameters(ctx context.Context, height int64) (*consensusAPI.Parameters, error) { - if err := t.ensureStarted(ctx); err != nil { +func (n *commonNode) GetParameters(ctx context.Context, height int64) (*consensusAPI.Parameters, error) { + if err := n.ensureStarted(ctx); err != nil { return nil, err } - params, err := t.client.ConsensusParams(&height) + tmHeight, err := n.heightToTendermintHeight(height) if err != nil { - return nil, fmt.Errorf("%w: tendermint: consensus params query failed: %s", consensusAPI.ErrVersionNotFound, err.Error()) + return nil, err } - meta, err := params.ConsensusParams.Marshal() + // Query consensus parameters directly from the state store, as fetching + // via tmcore.ConsensusParameters also tries fetching latest uncommitted + // block which wont work with the archive node setup. + consensusParams, err := tmstate.LoadConsensusParams(n.stateStore, tmHeight) + if err != nil { + return nil, fmt.Errorf("%w: tendermint: consensus params query failed: %s", consensusAPI.ErrVersionNotFound, err.Error()) + } + meta, err := consensusParams.Marshal() if err != nil { return nil, fmt.Errorf("tendermint: failed to marshal consensus params: %w", err) } return &consensusAPI.Parameters{ - Height: params.BlockHeight, + Height: tmHeight, Meta: meta, }, nil } // Implements LightClientBackend. -func (t *fullService) State() syncer.ReadSyncer { - return t.mux.State().Storage() +func (n *commonNode) State() syncer.ReadSyncer { + return n.mux.State().Storage() } // Implements LightClientBackend. func (t *fullService) SubmitTxNoWait(ctx context.Context, tx *transaction.SignedTransaction) error { return t.broadcastTxRaw(cbor.Marshal(tx)) } + +// Implements LightClientBackend. +func (srv *archiveService) SubmitTxNoWait(ctx context.Context, tx *transaction.SignedTransaction) error { + return consensusAPI.ErrUnsupported +} diff --git a/go/consensus/tendermint/full/services.go b/go/consensus/tendermint/full/services.go index dea8b275eba..13d1cc6354e 100644 --- a/go/consensus/tendermint/full/services.go +++ b/go/consensus/tendermint/full/services.go @@ -34,7 +34,13 @@ func (t *fullService) serviceClientWorker(ctx context.Context, svc api.ServiceCl }) queries = append(queries, nil) // General query for new block headers. - newBlockCh, newBlockSub := t.WatchTendermintBlocks() + newBlockCh, newBlockSub, err := t.WatchTendermintBlocks() + if err != nil { + logger.Error("failed to watch tendermint blocks", + "err", err, + ) + return + } defer newBlockSub.Close() const indexNewBlock = 1 diff --git a/go/consensus/tendermint/tendermint.go b/go/consensus/tendermint/tendermint.go index c98ac0a83a9..773e52a9415 100644 --- a/go/consensus/tendermint/tendermint.go +++ b/go/consensus/tendermint/tendermint.go @@ -21,17 +21,15 @@ const ( CfgMode = "consensus.tendermint.mode" ) -const ( - // ModeFull is the name of the full node consensus mode. - ModeFull = "full" - - // ModeSeed is the name of the seed-only node consensus mode. - ModeSeed = "seed" -) - // Flags has the configuration flags. var Flags = flag.NewFlagSet("", flag.ContinueOnError) +// Mode returns the configured tendermint mode. +func Mode() (mode consensusAPI.Mode, err error) { + err = mode.UnmarshalText([]byte(viper.GetString(CfgMode))) + return +} + // New creates a new Tendermint consensus backend. func New( ctx context.Context, @@ -40,20 +38,28 @@ func New( upgrader upgradeAPI.Backend, genesisProvider genesisAPI.Provider, ) (consensusAPI.Backend, error) { - switch mode := viper.GetString(CfgMode); mode { - case ModeFull: + var mode consensusAPI.Mode + if err := mode.UnmarshalText([]byte(viper.GetString(CfgMode))); err != nil { + return nil, err + } + + switch mode { + case consensusAPI.ModeFull: // Full node. return full.New(ctx, dataDir, identity, upgrader, genesisProvider) - case ModeSeed: + case consensusAPI.ModeSeed: // Seed-only node. return seed.New(dataDir, identity, genesisProvider) + case consensusAPI.ModeArchive: + // Archive node. + return full.NewArchive(ctx, dataDir, identity, genesisProvider) default: return nil, fmt.Errorf("tendermint: unsupported mode: %s", mode) } } func init() { - Flags.String(CfgMode, ModeFull, "tendermint mode (full, seed)") + Flags.String(CfgMode, consensusAPI.ModeFull.String(), "tendermint mode (full, seed, archive)") _ = viper.BindPFlags(Flags) Flags.AddFlagSet(common.Flags) diff --git a/go/oasis-node/cmd/node/node.go b/go/oasis-node/cmd/node/node.go index 26579711264..8a250409752 100644 --- a/go/oasis-node/cmd/node/node.go +++ b/go/oasis-node/cmd/node/node.go @@ -348,6 +348,27 @@ func (n *Node) initRuntimeWorkers() error { } n.svcMgr.Register(n.RegistrationWorker) + // Initialize the key manager worker. + n.KeymanagerWorker, err = workerKeymanager.New( + dataDir, + n.CommonWorker, + n.IAS, + n.RegistrationWorker, + n.Consensus.KeyManager(), + ) + if err != nil { + return err + } + n.svcMgr.Register(n.KeymanagerWorker) + + tmMode, err := tendermint.Mode() + if err != nil { + logger.Error("invalid tendermint mode", + "err", err, + ) + return err + } + // Initialize the storage worker. n.StorageWorker, err = workerStorage.New( n.grpcInternal, @@ -355,6 +376,7 @@ func (n *Node) initRuntimeWorkers() error { n.RegistrationWorker, n.Genesis, n.commonStore, + tmMode == consensusAPI.ModeArchive, ) if err != nil { return err @@ -584,21 +606,31 @@ func newNode(testNode bool) (node *Node, err error) { // nolint: gocyclo ) return nil, err } - - // Initialize upgrader backend and check if we can even launch. - node.Upgrader, err = upgrade.New(node.commonStore, cmdCommon.DataDir()) + // Initialize upgrader backend. + tmMode, err := tendermint.Mode() if err != nil { - logger.Error("failed to initialize upgrade backend", + logger.Error("invalid tendermint mode", "err", err, ) return nil, err } - if err = node.Upgrader.StartupUpgrade(); err != nil { - logger.Error("error occurred during startup upgrade", + isArchive := tmMode == consensusAPI.ModeArchive + node.Upgrader, err = upgrade.New(node.commonStore, cmdCommon.DataDir(), !isArchive) + if err != nil { + logger.Error("failed to initialize upgrade backend", "err", err, ) return nil, err } + // If not an archive mode, check if we can even launch. + if !isArchive { + if err = node.Upgrader.StartupUpgrade(); err != nil { + logger.Error("error occurred during startup upgrade", + "err", err, + ) + return nil, err + } + } // Generate/Load the node identity. signerFactory, err := cmdSigner.NewFactory(cmdSigner.Backend(), dataDir, signature.SignerNode, signature.SignerP2P, signature.SignerConsensus) diff --git a/go/oasis-test-runner/oasis/args.go b/go/oasis-test-runner/oasis/args.go index 442ff63066e..5788b780d81 100644 --- a/go/oasis-test-runner/oasis/args.go +++ b/go/oasis-test-runner/oasis/args.go @@ -13,6 +13,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/common" commonGrpc "github.com/oasisprotocol/oasis-core/go/common/grpc" "github.com/oasisprotocol/oasis-core/go/common/sgx" + consensusAPI "github.com/oasisprotocol/oasis-core/go/consensus/api" "github.com/oasisprotocol/oasis-core/go/consensus/tendermint" "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/abci" tendermintCommon "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/common" @@ -162,7 +163,7 @@ func (args *argBuilder) tendermintDisablePeerExchange() *argBuilder { } func (args *argBuilder) tendermintSeedMode() *argBuilder { - args.vec = append(args.vec, "--"+tendermint.CfgMode, tendermint.ModeSeed) + args.vec = append(args.vec, "--"+tendermint.CfgMode, consensusAPI.ModeSeed.String()) return args } diff --git a/go/upgrade/upgrade.go b/go/upgrade/upgrade.go index 69c8ca1452e..71034f67aab 100644 --- a/go/upgrade/upgrade.go +++ b/go/upgrade/upgrade.go @@ -264,7 +264,7 @@ func (u *upgradeManager) Close() { // New constructs and returns a new upgrade manager. It also checks for and loads any // pending upgrade descriptors; if this node is not the one intended to be run according // to the loaded descriptor, New will return an error. -func New(store *persistent.CommonStore, dataDir string) (api.Backend, error) { +func New(store *persistent.CommonStore, dataDir string, checkStatus bool) (api.Backend, error) { svcStore, err := store.GetServiceStore(api.ModuleName) if err != nil { return nil, err @@ -274,8 +274,10 @@ func New(store *persistent.CommonStore, dataDir string) (api.Backend, error) { logger: logging.GetLogger(api.ModuleName), } - if err := upgrader.checkStatus(dataDir); err != nil { - return nil, err + if checkStatus { + if err := upgrader.checkStatus(dataDir); err != nil { + return nil, err + } } return upgrader, nil diff --git a/go/worker/storage/worker.go b/go/worker/storage/worker.go index 52c04cadf04..5d8c369ebf2 100644 --- a/go/worker/storage/worker.go +++ b/go/worker/storage/worker.go @@ -56,6 +56,7 @@ func New( registration *registration.Worker, genesis genesis.Provider, commonStore *persistent.CommonStore, + isArchive bool, ) (*Worker, error) { s := &Worker{ enabled: viper.GetBool(CfgWorkerEnabled), @@ -95,7 +96,7 @@ func New( // Start storage node for every runtime. for _, rt := range s.commonWorker.GetRuntimes() { - if err := s.registerRuntime(commonWorker.DataDir, rt, checkpointerCfg); err != nil { + if err := s.registerRuntime(commonWorker.DataDir, rt, checkpointerCfg, isArchive); err != nil { return nil, err } } @@ -107,7 +108,7 @@ func New( return s, nil } -func (s *Worker) registerRuntime(dataDir string, commonNode *committeeCommon.Node, checkpointerCfg *checkpoint.CheckpointerConfig) error { +func (s *Worker) registerRuntime(dataDir string, commonNode *committeeCommon.Node, checkpointerCfg *checkpoint.CheckpointerConfig, isArchive bool) error { id := commonNode.Runtime.ID() s.logger.Info("registering new runtime", "runtime_id", id, @@ -129,6 +130,10 @@ func (s *Worker) registerRuntime(dataDir string, commonNode *committeeCommon.Nod } commonNode.Runtime.RegisterStorage(localStorage) + if isArchive { + return nil + } + node, err := committee.NewNode( commonNode, s.grpcPolicy,