feat: monitor empty consensus block (#89)

Add monitoring for empty consensus blocks. Important for chains where empty CL blocks aren't broadcasted to execution layer (e.g. Berachain), affecting validator's EL uptime. - Added empty_blocks prometheus counter - Added empty block indicator (🟡) in logs - Added tests
kilnfi · Nov 20, 2024 · c4d7ca1 · c4d7ca1
1 parent f09d0c2
commit c4d7ca1
Show file tree

Hide file tree

Showing 4 changed files with 46 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -116,6 +116,7 @@ Metrics (without prefix)        | Description
 `node_synced`                   | Set to 1 is the node is synced (ie. not catching-up)
 `proposal_end_time`             | Timestamp of the voting end time of a proposal
 `proposed_blocks`               | Number of proposed blocks per validator (for a bonded validator)
+`empty_blocks`                  | Number of empty blocks (blocks with zero transactions) proposed by validator
 `rank`                          | Rank of the validator
 `seat_price`                    | Min seat price to be in the active set (ie. bonded tokens of the latest validator)
 `signed_blocks_window`          | Number of blocks per signing window

diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
@@ -31,6 +31,7 @@ type Metrics struct {
 	SoloMissedBlocks        *prometheus.CounterVec
 	ConsecutiveMissedBlocks *prometheus.GaugeVec
 	MissedBlocksWindow      *prometheus.GaugeVec
+	EmptyBlocks             *prometheus.CounterVec
 	Tokens                  *prometheus.GaugeVec
 	IsBonded                *prometheus.GaugeVec
 	IsJailed                *prometheus.GaugeVec
@@ -125,6 +126,14 @@ func New(namespace string) *Metrics {
 			},
 			[]string{"chain_id", "address", "name"},
 		),
+		EmptyBlocks: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "empty_blocks",
+				Help:      "Number of empty blocks proposed by validator",
+			},
+			[]string{"chain_id", "address", "name"},
+		),
 		TrackedBlocks: prometheus.NewCounterVec(
 			prometheus.CounterOpts{
 				Namespace: namespace,
@@ -280,6 +289,7 @@ func (m *Metrics) Register() {
 	m.Registry.MustRegister(m.SoloMissedBlocks)
 	m.Registry.MustRegister(m.ConsecutiveMissedBlocks)
 	m.Registry.MustRegister(m.MissedBlocksWindow)
+	m.Registry.MustRegister(m.EmptyBlocks)
 	m.Registry.MustRegister(m.TrackedBlocks)
 	m.Registry.MustRegister(m.Transactions)
 	m.Registry.MustRegister(m.SkippedBlocks)

diff --git a/pkg/watcher/block.go b/pkg/watcher/block.go
@@ -31,6 +31,7 @@ type BlockWatcher struct {
 	validatorSet        atomic.Value // []*types.Validator
 	latestBlockHeight   int64
 	latestBlockProposer string
+	latestBlockTransactions int
 	webhook             *webhook.Webhook
 	customWebhooks      []BlockWebhook
 }
@@ -177,12 +178,13 @@ func (w *BlockWatcher) handleBlockInfo(ctx context.Context, block *BlockInfo) {
 		return
 	}
 
-	// Ensure to inititalize counters for each validator
+	// Ensure to initialize counters for each validator
 	for _, val := range w.trackedValidators {
 		w.metrics.ValidatedBlocks.WithLabelValues(chainId, val.Address, val.Name)
 		w.metrics.MissedBlocks.WithLabelValues(chainId, val.Address, val.Name)
 		w.metrics.SoloMissedBlocks.WithLabelValues(chainId, val.Address, val.Name)
 		w.metrics.ConsecutiveMissedBlocks.WithLabelValues(chainId, val.Address, val.Name)
+		w.metrics.EmptyBlocks.WithLabelValues(chainId, val.Address, val.Name)
 	}
 	w.metrics.SkippedBlocks.WithLabelValues(chainId)
 
@@ -202,7 +204,13 @@ func (w *BlockWatcher) handleBlockInfo(ctx context.Context, block *BlockInfo) {
 	for _, res := range block.ValidatorStatus {
 		icon := "⚪️"
 		if w.latestBlockProposer == res.Address {
-			icon = "👑"
+			// Check if this is an empty block
+			if w.latestBlockTransactions == 0 {
+				icon = "🟡"
+				w.metrics.EmptyBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc()
+			} else {
+				icon = "👑"
+			}
 			w.metrics.ProposedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc()
 			w.metrics.ValidatedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc()
 			w.metrics.ConsecutiveMissedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Set(0)
@@ -235,6 +243,7 @@ func (w *BlockWatcher) handleBlockInfo(ctx context.Context, block *BlockInfo) {
 
 	w.latestBlockHeight = block.Height
 	w.latestBlockProposer = block.ProposerAddress
+	w.latestBlockTransactions = block.Transactions
 }
 
 func (w *BlockWatcher) computeValidatorStatus(block *types.Block) []ValidatorStatus {

diff --git a/pkg/watcher/block_test.go b/pkg/watcher/block_test.go
@@ -103,6 +103,23 @@ func TestBlockWatcher(t *testing.T) {
 			{
 				ChainID:          chainID,
 				Height:           44,
+				Transactions:     0,
+				TotalValidators:  2,
+				SignedValidators: 2,
+				ProposerAddress:  kilnAddress,
+				ValidatorStatus: []ValidatorStatus{
+					{
+						Address: kilnAddress,
+						Label:   kilnName,
+						Bonded:  true,
+						Signed:  true,
+						Rank:    2,
+					},
+				},
+			},
+			{
+				ChainID:          chainID,
+				Height:           45,
 				Transactions:     7,
 				TotalValidators:  2,
 				SignedValidators: 2,
@@ -129,24 +146,27 @@ func TestBlockWatcher(t *testing.T) {
 				`#41   1/2 validators ✅ Kiln`,
 				`#42   2/2 validators ✅ Kiln`,
 				`#43   2/2 validators 👑 Kiln`,
+				`#44   2/2 validators 🟡 Kiln`,
 			}, "\n")+"\n",
 			blockWatcher.writer.(*bytes.Buffer).String(),
 		)
 
-		assert.Equal(t, float64(44), testutil.ToFloat64(blockWatcher.metrics.BlockHeight.WithLabelValues(chainID)))
+		assert.Equal(t, float64(45), testutil.ToFloat64(blockWatcher.metrics.BlockHeight.WithLabelValues(chainID)))
 		assert.Equal(t, float64(29), testutil.ToFloat64(blockWatcher.metrics.Transactions.WithLabelValues(chainID)))
 		assert.Equal(t, float64(2), testutil.ToFloat64(blockWatcher.metrics.ActiveSet.WithLabelValues(chainID)))
-		assert.Equal(t, float64(5), testutil.ToFloat64(blockWatcher.metrics.TrackedBlocks.WithLabelValues(chainID)))
+		assert.Equal(t, float64(6), testutil.ToFloat64(blockWatcher.metrics.TrackedBlocks.WithLabelValues(chainID)))
 		assert.Equal(t, float64(5), testutil.ToFloat64(blockWatcher.metrics.SkippedBlocks.WithLabelValues(chainID)))
 
 		assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.ValidatedBlocks))
 		assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.MissedBlocks))
 		assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.SoloMissedBlocks))
 		assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.ConsecutiveMissedBlocks))
-		assert.Equal(t, float64(1), testutil.ToFloat64(blockWatcher.metrics.ProposedBlocks.WithLabelValues(chainID, kilnAddress, kilnName)))
-		assert.Equal(t, float64(3), testutil.ToFloat64(blockWatcher.metrics.ValidatedBlocks.WithLabelValues(chainID, kilnAddress, kilnName)))
+		assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.EmptyBlocks))
+		assert.Equal(t, float64(2), testutil.ToFloat64(blockWatcher.metrics.ProposedBlocks.WithLabelValues(chainID, kilnAddress, kilnName)))
+		assert.Equal(t, float64(4), testutil.ToFloat64(blockWatcher.metrics.ValidatedBlocks.WithLabelValues(chainID, kilnAddress, kilnName)))
 		assert.Equal(t, float64(1), testutil.ToFloat64(blockWatcher.metrics.MissedBlocks.WithLabelValues(chainID, kilnAddress, kilnName)))
 		assert.Equal(t, float64(0), testutil.ToFloat64(blockWatcher.metrics.SoloMissedBlocks.WithLabelValues(chainID, kilnAddress, kilnName)))
 		assert.Equal(t, float64(0), testutil.ToFloat64(blockWatcher.metrics.ConsecutiveMissedBlocks.WithLabelValues(chainID, kilnAddress, kilnName)))
+		assert.Equal(t, float64(1), testutil.ToFloat64(blockWatcher.metrics.EmptyBlocks.WithLabelValues(chainID, kilnAddress, kilnName)))
 	})
 }