Skip to content

Commit

Permalink
Merge pull request #4 from nimiq/retrieve-size-from-ended-epochs
Browse files Browse the repository at this point in the history
Retrieve size from ended epochs
onmax authored Aug 31, 2024
2 parents c1849a3 + 5fc5b24 commit 6c77219
Showing 59 changed files with 1,023 additions and 14,779 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -44,16 +44,16 @@ jobs:

- name: Install
run: pnpm install --frozen-lockfile
working-directory: packages/nimiq-vts
working-directory: packages/nimiq-validators-score

- name: Lint
run: pnpm run lint
working-directory: packages/nimiq-vts
working-directory: packages/nimiq-validators-score

- name: Typecheck
run: pnpm run typecheck
working-directory: packages/nimiq-vts
working-directory: packages/nimiq-validators-score

- name: Test
run: pnpm run test
working-directory: packages/nimiq-vts
working-directory: packages/nimiq-validators-score
17 changes: 10 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
@@ -9,6 +9,9 @@
Details of validators in the Nimiq Blockchain and their scores, calculated using Nimiq's VTS algorithm.
<p>

> [!NOTE]
> If you're a validator and would like to add your data to the API, please open a pull request (PR) with your JSON file, following the structure of `./public/validators/.example` in the `./public/validators` directory.
## Validators Dashboard

https://validators-api.pages.dev/
@@ -17,19 +20,19 @@ The dashboard is a simple Nuxt application that displays all validators and thei

## Validators API

https://validators-api.pages.dev/api/vts
https://validators-api.pages.dev/api/scores

An endpoint that returns the list of validators and their scores.

https://validators-api.pages.dev/api/vts/health
# https://validators-api.pages.dev/api/scores/health

An endpoint that returns the internal status of the VTS. Basically if the server is synced or not with the chain.
An endpoint that returns the state of the database

## Validator Trust Score
## Validator Score

[Source code](./packages/nimiq-vts/)
[Source code](./packages/nimiq-validators-score/)

This is a npm package that calculates the Trust Score of a validator. You can read more about the Trust Score [here](https://validators-api.pages.dev/vts).
This is a npm package that calculates the Trust Score of a validator. You can read more about the Score [here](https://validators-api.pages.dev/scores).

## Validator Details

@@ -51,7 +54,7 @@ We use Drizzle to access the database.

To calculate the score, we need to run two processes: the fetcher and the score calculator. We do this using a Nitro Task, which is currently an experimental feature of Nitro. Nitro Task is a feature that allows you to trigger an action in the server programmatically or manually from the Nuxt Dev Center(go to tasks page).

Read more about the process of computing the score in the [nimiq-vts](./packages/nimiq-vts/README.md) package.
Read more about the process of computing the score in the [nimiq-validators-score](./packages/nimiq-validators-score/README.md) package.

#### Database

4 changes: 2 additions & 2 deletions app/app.vue
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@ const validatorDetail = computed(() => !!route.params.address)
const networkName = useRuntimeConfig().public.nimiqNetwork
const { data: health } = useFetch('/api/vts/health')
const { data: health } = useFetch('/api/scores/health')
</script>

<template>
@@ -22,7 +22,7 @@ const { data: health } = useFetch('/api/vts/health')
Go back
</NuxtLink>
<div flex-auto />
<NuxtLink to="/vts/health" :class="{ 'bg-green/10 text-green': health?.isSynced, 'bg-red/10 text-red': !health?.isSynced }" rounded-full px-12 py-4 text-11 flex="~ items-center gap-6">
<NuxtLink to="/scores/health" :class="{ 'bg-green/10 text-green': health?.isSynced, 'bg-red/10 text-red': !health?.isSynced }" rounded-full px-12 py-4 text-11 flex="~ items-center gap-6">
<div text-10 :class="health?.isSynced ? 'i-nimiq:check' : 'i-nimiq:alert'" />
<span font-semibold>{{ health?.isSynced ? 'Synced' : 'Not Synced' }}</span>
</NuxtLink>
2 changes: 1 addition & 1 deletion app/components/Donut.client.vue
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@ import { VisDonut, VisSingleContainer, VisTooltip } from '@unovis/vue'
import { Donut } from '@unovis/ts'
import { render } from 'vue'
import ScorePies from './ScorePies.vue'
import type { Validator } from '~~/server/api/vts/index.get'
import type { Validator } from '~~/server/api/scores/index.get'
defineProps<{ data: Validator[] }>()
4 changes: 2 additions & 2 deletions app/pages/index.vue
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<script setup lang="ts">
const { data, status, error } = useFetch('/api/vts')
const { data, status, error } = useFetch('/api/scores')
const validators = computed(() => data.value?.validators || [])
const averageScore = computed(() => {
@@ -31,7 +31,7 @@ const averageScore = computed(() => {
</Stat>
<Stat text-blue>
<template #value>
{{ data?.epochNumber }}
{{ data?.range.toEpoch }}
</template>
<template #description>
Epoch
10 changes: 5 additions & 5 deletions app/pages/vts/health.vue → app/pages/scores/health.vue
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<script setup lang="ts">
const { data: health, status, error } = useFetch('/api/vts/health')
const { data: health, status, error } = useFetch('/api/scores/health')
const network = useRuntimeConfig().public.nimiqNetwork
</script>

@@ -9,7 +9,7 @@ const network = useRuntimeConfig().public.nimiqNetwork
<h1 my-32 max-w-inherit>
Network <code rounded-6 px-4>{{ network }}</code>
</h1>
<NuxtLink to="/api/vts/health" target="_blank" text-12 op-80 nq-arrow nq-pill-tertiary>
<NuxtLink to="/api/scores/health" target="_blank" text-12 op-80 nq-arrow nq-pill-tertiary>
API
</NuxtLink>
</div>
@@ -32,15 +32,15 @@ const network = useRuntimeConfig().public.nimiqNetwork
</p>

<p v-if="health.flags.length > 0">
{{ JSON.stringify(health.flags) }}
Flags: {{ JSON.stringify(health.flags) }}
</p>

<div mt-32>
<p>
Range: [{{ health.range.fromEpoch }} - {{ health.range.toEpoch }}] ({{ health.range.epochCount }} epochs)
Range: [{{ health.range.fromEpoch }} ({{ (health.range.fromBlockNumber) }}) - {{ health.range.toEpoch }} ({{ health.range.toBlockNumber }})] ({{ health.range.epochCount }} epochs)
</p>
<p m-0 text-13 text-neutral-700>
The range of blocks used to compute the score of the validators. We don't consider the first epoch as it is an speacial epoch.
The range of blocks used to compute the score of the validators. We don't consider the first epoch (<code>epoch index = 0</code>) as it is an speacial epoch.
</p>
</div>

14 changes: 9 additions & 5 deletions app/pages/validator/[address].vue
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<script setup lang="ts">
import { ValidatorTag } from '~~/server/utils/drizzle'
const { data } = useFetch('/api/vts')
const { data } = useFetch('/api/scores')
const validators = computed(() => data.value?.validators || [])
const route = useRoute()
@@ -49,10 +49,14 @@ const validator = computed(() => {

<ScorePies :validator mt-64 text-28 />
<!-- <div self-stretch w-2 bg-neutral-300 mx-48 /> -->

<p mt-32 block max-w-700 text-neutral-900>
{{ validator }}
</p>
<details>
<summary text-neutral-900 font-semibold mt-32 w-full>
More details
</summary>
<code nq-prose mt-32 block max-w-700 text-neutral-900>
{{ JSON.stringify(validator, null, 2) }}
</code>
</details>
</div>
</div>

6 changes: 5 additions & 1 deletion nuxt.config.ts
Original file line number Diff line number Diff line change
@@ -59,7 +59,7 @@ export default defineNuxtConfig({
},

scheduledTasks: {
'0 */12 * * *': ['fetch', 'seed', 'score'],
'0 */12 * * *': ['fetch'],
},
},

@@ -82,6 +82,10 @@ export default defineNuxtConfig({
},
},

watch: [
'~~/packages/nimiq-validators-score',
],

features: {
// For UnoCSS
inlineStyles: false,
21 changes: 10 additions & 11 deletions package.json
Original file line number Diff line number Diff line change
@@ -2,30 +2,29 @@
"name": "nimiq-validators",
"type": "module",
"private": true,
"packageManager": "pnpm@9.7.1",
"packageManager": "pnpm@9.9.0",
"scripts": {
"build": "nuxt build",
"dev:local": "nuxt dev",
"dev": "nuxt dev --remote",
"generate": "nuxt generate",
"preview": "nuxt preview",
"postinstall": "nuxt prepare",
"build": "nuxi build",
"dev:packages": "pnpm --filter=./packages/* --parallel run dev",
"dev:local": "pnpm run dev:packages && nuxi dev",
"dev": "pnpm run dev:packages && nuxi dev --remote",
"generate": "nuxo generate",
"preview": "nuxi preview",
"postinstall": "nuxi prepare",
"lint": "eslint .",
"lint:fix": "eslint . --fix",
"db:generate": "drizzle-kit generate",
"db:delete": "drizzle-kit drop"
},
"dependencies": {
"@nimiq/core": "2.0.0-next.23.0",
"@nuxt/eslint": "^0.5.1",
"@nimiq/core": "next",
"@nuxthub/core": "^0.7.3",
"@unovis/ts": "^1.4.4",
"@unovis/vue": "^1.4.4",
"@vueuse/core": "^11.0.1",
"defu": "^6.1.4",
"drizzle-orm": "^0.33.0",
"nimiq-rpc-client-ts": "^0.4.0",
"nimiq-vts": "workspace:*",
"nimiq-validators-score": "workspace:*",
"nuxt": "^3.12.4",
"pinia": "^2.2.2",
"radix-vue": "^1.9.4",
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Nimiq Validator Trust Score
# Nimiq Validator Score

The algorithm to compute the Nimiq's Validator Trust Score. The VTS is a metric that helps to evaluate the trustworthiness of a validator. You can read more about the Algorithm of the Score [here](https://nimiq-validators.pages.dev/vts).
The algorithm to compute the Nimiq's Validator Score. It is a metric that helps to evaluate the trustworthiness of a validator. You can read more about the Algorithm of the Score [here](https://nimiq-validators.pages.dev/scores).

This package is the implementation of such algorithm. Anyone with access to a node should be able to run the same algorithm and get the same result.

File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
{
"name": "nimiq-vts",
"name": "nimiq-validators-score",
"type": "module",
"version": "0.0.0",
"description": "The algorithm to compute the Nimiq's Validator Trust Score",
"version": "0.0.1",
"packageManager": "pnpm@9.9.0",
"description": "The algorithm to compute the Nimiq's Validator Score",
"license": "MIT",
"homepage": "https://github.com/onmax/nimiq-validators#readme",
"repository": {
@@ -46,7 +47,6 @@
"lint:fix": "eslint . --fix"
},
"dependencies": {
"consola": "^3.2.3",
"defu": "^6.1.4",
"nimiq-rpc-client-ts": "^0.4.0",
"vitest": "^2.0.5"
File renamed without changes.
139 changes: 139 additions & 0 deletions packages/nimiq-validators-score/src/fetcher.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
import { type ElectionMacroBlock, InherentType, type NimiqRPCClient } from 'nimiq-rpc-client-ts'
import { getPolicyConstants } from './utils'
import type { EpochActivity, EpochsActivities } from './types'

// TODO remove Console log

/**
* For a given block number, fetches the validator slots assignation.
* The block number MUST be an election block otherwise it will throw an error.
*/
export async function fetchActivity(client: NimiqRPCClient, epochIndex: number) {
const { batchesPerEpoch, genesisBlockNumber, blocksPerBatch, slots: slotsCount, blocksPerEpoch } = await getPolicyConstants(client)

const electionBlock = genesisBlockNumber + (epochIndex * blocksPerEpoch)
const { data: block, error } = await client.blockchain.getBlockByNumber(electionBlock, { includeTransactions: true })
if (error || !block)
throw new Error(JSON.stringify({ epochIndex, error, block }))
if (!('isElectionBlock' in block))
throw new Error(JSON.stringify({ message: 'Block is not election block', epochIndex, block }))

const { data: currentEpoch, error: errorCurrentEpoch } = await client.blockchain.getEpochNumber()
if (errorCurrentEpoch || !currentEpoch)
throw new Error(`There was an error fetching current epoch: ${JSON.stringify({ epochIndex, errorCurrentEpoch, currentEpoch })}`)
if (epochIndex >= currentEpoch)
throw new Error(`You tried to fetch an epoch that is not finished yet: ${JSON.stringify({ epochIndex, currentEpoch })}`)

// The election block will be the first block of the epoch, since we only fetch finished epochs, we can assume that all the batches in this epoch can be fetched
// First, we need to know in which batch this block is. Batches start at 0
const firstBatchIndex = (electionBlock - genesisBlockNumber) / blocksPerBatch
if (firstBatchIndex % 1 !== 0)
// It should be an exact division since we are fetching election blocks
throw new Error(JSON.stringify({ message: 'Something happened calculating batchIndex', firstBatchIndex, electionBlock, block }))

// Initialize the list of validators and their activity in the epoch
const epochActivity: EpochActivity = {}
for (const { numSlots: likelihood, validator } of (block as ElectionMacroBlock).slots) {
epochActivity[validator] = { likelihood, missed: 0, rewarded: 0, sizeRatio: likelihood / slotsCount, sizeRatioViaSlots: true }
}

const maxBatchSize = 120
const minBatchSize = 10
let batchSize = maxBatchSize
for (let i = 0; i < batchesPerEpoch; i += batchSize) {
const batchPromises = Array.from({ length: Math.min(batchSize, batchesPerEpoch - i) }, (_, j) => createPromise(i + j))

let results = await Promise.allSettled(batchPromises)

let rejectedIndexes: number[] = results.reduce((acc: number[], result, index) => {
if (result.status === 'rejected') {
acc.push(index)
}
return acc
}, [])

if (rejectedIndexes.length > 0) {
// Lowering the batch size to prevent more rejections
batchSize = Math.max(minBatchSize, Math.floor(batchSize / 2))
}
else {
// Increasing the batch size to speed up the process
batchSize = Math.min(maxBatchSize, Math.floor(batchSize + batchSize / 2))
}

while (rejectedIndexes.length > 0) {
const retryPromises = rejectedIndexes.map(index => createPromise(i + index))
results = await Promise.allSettled(retryPromises)

rejectedIndexes = results.reduce((acc: number[], result, index) => {
if (result.status === 'rejected') {
acc.push(rejectedIndexes[index])
}
return acc
}, [])
}
}

async function createPromise(index: number) {
const { data: inherents, error: errorBatch } = await client.blockchain.getInherentsByBatchNumber(firstBatchIndex + index)
return new Promise<void>((resolve, reject) => {
if (errorBatch || !inherents) {
reject(JSON.stringify({ epochIndex, blockNumber: electionBlock, errorBatch, index, firstBatchIndex, currentIndex: firstBatchIndex + index }))
}
else {
for (const { type, validatorAddress } of inherents) {
if (validatorAddress === 'NQ07 0000 0000 0000 0000 0000 0000 0000 0000')
continue
if (!epochActivity[validatorAddress])
continue
epochActivity[validatorAddress].rewarded += type === InherentType.Reward ? 1 : 0
epochActivity[validatorAddress].missed += [InherentType.Penalize, InherentType.Jail].includes(type) ? 1 : 0
}
resolve()
}
})
}

return epochActivity
}

/**
* Fetches the activity for the given block numbers.
* This function is an asynchronous generator. It yields each activity one by one,
* allowing the caller to decide when to fetch the next activity.
*
* @param client - The client instance to use for fetching validator activities.
* @param epochsIndexes - An array of epoch block numbers to fetch the activities for.
* @returns An asynchronous generator yielding objects containing the address, epoch block number, and activity.
*
* Usage:
* const activitiesGenerator = fetchActivities(client, epochBlockNumbers);
* for await (const { key, activity } of activitiesGenerator) {
* console.log(`Address: ${key.address}, Epoch: ${key.epochBlockNumber}, Activity: ${activity}`);
* }
*/
export async function* fetchEpochs(client: NimiqRPCClient, epochsIndexes: number[]) {
for (const epochIndex of epochsIndexes) {
const validatorActivities = await fetchActivity(client, epochIndex)
for (const [address, activity] of Object.entries(validatorActivities)) {
yield { address, epochIndex, activity }
}
}
}

export async function fetchCurrentEpoch(client: NimiqRPCClient) {
const { data: currentEpoch, error } = await client.blockchain.getEpochNumber()
if (error || !currentEpoch)
throw new Error(JSON.stringify({ error, currentEpoch }))
const { data: activeValidators, error: errorValidators } = await client.blockchain.getActiveValidators()
if (errorValidators || !activeValidators)
throw new Error(JSON.stringify({ errorValidators, activeValidators }))
const totalBalance = Object.values(activeValidators).reduce((acc, { balance }) => acc + balance, 0)
const epochActivity: EpochsActivities = {
[currentEpoch]: Object.entries(activeValidators).reduce((acc, [, { address, balance }]) => {
acc[address] = { likelihood: -1, missed: -1, rewarded: -1, sizeRatio: balance / totalBalance, sizeRatioViaSlots: false }
return acc
}, {} as EpochActivity),
}
return epochActivity
}
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
import { defu } from 'defu'
import type { ScoreParams, ScoreValues } from './types'

export function getSize({ balance, threshold, steepness, totalBalance }: ScoreParams['size']) {
if (!balance || !threshold || !steepness || !totalBalance)
export function getSize({ threshold, steepness, sizeRatio }: ScoreParams['size']) {
if (!threshold || !steepness || !sizeRatio)
throw new Error('Balance, threshold, steepness, or total balance is not set')
if (balance < 0 || totalBalance < 0)
throw new Error('Balance or total balance is negative')
const size = balance / totalBalance
const s = Math.max(0, 1 - (size / threshold) ** steepness)
if (sizeRatio < 0 || sizeRatio > 1)
throw new Error(`Invalid size ratio: ${sizeRatio}`)
const s = Math.max(0, 1 - (sizeRatio / threshold) ** steepness)
return s
}

@@ -39,13 +38,13 @@ export function getReliability({ inherentsPerEpoch, weightFactor, curveCenter }:

let numerator = 0
let denominator = 0
const length = Object.keys(inherentsPerEpoch).length
const length = inherentsPerEpoch.size

for (const [epochIndex, inherents] of Object.entries(inherentsPerEpoch)) {
const { rewarded, missed } = inherents
for (const [epochIndex, { missed, rewarded }] of Array.from(inherentsPerEpoch.entries())) {
// console.log(epochIndex, { missed, rewarded }) // TODO Something missed and rewarded are -1, is that correct?
const totalBlocks = rewarded + missed

if (totalBlocks === 0)
if (totalBlocks <= 0)
continue

const r = rewarded / totalBlocks
@@ -72,9 +71,9 @@ export function getReliability({ inherentsPerEpoch, weightFactor, curveCenter }:
// The default values for the computeScore function
// Negative values and empty arrays are used to indicate that the user must provide the values or an error will be thrown
const defaultScoreParams: ScoreParams = {
size: { threshold: 0.1, steepness: 7.5, balance: -1, totalBalance: -1 },
size: { threshold: 0.1, steepness: 7.5, sizeRatio: -1 },
liveness: { weightFactor: 0.5, activeEpochStates: [] },
reliability: { weightFactor: 0.5, curveCenter: -0.16, inherentsPerEpoch: {} },
reliability: { weightFactor: 0.5, curveCenter: -0.16, inherentsPerEpoch: new Map() },
}

export function computeScore(params: ScoreParams) {
Original file line number Diff line number Diff line change
@@ -18,7 +18,7 @@ export interface ScoreParams {
* - 0: The validator was inactive in the epoch.
* The number of items in the array should match the total number of epochs being considered.
*/
activeEpochStates?: ValidatorEpochState[]
activeEpochStates: ValidatorEpochState[]
}

size: {
@@ -27,24 +27,19 @@ export interface ScoreParams {
* Validators with a stake percentage below this threshold receive a maximum score.
* @default 0.1
*/
threshold: number
threshold?: number

/**
* Controls how quickly the size score decreases once the validator's stake percentage surpasses the threshold.
* A higher value results in a steeper decline in the score.
* @default 7.5
*/
steepness: number
steepness?: number

/**
* The balance or stake amount of the validator.
* The proportion of the validator's stake relative to the rest of the stake in the epoch.
*/
balance: number

/**
* The total balance or stake amount across all validators in the network.
*/
totalBalance: number
sizeRatio: number
}

reliability: {
@@ -69,32 +64,39 @@ export interface ScoreParams {
* - `rewarded`: The number of blocks for which the validator received rewards in the epoch.
* - `missed`: The number of blocks the validator was expected to produce but did not.
*/
inherentsPerEpoch?: Record<number, {
rewarded: number
missed: number
}>
inherentsPerEpoch?: Map<number, { rewarded: number, missed: number }>
}
}

// The activity of the validator and their block production activity for a given election block
export interface Activity { likelihood: number, missed: number, rewarded: number }
export interface Activity { likelihood: number, missed: number, rewarded: number, sizeRatio: number, sizeRatioViaSlots: boolean }

// A map of validator addresses to their activities in a single epoch
export type ValidatorActivity = Record<string /* address */, Activity>
export type EpochActivity = Record<string /* address */, Activity>

// A map of validator addresses to their activities in a single epoch
export type ValidatorsActivities = Map<{ address: string, epochBlockNumber: number }, Activity>
export type EpochsActivities = Record<number /* election block */, EpochActivity>

export interface ScoreValues { liveness: number, reliability: number, size: number, total: number }

export interface Range {
// The first block number that we will consider
// The first epoch index that we will consider
fromEpoch: number
// The last block number that we will consider

// The first block number that we will consider
fromBlockNumber: number

// The last epoch index that we will consider
toEpoch: number

// The last block number that we will consider
toBlockNumber: number

// Given a block number, it returns the index in the array of epochs
blockNumberToIndex: (blockNumber: number) => number
blockNumberToEpochIndex: (blockNumber: number) => number

// Given an epoch index, it returns the block number
epochIndexToBlockNumber: (epochIndex: number) => number

blocksPerEpoch: number

Original file line number Diff line number Diff line change
@@ -2,45 +2,56 @@ import type { NimiqRPCClient, PolicyConstants } from 'nimiq-rpc-client-ts'
import type { Range } from './types'

interface GetRangeOptions {
// The last epoch number that we will consider. Default to the last finished epoch.
/**
* The last epoch number that we will consider. Default to the last finished epoch.
*/
toEpochIndex?: number
// The amount of milliseconds we want to consider. Default to 9 months.
/**
* The amount of milliseconds we want to consider. Default to 9 months.
* @default 1000 * 60 * 60 * 24 * 30 * 9 (9 months)
*/
durationMs?: number
}

export const DEFAULT_WINDOW_IN_DAYS = 30 * 9
export const DEFAULT_WINDOW_IN_MS = DEFAULT_WINDOW_IN_DAYS * 24 * 60 * 60 * 1000

/**
* Given the amount of milliseconds we want to consider, it returns an object with the epoch range we will consider.
*/
export async function getRange(client: NimiqRPCClient, options?: GetRangeOptions): Promise<Range> {
const { blockSeparationTime, blocksPerEpoch, genesisBlockNumber } = await getPolicyConstants(client)
const durationMs = options?.durationMs || 1000 * 60 * 60 * 24 * 30 * 9
const durationMs = options?.durationMs || DEFAULT_WINDOW_IN_MS
const epochsCount = Math.ceil(durationMs / (blockSeparationTime * blocksPerEpoch))

const { data: currentEpoch, error: errorCurrentEpoch } = await client.blockchain.getEpochNumber()
if (errorCurrentEpoch || !currentEpoch)
throw new Error(errorCurrentEpoch?.message || 'No current epoch')

const toEpochIndex = options?.toEpochIndex ?? currentEpoch - 1
const fromEpochIndex = Math.max(1, toEpochIndex - epochsCount)
const toEpoch = options?.toEpochIndex ?? currentEpoch - 1
const fromEpoch = Math.max(1, toEpoch - epochsCount)

const fromEpoch = genesisBlockNumber + blocksPerEpoch * fromEpochIndex
const toEpoch = genesisBlockNumber + blocksPerEpoch * toEpochIndex
const fromBlockNumber = genesisBlockNumber + blocksPerEpoch * fromEpoch
const toBlockNumber = genesisBlockNumber + blocksPerEpoch * toEpoch

if (fromEpoch < 0 || toEpoch < 0 || fromEpoch > toEpoch)
throw new Error(`Invalid epoch range: [${fromEpoch}, ${toEpoch}]`)
if (fromBlockNumber < 0 || toBlockNumber < 0 || fromBlockNumber > toBlockNumber)
throw new Error(`Invalid epoch range: [${fromBlockNumber}, ${toBlockNumber}]`)
if (fromEpoch === 0)
throw new Error(`Invalid epoch range: [${fromEpoch}, ${toEpoch}]. The range should start from epoch 1`)

const { data: head, error: headError } = await client.blockchain.getBlockNumber()
if (headError || !head)
throw new Error(headError?.message || 'No block number')
if (toEpoch >= head)
throw new Error(`Invalid epoch range: [${fromEpoch}, ${toEpoch}]. The current head is ${head}`)
if (toBlockNumber >= head)
throw new Error(`Invalid epoch range: [${fromBlockNumber}, ${toBlockNumber}]. The current head is ${head}`)

const blockNumberToIndex = (blockNumber: number) => Math.floor((blockNumber - genesisBlockNumber) / blocksPerEpoch) - fromEpochIndex
const epochCount = toEpochIndex - fromEpochIndex + 1
const blockNumberToEpochIndex = (blockNumber: number) =>
Math.floor((blockNumber - genesisBlockNumber) / blocksPerEpoch)
const epochIndexToBlockNumber = (epochIndex: number) =>
genesisBlockNumber + epochIndex * blocksPerEpoch
const epochCount = toEpoch - fromEpoch + 1

return { fromEpoch, toEpoch, blocksPerEpoch, blockNumberToIndex, epochCount }
return { fromEpoch, fromBlockNumber, toEpoch, toBlockNumber, blocksPerEpoch, blockNumberToEpochIndex, epochCount, epochIndexToBlockNumber }
}

export type PolicyConstantsPatch = PolicyConstants & { blockSeparationTime: number, genesisBlockNumber: number }
File renamed without changes.
124 changes: 0 additions & 124 deletions packages/nimiq-vts/src/fetcher.ts

This file was deleted.

14,119 changes: 0 additions & 14,119 deletions pnpm-lock.yaml

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions pnpm-workspace.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
packages:
- packages/*
catalog:
'@antfu/eslint-config': ^2.26.1
eslint: ^9.9.0
'@antfu/eslint-config': ^3.0.0
eslint: ^9.9.1
lint-staged: ^15.2.9
simple-git-hooks: ^2.11.1
7 changes: 7 additions & 0 deletions public/validators/.example.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"name": "Optional. The name of the validator, by default it is the address",
"address": "NQXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX",
"fee": -1,
"payoutType": "restake | direct",
"tag": "Optional field. Valid values are: 'community' | 'unkwown'"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"name": "Not-A-Pool",
"address": "NQ08 N4RH FQDL TE7S 8C66 65LT KYDU Q382 YG7U",
"fee": 0.11,
"payoutType": "restake",
"tag": "Nimiq"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"name": "Pooly McPoolface",
"address": "NQ24 DJE3 KX3U HG5X 1BXP 8XQ3 SK7S X364 N7G7",
"fee": 0.09,
"payoutType": "restake",
"tag": "Nimiq"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"name": "Swimming Pool",
"address": "NQ38 YX2J GTMX 5XAU LKFU H0GS A4AA U26L MDA3",
"fee": 0.1,
"payoutType": "restake",
"tag": "Nimiq"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"name": "Helvetia Staking",
"address": "NQ49 E4LQ FN9M B9BP 0FRE BCL5 MHFY TGQE D4XX",
"fee": 0.1,
"payoutType": "restake",
"description": "The Swiss-standard of NIMIQ Staking",
"icon": "data:image/svg+xml,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20xml%3Aspace%3D%22preserve%22%20viewBox%3D%220%200%201080%20960%22%3E%3Cpath%20fill%3D%22%23FF0D00%22%20d%3D%22M1081%20462c0%2012.354%200%2024.708-.367%2037.297-.764.738-1.326%201.182-1.531%201.754-2.688%207.476-4.442%2015.442-8.146%2022.368-9.609%2017.964-20.018%2035.503-30.204%2053.156-13.368%2023.168-26.87%2046.258-40.234%2069.428-9.289%2016.104-18.37%2032.328-27.675%2048.422-12.898%2022.307-25.987%2044.502-38.867%2066.82-12.147%2021.048-24.055%2042.235-36.202%2063.284-15.735%2027.264-31.625%2054.439-47.403%2081.677-7.31%2012.618-14.546%2025.145-26.362%2034.45-11.22%208.835-23.264%2015.541-37.329%2018.428-.663.136-1.125%201.255-1.68%201.916-162.354%200-324.708%200-487.297-.367-.738-.764-1.19-1.403-1.751-1.521-21.56-4.544-39.59-15.245-51.79-33.498-12.578-18.822-23.58-38.707-35.045-58.26-9.403-16.035-18.438-32.285-27.702-48.402-9.506-16.54-19.112-33.024-28.636-49.554-8.145-14.135-16.19-28.327-24.366-42.444-9.4-16.233-18.95-32.378-28.33-48.622-9.296-16.098-18.424-32.293-27.694-48.407-9.432-16.395-18.937-32.75-28.429-49.11-9.413-16.223-18.881-32.414-28.265-48.654-5.635-9.753-11.006-19.627-12.831-30.98C2.787%20500.7%201.645%20500.39%201%20500c0-12.354%200-24.708.367-37.297.786-1.07%201.281-1.875%201.612-2.743%202.94-7.726%205.044-15.897%208.932-23.107%208.894-16.492%2018.695-32.493%2028.053-48.736%208.3-14.407%2016.452-28.899%2024.757-43.303%209.363-16.24%2018.86-32.403%2028.233-48.639%209.29-16.092%2018.458-32.255%2027.75-48.346%209.373-16.235%2018.873-32.397%2028.245-48.632%209.29-16.093%2018.448-32.26%2027.74-48.352%209.374-16.234%2018.837-32.418%2028.268-48.62a5598.355%205598.355%200%200%201%2018.38-31.395c.378-.38.53-.64.63-.898.003.036-.065.014.308.016%201.715-.585%203.173-.994%204.384-1.783%2036.475-23.759%2073.858-45.935%20113.018-65.03C342.852%202.562%20343.895%201.718%20345%201c146.354%200%20292.708%200%20439.297.367.738.764%201.19%201.403%201.751%201.522%2021.56%204.543%2039.591%2015.244%2051.79%2033.498%2012.578%2018.821%2023.58%2038.707%2035.045%2058.26%209.403%2016.034%2018.439%2032.284%2027.702%2048.402%209.507%2016.54%2019.112%2033.023%2028.637%2049.553%208.145%2014.135%2016.19%2028.327%2024.365%2042.444%209.4%2016.233%2018.95%2032.378%2028.33%2048.622%209.296%2016.099%2018.424%2032.294%2027.694%2048.407%209.433%2016.396%2018.937%2032.75%2028.429%2049.11%209.413%2016.223%2018.882%2032.414%2028.265%2048.654%205.635%209.753%2011.006%2019.628%2012.831%2030.98.078.481%201.218.791%201.864%201.181m-447-96.5V169.319H448V381.556c0%206.442-.003%206.444-6.631%206.444H229.365v186h5.848c68.824%200%20137.649.054%20206.473-.112%205.035-.012%206.441%201.269%206.429%206.374-.171%2068.99-.115%20137.982-.115%20206.973v5.531h186.057v-218.85h218.667V387.858H634V365.5z%22%2F%3E%3Cpath%20fill%3D%22%23FF1C00%22%20d%3D%22M224.275%2069.948c7.288-12.449%2014.338-25.328%2022.77-37.228%209.736-13.739%2024.006-21.838%2039.595-27.604%203.605-1.333%207.32-2.372%2010.672-3.831C312.688%201%20328.375%201%20344.532%201c-.637.718-1.68%201.562-2.855%202.135-39.16%2019.095-76.543%2041.271-113.018%2065.03-1.211.789-2.669%201.198-4.384%201.783z%22%2F%3E%3Cpath%20fill%3D%22%23FFF%22%20d%3D%22M634%20366v21.858h218.724v186.058H634.057v218.85H448v-5.531c0-68.991-.056-137.982.115-206.973.012-5.105-1.394-6.386-6.429-6.374-68.824.166-137.649.112-206.473.112h-5.848V388h212.004c6.628%200%206.63-.002%206.63-6.444L448%20175.083v-5.764h186v196.68z%22%2F%3E%3Cpath%20fill%3D%22%23FF1C00%22%20d%3D%22M223.338%2070.83c-.069-.151.034-.452.381-.844.148.204-.004.465-.381.844z%22%2F%3E%3C%2Fsvg%3E",
"tag": "community"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"name": "Pool Billard",
"address": "NQ57 2F6C X3GB Y9B7 04U5 2BVA 4BVC M2T0 ELRL",
"fee": 0.105,
"payoutType": "restake",
"tag": "Nimiq"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"name": "Mint Pool",
"address": "NQ57 M1NT JRQA FGD2 HX1P FN2G 611P JNAE K7HN",
"fee": 0.1,
"payoutType": "restake",
"description": "Minting together",
"website": "https://mintpool.io",
"icon": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMEAAACvCAYAAABEvLq2AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAwaADAAQAAAABAAAArwAAAADR/xIVAAAfWklEQVR4Ae1d/XIbx5GfWYCU/kPeALw6J+dc4og09W1fga6KcycrsaQnIPwE4j2BVk9w1BMEfAKRcWzFUqpMVFKxvngA7VOi2Kk64QlO+I8fwMz9ehcgARDATi/2YwDsspbYj56Znp7u6Znenh4p5uBYqz1aaon80hxUta+KSui3f1n5Vb3vYXZzhgL5M09m5MGH+3/cFELfRnWKrRmpE7caOSG2kKbMTTdv8DMrBEqqutTy7rw1aLe+UohmTi1sdO9Nfq/XHi87Qv6IYPMiX99d+eitSbpphwGtZvf4oP7HupD6wuzWcHTN0LD3/3zhY3c0RP8bb8jo5P+3/ynutNyX6FCEFLu59sL2LAqGc6bSM/RAar2hxVz+NfMqv8lpymMnv6mJ5wdPdCJKyHWl5W+Pndb/fbD/ZPvD2pM1Tt62w860JiDiX9t/XEEl121viEjxk+Kzv/ziVxXTPImptSO+NoUnOAhL1VHC/fPKx7ucdDbCzrQmIIJr1XbRYs2zXRy9nMmzwREAopFyhEu/nAMdS4kE5zoMEGu1r715BCe9TbBTJQQf7j9xcb7hEP3pyo03Quvhqh4tMWtyoDSPoUkLEEOHZUop9N1j2d6lSXXYPNJONzVC8AGGNWDYeziLLae1wSHcolik8XGDk2YqYaWoPl0xHwZRHdGbVyauK+YNjiOnVhCmQghoXA/mX+9OcfERaOMqPoCZNh5ZNDR6yG76mf2loR/juP7t4zJoUYyCHmiTgnDEVAqC9UIwYmJbkE6O1eBeD4meksEj0wXqaYEbu6ZIe0NK5tDJIO+pFASrheBa7cmG0NAAUAODJz2/yjXVKQVtMHvzAKpTW0nWEPHIOSL4ogFjc0FIELY58zZuAVHDWysE3kRL6v8aV2HpaHfc+8F3mCTvSum5Egy+mup7qtMLho+Qz6A8oWESqAghqzDTpAZurRAohyw6AX9al67WHpdZ1IM2QLazZTKlOjGOjhYoMJKEAJW3puWjmpVCQIyN4U/JaOgi9eYyw05NJlMtVLCABQmgNe/Vfc8MbMimZFDQWt4bHF7Gcd+WumKIVqpg1gkBMTSYn3pr08F74bzwxrfGhHy+csMF8CyYTJtH4jyZf40PrkHBOOPhgMXrta/c4a/seWqdECyKgw301MWAgVBfPwzz3D2OydQnv0Q5pnJmJxyw2qwzPD09GsGgkCT7aSk3bJ8kWyUEPiPLe2EaCU5erB7x+cq/bwshp9ZkKqHJQmiBShjaTpimcOgcs9pmwvLYya0SAmLkCXrnWxdrj9Z4FIjVQsJDhQkNTenytAA8PzHPYhYTDTi0j81uFdYIATGw0vqWxgwt/ClYPY5nVtR6K3x5k+A6UdrGi5UbFQ6HSocsSBN0MROm1bD2cfBNEtYaIQDvT0wkDBEuXKo9KnMIeCwPNgDf5KRJG1ZLTTgbHyxrG3KNRVSgha7VvrptjHSCgFYIwfs+416IgvjwooTJ9OGPTGlYX7nzFq0+yTAsHqYZwYwKfvx7K59sm9bPg5M8z1JW3gxgTJKtdLtOXQiIYWUEWqCnLQp5dZ7VU75cnR6TqcNkaP9joi720CfFS108YJqzk0A2dSFw1DkXo+MCzsj+lNT3lhleph6hmUOMJBpnSBk7L+H6MeT50EeeaRIfE4e+TOuhFCwP4CTQTFUIiFExBLobx9dKqXhfK2mIATGsRieK0efUBgNxmIJ6XdAXHUxM4/xw+cLBznE59YgbNlUh4DIqkxil5Rc8k6mSsgx1ZBXHdPEBWlt1WiVneHhf3plCY5j1xGDo9OABzGubiQsdk0FqQrD84vM19JWl6PvL0xyFoypj6n7mVYfJHlgoB01MKt0zCI95cE4cEnzMTnJjEAh4pSzSBqkJAVRiJYBOUbwuLu/9njWEUI7HPFaZTLGOF+4R5lqg8+X9bleL2PgrleB7AEfBEUPySEUIiDGhEotxzAXO5ildrskUmsC1SBs0lXPEnNzaNeYewnfeI+hslgfwqHwmfZ64EBBDIpBTkkxWUO1Fl0Oo+uonYDoNL1MrRAFaAN8yDI9LiPqAbwnrVmAOnAPwKCxaYDJNXAhUe4EYDGPVAPJE+/4u22SqVNmQ7+IEa9RXb7qcAmjolDBtgd5EbRnCA5hDkWDYRIXgZ8+/WAa5Uuml2oo5Sb70m136Ont2eIUmRyWSONF8bnATnkL4FhddOn0yHVdcD+Coa5WoEOBr5+ZknQaqH7bTUbr0c1ikOAR0HKfMgY8Ydh9aoMLJM21m4uA6AHsrTZNpYkLw3svPb2OxTOnUgJn8FQhfGSD+2FuyyEDmUjGZSq02xiI38LLjOHhh4PHU3EKAK2khm4gQ+JNhK5zUitAGLofYTu4I8Bom07AqKEQ6xBCqYzjGwVMK3ncETt4JwRav1P7AEvyo8EpECFqtPFWuGBXSk+QDltzgmkwRm9+dpExuWocZPeIimAf1KoYQtyRFO7AszMFgzk4+uG/sQvDuNw8R3UDQd4FEJpMG5RQOj3mx+79d/fUm0N9PiMm2OFqAmAbrKBIVUq5QM+ALC/6XbkaSyUFjFwKZy7sUpzL5GcDoEiGO62Sp4pAPq88oGl7so6LcMS+8JAUmAFbWukdwaNyBvet/8Q6RMmSSWIXgXc8ao9dD4hZrMiXam5wC/sc3me7EKQcQsq36NXP3CD88zfSukx5F/7ZIxKXmpPhYhQDjHzdOppkw79JPn31++4QSBhcLrWP0uhOWOjp9M7dA+ZsfC/pgE8O/gsEQ0JahqCEeurSa4FLM2ITg3ee/K6PGJcNag7fAXAmfmqkN6tfuvAGL3jdnUxYkyz2Chgyg2HqMQgnkYxP4wLxhImZpahalB4BjEYIl+AdhDE2TyRTJaFK2LL77bMcdoMnY23y+RfVqRly3JuU7tuCBl61ol6QO5G7FbfHi3iM3CUxiEYLFI4dMdrataBoqkDDLbZDQmhLbX5jPi/YQmDcm3RwnuU58pVuB+U47ABYFcczZYasbuRCQSRTI3AuLUArpChBaVi/86vKnFYzc9iMZvSnRoPxY9Y5+cw1W8QkCFxY0L9ZqGNwiFwLshIjhwtT9rb/zzcM1DgHhB7VhMuAKgpGSt8dCRwuUOLhOMyxGFOuXajxzNre+kQoBMRLCft8aOu5AbWx+LpnL/chkCk0wqcm0ytUCKLNiOSkjb+a25mnqVIVASGcKJsMjZBGWrB9/87DMISD2SIY2kKGXYnLXDXeClBU5OM4CLL6IL8dZj3xUmS/RXECrbeRHp3dge9A61pK+7d5b/5vj4foaJtN/fbazCWa+F6Ju1ddcJ7n5mQv0kVPHHDUDQpYdk1CArBcHR7k6nOxYPXRO5FZeXb5ZNy2bzIVghjDCZlqErXCNvfdvLMWJXKRzgjgRtTVvz7TpeZlqoGh8bnEEgAQt7t7QVvqKBCIDZkIQQeu/9k2mxksxsd+qyynWQWxViNdUfHcx7gZAAANYfvBhDmE7sJkQhCDasCTScA0Axp/3aS4xLI9hz7wAAYitasQyJmw1RTDc4MPD6GfyLBMCEyoZwICxd9GzbQWANg8WeT4xEk6IU8S3UcpqlRN8OIDuY19nQjCWPLyXtCIMgjDOr2jzDSOGEGkBhK1f52ExG9DKkeWkapIJQYSU7gxzNodnqRtHTC3gMCNrDy93+p7CCMAKPjxpDTMhmJSCA+l9RteNM44jcI9gaQFE1IZWKeGcv9EQM/jwQBOwbzMhYJNsfAKf0fuXYmJI0/jh8p3K+JT9b6XD8ynqTz29d4igd58TfJg2Ipl0n+SJvhhT4S3RWjYhucxJWmNgBGuSX9IwUsr6n37xy22Tcr+/cmf7x892qlgkVPLgmbbu5Zdf3kY6P61JgbMDww4+3MYeyVrINyCBG5YMEwkBIQDfmXWTwiEAJmDWwhD+a7VH/7RrGiJdKkTYkDUhZfWHK7eNhKdbeUmR+ubwoDiqNYbhgFbXIWiXx3+4rjw1bZsB2oYeDtHmzIQAsfa8nMeOeagWDH/qMG9uCcPvB912Wd7ztqAtdu/n6LfBDUHvOPlKlz69191npr+hfYeu7z/ZReKSaUGzAgeHwI/+vPLxbhz18VZRqXN15D2HQiA/q6+ab1D+Ye3JGhw0v+5th7BtE2o4dO3br27TeFf3YjAn1zDf0VBlOZbqqsUN6NU5FACBEPSfVDg0xeItd7AHh1BQHkucfAiWPRyiybDUkhhhPg8pLlz/9nE56sp31tJCCObwYO4F8W/ohDF/KA0ZiBc/2H/CpiFbCI5oW9DEtlpCNaFurDuViH5ndrXogv1nKZKcqTSzgw+30QnTKGT4qV2uyZQlBF6sGzl7Ec9MW6sHrnAojt2e+4kuO7vo3J0ok2lNzDQcdLRwcUx1C0cOr21YQiAdL07mPPZWZ2kuRWQxM1UbkfrQrc3bqbBGmxN82OvhzVbXsdrGeGIMLbAGW/n6WW6Y4yf+NrRrk1Bg2dvCaj7p6uQc1vidhuJCy3Fa4KQpHCdXwc3ayYMxF+aaAIvox+Qzn6807cU72c7sKjefH8bwlZflJEdaAGu5aXHRiLnAwHO0zTXDeKZGQnC19riMwi8YI2CK6CzASa/HCdUJLCNqN/yKSqEST3eippPj7azTmYPxhuLSzIoZKAR++G+d3oZ71kueDr3NkGLGOppuvj/FHoKP4MPmIejJIINABmEMB8Xrta/c05KHXwUKAW0CAbdgqzbZOOOmDAWZ7h9/myHSAkC6ZL2Mg28ixrEp8kesoXXHIDOcgwOe0hAqyGQ6Vgg8CRShYuoEoDZzr9nbDHWsIo2Zo0RQhTwtcOdtEFj3PfmowWy2PoEoFg6d8UI3VgimeF/cLg0T+8UnfJZZjhBztCpP0LjIIeI+Ov78Gt9euukCceNDORGE+McSVU+YRpQ6Uggo8CtIfGvqyIyKpoUzd5uhjjbYGdE2M/eYG3bSs7xFZDggYRpF0JFC4KS4ufIoZO1/rkuXDc1y3bo4udxG93qWf6Ep979bvVlh1TFKw8EYk+lQIbi49+WG0rpIC0myk0cDxdxmyLOSaH0/Le2VVLlt2v2TcZBZHiq9xEgSCApNNNTn64wQeN6MCS90DsR+ugDY2ww5/lZNoaNbW08erK6jUPYcPOGy7kYvoLp4QF+dB44zQpBT56jweQz5F91cgrnNkB/PdHYdEzXTSe5K7Q9g1HjWVVBMV9/qeSoJfULgh/yTYT5KnOaYXREFCjnmNkOd8fJ+dJIILKLvSvl5KrHF0QL0cRbr1t0YccdXZ8elRuoefUKQM/PQ66bNfsdTgL3NEOZfZ1T1+CLsf5tre57Hxoie94crYNT4DvQN673a4EQIvPDfQt9O98vrbJVOiz84TUk9JnbTnHQLqBg7UZ4iACJb9Wvm7hGkBZTQ5KEQ+x/oXO62zYkQIPw31g1jZZMNKnR2cCit+jGEuvQO/F1ozYzJtJlbOGZpNrjoUKcRqxY4bQBZ7l6fCAGsodACPEnP4IPppZgxhPyeU98Pztl66sNJztw9ouOis95lzAR+i5fIJQPHiRBgR5C1BAqexyKK7+994XIq3tndfpr9ipqdOhhXG8MTFo2MMx4D6Ai1Rq89ISCrEPqVzCwKgsTRv8IHCzuzwx3Y8KAe1NuXwBDeNjAsmHGZWmDNNJJhlHXFzKNHE7RaS/E0fxwsNZV5Yq7FC7D7HW0BJUR1CmvbeHXp1zS2Nz7gc+WmU09niZD0h0OO9CTCGOsMkE8BuAN7awgYKeFv4zLA7QBlCjs5aqIDLqWDfK8mUBQxGqhkZ6w04K4k8z4ywcyYDoOEKrX6ChqMk1IKnhmZk7cBrGeJOp0YG6TIQCajAK0nXt77oszJpd1uuYCfCr8irua6VPOCD1/g0CMOWE8IMDOPZUKYzjjP7rog1o7bCblo1J60BRSYizXGNso4eiC+kxxokbaXMpGhMyfQb+1mnZkSp6Jq5Tc4PEjmRlCgYTMVhObFELq498gFDYocOsQA62lYTwiwQVw9hgKyLEdRAIu/Q5lMLZUCzCW3Xl2+acxDvru+YHUEo0g54XMPZ18T5PNvLKXvrA7TCu122+U0IE04MSyqctIkBav9eYtxcXl1ngQgIfeI0WiBnm/orScE/uomTL4ySUhyVLj+cwq7wjpoyGFXI4GRHnS2rjWqieceAZ9+I+CYgTAf6dEEKAyT4127yGtbc0ePD/Yz2+S0Mw05KHwhJ02csDBvNhcWPOuVcTHHNBm2xDsBO4TuEuL+cIgutNqOvpkzsRpLU60vvPf8d2VqCNPj3EILrsayaQVlsZ6a4x7RmQetm9Y1ZrjGixV/HnMiBLmF9jYIawdxUXsrGjkBPBS0Acdk6vsVKZYGiYWZsDfz4iIPDyzaSh/vDjHwzabSpcuJEHgSrQW0QXYkTIFCi2ky/euVWy56iUbCePYVRwvhOVrAd48Qt/oySfGm5e9v5mFwIgR01/k6mSJq81k0tN69d795uMSrfYoL8yGAr5nuEfSR0BbtjvlvX1j4PiHwZvlaPMh8iDAcQ4sleTrOAmuo8LcrvyGtXeUJTlTQPAFcrX15G9Qs4QQCqZ9N0mK9lOgTAnpBs32YvfAlLXVkgc0c4SDVLa7J1BE5TJITp1K1I4DELkaHxkaH9jSlPhMW/owQ0DhPSeyhheplZ7I0aOnR8TKHcRuZTGGmfDDsXWzPmIHZ3t/znOSKseHDy7ihhkSoPiMElOffLn1KkpvqxItXt9mAhga+8C7fZOpiVRY0dyJH9TUjkpxv9eItJoq3FsNXvA0VAkJEO7KcaYJkNQHRGwzNN5lCc3tJ4/7Xbpc5RThqkYZrRUv4qPrfqzcqw/AfKQQk8bCl7gxLlD2LkwK6cHicdzkl+JpbNWKeyW9x3CNIC+Cj3ganHnHCamd0YLORQkAIadXeiJmwKAT9RHb20UBrdZdtMlU6Vs0tmPFESQuAhVJ3kvMES5JJdLSX61gh6Ej+fUvU2XxN1P09kr02NPmHttoFXCwmU8xV7vO0AEUvsWabL5hED8dqpLFCQMQ/8D+NJzXxoiKzw6dACdpgjUMMqWj7p8iPZocHzDPGYnt7Ok4yiY4PAhYoBG88kyljE2WQyh4CTDcuypEVc84TwuutpYjUZAotsEk8YIqH5yQ32UZ7UXJQo7560w3CPVAIKIMf/E/k+0GZZe+jpoAs/svT341V5YMlHi4oF88i0ty68Zr8lDiHUhUOeKyw2mxibiQEhKhSCuau7C9pCiip3CVYWkyZxeu1tYxkOKJkfxz/IBw6cZVKQXAJva/WL36ybVKWsRD8gyZeSm9lYx2QNdnxXmHhsN/XJahh/34VHzuxUV4Q3Pj3utEZAYwH630b5UZ7vfmGuXacsmkyYyGgDHN+RLSIVK0pihkcPqCxTaYONPcklEPwhTInPcVTgqW7ZIO1G53UA2/JsGEFWEJAEy+sy9xMtiNMuuO1s7xWOJPpjiEf9IFhMlztmFz7no+7AfO7494n+K4pckcsXFhCQBVpnfdWBzUSrFRWFFFA69I/P3t4m0MMXxvwuyyYWl1OOZ2oekVOmrhgJVxIgkyig2VD6PnHO779eo2f0k+hHVGXSrwNm35e07UQIuQNtDGn/j959hCTZPMPV1Lone+v3DEWNs89or1IONnwdXh //+LNZQ59CDbPTUDw3iQZ0SnCpM3SJEuBo0WxuXDkhTgxYlJH8cKhqBbcI6QVAkDBIjbCUJc9HApTSJYmPQp4JlMF5zGTURGsfzz3CJhuLYkhBArv1Blu3r0tkglBLzVm9PqHa3cqsDAFmUybHeufMRVUe9HFZAUaxkTC4oVxcrxYqL2VzISglxozfA3P1LFDBWwPtcnTArT9lL5rBcm0vs8xiQ7inAnBIEVm9N6bx2l87Bx+NI98q9/wt0OeUizVePt2Y/3SdBC1ewiKxo8yITAm1fQD0nAHjDskwJp2OU5ynYAA61ZQBBG+uSbRQbwzIRikyAzfe8Ods4v5G/+4eofVk0pb3COkrH63erMyaZNlQjApBacs/eDHTu5WsZ4WUIghZMFYSDM/6o1qqkwIRlFmRp/TsOeU8eW+bzkyr6xkRtI2z5kJiY1BvI0NmcmGgYf6WDYsozVsVt0S+aVh78Y+c8Ta2PfZy7EUAFPW//SLX26PBRp4SYwPF4yyVLwoFe89Jyc5fWEguzRum/l2zo2q4MiE4AAY5RzxdVSIZfkYUkDrJja+WH66cuONYQoPrK1EmeuCoZMK7RJYESyZvMar77gsIxsOUSPAR+XBuMKyd9FTAEPzQs7h94pcAfjZi9/jO4MuGhsu45s0NDAMcqOkZGRCQEgtqhxMcBomuOwvWQqI9Q9rT9aiZIzevMhJDo51bu+ztK5lDG4akQrB7spHby3yJUmrnVIpVznxMSntn0AaB2d8/btZ3tXvLnrRuCOlcShX6iAMrtYfI1CssGECFYTqTL3HwpbPnq78qhJlpSgIWC6fryNP+AilfGhnhbNVrCm2kWqCk0INV/mfwGcXkVAAQwV3rfb1jyLJrJNJLpd3cWmBAPD2Su6gb/QTixA8Xfl4V0i9k+y4OCsNFCgeiCNMYKM5vFCQUqxHk9tEudAumZHVaxCTWISACsHXvNiQHqxEdt9DAUwcvb2Cex6FvczlebvnhC0nON3wkOrB6cwgYhMCz26tRRbHFO2Q8ISyICLw7SH3CGiWWwnjPoxWjVeXfr1pxs7hoGITAkLnQCxisw+JkOG4yc7EaIDtkdahDdaoDcIeXvQIC9oMDFoOWwfTdLFYh3oLv1R7VMaWQr/tfZZdJ0KB/ecr/7EcpiTSAkrrr8OkjTINmLP66vKna1HmOSyvWDUBFfhihXYHkdVhhWfPYqXABeqAwpSQz7doL7TUg6xh4U4o/Ll1jl0ICCEllGuBZp27ERk6n1BjaVqkguWYKQdZkw84yz25jN8Ln4gQvFy5sYsB8VZvwdl1IhQoXK49csOU9FeKRp3a5o2yeQ5bCYfBO0yaRISAEDsWBxtww23iRDC17EyKBthJ/l54k6lZaPMwjDcuDbhj4iWT4/IffJeYEHjrQLUXwnEQh+w+Zgq0QtK9s2l3ovM5DJv3X/v7YcRMldPsExMCKvLl6g2aGzSy+UFi1tLuPOjWxZAmU0fkkv3oiYXzp+yZzFWiQuBVSY7eSjOZKs9nKRiBVsLUnBzWsE5kK4mOCxK7w9ksPEx9hqVJXAj2Vj7ZBkGrSRA1K6NP4xTf3/syVC973vPbidtkKpvelsHDuDTmZ4kLAdUHWxBl+yOja4aFINkTcYdogQyXp/z5nNrkpmPBwySblEl0EK9UhKCzsfKDrKfu66m74/c4fwuOOucOMoHJvbeBn1ZwgYlDcFXjvL9VsAkqkcOkIgRUC+UcuvhJ/ask4TJPB1j4rrfNaphKYwunODougQ0CJ40iF6Y63TSpCYH3VdIPCxhnz5fljZYeZFxswFjpMgDnF8OVXcBHbTKtJm0SHaxzakJAiNRXP8E4UzfONtNgs2X3EdOo1NludZAfAu+xlVM5EIgBwN0aipG1MWiqQuBhCaJmLB4xi4OwgTRlbgTY5Shv8ipFJKF1gCNtCrLbzTut39SFgHYXATGqwa0GEgW2bAbDoFFxeY9iCfGPwwVvY79J53PNo0U7Vh+mLgTUBA42Xs74O3kZV1qGMpl6YdyRdsI22+SEg+eLqnkKK4TA22VEq/vJs8GEzcjodi2tW0G1w60j/vvVTzvzOXNmO4XUje/JS9WSwwohIFrQbiNgySEbSFjKPsB5RkRo/WfPv1gOw4/cXe9Py0jYH+m04KFX1giBZydWcJ6aEc6apno4Mpx3L01qpWcyNW80gv/+SvRR5IZyt+FDa4SA8P3u8s2KlmLfnKQz0xunLDO69N7Lz28b8kwfGJlMOe3lucz05ZD+jVVC4JFDk5cph6wZbBT0gjcExvj8wzOZav3ArMn0gx8u36nzS4k3hXVCQLuPwHUX0euiaNosDwYdi4gy4YZht+NzgtIFmUybHbgwRcSaxjohoNq2W8fQBoFEjZUw85g5BGYjjJepbzIdv04ESyZZO2QmSX8rhYBULBok5WgHc6lFCofH+VDDItoCChp81HyOvUPm3AsBEWCRNmhW8CuKxXUXIpblO5QGUujQJlMEoCUNfvbQunz2oT1PrNQERB4ymSqEGmeMaUUGG432UqIdShv8w/MD0ju97I2dMrf8571P7bq2VgiITB0X26pdJJsLbEo/fRbOZJrXokcbyP3j8733dtLOaiEgklG0g6yHj6aH59FRb4aZJHdMpuQC05SOKtviHzRO/KwXAop2gB1Y/nNcJbJ3MVBAiuLBkdPTq5uX0TovNiX2p7bxm8CwWuAr9nQcP3m2UwGy69OB7cxg2cQX4WWvd5+ZKp2tiPWaoIsyvA7LUOdb3fvsNxEKFFQEG34kgukEhUyNJujWkTQCrjON0CVIAr+OUh9BG+wmUFQqRUyNJuhShzQCru9377Pf+CmgZ1wbTJ0m6Db5O988XHMcia+Uoth9lv3GSAGpP8NEtxJjCallPbVCQBRbQjS1xQOxof0grunvtZtaM8ZfMBilcXROL0+DyZNLjakWgm5lSRjyh7TBmxfRONMMXcJE99vEl99t+vCVCUF0RI0tpx8/f7is2+I2Vu+vwT9oGQVlGiIMtaWsYsumupBit70odmeR+btkmQlN0K3MqN+lbx4u5TF6GvU+e35KAdv9fE4xje7q/wHCbOoZADNaqwAAAABJRU5ErkJggg==",
"tag": "community"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"name": "Kiddie Pool",
"address": "NQ57 UQJL 5A3H N45M 1FHS 2454 C7L5 BTE6 KEU1",
"fee": 0.1,
"payoutType": "restake",
"tag": "Nimiq"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"name": "AceStaking",
"address": "NQ65 DHN8 4BSR 5YSX FC3V BB5J GKM2 GB2L H17C",
"fee": 0.1,
"payoutType": "direct",
"description": "The Ace in staking",
"tag": "Community"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"name": "Monopooly",
"address": "NQ71 CK94 3V7U H62Y 4L0F GUUK DPA4 6SA6 DKKM",
"fee": 0.11,
"payoutType": "direct",
"tag": "Nimiq"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"name": "Puddle",
"address": "NQ82 BHPS UR9K 07X1 X6QH 3DY3 J325 UCSP UHV3",
"fee": 0.095,
"payoutType": "direct",
"tag": "Nimiq"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"name": "Cool Pool",
"address": "NQ87 FEGQ 01TF M29N T03J 3YCB JB5M X5VM XP8Q",
"fee": 0.09,
"payoutType": "direct",
"tag": "Nimiq"
}
8 changes: 8 additions & 0 deletions server/api/index.get.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import { poolQuerySchema } from '../utils/schemas'
import { fetchValidators } from '../utils/validators'

export default defineEventHandler(async (event) => {
const { onlyPools } = await getValidatedQuery(event, poolQuerySchema.parse)
const validators = await fetchValidators({ onlyPools })
return { validators }
})
28 changes: 17 additions & 11 deletions server/api/vts/health.get.ts → server/api/scores/health.get.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
import { count, max } from 'drizzle-orm'
import type { Range } from 'nimiq-vts'
import { getRange } from 'nimiq-vts'
import type { Range } from 'nimiq-validators-score'
import { getRange } from 'nimiq-validators-score'
import { consola } from 'consola'
import { getMissingEpochs } from '~~/server/database/utils'
import { getRpcClient } from '~~/server/lib/client'
import { findMissingEpochs } from '~~/server/utils/activities'

export enum HealthFlag {
MissingEpochs = 'missing-epochs',
NoValidators = 'no-validators',
// TODO
// ScoreNotComputed = 'score-not-computed',
}
@@ -39,7 +40,7 @@ export default defineEventHandler(async (event) => {

// Get the latest epoch number in the activity table
const latestActivityBlock = await useDrizzle()
.select({ epoch: max(tables.activity.epochBlockNumber) })
.select({ epoch: max(tables.activity.epochNumber) })
.from(tables.activity)
.get()
.then(row => row?.epoch ?? -1)
@@ -56,9 +57,9 @@ export default defineEventHandler(async (event) => {
.then(row => row?.count ?? 0)

const fetchedEpochs = await useDrizzle()
.selectDistinct({ epoch: tables.activity.epochBlockNumber })
.selectDistinct({ epoch: tables.activity.epochNumber })
.from(tables.activity)
.orderBy(tables.activity.epochBlockNumber)
.orderBy(tables.activity.epochNumber)
.all()
.then(rows => rows.map(row => row.epoch))

@@ -70,14 +71,20 @@ export default defineEventHandler(async (event) => {
if (errorCurrentEpoch)
return err(errorCurrentEpoch)

const flags: HealthFlag[] = []
const range = await getRange(rpcClient)
const missingEpochs = await getMissingEpochs(range)

const isSynced = missingEpochs.length === 0
const flags: HealthFlag[] = []
if (!isSynced)
const missingEpochs = await findMissingEpochs(range)
if (missingEpochs.length > 0)
flags.push(HealthFlag.MissingEpochs)

if (totalValidators === 0)
flags.push(HealthFlag.NoValidators)

const isSynced = flags.length === 0

// TODO Add component to see how much time left for next epoch and length of current epoch

// Combine all the data into a HealthStatus object
const healthStatus: HealthStatus = {
latestFetchedEpoch,
@@ -91,7 +98,6 @@ export default defineEventHandler(async (event) => {
fetchedEpochs,
network: networkName,
}
consola.info('Health Status:', healthStatus)

// Return the health status
setResponseStatus(event, 200)
60 changes: 24 additions & 36 deletions server/api/vts/index.get.ts → server/api/scores/index.get.ts
Original file line number Diff line number Diff line change
@@ -1,46 +1,43 @@
import { desc, eq, isNotNull, max } from 'drizzle-orm'
import { consola } from 'consola'
import { getRange } from '~~/packages/nimiq-vts/src'
import { getMissingEpochs } from '~~/server/database/utils'
import { fetchVtsData } from '~~/server/lib/fetch'
import { desc, isNotNull } from 'drizzle-orm'
import { getRpcClient } from '~~/server/lib/client'

export interface Validator {
id: number
name: string
address: string
fee: number
payoutType: string
description: string
icon: string
tag: string
website: string
total: number
liveness: number
size: number
reliability: number
}
import { fetchParams } from '~~/server/lib/fetch'
import { findMissingEpochs } from '~~/server/utils/activities'
import { extractRangeFromRequest } from '~~/server/utils/range'
import { calculateScores, checkIfScoreExistsInDb } from '~~/server/utils/scores'
import type { ValidatorScore } from '~~/server/utils/types'

function err(error: any) {
consola.error(error)
return createError(error)
}

// TODO What if the selected epoch does not have activity for the validator?
// Code now will return a range where the last epoch is the last epoch with activity
// but we should maybe add a flag to the return?

export default defineEventHandler(async (event) => {
const networkName = useRuntimeConfig().public.nimiqNetwork

const rpcClient = getRpcClient()

// TODO Remove this block once scheduled tasks are implemented in NuxtHub
const { data: range, error: errorRange } = await extractRangeFromRequest(rpcClient, event)
if (errorRange || !range)
return err(errorRange || 'No range')

// TODO Remove this block once scheduled tasks are implemented in NuxtHub and the data is being
// fetched periodically
// This is just a workaround to sync the data before the request in case of missing epochs
const range = await getRange(rpcClient)
const missingEpochs = await getMissingEpochs(range)
const missingEpochs = await findMissingEpochs(range)
if (missingEpochs.length > 0) {
consola.warn(`Missing epochs: ${JSON.stringify(missingEpochs)}. Fetching data...`)
await fetchVtsData(rpcClient)
await fetchParams(rpcClient)
}
// End of workaround

if (!(await checkIfScoreExistsInDb(range)))
await calculateScores(range)

const validators = await useDrizzle()
.select({
id: tables.validators.id,
@@ -52,8 +49,8 @@ export default defineEventHandler(async (event) => {
icon: tables.validators.icon,
tag: tables.validators.tag,
website: tables.validators.website,
total: tables.scores.total,
liveness: tables.scores.liveness,
total: tables.scores.total,
size: tables.scores.size,
reliability: tables.scores.reliability,
})
@@ -62,17 +59,8 @@ export default defineEventHandler(async (event) => {
.where(isNotNull(tables.scores.validatorId))
.groupBy(tables.validators.id)
.orderBy(desc(tables.scores.total))
.all() as Validator[]

const epochBlockNumber = await useDrizzle()
.select({ epoch: max(tables.activity.epochBlockNumber) })
.from(tables.activity)
.get().then(row => row?.epoch ?? -1)

const { data: epochNumber, error: epochNumberError } = await rpcClient.policy.getEpochAt(epochBlockNumber)
if (epochNumberError)
return err(epochNumberError)
.all() as ValidatorScore[]

setResponseStatus(event, 200)
return { validators, epochNumber, network: networkName } as const
return { validators, range, network: networkName } as const
})
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,26 +1,30 @@
CREATE TABLE `activity` (
`validator_id` integer NOT NULL,
`epoch_block_number` integer NOT NULL,
`epoch_number` integer NOT NULL,
`likelihood` integer NOT NULL,
`rewarded` integer NOT NULL,
`missed` integer NOT NULL,
PRIMARY KEY(`epoch_block_number`, `validator_id`),
`size_ratio` integer NOT NULL,
`size_ratio_via_slots` integer NOT NULL,
PRIMARY KEY(`validator_id`, `epoch_number`),
FOREIGN KEY (`validator_id`) REFERENCES `validators`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
CREATE TABLE `scores` (
`score_id` integer PRIMARY KEY NOT NULL,
`validator_id` integer NOT NULL,
`from_epoch` integer NOT NULL,
`to_epoch` integer NOT NULL,
`total` real NOT NULL,
`liveness` real NOT NULL,
`size` real NOT NULL,
`reliability` real NOT NULL,
PRIMARY KEY(`validator_id`, `from_epoch`, `to_epoch`),
FOREIGN KEY (`validator_id`) REFERENCES `validators`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
CREATE TABLE `validators` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`name` text DEFAULT 'Unknown validator',
`name` text DEFAULT 'Unknown validator' NOT NULL,
`address` text NOT NULL,
`fee` real DEFAULT -1,
`payout_type` text DEFAULT 'unknown',
@@ -30,5 +34,6 @@ CREATE TABLE `validators` (
`website` text
);
--> statement-breakpoint
CREATE UNIQUE INDEX `scores_validator_id_unique` ON `scores` (`validator_id`);--> statement-breakpoint
CREATE INDEX `idx_election_block` ON `activity` (`epoch_number`);--> statement-breakpoint
CREATE INDEX `idx_validator_id` ON `scores` (`validator_id`);--> statement-breakpoint
CREATE UNIQUE INDEX `validators_address_unique` ON `validators` (`address`);
77 changes: 59 additions & 18 deletions server/database/migrations/meta/0000_snapshot.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"version": "6",
"dialect": "sqlite",
"id": "a8b4c590-767a-454e-9a04-930d60dcf51b",
"id": "2a9191cd-1bc2-4f2b-a890-4663de575240",
"prevId": "00000000-0000-0000-0000-000000000000",
"tables": {
"activity": {
@@ -14,8 +14,8 @@
"notNull": true,
"autoincrement": false
},
"epoch_block_number": {
"name": "epoch_block_number",
"epoch_number": {
"name": "epoch_number",
"type": "integer",
"primaryKey": false,
"notNull": true,
@@ -41,9 +41,31 @@
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"size_ratio": {
"name": "size_ratio",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"size_ratio_via_slots": {
"name": "size_ratio_via_slots",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"idx_election_block": {
"name": "idx_election_block",
"columns": [
"epoch_number"
],
"isUnique": false
}
},
"indexes": {},
"foreignKeys": {
"activity_validator_id_validators_id_fk": {
"name": "activity_validator_id_validators_id_fk",
@@ -60,28 +82,35 @@
}
},
"compositePrimaryKeys": {
"activity_validator_id_epoch_block_number_pk": {
"activity_validator_id_epoch_number_pk": {
"columns": [
"epoch_block_number",
"validator_id"
"validator_id",
"epoch_number"
],
"name": "activity_validator_id_epoch_block_number_pk"
"name": "activity_validator_id_epoch_number_pk"
}
},
"uniqueConstraints": {}
},
"scores": {
"name": "scores",
"columns": {
"score_id": {
"name": "score_id",
"validator_id": {
"name": "validator_id",
"type": "integer",
"primaryKey": true,
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"validator_id": {
"name": "validator_id",
"from_epoch": {
"name": "from_epoch",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"to_epoch": {
"name": "to_epoch",
"type": "integer",
"primaryKey": false,
"notNull": true,
@@ -117,12 +146,12 @@
}
},
"indexes": {
"scores_validator_id_unique": {
"name": "scores_validator_id_unique",
"idx_validator_id": {
"name": "idx_validator_id",
"columns": [
"validator_id"
],
"isUnique": true
"isUnique": false
}
},
"foreignKeys": {
@@ -140,7 +169,16 @@
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"compositePrimaryKeys": {
"scores_validator_id_from_epoch_to_epoch_pk": {
"columns": [
"validator_id",
"from_epoch",
"to_epoch"
],
"name": "scores_validator_id_from_epoch_to_epoch_pk"
}
},
"uniqueConstraints": {}
},
"validators": {
@@ -157,7 +195,7 @@
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": false,
"notNull": true,
"autoincrement": false,
"default": "'Unknown validator'"
},
@@ -233,5 +271,8 @@
"schemas": {},
"tables": {},
"columns": {}
},
"internal": {
"indexes": {}
}
}
6 changes: 3 additions & 3 deletions server/database/migrations/meta/_journal.json
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
{
"version": "6",
"version": "7",
"dialect": "sqlite",
"entries": [
{
"idx": 0,
"version": "6",
"when": 1718092093045,
"tag": "0000_orange_gargoyle",
"when": 1725108191944,
"tag": "0000_lush_jocasta",
"breakpoints": true
}
]
32 changes: 22 additions & 10 deletions server/database/schema.ts
Original file line number Diff line number Diff line change
@@ -1,32 +1,44 @@
import { integer, primaryKey, real, sqliteTable, text } from 'drizzle-orm/sqlite-core'
import { index, integer, primaryKey, real, sqliteTable, text, uniqueIndex } from 'drizzle-orm/sqlite-core'

// TODO
// Is delete on cascade ok?

export const validators = sqliteTable('validators', {
id: integer('id').primaryKey({ autoIncrement: true, onConflict: 'replace' }),
name: text('name').default('Unknown validator'),
name: text('name').default('Unknown validator').notNull(),
address: text('address').notNull().unique(),
fee: real('fee').default(-1),
payoutType: text('payout_type').default('unknown'),
description: text('description'),
icon: text('icon').notNull(),
tag: text('tag').default('unknown'),
website: text('website'),
})
}, table => ({
uniqueAddress: uniqueIndex('validators_address_unique').on(table.address),
}))

// TODO The score is calculated based on a window of epochs (default 9 months)
// Maybe we could add those two parameters (fromEpochNumber and toEpochNumber) to the scores table to have more context
export const scores = sqliteTable('scores', {
id: integer('score_id').notNull().primaryKey(),
validatorId: integer('validator_id').notNull().references(() => validators.id).unique(),
validatorId: integer('validator_id').notNull().references(() => validators.id),
fromEpoch: integer('from_epoch').notNull(),
toEpoch: integer('to_epoch').notNull(),
total: real('total').notNull(),
liveness: real('liveness').notNull(),
size: real('size').notNull(),
reliability: real('reliability').notNull(),
})
}, table => ({
idxValidatorId: index('idx_validator_id').on(table.validatorId),
compositePrimaryKey: primaryKey({ columns: [table.validatorId, table.fromEpoch, table.toEpoch] }),
}))

export const activity = sqliteTable('activity', {
validatorId: integer('validator_id').notNull().references(() => validators.id),
epochBlockNumber: integer('epoch_block_number').notNull(),
epochNumber: integer('epoch_number').notNull(),
likelihood: integer('likelihood').notNull(),
rewarded: integer('rewarded').notNull(),
missed: integer('missed').notNull(),
}, ({ epochBlockNumber, validatorId }) => ({ pk: primaryKey({ columns: [validatorId, epochBlockNumber] }) }))
sizeRatio: integer('size_ratio').notNull(),
sizeRatioViaSlots: integer('size_ratio_via_slots').notNull(),
}, table => ({
idxElectionBlock: index('idx_election_block').on(table.epochNumber),
compositePrimaryKey: primaryKey({ columns: [table.validatorId, table.epochNumber] }),
}))
101 changes: 0 additions & 101 deletions server/database/seed.ts

This file was deleted.

165 changes: 0 additions & 165 deletions server/database/utils.ts

This file was deleted.

100 changes: 67 additions & 33 deletions server/lib/fetch.ts
Original file line number Diff line number Diff line change
@@ -1,33 +1,43 @@
import type { NimiqRPCClient } from 'nimiq-rpc-client-ts'
import type { ValidatorsActivities } from 'nimiq-vts'
import { fetchValidatorsActivities, getRange } from 'nimiq-vts'
import { consola } from 'consola'
import { getMissingEpochs, getMissingValidators, storeActivities, storeValidator } from '../database/utils'
import type { EpochsActivities } from 'nimiq-validators-score'
import { fetchCurrentEpoch, fetchEpochs, getRange } from 'nimiq-validators-score'
import { findMissingValidators, storeValidator } from '../utils/validators'
import { findMissingEpochs, storeActivities } from '../utils/activities'

const EPOCHS_IN_PARALLEL = 2
const EPOCHS_IN_PARALLEL = 3

let running = false

export async function fetchVtsData(client: NimiqRPCClient) {
export async function getActiveValidators(client: NimiqRPCClient) {
const { data: activeValidators, error: errorActiveValidators } = await client.blockchain.getActiveValidators()
if (errorActiveValidators || !activeValidators)
throw new Error(errorActiveValidators.message || 'No active validators')
return activeValidators
}

/**
* Fetches the required data for computing the score.
* The size ratio parameter can be obtained via two different methods:
* 1. Knowing the slots assignation of the validator relative to the total amount of slots in the epoch
* We can retrieve this data from the election blocks
* 2. From the balance of the validators relative to the other validators in the epoch
* We can retrieve this data only when the epoch is active, and this is the prefered method as it is more precise but
* it is not reliable
*
* @param client
*/
export async function fetchParams(client: NimiqRPCClient) {
if (running) {
consola.info('Task is already running')
return
}

try {
running = true
consola.info('Fetching data for VTS...')
const epochBlockNumbers = await fetchEpochs(client)

// We need to fetch the data of the active validators that are active in the current epoch
// but we don't have the data yet.
const { data: activeValidators, error: errorActiveValidators } = await client.blockchain.getActiveValidators()
if (errorActiveValidators || !activeValidators)
throw new Error(errorActiveValidators.message || 'No active validators')
const addressesCurrentValidators = activeValidators.map(v => v.address)
const missingValidators = await getMissingValidators(addressesCurrentValidators)
await Promise.all(missingValidators.map(missingValidator => storeValidator(missingValidator)))
return { epochBlockNumbers, missingValidators, addressesCurrentValidators }
const missingEpochs = await fetchMissingEpochs(client)
const { missingValidators, addresses: addressesCurrentValidators } = await fetchActiveEpoch(client)
return { missingEpochs, missingValidators, addressesCurrentValidators }
}
catch (error) {
consola.error(error)
@@ -38,38 +48,62 @@ export async function fetchVtsData(client: NimiqRPCClient) {
}
}

async function fetchEpochs(client: NimiqRPCClient) {
/**
* Fetches the activities of the epochs that have finished and are missing in the database.
*/
async function fetchMissingEpochs(client: NimiqRPCClient) {
// The range that we will consider
const range = await getRange(client)
consola.info(`Fetching data for range: ${JSON.stringify(range)}`)

// Only fetch the missing epochs that are not in the database
const epochBlockNumbers = await getMissingEpochs(range)
consola.info(`Fetching data for epochs: ${JSON.stringify(epochBlockNumbers)}`)
if (epochBlockNumbers.length === 0)
const missingEpochs = await findMissingEpochs(range)
const fetchedEpochs = []
consola.info(`Fetching missing epochs: ${JSON.stringify(missingEpochs)}`)
if (missingEpochs.length === 0)
return []

const activitiesGenerator = fetchValidatorsActivities(client, epochBlockNumbers)
const epochGenerator = fetchEpochs(client, missingEpochs)

// We fetch epochs 3 by 3 in parallel and store them in the database
while (true) {
const start = globalThis.performance.now()
const epochsActivities: ValidatorsActivities = new Map()
const epochsActivities: EpochsActivities = {}

// Fetch the activities in parallel
for (let i = 0; i < EPOCHS_IN_PARALLEL; i++) {
const { value: pair, done } = await activitiesGenerator.next()
if (done)
const { value: pair, done } = await epochGenerator.next()
if (done || !pair)
break
epochsActivities.set(pair.key, pair.activity)
if (!epochsActivities[`${pair.epochIndex}`])
epochsActivities[`${pair.epochIndex}`] = {}
const epoch = epochsActivities[`${pair.epochIndex}`]
if (!epoch[pair.address])
epoch[pair.address] = pair.activity
}

const end = globalThis.performance.now()
const seconds = (end - start) / 1000
consola.info(`Fetched ${epochsActivities.size} epochs in ${seconds} seconds`)
const epochs = Object.keys(epochsActivities).map(Number)
fetchedEpochs.push(...epochs)
const newestEpoch = Math.max(...fetchedEpochs)
const percentage = Math.round((fetchedEpochs.length / missingEpochs.length) * 100).toFixed(2)
consola.info(`Fetched ${newestEpoch} epochs. ${percentage}%`)

if (epochsActivities.size === 0)
if (epochs.length === 0)
break
await storeActivities(epochsActivities)
}

return epochBlockNumbers
return missingEpochs
}

async function fetchActiveEpoch(client: NimiqRPCClient) {
// We need to fetch the data of the active validators that are active in the current epoch
// but we don't have the data yet.
const { data: activeValidators, error: errorActiveValidators } = await client.blockchain.getActiveValidators()
if (errorActiveValidators || !activeValidators)
throw new Error(errorActiveValidators.message || 'No active validators')
const addresses = activeValidators.map(v => v.address)
const activity = await fetchCurrentEpoch(client)
const missingValidators = await findMissingValidators(addresses)
await Promise.all(missingValidators.map(missingValidator => storeValidator(missingValidator)))
await storeActivities(activity)
return { missingValidators, addresses }
}
10 changes: 9 additions & 1 deletion server/plugins/migrations.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import { migrate } from 'drizzle-orm/d1/migrator'
import { consola } from 'consola'
import { importValidatorsFromFiles } from '../utils/validators'

let validatorImported = false

export default defineNitroPlugin(async () => {
if (!import.meta.dev)
@@ -11,7 +14,12 @@ export default defineNitroPlugin(async () => {
consola.success('Database migrations done')
})
.catch((err) => {
consola.error('Database migrations failed', err)
consola.error('Database migrations failed', JSON.stringify(err))
})
// TODO Find a way to only run this once
if (!validatorImported) {
await importValidatorsFromFiles('./public/validators')
validatorImported = true
}
})
})
6 changes: 3 additions & 3 deletions server/tasks/fetch.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { consola } from 'consola'
import { fetchVtsData } from '../lib/fetch'
import { fetchParams } from '../lib/fetch'
import { getRpcClient } from '../lib/client'

export default defineTask({
@@ -10,9 +10,9 @@ export default defineTask({
async run() {
consola.info('Running fetch task...')
const client = getRpcClient()
const res = await fetchVtsData(client)
const res = await fetchParams(client)
if (!res)
return { result: 'No new epochs fetched' }
return { result: `New ${res.epochBlockNumbers.length} epochs fetched and ${res.missingValidators.length} validators of the current epoch stored ${JSON.stringify(res.addressesCurrentValidators)}` }
return { result: `New ${res.missingEpochs.length} epochs fetched and ${res.missingValidators.length} validators of the current epoch stored ${JSON.stringify(res.addressesCurrentValidators)}` }
},
})
10 changes: 5 additions & 5 deletions server/tasks/reset.ts
Original file line number Diff line number Diff line change
@@ -9,16 +9,16 @@ export default defineTask({
async run({ payload }: { payload: { latest?: string, epoch_block_number?: string } }) {
consola.info('Deleting DB...')
if (payload.latest && payload.latest === 'true') {
const latest = await useDrizzle().selectDistinct({ epochBlockNumber: tables.activity.epochBlockNumber }).from(tables.activity).orderBy(desc(tables.activity.epochBlockNumber)).limit(1)
const latest = await useDrizzle().selectDistinct({ electionBlockNumber: tables.activity.epochNumber }).from(tables.activity).orderBy(desc(tables.activity.epochNumber)).limit(1)
if (latest.length > 0) {
consola.info('Deleting data from latest epoch block number:', latest[0].epochBlockNumber)
await useDrizzle().delete(tables.activity).where(eq(tables.activity.epochBlockNumber, latest[0].epochBlockNumber)).get()
return { result: `Deleted epoch_block_number: ${latest[0].epochBlockNumber}` }
consola.info('Deleting data from latest epoch block number:', latest[0].electionBlockNumber)
await useDrizzle().delete(tables.activity).where(eq(tables.activity.epochNumber, latest[0].electionBlockNumber)).get()
return { result: `Deleted epoch_block_number: ${latest[0].electionBlockNumber}` }
}
}

if (payload.epoch_block_number && !Number.isNaN(Number(payload.epoch_block_number))) {
await useDrizzle().delete(tables.activity).where(eq(tables.activity.epochBlockNumber, Number(payload.epoch_block_number))).get()
await useDrizzle().delete(tables.activity).where(eq(tables.activity.epochNumber, Number(payload.epoch_block_number))).get()
return { result: `Deleted epoch_block_number: ${payload.epoch_block_number}` }
}

39 changes: 0 additions & 39 deletions server/tasks/score.ts

This file was deleted.

14 changes: 0 additions & 14 deletions server/tasks/seed.ts

This file was deleted.

78 changes: 78 additions & 0 deletions server/utils/activities.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
import { gte, lte } from 'drizzle-orm'
import type { Activity, EpochsActivities, Range } from 'nimiq-validators-score'
import type { NewActivity } from './drizzle'
import { storeValidator } from './validators'

/**
* Given a range, it returns the epochs that are missing in the database.
*/
export async function findMissingEpochs(range: Range) {
const existingEpochs = await useDrizzle()
.selectDistinct({ epochBlockNumber: tables.activity.epochNumber })
.from(tables.activity)
.where(and(
gte(tables.activity.epochNumber, range.fromEpoch),
lte(tables.activity.epochNumber, range.toEpoch),
))
.execute().then(r => r.map(r => r.epochBlockNumber))

const missingEpochs = []
for (let i = range.fromEpoch; i <= range.toEpoch; i++) {
if (!existingEpochs.includes(i))
missingEpochs.push(i)
}
return missingEpochs
}

/**
* We loop over all the pairs activities/epochBlockNumber and store the validator activities.
*/
export async function storeActivities(epochs: EpochsActivities) {
const promises = Object.entries(epochs).map(async ([_epochNumber, activities]) => {
const epochNumber = Number(_epochNumber)
const activityPromises = Object.entries(activities).map(async ([address, activity]) => storeSingleActivity({ address, activity, epochNumber }))
return await Promise.all(activityPromises)
})
await Promise.all(promises)
}

interface StoreActivityParams {
address: string
activity: Activity
epochNumber: number
}

async function storeSingleActivity({ address, activity, epochNumber }: StoreActivityParams) {
const validatorId = await storeValidator(address)
if (!validatorId)
return
// If we ever move out of cloudflare we could use transactions to avoid inconsistencies and improve performance
// Cloudflare D1 does not support transactions: https://github.com/cloudflare/workerd/blob/e78561270004797ff008f17790dae7cfe4a39629/src/workerd/api/sql-test.js#L252-L253
const existingActivity = await useDrizzle()
.select({ sizeRatioViaSlots: tables.activity.sizeRatioViaSlots, sizeRatio: tables.activity.sizeRatio })
.from(tables.activity)
.where(and(
eq(tables.activity.epochNumber, epochNumber),
eq(tables.activity.validatorId, validatorId),
))

const { likelihood, rewarded, missed, sizeRatio: _sizeRatio, sizeRatioViaSlots: _sizeRatioViaSlots } = activity

// We always want to update db except the columns `sizeRatio` and `sizeRatioViaSlots`.
// If we have `sizeRatioViaSlots` as false and `sizeRatio` > 0, then we won't update only those columns
// As we want to keep the values from the first time we inserted the activity as they are more accurate
const viaSlotsDb = Boolean(existingActivity.at(0)?.sizeRatioViaSlots)
const sizeRatioDb = existingActivity.at(0)?.sizeRatio || 0
const updateSizeColumns = viaSlotsDb !== false || sizeRatioDb <= 0
const sizeRatio = updateSizeColumns ? _sizeRatio : sizeRatioDb
const sizeRatioViaSlotsBool = updateSizeColumns ? _sizeRatioViaSlots : viaSlotsDb
const sizeRatioViaSlots = sizeRatioViaSlotsBool ? 1 : 0

await useDrizzle().delete(tables.activity)
.where(and(
eq(tables.activity.epochNumber, epochNumber),
eq(tables.activity.validatorId, validatorId),
))
const activityDb: NewActivity = { likelihood, rewarded, missed, epochNumber, validatorId, sizeRatio, sizeRatioViaSlots }
await useDrizzle().insert(tables.activity).values(activityDb)
}
55 changes: 55 additions & 0 deletions server/utils/range.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import type { Range } from 'nimiq-validators-score'
import { getRange } from 'nimiq-validators-score'
import type { NimiqRPCClient } from 'nimiq-rpc-client-ts'
import type { EventHandlerRequest, H3Event } from 'h3'
import { consola } from 'consola'
import type { Result } from './types'
import { rangeQuerySchema } from './schemas'

/**
* To compute the score for a validator for a given range, it is mandatory that we have the activity for that validator
* in the last epoch of the range. If we don't have the activity for that epoch, we can't compute the score.
* Instead of throwing an error, we will modify the range so the last epoch is the last epoch where we have activity.
*/
// export async function adjustRangeForAvailableData(expectedRange: Range): Result<Range> {
// const highestScoreEpoch = await useDrizzle()
// .select({ toEpoch: max(tables.scores.toEpoch) })
// .from(tables.scores)
// .where(and(
// gte(tables.scores.fromEpoch, expectedRange.fromEpoch),
// lte(tables.scores.toEpoch, expectedRange.toEpoch),
// ))
// .get()
// .then(r => r?.toEpoch)
// consola.info({ highestScoreEpoch })
// if (!highestScoreEpoch)
// return { error: `No scores found between epochs ${expectedRange.fromEpoch} and ${expectedRange.toEpoch}. Run the fetch task first.`, data: undefined }

// const toEpoch = Math.min(highestScoreEpoch, expectedRange.toEpoch)
// const toBlockNumber = expectedRange.epochIndexToBlockNumber(toEpoch)
// const range: Range = { ...expectedRange, toEpoch, toBlockNumber }
// return { data: range, error: undefined }
// }

export async function extractRangeFromRequest(rpcClient: NimiqRPCClient, event: H3Event<EventHandlerRequest>): Result<Range> {
const { data: currentEpoch, error: currentEpochError } = await rpcClient.blockchain.getEpochNumber()
if (currentEpochError || !currentEpoch)
return { error: JSON.stringify(currentEpochError), data: undefined }
const { epoch: userEpoch } = await getValidatedQuery(event, rangeQuerySchema.parse)

let epoch
if (userEpoch === 'latest')
epoch = currentEpoch - 1
else if (currentEpoch <= userEpoch)
return { error: `Epoch ${epoch} is in the future or it didn't finished yet. The newest epoch you can fetch is ${currentEpoch - 1}.`, data: undefined }
else
epoch = userEpoch
consola.info(`Fetching data for epoch ${epoch}`)
let range: Range
consola.info(`Fetching data for epoch ${epoch}`)
try {
range = await getRange(rpcClient, { toEpochIndex: epoch })
}
catch (error: unknown) { return { error: JSON.stringify(error), data: undefined } }
return { data: range, error: undefined }
}
27 changes: 27 additions & 0 deletions server/utils/schemas.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import { z } from 'zod'
import { DEFAULT_WINDOW_IN_DAYS, DEFAULT_WINDOW_IN_MS } from '~~/packages/nimiq-validators-score/src'

export const rangeQuerySchema = z.object({
epoch: z.literal('latest').or(z.number().min(1)).default('latest'),
epochsCount: z.number().min(1).default(DEFAULT_WINDOW_IN_DAYS),
durationWindowMs: z.number().min(1).default(DEFAULT_WINDOW_IN_MS),
}).refine(({ epochsCount, durationWindowMs }) => {
const defaultCounts = epochsCount === DEFAULT_WINDOW_IN_DAYS
const defaultWindow = durationWindowMs === DEFAULT_WINDOW_IN_MS
return (!epochsCount || !durationWindowMs) || (defaultCounts && defaultWindow) || (!defaultCounts && !defaultWindow)
})

export const validatorSchema = z.object({
name: z.string().optional(),
address: z.string().regex(/^NQ\d{2}(\s\w{4}){8}$/, 'Invalid Nimiq address format'),
fee: z.number().min(0).max(1),
payoutType: z.nativeEnum(PayoutType),
tag: z.nativeEnum(ValidatorTag),
description: z.string().optional(),
website: z.string().url().optional(),
icon: z.string().optional(),
})

export const poolQuerySchema = z.object({
onlyPools: z.literal('true').or(z.literal('false')).default('false').transform(v => v === 'true'),
})
123 changes: 123 additions & 0 deletions server/utils/scores.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
import { gte, inArray, lte } from 'drizzle-orm'
import type { Range, ScoreParams } from 'nimiq-validators-score'
import { computeScore } from 'nimiq-validators-score'
import type { NewScore } from './drizzle'
import type { Result, ValidatorScore } from './types'
import { fetchValidatorsScoreByIds } from './validators'
import { findMissingEpochs } from './activities'

interface GetScoresResult {
validators: ValidatorScore[]
range: Range
}

/**
* Given a range of epochs, it returns the scores for the validators in that range.
*/
export async function calculateScores(range: Range): Result<GetScoresResult> {
const missingEpochs = await findMissingEpochs(range)
if (missingEpochs.length > 0)
throw new Error(`Missing epochs in database: ${missingEpochs.join(', ')}. Run the fetch task first.`)

// TODO Decide how we want to handle the case of missing activity
// const { data: range, error: rangeError } = await adjustRangeForAvailableData(expectedRange)
// consola.info({ range, rangeError })
// if (rangeError || !range)
// return { error: rangeError, data: undefined }

// TODO Check if we already have scores for the given range

const sizeLastEpoch = await useDrizzle()
.select({
sizeRatio: tables.activity.sizeRatio,
sizeRatioViaSlots: tables.activity.sizeRatioViaSlots,
validatorId: tables.activity.validatorId,
})
.from(tables.activity)
.where(and(
eq(tables.activity.epochNumber, range.toEpoch),
))

const sizeLastEpochByValidator = new Map<number, { sizeRatio: number, sizeRatioViaSlots: boolean }>()
sizeLastEpoch.forEach(({ validatorId, sizeRatio, sizeRatioViaSlots }) =>
sizeLastEpochByValidator.set(validatorId, { sizeRatio, sizeRatioViaSlots: Boolean(sizeRatioViaSlots) }))
const validatorsIds = Array.from(sizeLastEpochByValidator.keys())

const _activities = await useDrizzle()
.select({
epoch: tables.activity.epochNumber,
validatorId: tables.validators.id,
rewarded: tables.activity.rewarded,
missed: tables.activity.missed,
})
.from(tables.activity)
.innerJoin(tables.validators, eq(tables.activity.validatorId, tables.validators.id))
.where(and(
gte(tables.activity.epochNumber, range.fromEpoch),
lte(tables.activity.epochNumber, range.toEpoch),
inArray(tables.activity.validatorId, validatorsIds),
))
.orderBy(tables.activity.epochNumber)
.execute()

type Activity = Map<number /* validatorId */, { inherentsPerEpoch: Map<number /* epoch */, { rewarded: number, missed: number }>, sizeRatio: number, sizeRatioViaSlots: boolean }>

const validatorsParams: Activity = new Map()

for (const { epoch, missed, rewarded, validatorId } of _activities) {
if (!validatorsParams.has(validatorId)) {
const { sizeRatio, sizeRatioViaSlots } = sizeLastEpochByValidator.get(validatorId) ?? { sizeRatio: -1, sizeRatioViaSlots: false }
if (sizeRatio === -1)
return { error: `Missing size ratio for validator ${validatorId}. Range: ${range.fromEpoch}-${range.toEpoch}`, data: undefined }
validatorsParams.set(validatorId, { sizeRatio, sizeRatioViaSlots, inherentsPerEpoch: new Map() })
}
const validatorInherents = validatorsParams.get(validatorId)!.inherentsPerEpoch
if (!validatorInherents.has(epoch))
validatorInherents.set(epoch, { rewarded: 0, missed: 0 })
const { missed: accMissed, rewarded: accRewarded } = validatorInherents.get(epoch)!
validatorInherents.set(epoch, { rewarded: accRewarded + rewarded, missed: accMissed + missed })
}
const scores = Array.from(validatorsParams.entries()).map(([validatorId, { inherentsPerEpoch }]) => {
const activeEpochStates = Array.from({ length: range.toEpoch - range.fromEpoch + 1 }, (_, i) => inherentsPerEpoch.has(range.fromEpoch + i) ? 1 : 0)
const size: ScoreParams['size'] = { sizeRatio: sizeLastEpochByValidator.get(validatorId)?.sizeRatio ?? -1 }
const liveness: ScoreParams['liveness'] = { activeEpochStates }
const reliability: ScoreParams['reliability'] = { inherentsPerEpoch }
const score = computeScore({ liveness, size, reliability })
const newScore: NewScore = { validatorId: Number(validatorId), fromEpoch: range.fromEpoch, toEpoch: range.toEpoch, ...score }
return newScore
})

await persistScores(scores)
const { data: validators, error: errorValidators } = await fetchValidatorsScoreByIds(scores.map(s => s.validatorId))
if (errorValidators || !validators)
return { error: errorValidators, data: undefined }
return { data: { validators, range }, error: undefined }
}

/**
* Insert the scores into the database. To avoid inconsistencies, it deletes all the scores for the given validators and then inserts the new scores.
*/
export async function persistScores(scores: NewScore[]) {
await useDrizzle().delete(tables.scores).where(or(...scores.map(({ validatorId }) => eq(tables.scores.validatorId, validatorId))))
await Promise.all(scores.map(async score => await useDrizzle().insert(tables.scores).values(score)))

// If we ever move out of cloudflare we could use transactions to avoid inconsistencies
// Cloudflare D1 does not support transactions: https://github.com/cloudflare/workerd/blob/e78561270004797ff008f17790dae7cfe4a39629/src/workerd/api/sql-test.js#L252-L253
// await useDrizzle().transaction(async (tx) => {
// await tx.delete(tables.scores).where(or(...scores.map(({ validatorId }) => eq(tables.scores.validatorId, validatorId))))
// await tx.insert(tables.scores).values(scores.map(s => ({ ...s, updatedAt })))
// })
}

export async function checkIfScoreExistsInDb(range: Range) {
const scoreAlreadyInDb = await useDrizzle()
.select({ validatorId: tables.scores.validatorId })
.from(tables.scores)
.where(and(
eq(tables.scores.toEpoch, range.toEpoch),
eq(tables.scores.fromEpoch, range.fromEpoch),
))
.get()
.then(r => Boolean(r?.validatorId))
return scoreAlreadyInDb
}
9 changes: 9 additions & 0 deletions server/utils/types.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import type { Score, Validator } from './drizzle'
import type { Range } from '~~/packages/nimiq-validators-score/src'

export type Result<T> = Promise<{ data: T, error: undefined } | { data: undefined, error: string }>

export type ValidatorScore =
Pick<Validator, 'id' | 'name' | 'address' | 'fee' | 'payoutType' | 'description' | 'icon' | 'tag' | 'website'>
& Pick<Score, 'total' | 'liveness' | 'size' | 'reliability'>
& { range: Range, sizeRatioViaSlots: number }
161 changes: 161 additions & 0 deletions server/utils/validators.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
import { readFile, readdir } from 'node:fs/promises'
import path from 'node:path'
import { desc, inArray } from 'drizzle-orm'
// @ts-expect-error no types in the package
import Identicons from '@nimiq/identicons'
import { consola } from 'consola'
import { Address } from '@nimiq/core'
import type { NewValidator, Validator } from './drizzle'
import type { Result, ValidatorScore } from './types'
import { validatorSchema } from './schemas'

/**
* Given a list of validator addresses, it returns the addresses that are missing in the database.
* This is useful when we are fetching the activity for a range of epochs and we need to check if the validators are already in the database.
* They should be present in the database because the fetch function needs to be run in order to compute the score.
*/
export async function findMissingValidators(addresses: string[]) {
const existingAddresses = await useDrizzle()
.select({ address: tables.validators.address })
.from(tables.validators)
.where(inArray(tables.validators.address, addresses))
.execute().then(r => r.map(r => r.address))

const missingAddresses = addresses.filter(a => !existingAddresses.includes(a))
return missingAddresses
}

const validators = new Map<string, number>()

interface StoreValidatorOptions {
/**
* If true, it will store the validator even if it already exists in the database.
* @default false
*/
force?: boolean
}

export async function storeValidator(
address: string,
rest: Omit<NewValidator, 'address' | 'icon'> = {},
options: StoreValidatorOptions = {},
): Promise<number | undefined> {
try {
Address.fromString(address)
}
catch (error: unknown) {
consola.error(`Invalid address: ${address}. Error: ${JSON.stringify(error)}`)
return
}

const { force = false } = options

// If the validator is cached and force is not true, return it
if (!force && validators.has(address)) {
return validators.get(address)
}

// Check if the validator already exists in the database
let validatorId = await useDrizzle()
.select({ id: tables.validators.id })
.from(tables.validators)
.where(eq(tables.validators.address, address))
.get()
.then(r => r?.id)

// If the validator exists and force is not true, return it
if (validatorId && !force) {
validators.set(address, validatorId)
return validatorId
}

consola.info(`${force ? 'Updating' : 'Storing'} validator ${address}`)

const icon = (await Identicons.default?.toDataUrl(address)) || ''
if (validatorId) {
await useDrizzle()
.update(tables.validators)
.set({ ...rest, icon })
.where(eq(tables.validators.id, validatorId))
.execute()
}
else {
validatorId = await useDrizzle()
.insert(tables.validators)
.values({ ...rest, address, icon })
.returning()
.get().then(r => r.id)
}

validators.set(address, validatorId!)
return validatorId
}

export async function fetchValidatorsScoreByIds(validatorIds: number[]): Result<ValidatorScore[]> {
const validators = await useDrizzle()
.select({
id: tables.validators.id,
name: tables.validators.name,
address: tables.validators.address,
fee: tables.validators.fee,
payoutType: tables.validators.payoutType,
description: tables.validators.description,
icon: tables.validators.icon,
tag: tables.validators.tag,
website: tables.validators.website,
liveness: tables.scores.liveness,
total: tables.scores.total,
size: tables.scores.size,
reliability: tables.scores.reliability,
})
.from(tables.validators)
.leftJoin(tables.scores, eq(tables.validators.id, tables.scores.validatorId))
.where(inArray(tables.validators.id, validatorIds))
.groupBy(tables.validators.id)
.orderBy(desc(tables.scores.total))
.all() as ValidatorScore[]
return { data: validators, error: undefined }
}

export interface FetchValidatorsOptions {
onlyPools: boolean
}

export async function fetchValidators({ onlyPools }: FetchValidatorsOptions): Result<Validator[]> {
const validators = await useDrizzle()
.select()
.from(tables.validators)
.where(onlyPools ? eq(tables.validators.payoutType, PayoutType.Restake) : undefined)
.groupBy(tables.validators.id)
.all()
return { data: validators, error: undefined }
}

/**
* Import validators from a folder containing .json files.
*
* This function is expected to be used when initializing the database with validators, so it will throw
* an error if the files are not valid and the program should stop.
*/
export async function importValidatorsFromFiles(folderPath: string) {
const allFiles = await readdir(folderPath)
const files = allFiles
.filter(f => path.extname(f) === '.json')
.filter(f => !f.endsWith('.example.json'))

for (const file of files) {
const filePath = path.join(folderPath, file)
const fileContent = await readFile(filePath, 'utf8')

// Validate the file content
const jsonData = JSON.parse(fileContent)
validatorSchema.safeParse(jsonData)

// Check if the address in the title matches the address in the body
const fileNameAddress = path.basename(file, '.json')
if (jsonData.address !== fileNameAddress)
throw new Error(`Address mismatch in file: ${file}`)

await storeValidator(jsonData.address, jsonData, { force: true })
}
}

0 comments on commit 6c77219

Please sign in to comment.