Improve Prometheus Metrics

- Added `antelope_token_api` prefix
- More descriptive names and description for all metrics
- Added configurable large queries counter monitoring

index.ts

@@ -5,8 +5,8 @@ import { SafeParseSuccess, z } from 'zod';
 
 import client from './src/clickhouse/client.js';
 import openapi from "./static/@typespec/openapi3/openapi.json";
-import * as prometheus from './src/prometheus.js';
 import { APP_VERSION, config } from "./src/config.js";
+import * as prometheus from './src/prometheus.js';
 import { logger } from './src/logger.js';
 import { makeUsageQuery } from "./src/usage.js";
 import { APIErrorResponse } from "./src/utils.js";
@@ -20,7 +20,7 @@ async function AntelopeTokenAPI() {
     app.use(async (ctx: Context, next) => {
         const pathname = ctx.req.path;
         logger.trace(`Incoming request: [${pathname}]`);
-        prometheus.request.inc({ pathname });
+        prometheus.requests.inc({ pathname });
 
         await next();
     });

src/clickhouse/makeQuery.ts

@@ -1,25 +1,40 @@
+import * as crypto from 'node:crypto'
 import client from "./client.js";
 
 import { logger } from "../logger.js";
 import * as prometheus from "../prometheus.js";
 
 import type { ResponseJSON } from "@clickhouse/client-web";
 import type { ValidQueryParams } from "../types/api.js";
+import { config } from "../config.js";
 
 export async function makeQuery<T = unknown>(query: string, query_params: ValidQueryParams) {
-    logger.trace({ query, query_params });
+    const query_id = crypto.randomUUID();
+    logger.trace({ query_id, query, query_params });
 
-    const response = await client.query({ query, query_params, format: "JSON" });
+    const response = await client.query({ query, query_params, format: "JSON", query_id });
     const data: ResponseJSON<T> = await response.json();
+    prometheus.queries.inc();
 
-    prometheus.query.inc();
-    if ( data.statistics ) {
+    if (response.query_id !== query_id) throw new Error(`Wrong query ID for query: sent ${query_id} / received ${response.query_id}`);
+
+    if (data.statistics) {
         prometheus.bytes_read.observe(data.statistics.bytes_read);
         prometheus.rows_read.observe(data.statistics.rows_read);
-        prometheus.elapsed.observe(data.statistics.elapsed);
+        prometheus.elapsed_seconds.observe(data.statistics.elapsed);
+
+        if (data.statistics.rows_read > config.maxRowsTrigger || data.statistics.bytes_read > config.maxBytesTrigger)
+            prometheus.large_queries.inc({
+                query_id,
+                query,
+                query_params: JSON.stringify(query_params),
+                bytes_read: data.statistics.bytes_read,
+                rows_read: data.statistics.rows_read,
+                elapsed_seconds: data.statistics.elapsed
+            });
     }
 
-    logger.trace({ statistics: data.statistics, rows: data.rows, rows_before_limit_at_least: data.rows_before_limit_at_least });
+    logger.trace({ query_id: response.query_id, statistics: data.statistics, rows: data.rows, rows_before_limit_at_least: data.rows_before_limit_at_least });
 
     return data;
 }

src/config.ts

@@ -11,6 +11,8 @@ export const DEFAULT_DATABASE = "default";
 export const DEFAULT_USERNAME = "default";
 export const DEFAULT_PASSWORD = "";
 export const DEFAULT_MAX_LIMIT = 10000;
+export const DEFAULT_LARGE_QUERIES_ROWS_TRIGGER = 10_000_000; // 10M rows
+export const DEFAULT_LARGE_QUERIES_BYTES_TRIGGER = 1_000_000_000; // 1Gb
 export const DEFAULT_IDLE_TIMEOUT = 60;
 export const DEFAULT_VERBOSE = false;
 export const DEFAULT_SORT_BY = "DESC";
@@ -33,6 +35,8 @@ const opts = program
     .addOption(new Option("--username <string>", "Database user").env("USERNAME").default(DEFAULT_USERNAME))
     .addOption(new Option("--password <string>", "Password associated with the specified username").env("PASSWORD").default(DEFAULT_PASSWORD))
     .addOption(new Option("--max-limit <number>", "Maximum LIMIT queries").env("MAX_LIMIT").default(DEFAULT_MAX_LIMIT))
+    .addOption(new Option("--max-rows-trigger <number>", "Queries returning rows above this treshold will be considered large queries for metrics").env("LARGE_QUERIES_ROWS_TRIGGER").default(DEFAULT_LARGE_QUERIES_ROWS_TRIGGER))
+    .addOption(new Option("--max-bytes-trigger <number>", "Queries processing bytes above this treshold will be considered large queries for metrics").env("LARGE_QUERIES_BYTES_TRIGGER").default(DEFAULT_LARGE_QUERIES_BYTES_TRIGGER))
     .addOption(new Option("--request-idle-timeout <number>", "Bun server request idle timeout (seconds)").env("BUN_IDLE_REQUEST_TIMEOUT").default(DEFAULT_IDLE_TIMEOUT))
     .addOption(new Option("-v, --verbose <boolean>", "Enable verbose logging").choices(["true", "false"]).env("VERBOSE").default(DEFAULT_VERBOSE))
     .parse()
@@ -46,6 +50,8 @@ export const config = z.object({
     username: z.string(),
     password: z.string(),
     maxLimit: z.coerce.number(),
+    maxRowsTrigger: z.coerce.number(),
+    maxBytesTrigger: z.coerce.number(),
     requestIdleTimeout: z.coerce.number(),
     // `z.coerce.boolean` doesn't parse boolean string values as expected (see https://github.com/colinhacks/zod/issues/1630)
     verbose: z.coerce.string().transform((val) => val.toLowerCase() === "true"),

src/prometheus.ts

@@ -1,12 +1,18 @@
 // From https://github.com/pinax-network/substreams-sink-websockets/blob/main/src/prometheus.ts
 import client, { Counter, CounterConfiguration, Gauge, GaugeConfiguration, Histogram, HistogramConfiguration } from 'prom-client';
 import { logger } from "./logger.js";
+import { config } from './config.js';
 
 export const registry = new client.Registry();
 
+function prefixMetric(name: string): string {
+    return `antelope_token_api_${name}`;
+}
+
 // Metrics
 export function registerCounter(name: string, help = "help", labelNames: string[] = [], config?: Partial<CounterConfiguration<string>>) {
     try {
+        name = prefixMetric(name);
         registry.registerMetric(new Counter({ name, help, labelNames, ...config }));
         logger.debug(`Registered new counter metric: ${name}`);
         return registry.getSingleMetric(name) as Counter;
@@ -18,6 +24,7 @@ export function registerCounter(name: string, help = "help", labelNames: string[
 
 export function registerGauge(name: string, help = "help", labelNames: string[] = [], config?: Partial<GaugeConfiguration<string>>) {
     try {
+        name = prefixMetric(name);
         registry.registerMetric(new Gauge({ name, help, labelNames, ...config }));
         logger.debug(`Registered new gauge metric: ${name}`);
         return registry.getSingleMetric(name) as Gauge;
@@ -29,6 +36,7 @@ export function registerGauge(name: string, help = "help", labelNames: string[]
 
 export function registerHistogram(name: string, help = "help", labelNames: string[] = [], config?: Partial<HistogramConfiguration<string>>) {
     try {
+        name = prefixMetric(name);
         registry.registerMetric(new Histogram({ name, help, labelNames, ...config }));
         logger.debug(`Registered new histogram metric: ${name}`);
         return registry.getSingleMetric(name) as Histogram;
@@ -38,12 +46,6 @@ export function registerHistogram(name: string, help = "help", labelNames: strin
     }
 }
 
-export async function getSingleMetric(name: string) {
-    const metric = registry.getSingleMetric(name);
-    const get = await metric?.get();
-    return get?.values[0]?.value;
-}
-
 function createBucket(lowExponentBound: number, highExponentBound: number): number[] {
     if (lowExponentBound > highExponentBound)
         return createBucket(highExponentBound, lowExponentBound);
@@ -63,20 +65,25 @@ function createBucket(lowExponentBound: number, highExponentBound: number): numb
 }
 
 // REST API metrics
-export const request_error = registerCounter('request_error', 'Total Requests errors', ['pathname', 'status']);
-export const request = registerCounter('request', 'Total Requests', ['pathname']);
-export const query = registerCounter('query', 'Clickhouse DB queries made');
-export const bytes_read = registerHistogram('bytes_read', 'Clickhouse DB Statistics bytes read', [],
-    { 
-        buckets: createBucket(3, 9) // 1Kb to 1Gb buckets, each divided by 10
+export const requests_errors = registerCounter('http_requests_errors_total', 'Total Requests errors by path and status', ['pathname', 'status']);
+export const requests = registerCounter('http_requests_total', 'Total Requests by path', ['pathname']);
+export const queries = registerCounter('ch_queries_total', 'Total ClickHouse DB queries made');
+export const large_queries = registerCounter('ch_large_queries', 'Large ClickHouse DB queries (>10M rows or >1GB read)',
+    ['query_id', 'query', 'query_params', 'bytes_read', 'rows_read', 'elapsed_seconds']
+);
+export const bytes_read = registerHistogram('ch_bytes_read', 'ClickHouse DB aggregated query statistics bytes read', [],
+    {
+        // Create buckets based on large queries trigger setting with a range of 10 values for 6 powers of 10.
+        buckets: createBucket(Math.floor(Math.log10(config.maxBytesTrigger)) - 6, Math.floor(Math.log10(config.maxBytesTrigger)))
     }
 );
-export const rows_read = registerHistogram('rows_read', 'Clickhouse DB Statistics rows read', [],
-    { 
-        buckets: createBucket(2, 7) // 100 to 10M, each divided by 10
+export const rows_read = registerHistogram('ch_rows_read', 'ClickHouse DB aggregated query statistics rows read', [],
+    {
+        // Create buckets based on large queries trigger setting with a range of 10 values for 5 powers of 10.
+        buckets: createBucket(Math.floor(Math.log10(config.maxRowsTrigger)) - 5, Math.floor(Math.log10(config.maxRowsTrigger)))
     }
 );
-export const elapsed = registerHistogram('elapsed', 'Clickhouse DB Statistics query elapsed time (seconds)', [],
+export const elapsed_seconds = registerHistogram('ch_elapsed_seconds', 'ClickHouse DB aggregated query statistics query elapsed time (seconds)', [],
     { 
         buckets: createBucket(-4, 1) // 0.1ms to 10s, each divided by 10
     }

src/usage.ts

@@ -97,7 +97,7 @@ export async function makeUsageQuery(ctx: Context, endpoint: UsageEndpoints, use
             `SELECT * FROM`
             + ` ((SELECT DISTINCT * FROM transfers_from WHERE (from = {account: String}))`
             + ` UNION ALL (SELECT DISTINCT * FROM transfers_to WHERE (to = {account: String})))`
-            + ` ${filters} ORDER BY block_num DESC`;
+            + ` ${filters}`;
     } else if (endpoint == "/transfers/id") {
         query += `SELECT * FROM transfer_events ${filters} ORDER BY action_index`;
     } else if (endpoint == "/tokens/holders") {

src/utils.ts

@@ -23,7 +23,7 @@ export function APIErrorResponse(ctx: Context, status: ApiErrorSchema["status"],
     };
 
     logger.error(api_error);
-    prometheus.request_error.inc({ pathname: ctx.req.path, status });
+    prometheus.requests_errors.inc({ pathname: ctx.req.path, status });
 
     return ctx.json<ApiErrorSchema, typeof status>(api_error, status);
 }