From 0b0db9a4fb5143d50df6ddb537cb44fa47d3eb7f Mon Sep 17 00:00:00 2001 From: Robert Jansen Date: Sun, 5 Jan 2025 17:15:41 +0000 Subject: [PATCH] ratelimits for telemetry --- .env.example | 1 + package.json | 3 +- pnpm-lock.yaml | 67 ++++++++++++++++++++++++++++++ src/api/routes/global/telemetry.ts | 3 +- src/globals/cache.ts | 59 ++++++++++++++++++++++---- src/globals/env.ts | 14 ++++++- src/globals/telemetry.ts | 19 ++++++++- 7 files changed, 154 insertions(+), 12 deletions(-) diff --git a/.env.example b/.env.example index 61cb46c..8f3527f 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,4 @@ +REDIS_URL="redis://localhost" DATABASE_URL="postgresql://blueprint_api:local@db:5432/blueprint_api" UPDATE_PRICES=true diff --git a/package.json b/package.json index 1b1c66c..6e02ffc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "api", - "version": "1.4.1", + "version": "1.4.2", "scripts": { "build": "rm -rf lib && esbuild `find src \\( -name '*.ts' -o -name '*.tsx' \\)` --platform='node' --sourcemap --ignore-annotations --format='cjs' --target='es2022' --outdir='lib' && esbuild src/index.ts --platform='node' --sourcemap --ignore-annotations --format='cjs' --target='es2022' --outdir='lib' --banner:js='require(\"module-alias\").addAlias(\"@\", __dirname);'", "kit": "drizzle-kit", @@ -18,6 +18,7 @@ "ansi-colors": "^4.1.3", "drizzle-kit": "^0.28.1", "drizzle-orm": "^0.36.4", + "ioredis": "^5.4.2", "module-alias": "^2.2.3", "node-cron": "^3.0.3", "pg": "^8.13.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 32d529e..241f35e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,6 +29,9 @@ importers: drizzle-orm: specifier: ^0.36.4 version: 0.36.4(@opentelemetry/api@1.9.0)(@types/pg@8.11.10)(pg@8.13.1) + ioredis: + specifier: ^5.4.2 + version: 5.4.2 module-alias: specifier: ^2.2.3 version: 2.2.3 @@ -498,6 +501,9 @@ packages: resolution: {integrity: sha512-BXvGj0ehzrngHTPTDqUoDT3NXL8U0RxUk2zJm2A66RhCEIWdtU1v6GuUqNAgArW4PQ9CinqIWyHdQgdwOj06zQ==} engines: {node: '>=18'} + '@ioredis/commands@1.2.0': + resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==} + '@opentelemetry/api-logs@0.53.0': resolution: {integrity: sha512-8HArjKx+RaAI8uEIgcORbZIPklyh1YLjPSBus8hjRmvLi6DeFzgOcdZ7KwPabKj8mXF8dX0hyfAyGfycz0DbFw==} engines: {node: '>=14'} @@ -844,6 +850,10 @@ packages: resolution: {integrity: sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==} engines: {node: '>=0.8'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -867,6 +877,10 @@ packages: defaults@1.0.4: resolution: {integrity: sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==} + denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + drizzle-kit@0.28.1: resolution: {integrity: sha512-JimOV+ystXTWMgZkLHYHf2w3oS28hxiH1FR0dkmJLc7GHzdGJoJAQtQS5DRppnabsRZwE2U1F6CuezVBgmsBBQ==} hasBin: true @@ -1032,6 +1046,10 @@ packages: resolution: {integrity: sha512-LJKFHCSeIRq9hanN14IlOtPSTe3lNES7TYDTE2xxdAy1LS5rYphajK1qtwvj3YmQXvvk0U2Vbmcni8P9EIQW9w==} engines: {node: '>=18'} + ioredis@5.4.2: + resolution: {integrity: sha512-0SZXGNGZ+WzISQ67QDyZ2x0+wVxjjUndtD8oSeik/4ajifeiRufed8fCb8QW8VMyi4MXcS+UO1k/0NGhvq1PAg==} + engines: {node: '>=12.22.0'} + is-core-module@2.16.1: resolution: {integrity: sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==} engines: {node: '>= 0.4'} @@ -1048,6 +1066,12 @@ packages: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} + lodash.defaults@4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + + lodash.isarguments@3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + log-symbols@4.1.0: resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==} engines: {node: '>=10'} @@ -1179,6 +1203,14 @@ packages: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} @@ -1239,6 +1271,9 @@ packages: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -1564,6 +1599,8 @@ snapshots: '@inquirer/figures@1.0.9': {} + '@ioredis/commands@1.2.0': {} + '@opentelemetry/api-logs@0.53.0': dependencies: '@opentelemetry/api': 1.9.0 @@ -2020,6 +2057,8 @@ snapshots: clone@1.0.4: {} + cluster-key-slot@1.1.2: {} + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -2038,6 +2077,8 @@ snapshots: dependencies: clone: 1.0.4 + denque@2.1.0: {} + drizzle-kit@0.28.1: dependencies: '@drizzle-team/brocli': 0.10.2 @@ -2195,6 +2236,20 @@ snapshots: wrap-ansi: 6.2.0 yoctocolors-cjs: 2.1.2 + ioredis@5.4.2: + dependencies: + '@ioredis/commands': 1.2.0 + cluster-key-slot: 1.1.2 + debug: 4.4.0 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + is-core-module@2.16.1: dependencies: hasown: 2.0.2 @@ -2205,6 +2260,10 @@ snapshots: is-unicode-supported@0.1.0: {} + lodash.defaults@4.2.0: {} + + lodash.isarguments@3.1.0: {} + log-symbols@4.1.0: dependencies: chalk: 4.1.2 @@ -2327,6 +2386,12 @@ snapshots: string_decoder: 1.3.0 util-deprecate: 1.0.2 + redis-errors@1.2.0: {} + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + require-directory@2.1.1: {} require-in-the-middle@7.4.0: @@ -2384,6 +2449,8 @@ snapshots: split2@4.2.0: {} + standard-as-callback@2.1.0: {} + string-width@4.2.3: dependencies: emoji-regex: 8.0.0 diff --git a/src/api/routes/global/telemetry.ts b/src/api/routes/global/telemetry.ts index a263ab3..7d641e1 100644 --- a/src/api/routes/global/telemetry.ts +++ b/src/api/routes/global/telemetry.ts @@ -6,7 +6,8 @@ export = new globalAPIRouter.Path('/') const [ data, error ] = await ctr.bindBody(ctr["@"].telemetry.telemetrySchema) if (!data) return ctr.status(ctr.$status.BAD_REQUEST).print({ errors: error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`) }) - ctr["@"].telemetry.log(ctr.client.ip, data) + const telemetry = await ctr["@"].telemetry.log(ctr.client.ip, data, ctr.headers) + if (!telemetry) return ctr.status(ctr.$status.TOO_MANY_REQUESTS).print({ errors: ['You are making too many requests! Slow down.'] }) return ctr.print({}) }) diff --git a/src/globals/cache.ts b/src/globals/cache.ts index 8d87cb3..4b0ad06 100644 --- a/src/globals/cache.ts +++ b/src/globals/cache.ts @@ -1,19 +1,62 @@ +import { Redis } from "ioredis" +import { version } from "ioredis/package.json" +import logger from "@/globals/logger" +import env from "@/globals/env" import { time } from "@rjweb/utils" -const localCache = new Map() +const startTime = performance.now(), + localCache = new Map() -export default { +const redis = env.REDIS_MODE === 'redis' + ? new Redis(env.REDIS_URL) + : new Redis({ + sentinels: env.REDIS_SENTINEL_NODES.map(([host, port]) => ({ host, port })), + name: 'mymaster' + }) + +redis.once('connect', () => { + logger() + .text('Cache', (c) => c.yellow) + .text(`(${version}) Connection established!`) + .text(`(${(performance.now() - startTime).toFixed(1)}ms)`, (c) => c.gray) + .info() +}) + +export default Object.assign(redis, { async use Promise | any>(key: string, run: Run, expire: number = time(3).s()): Promise>> { const mapResult = localCache.get(`internal-middlewares::cache::${key}`) if (mapResult) return mapResult + const redisResult = await redis.get(`internal-middlewares::cache::${key}`) + if (redisResult) return JSON.parse(redisResult) + const runResult = await Promise.resolve(run()) - localCache.set(`internal-middlewares::cache::${key}`, runResult) + if (!expire) await redis.set(`internal-middlewares::cache::${key}`, JSON.stringify(runResult)) + else if (expire > time(15).s()) await redis.set(`internal-middlewares::cache::${key}`, JSON.stringify(runResult), 'EX', Math.ceil(expire / 1000)) + else { + localCache.set(`internal-middlewares::cache::${key}`, runResult) - setTimeout(() => { - localCache.delete(`internal-middlewares::cache::${key}`) - }, expire) + setTimeout(() => { + localCache.delete(`internal-middlewares::cache::${key}`) + }, expire) + } return runResult - } -} \ No newline at end of file + }, + + local: { + use any>(key: string, run: Run, expire: number = time(3).s()): ReturnType { + const mapResult = localCache.get(`internal-middlewares::cache::${key}`) + if (mapResult) return mapResult + + const runResult = run() + localCache.set(`internal-middlewares::cache::${key}`, runResult) + + if (!!expire) setTimeout(() => { + localCache.delete(`internal-middlewares::cache::${key}`) + }, expire) + + return runResult + } + } as const +}) \ No newline at end of file diff --git a/src/globals/env.ts b/src/globals/env.ts index 93ae4ac..f2e0c9b 100644 --- a/src/globals/env.ts +++ b/src/globals/env.ts @@ -12,7 +12,7 @@ try { } } -const infos = z.object({ +const base = z.object({ DATABASE_URL: z.string(), DATABASE_URL_PRIMARY: z.string().optional(), SENTRY_URL: z.string().optional(), @@ -20,6 +20,7 @@ const infos = z.object({ UPDATE_PRICES: z.enum(['true', 'false']).transform((str) => str === 'true'), PORT: z.string().transform((str) => parseInt(str)).optional(), + RATELIMIT_PER_DAY: z.string().transform((str) => parseInt(str)).optional().default('2'), INTERNAL_KEY: z.string(), SXC_TOKEN: z.string().optional(), @@ -32,6 +33,17 @@ const infos = z.object({ SERVER_NAME: z.string().optional() }) +const infos = z.union([ + z.object({ + REDIS_MODE: z.literal('redis').default('redis'), + REDIS_URL: z.string() + }).merge(base), + z.object({ + REDIS_MODE: z.literal('sentinel'), + REDIS_SENTINEL_NODES: z.string().transform((str) => str.split(',').map((node) => node.trim().split(':').map((part, i) => i === 1 ? parseInt(part) : part)) as [string, number][]), + }).merge(base) +]) + export type Environment = z.infer export default infos.parse(env) \ No newline at end of file diff --git a/src/globals/telemetry.ts b/src/globals/telemetry.ts index d973198..0f2a608 100644 --- a/src/globals/telemetry.ts +++ b/src/globals/telemetry.ts @@ -4,7 +4,9 @@ import { network, object, string, time } from "@rjweb/utils" import * as schema from "@/schema" import { lookup } from "@/globals/ip" import cache from "@/globals/cache" +import env from "@/globals/env" import { z } from "zod" +import { Content, ValueCollection } from "rjweb-server" export const telemetrySchema = z.object({ id: z.string().uuid(), @@ -63,7 +65,22 @@ const processing: Telemetry[] = [] /** * Log a new Telemetry * @since 1.0.0 -*/ export function log(ip: network.IPAddress, telemetry: z.infer): Telemetry { +*/ export async function log(ip: network.IPAddress, telemetry: z.infer, headers: ValueCollection): Promise { + let ratelimitKey = 'ratelimit::' + if (ip['type'] === 4) ratelimitKey += ip.long() + else ratelimitKey += ip.rawData.slice(0, 4).join(':') + + const count = await cache.incr(ratelimitKey) + if (count === 1) await cache.expire(ratelimitKey, Math.floor(time(1).d() / 1000)) + + const expires = await cache.ttl(ratelimitKey) + + headers.set('X-RateLimit-Limit', env.RATELIMIT_PER_DAY) + headers.set('X-RateLimit-Remaining', env.RATELIMIT_PER_DAY - count) + headers.set('X-RateLimit-Reset', expires) + + if (count > env.RATELIMIT_PER_DAY) return null + const data: Telemetry = { panelId: telemetry.id, telemetryVersion: telemetry.telemetry_version,