From 6a98e09a4bc9b597eb5c106eab1827d4ef3c3ea5 Mon Sep 17 00:00:00 2001 From: Steven Tey Date: Sun, 23 Feb 2025 18:27:48 -0800 Subject: [PATCH 1/3] improve /track/click and /track/visit --- apps/web/app/api/track/click/route.ts | 43 ++++----- apps/web/app/api/track/visit/route.ts | 101 +++++++++++++++++++++ apps/web/lib/middleware/link.ts | 28 ++++-- apps/web/lib/rewardful/import-referrals.ts | 1 - apps/web/lib/tinybird/record-click.ts | 17 ++-- 5 files changed, 149 insertions(+), 41 deletions(-) create mode 100644 apps/web/app/api/track/visit/route.ts diff --git a/apps/web/app/api/track/click/route.ts b/apps/web/app/api/track/click/route.ts index 015d49fead..e1d3b08147 100644 --- a/apps/web/app/api/track/click/route.ts +++ b/apps/web/app/api/track/click/route.ts @@ -3,7 +3,7 @@ import { DubApiError, handleAndReturnErrorResponse } from "@/lib/api/errors"; import { parseRequestBody } from "@/lib/api/utils"; import { getLinkWithAllowedHostnames } from "@/lib/planetscale/get-link-with-allowed-hostnames"; import { recordClick } from "@/lib/tinybird"; -import { ratelimit, redis } from "@/lib/upstash"; +import { redis } from "@/lib/upstash"; import { isValidUrl, LOCALHOST_IP, nanoid } from "@dub/utils"; import { ipAddress, waitUntil } from "@vercel/functions"; import { AxiomRequest, withAxiom } from "next-axiom"; @@ -21,7 +21,7 @@ const CORS_HEADERS = { export const POST = withAxiom( async (req: AxiomRequest) => { try { - const { domain, key, url } = await parseRequestBody(req); + const { domain, key, url, referrer } = await parseRequestBody(req); if (!domain || !key) { throw new DubApiError({ @@ -32,34 +32,25 @@ export const POST = withAxiom( const ip = process.env.VERCEL === "1" ? ipAddress(req) : LOCALHOST_IP; - const { success } = await ratelimit().limit( - `track-click:${domain}-${key}:${ip}`, - ); - - if (!success) { - throw new DubApiError({ - code: "rate_limit_exceeded", - message: "Don't DDoS me pls 🥺", - }); - } + const cacheKey = `recordClick:${domain}:${key}:${ip}`; + let clickId = await redis.get(cacheKey); - const link = await getLinkWithAllowedHostnames(domain, key); + // only generate + record a new click ID if it's not already cached in Redis + if (!clickId) { + clickId = nanoid(16); - if (!link) { - throw new DubApiError({ - code: "not_found", - message: `Link not found for domain: ${domain} and key: ${key}.`, - }); - } + const link = await getLinkWithAllowedHostnames(domain, key); - const allowedHostnames = link.allowedHostnames; - verifyAnalyticsAllowedHostnames({ allowedHostnames, req }); + if (!link) { + throw new DubApiError({ + code: "not_found", + message: `Link not found for domain: ${domain} and key: ${key}.`, + }); + } - const cacheKey = `recordClick:${link.id}:${ip}`; - let clickId = await redis.get(cacheKey); + const allowedHostnames = link.allowedHostnames; + verifyAnalyticsAllowedHostnames({ allowedHostnames, req }); - if (!clickId) { - clickId = nanoid(16); const finalUrl = isValidUrl(url) ? url : link.url; waitUntil( @@ -68,8 +59,8 @@ export const POST = withAxiom( clickId, linkId: link.id, url: finalUrl, - skipRatelimit: true, workspaceId: link.projectId, + ...(referrer && { referrer }), }), ); } diff --git a/apps/web/app/api/track/visit/route.ts b/apps/web/app/api/track/visit/route.ts new file mode 100644 index 0000000000..0b64075adc --- /dev/null +++ b/apps/web/app/api/track/visit/route.ts @@ -0,0 +1,101 @@ +import { verifyAnalyticsAllowedHostnames } from "@/lib/analytics/verify-analytics-allowed-hostnames"; +import { DubApiError, handleAndReturnErrorResponse } from "@/lib/api/errors"; +import { parseRequestBody } from "@/lib/api/utils"; +import { getLinkWithAllowedHostnames } from "@/lib/planetscale/get-link-with-allowed-hostnames"; +import { recordClick } from "@/lib/tinybird"; +import { redis } from "@/lib/upstash"; +import { isValidUrl, LOCALHOST_IP, nanoid } from "@dub/utils"; +import { ipAddress, waitUntil } from "@vercel/functions"; +import { AxiomRequest, withAxiom } from "next-axiom"; +import { NextResponse } from "next/server"; + +export const runtime = "edge"; + +const CORS_HEADERS = { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type, Authorization", +}; + +// POST /api/track/visit – Track a visit event from the client-side +export const POST = withAxiom( + async (req: AxiomRequest) => { + try { + const { domain, url, referrer } = await parseRequestBody(req); + + if (!domain || !url) { + throw new DubApiError({ + code: "bad_request", + message: "Missing domain or url", + }); + } + + const urlObj = new URL(url); + + let linkKey = urlObj.pathname.slice(1); + if (linkKey === "") { + linkKey = "_root"; + } + + const ip = process.env.VERCEL === "1" ? ipAddress(req) : LOCALHOST_IP; + const cacheKey = `recordClick:${domain}:${linkKey}:${ip}`; + + let clickId = await redis.get(cacheKey); + + // only generate + record a new click ID if it's not already cached in Redis + if (!clickId) { + clickId = nanoid(16); + + let link = await getLinkWithAllowedHostnames(domain, linkKey); + + if (!link) { + return NextResponse.json( + { + clickId: null, + }, + { + headers: CORS_HEADERS, + }, + ); + } + + const allowedHostnames = link.allowedHostnames; + verifyAnalyticsAllowedHostnames({ allowedHostnames, req }); + + const finalUrl = isValidUrl(url) ? url : link.url; + + waitUntil( + recordClick({ + req, + clickId, + linkId: link.id, + url: finalUrl, + workspaceId: link.projectId, + ...(referrer && { referrer }), + }), + ); + } + + return NextResponse.json( + { + clickId, + }, + { + headers: CORS_HEADERS, + }, + ); + } catch (error) { + return handleAndReturnErrorResponse(error, CORS_HEADERS); + } + }, + { + logRequestDetails: ["body", "nextUrl"], + }, +); + +export const OPTIONS = () => { + return new Response(null, { + status: 204, + headers: CORS_HEADERS, + }); +}; diff --git a/apps/web/lib/middleware/link.ts b/apps/web/lib/middleware/link.ts index bc384fb1ea..a102091ba5 100644 --- a/apps/web/lib/middleware/link.ts +++ b/apps/web/lib/middleware/link.ts @@ -196,8 +196,10 @@ export default async function LinkMiddleware( ev.waitUntil( recordClick({ req, - linkId, clickId, + linkId, + domain, + key, url, webhookIds, workspaceId, @@ -241,8 +243,10 @@ export default async function LinkMiddleware( ev.waitUntil( recordClick({ req, - linkId, clickId, + linkId, + domain, + key, url, webhookIds, workspaceId, @@ -275,8 +279,10 @@ export default async function LinkMiddleware( ev.waitUntil( recordClick({ req, - linkId, clickId, + linkId, + domain, + key, url, webhookIds, workspaceId, @@ -311,8 +317,10 @@ export default async function LinkMiddleware( ev.waitUntil( recordClick({ req, - linkId, clickId, + linkId, + domain, + key, url: ios, webhookIds, workspaceId, @@ -341,8 +349,10 @@ export default async function LinkMiddleware( ev.waitUntil( recordClick({ req, - linkId, clickId, + linkId, + domain, + key, url: android, webhookIds, workspaceId, @@ -371,8 +381,10 @@ export default async function LinkMiddleware( ev.waitUntil( recordClick({ req, - linkId, clickId, + linkId, + domain, + key, url: geo[country], webhookIds, workspaceId, @@ -401,8 +413,10 @@ export default async function LinkMiddleware( ev.waitUntil( recordClick({ req, - linkId, clickId, + linkId, + domain, + key, url, webhookIds, workspaceId, diff --git a/apps/web/lib/rewardful/import-referrals.ts b/apps/web/lib/rewardful/import-referrals.ts index e8416a49d0..6bd2ebc226 100644 --- a/apps/web/lib/rewardful/import-referrals.ts +++ b/apps/web/lib/rewardful/import-referrals.ts @@ -172,7 +172,6 @@ async function createReferral({ url: link.url, workspaceId: workspace.id, timestamp: new Date(referral.created_at).toISOString(), - skipRatelimit: true, }); const clickEvent = clickEventSchemaTB.parse({ diff --git a/apps/web/lib/tinybird/record-click.ts b/apps/web/lib/tinybird/record-click.ts index 7833e4816f..d0ed96c4ac 100644 --- a/apps/web/lib/tinybird/record-click.ts +++ b/apps/web/lib/tinybird/record-click.ts @@ -26,21 +26,23 @@ import { transformClickEventData } from "../webhook/transform"; **/ export async function recordClick({ req, - linkId, clickId, + linkId, + domain, + key, url, webhookIds, - skipRatelimit, workspaceId, timestamp, referrer, }: { req: Request; - linkId: string; clickId: string; + linkId: string; + domain?: string; + key?: string; url?: string; webhookIds?: string[]; - skipRatelimit?: boolean; workspaceId: string | undefined; timestamp?: string; referrer?: string; @@ -61,10 +63,11 @@ export async function recordClick({ const ip = process.env.VERCEL === "1" ? ipAddress(req) : LOCALHOST_IP; - const cacheKey = `recordClick:${linkId}:${ip}`; + const cacheKey = `recordClick:${domain}:${key}:${ip}`; - if (!skipRatelimit) { - // by default, we deduplicate clicks from the same IP address + link ID – only record 1 click per hour + // by default, we deduplicate clicks for a domain + key pair from the same IP address – only record 1 click per hour + // we only need to do these if domain + key are defined (only in middleware/link and not /api/track/:path endpoints) + if (domain && key) { // here, we check if the clickId is cached in Redis within the last hour const cachedClickId = await redis.get(cacheKey); if (cachedClickId) { From d2b8214c360a13a3179008f8fff29853dd65dd5b Mon Sep 17 00:00:00 2001 From: Steven Tey Date: Sun, 23 Feb 2025 19:27:19 -0800 Subject: [PATCH 2/3] add back skipRatelimit --- apps/web/app/api/track/click/route.ts | 3 +++ apps/web/app/api/track/visit/route.ts | 13 ++++++++----- apps/web/lib/tinybird/record-click.ts | 6 ++++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/apps/web/app/api/track/click/route.ts b/apps/web/app/api/track/click/route.ts index e1d3b08147..2f09158d9b 100644 --- a/apps/web/app/api/track/click/route.ts +++ b/apps/web/app/api/track/click/route.ts @@ -58,8 +58,11 @@ export const POST = withAxiom( req, clickId, linkId: link.id, + domain, + key, url: finalUrl, workspaceId: link.projectId, + skipRatelimit: true, ...(referrer && { referrer }), }), ); diff --git a/apps/web/app/api/track/visit/route.ts b/apps/web/app/api/track/visit/route.ts index 0b64075adc..4d751d06ef 100644 --- a/apps/web/app/api/track/visit/route.ts +++ b/apps/web/app/api/track/visit/route.ts @@ -32,13 +32,13 @@ export const POST = withAxiom( const urlObj = new URL(url); - let linkKey = urlObj.pathname.slice(1); - if (linkKey === "") { - linkKey = "_root"; + let key = urlObj.pathname.slice(1); + if (key === "") { + key = "_root"; } const ip = process.env.VERCEL === "1" ? ipAddress(req) : LOCALHOST_IP; - const cacheKey = `recordClick:${domain}:${linkKey}:${ip}`; + const cacheKey = `recordClick:${domain}:${key}:${ip}`; let clickId = await redis.get(cacheKey); @@ -46,7 +46,7 @@ export const POST = withAxiom( if (!clickId) { clickId = nanoid(16); - let link = await getLinkWithAllowedHostnames(domain, linkKey); + let link = await getLinkWithAllowedHostnames(domain, key); if (!link) { return NextResponse.json( @@ -69,8 +69,11 @@ export const POST = withAxiom( req, clickId, linkId: link.id, + domain, + key, url: finalUrl, workspaceId: link.projectId, + skipRatelimit: true, ...(referrer && { referrer }), }), ); diff --git a/apps/web/lib/tinybird/record-click.ts b/apps/web/lib/tinybird/record-click.ts index d0ed96c4ac..b6cc34e726 100644 --- a/apps/web/lib/tinybird/record-click.ts +++ b/apps/web/lib/tinybird/record-click.ts @@ -33,6 +33,7 @@ export async function recordClick({ url, webhookIds, workspaceId, + skipRatelimit, timestamp, referrer, }: { @@ -44,6 +45,7 @@ export async function recordClick({ url?: string; webhookIds?: string[]; workspaceId: string | undefined; + skipRatelimit?: boolean; timestamp?: string; referrer?: string; }) { @@ -66,8 +68,8 @@ export async function recordClick({ const cacheKey = `recordClick:${domain}:${key}:${ip}`; // by default, we deduplicate clicks for a domain + key pair from the same IP address – only record 1 click per hour - // we only need to do these if domain + key are defined (only in middleware/link and not /api/track/:path endpoints) - if (domain && key) { + // we only need to do these if skipRatelimit is not true (we skip it in /api/track/:path endpoints) + if (!skipRatelimit) { // here, we check if the clickId is cached in Redis within the last hour const cachedClickId = await redis.get(cacheKey); if (cachedClickId) { From 64fe3dee93238dfbad89432e4ee9a14a506300b3 Mon Sep 17 00:00:00 2001 From: Steven Tey Date: Sun, 23 Feb 2025 19:33:42 -0800 Subject: [PATCH 3/3] make domain and key mandatory --- apps/web/lib/rewardful/import-referrals.ts | 3 +++ apps/web/lib/tinybird/record-click.ts | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/web/lib/rewardful/import-referrals.ts b/apps/web/lib/rewardful/import-referrals.ts index 6bd2ebc226..b3948001dc 100644 --- a/apps/web/lib/rewardful/import-referrals.ts +++ b/apps/web/lib/rewardful/import-referrals.ts @@ -170,7 +170,10 @@ async function createReferral({ linkId: link.id, clickId: nanoid(16), url: link.url, + domain: link.domain, + key: link.key, workspaceId: workspace.id, + skipRatelimit: true, timestamp: new Date(referral.created_at).toISOString(), }); diff --git a/apps/web/lib/tinybird/record-click.ts b/apps/web/lib/tinybird/record-click.ts index b6cc34e726..ed9d14b40d 100644 --- a/apps/web/lib/tinybird/record-click.ts +++ b/apps/web/lib/tinybird/record-click.ts @@ -40,8 +40,8 @@ export async function recordClick({ req: Request; clickId: string; linkId: string; - domain?: string; - key?: string; + domain: string; + key: string; url?: string; webhookIds?: string[]; workspaceId: string | undefined;