diff --git a/lib/handler.js b/lib/handler.js index b2cf5ad..c077d70 100644 --- a/lib/handler.js +++ b/lib/handler.js @@ -1,3 +1,6 @@ +import qs from 'node:querystring' +import assert from 'http-assert' +import { json } from 'http-responders' import Sentry from '@sentry/node' /** @@ -28,8 +31,28 @@ export const createHandler = ({ * @param {import('pg').Pool} pgPool */ const handler = async (req, res, pgPool) => { - // TBD - notFound(res) + const [pathname, search] = parseRequestUrl(req.url) + const segs = pathname.split('/').filter(Boolean) + if (req.method === 'GET' && segs[0] === 'retrieval-success-rate' && segs.length === 1) { + await getRetrievalSuccessRate(pathname, search, res, pgPool) + } else { + notFound(res) + } +} + +/** + * @param {string} url + * @returns {[string, string]} [pathname, search] + */ +const parseRequestUrl = (url) => { + // Split the url in the format "/path?query" into two parts + // We need to take into account that query can contain '?' characters + const ix = url.indexOf('?') + if (ix < 0) return [url, ''] + return [ + url.slice(0, ix), + url.slice(ix + 1) // +1 to skip '?' + ] } const errorHandler = (res, err, logger) => { @@ -54,3 +77,105 @@ const notFound = (res) => { res.statusCode = 404 res.end('Not Found') } + +/** + * @param {string} pathname + * @param {string} search + * @param {import('node:http').ServerResponse} res + * @param {string} querystring + * @param {import('pg').Pool} pgPool + */ +const getRetrievalSuccessRate = async (pathname, search, res, pgPool) => { + const filter = parseAndValidateFilter(pathname, search, res) + if (res.headersSent) return + + setCacheControlForStatsResponse(res, filter) + + // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres for converting it into + // a JavaScript Date with a timezone, as that could change the date one day forward or back. + const { rows } = await pgPool.query( + 'SELECT day::text, total, successful FROM retrieval_stats WHERE day >= $1 AND day <= $2', + [filter.from, filter.to] + ) + const stats = rows.map(r => ({ + day: r.day, + success_rate: r.total > 0 ? r.successful / r.total : null + })) + json(res, stats) +} + +export const today = () => new Date().toISOString().split('T')[0] + +/** + * @param {string} pathname + * @param {string} search + * @param {import('node:http').ServerResponse} res + * @returns {{from: string | undefined; to: string | undefined}} + */ +export const parseAndValidateFilter = (pathname, search, res) => { + let { from, to } = qs.parse(search) + let shouldRedirect = false + + // Provide default values for "from" and "to" when not specified + + if (!to) { + to = today() + shouldRedirect = true + } + if (!from) { + from = to + shouldRedirect = true + } + if (shouldRedirect) { + res.setHeader('cache-control', `public, max-age=${600 /* 10min */}`) + res.setHeader('location', `${pathname}?${qs.stringify({ from, to })}`) + res.writeHead(302) // Found + res.end() + return { from, to } + } + + // Trim time from date-time values that are typically provided by Grafana + + const matchFrom = from.match(/^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}\.\d{3}Z)?$/) + assert(matchFrom, 400, '"from" must have format YYYY-MM-DD or YYYY-MM-DDThh:mm:ss.sssZ') + if (matchFrom[2]) { + from = matchFrom[1] + shouldRedirect = true + } + + const matchTo = to.match(/^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}\.\d{3}Z)?$/) + assert(matchTo, 400, '"to" must have format YYYY-MM-DD or YYYY-MM-DDThh:mm:ss.sssZ') + if (matchTo[2]) { + to = matchTo[1] + shouldRedirect = true + } + + if (shouldRedirect) { + res.setHeader('cache-control', `public, max-age=${24 * 3600 /* one day */}`) + res.setHeader('location', `${pathname}?${qs.stringify({ from, to })}`) + res.writeHead(301) // Found + res.end() + return { from, to } + } + + // We have well-formed from & to dates now + return { from, to } +} + +/** + * @param {import('node:http').ServerResponse} res + * @param {{from: string, to: string}} filter + */ +const setCacheControlForStatsResponse = (res, filter) => { + // We cannot simply compare filter.to vs today() because there may be a delay in finalizing + // stats for the previous day. Let's allow up to one hour for the finalization. + const boundary = new Date(Date.now() - 3600_000).toISOString().split('T')[0] + + if (filter.to >= boundary) { + // response includes partial data for today, cache it for 10 minutes only + res.setHeader('cache-control', 'public, max-age=600') + } else { + // historical data should never change, cache it for one year + res.setHeader('cache-control', `public, max-age=${365 * 24 * 3600}, immutable`) + } +} diff --git a/package-lock.json b/package-lock.json index 6cf3ffc..79ee003 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,8 @@ "dependencies": { "@sentry/node": "^7.93.0", "debug": "^4.3.4", + "http-assert": "^1.5.0", + "http-responders": "^2.0.2", "pg": "^8.11.3" }, "devDependencies": { @@ -745,6 +747,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/deep-equal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", + "integrity": "sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==" + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -782,6 +789,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/diff": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", @@ -1939,6 +1954,38 @@ "he": "bin/he" } }, + "node_modules/http-assert": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/http-assert/-/http-assert-1.5.0.tgz", + "integrity": "sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==", + "dependencies": { + "deep-equal": "~1.0.1", + "http-errors": "~1.8.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/http-responders": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/http-responders/-/http-responders-2.0.2.tgz", + "integrity": "sha512-3Q0cXn81VLyjVOcKmw5RhZHTl7gioVcv54T9lNc/UFx2uraI+chNX/JLRUhBNgE29o3tqfTtAqo306zNq4F2Lw==" + }, "node_modules/https-proxy-agent": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", @@ -1998,8 +2045,7 @@ "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "node_modules/internal-slot": { "version": "1.0.6", @@ -3448,6 +3494,11 @@ "node": ">= 0.4" } }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -3557,6 +3608,14 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -3714,6 +3773,14 @@ "node": ">=8.0" } }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "engines": { + "node": ">=0.6" + } + }, "node_modules/tsconfig-paths": { "version": "3.15.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz", diff --git a/package.json b/package.json index f7fd9ce..0010878 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,8 @@ "dependencies": { "@sentry/node": "^7.93.0", "debug": "^4.3.4", + "http-assert": "^1.5.0", + "http-responders": "^2.0.2", "pg": "^8.11.3" }, "standard": { diff --git a/test/handler.test.js b/test/handler.test.js index 152414c..7488328 100644 --- a/test/handler.test.js +++ b/test/handler.test.js @@ -1,10 +1,10 @@ import http from 'node:http' import { once } from 'node:events' -import { AssertionError } from 'node:assert' +import assert, { AssertionError } from 'node:assert' import pg from 'pg' import createDebug from 'debug' -import { createHandler } from '../lib/handler.js' +import { createHandler, today } from '../lib/handler.js' import { DATABASE_URL } from '../lib/config.js' const debug = createDebug('test') @@ -29,7 +29,6 @@ describe('HTTP request handler', () => { } }) - // server = http.createServer((req, res) => { console.log(req.method, req.url); res.end('hello') }) server = http.createServer(handler) server.listen() await once(server, 'listening') @@ -46,6 +45,83 @@ describe('HTTP request handler', () => { const res = await fetch(new URL('/unknown-path', baseUrl)) assertResponseStatus(res, 404) }) + + describe('GET /retrieval-success-rate', () => { + beforeEach(async () => { + await pgPool.query('DELETE FROM retrieval_stats') + }) + + it('returns today stats for no query string', async () => { + const day = today() + await givenRetrievalStats(pgPool, { day, total: 10, successful: 1 }) + const res = await fetch(new URL('/retrieval-success-rate', baseUrl), { redirect: 'follow' }) + await assertResponseStatus(res, 200) + const stats = await res.json() + assert.deepStrictEqual(stats, [ + { day, success_rate: 0.1 } + ]) + }) + + it('applies from & to in YYYY-MM-DD format', async () => { + await givenRetrievalStats(pgPool, { day: '2024-01-10', total: 10, successful: 1 }) + await givenRetrievalStats(pgPool, { day: '2024-01-11', total: 20, successful: 1 }) + await givenRetrievalStats(pgPool, { day: '2024-01-12', total: 30, successful: 3 }) + await givenRetrievalStats(pgPool, { day: '2024-01-13', total: 40, successful: 1 }) + + const res = await fetch( + new URL( + '/retrieval-success-rate?from=2024-01-11&to=2024-01-12', + baseUrl + ), { + redirect: 'manual' + } + ) + await assertResponseStatus(res, 200) + const stats = await res.json() + assert.deepStrictEqual(stats, [ + { day: '2024-01-11', success_rate: 0.05 }, + { day: '2024-01-12', success_rate: 0.1 } + ]) + }) + + it('redirects when from & to is in YYYY-MM-DDThh:mm:ss.sssZ format', async () => { + const res = await fetch( + new URL( + '/retrieval-success-rate?from=2024-01-10T13:44:44.289Z&to=2024-01-15T09:44:44.289Z', + baseUrl + ), { + redirect: 'manual' + } + ) + await assertResponseStatus(res, 301) + assert.strictEqual( + res.headers.get('location'), + '/retrieval-success-rate?from=2024-01-10&to=2024-01-15' + ) + }) + + it('caches data including today for short time', async () => { + const res = await fetch( + new URL(`/retrieval-success-rate?from=2024-01-01&to=${today()}`, baseUrl), + { + redirect: 'manual' + } + ) + await assertResponseStatus(res, 200) + assert.strictEqual(res.headers.get('cache-control'), 'public, max-age=600') + }) + + it('caches historical including for long time & marks them immutable', async () => { + const res = await fetch( + new URL('/retrieval-success-rate?from=2023-01-01&to=2023-12-31', baseUrl), + { + redirect: 'manual' + } + ) + await assertResponseStatus(res, 200) + assert.strictEqual(res.headers.get('cache-control'), 'public, max-age=31536000, immutable') + }) + }) }) const assertResponseStatus = async (res, status) => { @@ -57,3 +133,10 @@ const assertResponseStatus = async (res, status) => { }) } } + +const givenRetrievalStats = async (pgPool, { day, total, successful }) => { + await pgPool.query( + 'INSERT INTO retrieval_stats (day, total, successful) VALUES ($1, $2, $3)', + [day, total, successful] + ) +}