feat: GET /retrieval-success-rate

Signed-off-by: Miroslav Bajtoš <[email protected]>

lib/handler.js

@@ -1,3 +1,6 @@
+import qs from 'node:querystring'
+import assert from 'http-assert'
+import { json } from 'http-responders'
 import Sentry from '@sentry/node'
 
 /**
@@ -28,8 +31,28 @@ export const createHandler = ({
  * @param {import('pg').Pool} pgPool
  */
 const handler = async (req, res, pgPool) => {
-  // TBD
-  notFound(res)
+  const [pathname, search] = parseRequestUrl(req.url)
+  const segs = pathname.split('/').filter(Boolean)
+  if (req.method === 'GET' && segs[0] === 'retrieval-success-rate' && segs.length === 1) {
+    await getRetrievalSuccessRate(pathname, search, res, pgPool)
+  } else {
+    notFound(res)
+  }
+}
+
+/**
+ * @param {string} url
+ * @returns {[string, string]} [pathname, search]
+ */
+const parseRequestUrl = (url) => {
+  // Split the url in the format "/path?query" into two parts
+  // We need to take into account that query can contain '?' characters
+  const ix = url.indexOf('?')
+  if (ix < 0) return [url, '']
+  return [
+    url.slice(0, ix),
+    url.slice(ix + 1) // +1 to skip '?'
+  ]
 }
 
 const errorHandler = (res, err, logger) => {
@@ -54,3 +77,105 @@ const notFound = (res) => {
   res.statusCode = 404
   res.end('Not Found')
 }
+
+/**
+ * @param {string} pathname
+ * @param {string} search
+ * @param {import('node:http').ServerResponse} res
+ * @param {string} querystring
+ * @param {import('pg').Pool} pgPool
+ */
+const getRetrievalSuccessRate = async (pathname, search, res, pgPool) => {
+  const filter = parseAndValidateFilter(pathname, search, res)
+  if (res.headersSent) return
+
+  setCacheControlForStatsResponse(res, filter)
+
+  // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres for converting it into
+  // a JavaScript Date with a timezone, as that could change the date one day forward or back.
+  const { rows } = await pgPool.query(
+    'SELECT day::text, total, successful FROM retrieval_stats WHERE day >= $1 AND day <= $2',
+    [filter.from, filter.to]
+  )
+  const stats = rows.map(r => ({
+    day: r.day,
+    success_rate: r.total > 0 ? r.successful / r.total : null
+  }))
+  json(res, stats)
+}
+
+export const today = () => new Date().toISOString().split('T')[0]
+
+/**
+ * @param {string} pathname
+ * @param {string} search
+ * @param {import('node:http').ServerResponse} res
+ * @returns {{from: string | undefined; to: string | undefined}}
+ */
+export const parseAndValidateFilter = (pathname, search, res) => {
+  let { from, to } = qs.parse(search)
+  let shouldRedirect = false
+
+  // Provide default values for "from" and "to" when not specified
+
+  if (!to) {
+    to = today()
+    shouldRedirect = true
+  }
+  if (!from) {
+    from = to
+    shouldRedirect = true
+  }
+  if (shouldRedirect) {
+    res.setHeader('cache-control', `public, max-age=${600 /* 10min */}`)
+    res.setHeader('location', `${pathname}?${qs.stringify({ from, to })}`)
+    res.writeHead(302) // Found
+    res.end()
+    return { from, to }
+  }
+
+  // Trim time from date-time values that are typically provided by Grafana
+
+  const matchFrom = from.match(/^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}\.\d{3}Z)?$/)
+  assert(matchFrom, 400, '"from" must have format YYYY-MM-DD or YYYY-MM-DDThh:mm:ss.sssZ')
+  if (matchFrom[2]) {
+    from = matchFrom[1]
+    shouldRedirect = true
+  }
+
+  const matchTo = to.match(/^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}\.\d{3}Z)?$/)
+  assert(matchTo, 400, '"to" must have format YYYY-MM-DD or YYYY-MM-DDThh:mm:ss.sssZ')
+  if (matchTo[2]) {
+    to = matchTo[1]
+    shouldRedirect = true
+  }
+
+  if (shouldRedirect) {
+    res.setHeader('cache-control', `public, max-age=${24 * 3600 /* one day */}`)
+    res.setHeader('location', `${pathname}?${qs.stringify({ from, to })}`)
+    res.writeHead(301) // Found
+    res.end()
+    return { from, to }
+  }
+
+  // We have well-formed from & to dates now
+  return { from, to }
+}
+
+/**
+ * @param {import('node:http').ServerResponse} res
+ * @param {{from: string, to: string}} filter
+ */
+const setCacheControlForStatsResponse = (res, filter) => {
+  // We cannot simply compare filter.to vs today() because there may be a delay in finalizing
+  // stats for the previous day. Let's allow up to one hour for the finalization.
+  const boundary = new Date(Date.now() - 3600_000).toISOString().split('T')[0]
+
+  if (filter.to >= boundary) {
+    // response includes partial data for today, cache it for 10 minutes only
+    res.setHeader('cache-control', 'public, max-age=600')
+  } else {
+    // historical data should never change, cache it for one year
+    res.setHeader('cache-control', `public, max-age=${365 * 24 * 3600}, immutable`)
+  }
+}

package.json

@@ -12,6 +12,8 @@
   "dependencies": {
     "@sentry/node": "^7.93.0",
     "debug": "^4.3.4",
+    "http-assert": "^1.5.0",
+    "http-responders": "^2.0.2",
     "pg": "^8.11.3"
   },
   "standard": {

test/handler.test.js

@@ -1,10 +1,10 @@
 import http from 'node:http'
 import { once } from 'node:events'
-import { AssertionError } from 'node:assert'
+import assert, { AssertionError } from 'node:assert'
 import pg from 'pg'
 import createDebug from 'debug'
 
-import { createHandler } from '../lib/handler.js'
+import { createHandler, today } from '../lib/handler.js'
 import { DATABASE_URL } from '../lib/config.js'
 
 const debug = createDebug('test')
@@ -29,7 +29,6 @@ describe('HTTP request handler', () => {
       }
     })
 
-    // server = http.createServer((req, res) => { console.log(req.method, req.url); res.end('hello') })
     server = http.createServer(handler)
     server.listen()
     await once(server, 'listening')
@@ -46,6 +45,83 @@ describe('HTTP request handler', () => {
     const res = await fetch(new URL('/unknown-path', baseUrl))
     assertResponseStatus(res, 404)
   })
+
+  describe('GET /retrieval-success-rate', () => {
+    beforeEach(async () => {
+      await pgPool.query('DELETE FROM retrieval_stats')
+    })
+
+    it('returns today stats for no query string', async () => {
+      const day = today()
+      await givenRetrievalStats(pgPool, { day, total: 10, successful: 1 })
+      const res = await fetch(new URL('/retrieval-success-rate', baseUrl), { redirect: 'follow' })
+      await assertResponseStatus(res, 200)
+      const stats = await res.json()
+      assert.deepStrictEqual(stats, [
+        { day, success_rate: 0.1 }
+      ])
+    })
+
+    it('applies from & to in YYYY-MM-DD format', async () => {
+      await givenRetrievalStats(pgPool, { day: '2024-01-10', total: 10, successful: 1 })
+      await givenRetrievalStats(pgPool, { day: '2024-01-11', total: 20, successful: 1 })
+      await givenRetrievalStats(pgPool, { day: '2024-01-12', total: 30, successful: 3 })
+      await givenRetrievalStats(pgPool, { day: '2024-01-13', total: 40, successful: 1 })
+
+      const res = await fetch(
+        new URL(
+          '/retrieval-success-rate?from=2024-01-11&to=2024-01-12',
+          baseUrl
+        ), {
+          redirect: 'manual'
+        }
+      )
+      await assertResponseStatus(res, 200)
+      const stats = await res.json()
+      assert.deepStrictEqual(stats, [
+        { day: '2024-01-11', success_rate: 0.05 },
+        { day: '2024-01-12', success_rate: 0.1 }
+      ])
+    })
+
+    it('redirects when from & to is in YYYY-MM-DDThh:mm:ss.sssZ format', async () => {
+      const res = await fetch(
+        new URL(
+          '/retrieval-success-rate?from=2024-01-10T13:44:44.289Z&to=2024-01-15T09:44:44.289Z',
+          baseUrl
+        ), {
+          redirect: 'manual'
+        }
+      )
+      await assertResponseStatus(res, 301)
+      assert.strictEqual(
+        res.headers.get('location'),
+        '/retrieval-success-rate?from=2024-01-10&to=2024-01-15'
+      )
+    })
+
+    it('caches data including today for short time', async () => {
+      const res = await fetch(
+        new URL(`/retrieval-success-rate?from=2024-01-01&to=${today()}`, baseUrl),
+        {
+          redirect: 'manual'
+        }
+      )
+      await assertResponseStatus(res, 200)
+      assert.strictEqual(res.headers.get('cache-control'), 'public, max-age=600')
+    })
+
+    it('caches historical including for long time & marks them immutable', async () => {
+      const res = await fetch(
+        new URL('/retrieval-success-rate?from=2023-01-01&to=2023-12-31', baseUrl),
+        {
+          redirect: 'manual'
+        }
+      )
+      await assertResponseStatus(res, 200)
+      assert.strictEqual(res.headers.get('cache-control'), 'public, max-age=31536000, immutable')
+    })
+  })
 })
 
 const assertResponseStatus = async (res, status) => {
@@ -57,3 +133,10 @@ const assertResponseStatus = async (res, status) => {
     })
   }
 }
+
+const givenRetrievalStats = async (pgPool, { day, total, successful }) => {
+  await pgPool.query(
+    'INSERT INTO retrieval_stats (day, total, successful) VALUES ($1, $2, $3)',
+    [day, total, successful]
+  )
+}