From ee3ec2b97418945e007429d631e7381c9fd9e554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Fri, 21 Jun 2024 16:51:29 +0200 Subject: [PATCH 1/3] feat: `GET /deals/summary` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Return a summary of deals for the selected interval, the last day, the last seven days and the last month. These values are a bit difficult to calculate on the Grafana side, I find it easier to build them in our backend. Signed-off-by: Miroslav Bajtoš --- README.md | 5 ++++ stats/lib/handler.js | 5 +++- stats/lib/stats-fetchers.js | 52 +++++++++++++++++++++++++++++++++++ stats/test/handler.test.js | 55 +++++++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8198186..2b63578 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,11 @@ Base URL: http://stats.filspark.com/ http://stats.filspark.com/deals/daily +- `GET /deals/summary?from=2024-01-01&to=2024-01-31` + + http://stats.filspark.com/deals/summary + + ## Development ### Database diff --git a/stats/lib/handler.js b/stats/lib/handler.js index 2b18f0d..f536230 100644 --- a/stats/lib/handler.js +++ b/stats/lib/handler.js @@ -10,7 +10,8 @@ import { fetchParticipantChangeRates, fetchParticipantScheduledRewards, fetchParticipantRewardTransfers, - fetchRetrievalSuccessRate + fetchRetrievalSuccessRate, + fetchDealSummary } from './stats-fetchers.js' import { handlePlatformRoutes } from './platform-routes.js' @@ -83,6 +84,8 @@ const handler = async (req, res, pgPools) => { if (req.method === 'GET' && url === '/deals/daily') { await respond(fetchDailyDealStats) + } else if (req.method === 'GET' && url === '/deals/summary') { + await respond(fetchDealSummary) } else if (req.method === 'GET' && url === '/retrieval-success-rate') { await respond(fetchRetrievalSuccessRate) } else if (req.method === 'GET' && url === '/participants/daily') { diff --git a/stats/lib/stats-fetchers.js b/stats/lib/stats-fetchers.js index 48e7d19..2422f68 100644 --- a/stats/lib/stats-fetchers.js +++ b/stats/lib/stats-fetchers.js @@ -46,6 +46,58 @@ export const fetchDailyDealStats = async (pgPools, filter) => { return rows } +/** + * @param {import('@filecoin-station/spark-stats-db').PgPools} pgPools + * @param {import('./typings.js').DateRangeFilter} filter + */ +export const fetchDealSummary = async (pgPools, filter) => { + const { rows: lastDayRows } = await pgPools.evaluate.query(` + SELECT + SUM(total) as total, + SUM(indexed) as indexed, + SUM(retrievable) as retrievable + FROM daily_deals + WHERE day = date_trunc('day', $1::DATE) + `, [filter.to]) + + const { rows: lastSevenDaysRows } = await pgPools.evaluate.query(` + SELECT + SUM(total) as total, + SUM(indexed) as indexed, + SUM(retrievable) as retrievable + FROM daily_deals + WHERE day > date_trunc('day', $1::DATE) - INTERVAL '7 day' + AND day <= date_trunc('day', $1) + `, [filter.to]) + + const { rows: lastThirtyDaysRows } = await pgPools.evaluate.query(` + SELECT + SUM(total) as total, + SUM(indexed) as indexed, + SUM(retrievable) as retrievable + FROM daily_deals + WHERE day > date_trunc('day', $1::DATE) - INTERVAL '30 day' + AND day <= date_trunc('day', $1) + `, [filter.to]) + + const { rows: requestedIntervalRows } = await pgPools.evaluate.query(` + SELECT + SUM(total) as total, + SUM(indexed) as indexed, + SUM(retrievable) as retrievable + FROM daily_deals + WHERE day >= date_trunc('day', $1::DATE) + AND day <= date_trunc('day', $2::DATE) + `, [filter.from, filter.to]) + + return { + requestedInterval: requestedIntervalRows[0], + lastDay: lastDayRows[0], + lastSevenDays: lastSevenDaysRows[0], + lastThirtyDays: lastThirtyDaysRows[0] + } +} + export const fetchDailyParticipants = async (pgPools, filter) => { return await getDailyDistinctCount({ pgPool: pgPools.evaluate, diff --git a/stats/test/handler.test.js b/stats/test/handler.test.js index 9237561..fafb69d 100644 --- a/stats/test/handler.test.js +++ b/stats/test/handler.test.js @@ -457,6 +457,61 @@ describe('HTTP request handler', () => { }) }) + describe('GET /deals/summary', () => { + it('returns deal summary for the given date range (including the end day)', async () => { + // last year - requested via the filter + await givenDailyDealStats(pgPools.evaluate, { day: '2024-01-01', total: 3, indexed: 2, retrievable: 1 }) + // filter.to - 30 days -> should be excluded + await givenDailyDealStats(pgPools.evaluate, { day: '2024-02-29', total: 3, indexed: 2, retrievable: 1 }) + // last 30 days + await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-01', total: 100, indexed: 51, retrievable: 1 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-12', total: 200, indexed: 52, retrievable: 2 }) + // filter.to - 7 days -> should be excluded + await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-23', total: 300, indexed: 53, retrievable: 3 }) + // last 7 days + await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-24', total: 400, indexed: 54, retrievable: 4 }) + await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-29', total: 500, indexed: 55, retrievable: 5 }) + // `filter.to` (e.g. today) - should be included + await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-30', total: 6000, indexed: 600, retrievable: 60 }) + // after the requested range + await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-31', total: 70000, indexed: 7000, retrievable: 700 }) + + const res = await fetch( + new URL( + '/deals/summary?from=2024-01-01&to=2024-03-30', + baseUrl + ), { + redirect: 'manual' + } + ) + await assertResponseStatus(res, 200) + const stats = await res.json() + + assert.deepStrictEqual(stats, { + lastDay: { + total: '6000', + indexed: '600', + retrievable: '60' + }, + lastSevenDays: { + total: '6900', + indexed: '709', + retrievable: '69' + }, + lastThirtyDays: { + total: '7500', + indexed: '865', + retrievable: '75' + }, + requestedInterval: { + total: '7506', + indexed: '869', + retrievable: '77' + } + }) + }) + }) + describe('CORS', () => { it('sets CORS headers for requests from Station Desktop in production', async () => { const res = await fetch(new URL('/', baseUrl), { From 42fde74bd29cfb9cccb87432f2443d9e3ca3ccf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Mon, 24 Jun 2024 14:06:23 +0200 Subject: [PATCH 2/3] fixup! simplify - return only one set of stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- stats/lib/stats-fetchers.js | 43 ++++----------------------------- stats/test/handler.test.js | 48 +++++++++++++++++-------------------- 2 files changed, 27 insertions(+), 64 deletions(-) diff --git a/stats/lib/stats-fetchers.js b/stats/lib/stats-fetchers.js index 2422f68..ecc5672 100644 --- a/stats/lib/stats-fetchers.js +++ b/stats/lib/stats-fetchers.js @@ -1,4 +1,5 @@ import { getDailyDistinctCount, getMonthlyDistinctCount } from './request-helpers.js' +import assert from 'http-assert' /** @typedef {import('@filecoin-station/spark-stats-db').PgPools} PgPools */ /** @@ -51,36 +52,7 @@ export const fetchDailyDealStats = async (pgPools, filter) => { * @param {import('./typings.js').DateRangeFilter} filter */ export const fetchDealSummary = async (pgPools, filter) => { - const { rows: lastDayRows } = await pgPools.evaluate.query(` - SELECT - SUM(total) as total, - SUM(indexed) as indexed, - SUM(retrievable) as retrievable - FROM daily_deals - WHERE day = date_trunc('day', $1::DATE) - `, [filter.to]) - - const { rows: lastSevenDaysRows } = await pgPools.evaluate.query(` - SELECT - SUM(total) as total, - SUM(indexed) as indexed, - SUM(retrievable) as retrievable - FROM daily_deals - WHERE day > date_trunc('day', $1::DATE) - INTERVAL '7 day' - AND day <= date_trunc('day', $1) - `, [filter.to]) - - const { rows: lastThirtyDaysRows } = await pgPools.evaluate.query(` - SELECT - SUM(total) as total, - SUM(indexed) as indexed, - SUM(retrievable) as retrievable - FROM daily_deals - WHERE day > date_trunc('day', $1::DATE) - INTERVAL '30 day' - AND day <= date_trunc('day', $1) - `, [filter.to]) - - const { rows: requestedIntervalRows } = await pgPools.evaluate.query(` + const { rows: [summary] } = await pgPools.evaluate.query(` SELECT SUM(total) as total, SUM(indexed) as indexed, @@ -88,14 +60,9 @@ export const fetchDealSummary = async (pgPools, filter) => { FROM daily_deals WHERE day >= date_trunc('day', $1::DATE) AND day <= date_trunc('day', $2::DATE) - `, [filter.from, filter.to]) - - return { - requestedInterval: requestedIntervalRows[0], - lastDay: lastDayRows[0], - lastSevenDays: lastSevenDaysRows[0], - lastThirtyDays: lastThirtyDaysRows[0] - } + `, [filter.from, filter.to] + ) + return summary } export const fetchDailyParticipants = async (pgPools, filter) => { diff --git a/stats/test/handler.test.js b/stats/test/handler.test.js index fafb69d..8ba3615 100644 --- a/stats/test/handler.test.js +++ b/stats/test/handler.test.js @@ -459,12 +459,6 @@ describe('HTTP request handler', () => { describe('GET /deals/summary', () => { it('returns deal summary for the given date range (including the end day)', async () => { - // last year - requested via the filter - await givenDailyDealStats(pgPools.evaluate, { day: '2024-01-01', total: 3, indexed: 2, retrievable: 1 }) - // filter.to - 30 days -> should be excluded - await givenDailyDealStats(pgPools.evaluate, { day: '2024-02-29', total: 3, indexed: 2, retrievable: 1 }) - // last 30 days - await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-01', total: 100, indexed: 51, retrievable: 1 }) await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-12', total: 200, indexed: 52, retrievable: 2 }) // filter.to - 7 days -> should be excluded await givenDailyDealStats(pgPools.evaluate, { day: '2024-03-23', total: 300, indexed: 53, retrievable: 3 }) @@ -478,7 +472,7 @@ describe('HTTP request handler', () => { const res = await fetch( new URL( - '/deals/summary?from=2024-01-01&to=2024-03-30', + '/deals/summary?from=2024-03-24&to=2024-03-30', baseUrl ), { redirect: 'manual' @@ -488,26 +482,28 @@ describe('HTTP request handler', () => { const stats = await res.json() assert.deepStrictEqual(stats, { - lastDay: { - total: '6000', - indexed: '600', - retrievable: '60' - }, - lastSevenDays: { - total: '6900', - indexed: '709', - retrievable: '69' - }, - lastThirtyDays: { - total: '7500', - indexed: '865', - retrievable: '75' - }, - requestedInterval: { - total: '7506', - indexed: '869', - retrievable: '77' + total: '6900', + indexed: '709', + retrievable: '69' + }) + }) + + it('handles query for future date with no recorded stats', async () => { + const res = await fetch( + new URL( + '/deals/summary?from=3024-04-24&to=3024-03-30', + baseUrl + ), { + redirect: 'manual' } + ) + await assertResponseStatus(res, 200) + const stats = await res.json() + + assert.deepStrictEqual(stats, { + total: null, + indexed: null, + retrievable: null }) }) }) From 148fa35d27657c93a6983c7552b05396bc4e5822 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Mon, 24 Jun 2024 14:11:14 +0200 Subject: [PATCH 3/3] fixup! linter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- stats/lib/stats-fetchers.js | 1 - 1 file changed, 1 deletion(-) diff --git a/stats/lib/stats-fetchers.js b/stats/lib/stats-fetchers.js index ecc5672..2a61f5d 100644 --- a/stats/lib/stats-fetchers.js +++ b/stats/lib/stats-fetchers.js @@ -1,5 +1,4 @@ import { getDailyDistinctCount, getMonthlyDistinctCount } from './request-helpers.js' -import assert from 'http-assert' /** @typedef {import('@filecoin-station/spark-stats-db').PgPools} PgPools */ /**