diff --git a/lib/preprocess.js b/lib/preprocess.js index 8073015..e2527b3 100644 --- a/lib/preprocess.js +++ b/lib/preprocess.js @@ -35,6 +35,7 @@ export class Measurement { this.first_byte_at = parseDateTime(m.first_byte_at) this.end_at = parseDateTime(m.end_at) this.status_code = m.status_code + this.head_status_code = m.head_status_code this.timeout = m.timeout this.indexerResult = pointerize(m.indexer_result) this.stationId = pointerize(m.station_id) @@ -266,6 +267,8 @@ export const assertValidMeasurement = measurement => { assert(measurement.end_at >= measurement.start_at, 'end_at must be greater than or equal to start_at') assert(measurement.end_at >= measurement.first_byte_at, 'end_at must be greater than or equal to first_byte_at') assert(measurement.first_byte_at >= measurement.start_at, 'first_byte_at must be greater than or equal to start_at') + + assert.strictEqual(typeof measurement.head_status_code, 'number', '`head_status_code` must be a number') } } diff --git a/lib/provider-retrieval-result-stats.js b/lib/provider-retrieval-result-stats.js index 69f66ee..77e2f89 100644 --- a/lib/provider-retrieval-result-stats.js +++ b/lib/provider-retrieval-result-stats.js @@ -16,17 +16,22 @@ const withPgClient = fn => async ({ createPgClient, ...args }) => { } export const build = committees => { - /** @type {Map} */ + /** @type {Map} */ const providerRetrievalResultStats = new Map() for (const c of committees) { // IMPORTANT: include minority results in the calculation for (const m of c.measurements) { const minerId = m.minerId - const retrievalStats = providerRetrievalResultStats.get(minerId) ?? { total: 0, successful: 0, successfulHttp: 0 } + const retrievalStats = providerRetrievalResultStats.get(minerId) ?? { total: 0, successful: 0, successfulHttp: 0, successfulHttpHead: 0 } retrievalStats.total++ if (m.retrievalResult === 'OK') { retrievalStats.successful++ - if (m.protocol && m.protocol === 'http') { retrievalStats.successfulHttp++ } + if (m.protocol && m.protocol === 'http') { + retrievalStats.successfulHttp++ + if (typeof m.head_status_code === 'number' && m.head_status_code >= 200 && m.head_status_code < 300) { + retrievalStats.successfulHttpHead++ + } + } } providerRetrievalResultStats.set(minerId, retrievalStats) } diff --git a/lib/public-stats.js b/lib/public-stats.js index 07af4a3..1d8b8e0 100644 --- a/lib/public-stats.js +++ b/lib/public-stats.js @@ -40,23 +40,26 @@ export const updatePublicStats = async ({ createPgClient, committees, allMeasure * @param {number} stats.total * @param {number} stats.successful * @param {number} stats.successfulHttp + * @param {number} stats.successfulHttpHead */ -const updateRetrievalStats = async (pgClient, minerId, { total, successful, successfulHttp }) => { - debug('Updating public retrieval stats for miner %s: total += %s successful += %s, successful_http += %s', minerId, total, successful, successfulHttp) +const updateRetrievalStats = async (pgClient, minerId, { total, successful, successfulHttp, successfulHttpHead }) => { + debug('Updating public retrieval stats for miner %s: total += %s successful += %s, successful_http += %s, successful_http_head += %s', minerId, total, successful, successfulHttp, successfulHttpHead) await pgClient.query(` INSERT INTO retrieval_stats - (day, miner_id, total, successful, successful_http) + (day, miner_id, total, successful, successful_http, successful_http_head) VALUES - (now(), $1, $2, $3, $4) + (now(), $1, $2, $3, $4, $5) ON CONFLICT(day, miner_id) DO UPDATE SET total = retrieval_stats.total + $2, successful = retrieval_stats.successful + $3, - successful_http = retrieval_stats.successful_http + $4 + successful_http = retrieval_stats.successful_http + $4, + successful_http_head = retrieval_stats.successful_http_head + $5 `, [ minerId, total, successful, - successfulHttp + successfulHttp, + successfulHttpHead ]) } diff --git a/lib/typings.d.ts b/lib/typings.d.ts index d4e217d..429e6f4 100644 --- a/lib/typings.d.ts +++ b/lib/typings.d.ts @@ -87,6 +87,7 @@ export interface RawMeasurement { finished_at: string; status_code: number | undefined | null; + head_status_code: number | undefined | null; timeout: boolean; byte_length: number; car_too_large: boolean; diff --git a/migrations/024.do.successful-http-head.sql b/migrations/024.do.successful-http-head.sql new file mode 100644 index 0000000..074a1fa --- /dev/null +++ b/migrations/024.do.successful-http-head.sql @@ -0,0 +1 @@ +ALTER TABLE retrieval_stats ADD COLUMN successful_http_head INT; diff --git a/test/evaluate.js b/test/evaluate.js index 175447e..467ca93 100644 --- a/test/evaluate.js +++ b/test/evaluate.js @@ -98,7 +98,8 @@ describe('evaluate', async function () { total: 1, successful: 1, // None of the measurements use http - successful_http: 0 + successful_http: 0, + successful_http_head: 0 }]) }) it('handles empty rounds', async () => { diff --git a/test/helpers/test-data.js b/test/helpers/test-data.js index 3b196a6..640b29e 100644 --- a/test/helpers/test-data.js +++ b/test/helpers/test-data.js @@ -27,6 +27,7 @@ export const VALID_MEASUREMENT = { stationId: VALID_STATION_ID, inet_group: VALID_INET_GROUP, status_code: 200, + head_status_code: 200, // TODO: these fields are not part of the Measurement object yet // timeout: false, // car_too_large: false, diff --git a/test/preprocess.js b/test/preprocess.js index 79d56fc..f13e7ad 100644 --- a/test/preprocess.js +++ b/test/preprocess.js @@ -310,6 +310,22 @@ describe('assertValidMeasurement', () => { /field `indexerResult` must be set/ ) }) + it('rejects measurements with OK retrieval result but head_status_code is not a number', () => { + assert.throws( + () => assertValidMeasurement({ + ...VALID_MEASUREMENT, + head_status_code: null + }), + /`head_status_code` must be a number/ + ) + assert.throws( + () => assertValidMeasurement({ + ...VALID_MEASUREMENT, + head_status_code: /** @type {any} */ ('200') + }), + /`head_status_code` must be a number/ + ) + }) it('should throw an error for invalid start_at', () => { const measurement = { diff --git a/test/provider-retrieval-result-stats.test.js b/test/provider-retrieval-result-stats.test.js index 35e4fd6..302b2bb 100644 --- a/test/provider-retrieval-result-stats.test.js +++ b/test/provider-retrieval-result-stats.test.js @@ -38,7 +38,8 @@ describe('Provider Retrieval Result Stats', () => { { minerId: '0', retrievalResult: 'OK', - protocol: 'http' + protocol: 'http', + head_status_code: 200 }, { minerId: '1', @@ -62,8 +63,8 @@ describe('Provider Retrieval Result Stats', () => { } ]) assert.deepStrictEqual(stats, new Map([ - ['0', { total: 2, successful: 2, successfulHttp: 1 }], - ['1', { total: 2, successful: 0, successfulHttp: 0 }] + ['0', { total: 2, successful: 2, successfulHttp: 1, successfulHttpHead: 1 }], + ['1', { total: 2, successful: 0, successfulHttp: 0, successfulHttpHead: 0 }] ])) }) }) @@ -142,7 +143,8 @@ describe('Provider Retrieval Result Stats', () => { { minerId: '0', retrievalResult: 'OK', - protocol: 'http' + protocol: 'http', + head_status_code: 200 }, { minerId: '1', @@ -176,8 +178,8 @@ describe('Provider Retrieval Result Stats', () => { contract_address: ieContractAddress, measurement_batches: round.measurementBatches, provider_retrieval_result_stats: { - 0: { successful: 2, total: 2, successfulHttp: 1 }, - 1: { successful: 0, total: 2, successfulHttp: 0 } + 0: { successful: 2, total: 2, successfulHttp: 1, successfulHttpHead: 1 }, + 1: { successful: 0, total: 2, successfulHttp: 0, successfulHttpHead: 0 } }, round_details: 'baguqeerawg5jfpiy2g5xp5d422uwa3mpyzkmiguoeecesds7q65mn2hdoa4q', round_index: String(round.index), diff --git a/test/public-stats.test.js b/test/public-stats.test.js index 983ef73..094e674 100644 --- a/test/public-stats.test.js +++ b/test/public-stats.test.js @@ -90,7 +90,7 @@ describe('public-stats', () => { it('calculates successful http retrievals correctly', async () => { /** @type {Measurement[]} */ const honestMeasurements = [ - { ...VALID_MEASUREMENT, protocol: 'http', retrievalResult: 'OK' }, + { ...VALID_MEASUREMENT, protocol: 'http', retrievalResult: 'OK', head_status_code: 200 }, { ...VALID_MEASUREMENT, protocol: 'graphsync', retrievalResult: 'OK' }, { ...VALID_MEASUREMENT, protocol: 'http', retrievalResult: 'HTTP_500' }, { ...VALID_MEASUREMENT, protocol: 'graphsync', retrievalResult: 'LASSIE_500' } @@ -106,14 +106,14 @@ describe('public-stats', () => { }) const { rows: created } = await pgClient.query( - 'SELECT day::TEXT, total, successful, successful_http FROM retrieval_stats' + 'SELECT day::TEXT, total, successful, successful_http, successful_http_head FROM retrieval_stats' ) assert.deepStrictEqual(created, [ - { day: today, total: 4, successful: 2, successful_http: 1 } + { day: today, total: 4, successful: 2, successful_http: 1, successful_http_head: 1 } ]) // Let's add another successful http retrieval to make sure the updating process works as expected - honestMeasurements.push({ ...VALID_MEASUREMENT, retrievalResult: 'OK', protocol: 'http' }) + honestMeasurements.push({ ...VALID_MEASUREMENT, retrievalResult: 'OK', protocol: 'http', head_status_code: 200 }) committees = buildEvaluatedCommitteesFromMeasurements(honestMeasurements) await updatePublicStats({ createPgClient, @@ -123,10 +123,10 @@ describe('public-stats', () => { }) const { rows: updated } = await pgClient.query( - 'SELECT day::TEXT, total, successful, successful_http FROM retrieval_stats' + 'SELECT day::TEXT, total, successful, successful_http, successful_http_head FROM retrieval_stats' ) assert.deepStrictEqual(updated, [ - { day: today, total: 4 + 5, successful: 2 + 3, successful_http: 1 + 2 } + { day: today, total: 4 + 5, successful: 2 + 3, successful_http: 1 + 2, successful_http_head: 1 + 2 } ]) })