diff --git a/bin/dry-run.js b/bin/dry-run.js index 84e9742f..d4eefcde 100644 --- a/bin/dry-run.js +++ b/bin/dry-run.js @@ -148,7 +148,7 @@ if (ignoredErrors.length) { } if (DUMP) { - const props = ['cid', 'minerId', 'participantAddress', 'inet_group', 'retrievalResult', 'fraudAssessment'] + const props = ['cid', 'minerId', 'participantAddress', 'inet_group', 'retrievalResult', 'taskingEvaluation', 'consensusEvaluation'] for (const k of Object.keys(round.measurements[0])) { if (!props.includes(k)) props.push(k) } diff --git a/bin/evaluate-measurements.js b/bin/evaluate-measurements.js index deac23e9..5d016309 100644 --- a/bin/evaluate-measurements.js +++ b/bin/evaluate-measurements.js @@ -42,7 +42,7 @@ const EVALUATION_NDJSON_FILE = `${basename(measurementsPath, '.ndjson')}.evaluat const evaluationTxtWriter = fs.createWriteStream(EVALUATION_TXT_FILE) const evaluationNdjsonWriter = fs.createWriteStream(EVALUATION_NDJSON_FILE) -evaluationTxtWriter.write(formatHeader({ includeFraudAssesment: keepRejected }) + '\n') +evaluationTxtWriter.write(formatHeader({ includeEvaluation: keepRejected }) + '\n') const resultCounts = { total: 0 @@ -98,7 +98,9 @@ async function processRound (roundIndex, measurements, resultCounts) { }) for (const m of round.measurements) { - if (m.fraudAssessment !== 'OK') continue + // FIXME: we should include non-majority measurements too + // See https://github.com/filecoin-station/spark-evaluate/pull/396 + if (m.taskingEvaluation !== 'OK' && m.consensusEvaluation === 'MAJORITY_RESULT') continue resultCounts.total++ resultCounts[m.retrievalResult] = (resultCounts[m.retrievalResult] ?? 0) + 1 } @@ -106,14 +108,16 @@ async function processRound (roundIndex, measurements, resultCounts) { if (!keepRejected) { round.measurements = round.measurements // Keep accepted measurements only - .filter(m => m.fraudAssessment === 'OK') - // Remove the fraudAssessment field as all accepted measurements have the same 'OK' value - .map(m => ({ ...m, fraudAssessment: undefined })) + // FIXME: we should include non-majority measurements too + // See https://github.com/filecoin-station/spark-evaluate/pull/396 + .filter(m => m.taskingEvaluation === 'OK' && m.consensusEvaluation === 'MAJORITY_RESULT') + // Remove the taskingEvaluation and consensusEvaluation fields as all accepted measurements have the same value + .map(m => ({ ...m, taskingEvaluation: undefined, majorityEvaluation: undefined })) } evaluationTxtWriter.write( round.measurements - .map(m => formatMeasurement(m, { includeFraudAssesment: keepRejected }) + '\n') + .map(m => formatMeasurement(m, { includeEvaluation: keepRejected }) + '\n') .join('') ) evaluationNdjsonWriter.write( @@ -144,17 +148,19 @@ function isFlagEnabled (envVarValue) { /** * @param {import('../lib/preprocess.js').Measurement} m * @param {object} options - * @param {boolean} [options.includeFraudAssesment] + * @param {boolean} [options.includeEvaluation] */ -function formatMeasurement (m, { includeFraudAssesment } = {}) { +function formatMeasurement (m, { includeEvaluation } = {}) { const fields = [ new Date(m.finished_at).toISOString(), (m.cid ?? '').padEnd(70), (m.protocol ?? '').padEnd(10) ] - if (includeFraudAssesment) { - fields.push((m.fraudAssessment === 'OK' ? '🫡 ' : '🙅 ')) + if (includeEvaluation) { + // FIXME: we should distinguish tasking and majority evaluation + // See https://github.com/filecoin-station/spark-evaluate/pull/396 + fields.push((m.taskingEvaluation === 'OK' && m.consensusEvaluation === 'MAJORITY_RESULT' ? '🫡 ' : '🙅 ')) } fields.push((m.retrievalResult ?? '')) @@ -164,16 +170,16 @@ function formatMeasurement (m, { includeFraudAssesment } = {}) { /** * @param {object} options - * @param {boolean} [options.includeFraudAssesment] + * @param {boolean} [options.includeEvaluation] */ -function formatHeader ({ includeFraudAssesment } = {}) { +function formatHeader ({ includeEvaluation } = {}) { const fields = [ 'Timestamp'.padEnd(new Date().toISOString().length), 'CID'.padEnd(70), 'Protocol'.padEnd(10) ] - if (includeFraudAssesment) { + if (includeEvaluation) { fields.push('🕵️ ') } diff --git a/lib/committee.js b/lib/committee.js index 83aeb218..8b9c8959 100644 --- a/lib/committee.js +++ b/lib/committee.js @@ -3,18 +3,18 @@ import createDebug from 'debug' import { getTaskId } from './retrieval-stats.js' /** @import {Measurement} from './preprocess.js' */ -/** @import {RetrievalResult, CommitteeCheckError} from './typings.js' */ +/** @import {RetrievalResult, ConsensusNotFoundReason} from './typings.js' */ const debug = createDebug('spark:committee') /** @typedef {Map} TaskIdToCommitteeMap */ /** @typedef {{ - hasIndexMajority: boolean; - indexerResult: string | CommitteeCheckError; - hasRetrievalMajority: boolean; + indexMajorityFound: boolean; + indexerResult: string | ConsensusNotFoundReason; + retrievalMajorityFound: boolean; retrievalResult: RetrievalResult - }} CommitteeEvaluation + }} CommitteeDecision */ export class Committee { /** @type {Measurement[]} */ @@ -28,8 +28,8 @@ export class Committee { this.#measurements = [] - /** @type {CommitteeEvaluation | undefined} */ - this.evaluation = undefined + /** @type {CommitteeDecision | undefined} */ + this.decision = undefined } get size () { @@ -48,7 +48,7 @@ export class Committee { addMeasurement (m) { assert.strictEqual(m.cid, this.retrievalTask.cid, 'cid must match') assert.strictEqual(m.minerId, this.retrievalTask.minerId, 'minerId must match') - assert.strictEqual(m.fraudAssessment, 'OK', 'only accepted measurements can be added') + assert.strictEqual(m.taskingEvaluation, 'OK', 'only measurements accepted by task evaluation can be added') this.#measurements.push(m) } @@ -69,13 +69,13 @@ export class Committee { this.#measurements.length, requiredCommitteeSize ) - this.evaluation = { - hasIndexMajority: false, + this.decision = { + indexMajorityFound: false, indexerResult: 'COMMITTEE_TOO_SMALL', - hasRetrievalMajority: false, + retrievalMajorityFound: false, retrievalResult: 'COMMITTEE_TOO_SMALL' } - for (const m of this.#measurements) m.fraudAssessment = 'COMMITTEE_TOO_SMALL' + for (const m of this.#measurements) m.consensusEvaluation = 'COMMITTEE_TOO_SMALL' return } @@ -88,7 +88,7 @@ export class Committee { 'protocol' ] const indexerResultMajority = this.#findMajority(indexerResultProps) - const hasIndexMajority = !!indexerResultMajority + const indexMajorityFound = !!indexerResultMajority const indexerResult = indexerResultMajority ? indexerResultMajority.majorityValue.indexerResult : 'MAJORITY_NOT_FOUND' @@ -110,23 +110,26 @@ export class Committee { ] const retrievalResultMajority = this.#findMajority(retrievalResultProps) - const hasRetrievalMajority = !!retrievalResultMajority - /** @type {CommitteeEvaluation['retrievalResult']} */ + const retrievalMajorityFound = !!retrievalResultMajority + /** @type {CommitteeDecision['retrievalResult']} */ let retrievalResult if (retrievalResultMajority) { retrievalResult = retrievalResultMajority.majorityValue.retrievalResult + for (const m of retrievalResultMajority.majorityMeasurements) { + m.consensusEvaluation = 'MAJORITY_RESULT' + } for (const m of retrievalResultMajority.minorityMeasurements) { - m.fraudAssessment = 'MINORITY_RESULT' + m.consensusEvaluation = 'MINORITY_RESULT' } } else { retrievalResult = 'MAJORITY_NOT_FOUND' - for (const m of this.#measurements) m.fraudAssessment = 'MAJORITY_NOT_FOUND' + for (const m of this.#measurements) m.consensusEvaluation = 'MAJORITY_NOT_FOUND' } - this.evaluation = { - hasIndexMajority, + this.decision = { + indexMajorityFound, indexerResult, - hasRetrievalMajority, + retrievalMajorityFound, retrievalResult } } diff --git a/lib/evaluate.js b/lib/evaluate.js index 1c82301f..cfcda852 100644 --- a/lib/evaluate.js +++ b/lib/evaluate.js @@ -62,12 +62,14 @@ export const evaluate = async ({ requiredCommitteeSize, logger }) - const honestMeasurements = measurements.filter(m => m.fraudAssessment === 'OK') + const measurementsToReward = measurements.filter( + m => m.taskingEvaluation === 'OK' && m.consensusEvaluation === 'MAJORITY_RESULT' + ) // Calculate reward shares const participants = {} let sum = 0n - for (const measurement of honestMeasurements) { + for (const measurement of measurementsToReward) { if (!participants[measurement.participantAddress]) { participants[measurement.participantAddress] = 0n } @@ -76,7 +78,7 @@ export const evaluate = async ({ for (const [participantAddress, participantTotal] of Object.entries(participants)) { const score = participantTotal * MAX_SCORE / - BigInt(honestMeasurements.length) + BigInt(measurementsToReward.length) participants[participantAddress] = score sum += score } @@ -88,24 +90,24 @@ export const evaluate = async ({ logger.log('EVALUATE ROUND %s: added %s as rounding to MAX_SCORE', roundIndex, delta) } - // Calculate aggregates per fraud detection outcome + // Calculate aggregates per evaluation outcome // This is used for logging and telemetry - /** @type {Partial>} */ - const fraudAssessments = { + /** @type {Partial>} */ + const evaluationOutcomes = { OK: 0, TASK_NOT_IN_ROUND: 0, DUP_INET_GROUP: 0, TOO_MANY_TASKS: 0 } for (const m of measurements) { - fraudAssessments[m.fraudAssessment] = (fraudAssessments[m.fraudAssessment] ?? 0) + 1 + evaluationOutcomes[m.taskingEvaluation] = (evaluationOutcomes[m.taskingEvaluation] ?? 0) + 1 } logger.log( - 'EVALUATE ROUND %s: Evaluated %s measurements, found %s honest entries.\n%o', + 'EVALUATE ROUND %s: Evaluated %s measurements, rewarding %s entries.\n%o', roundIndex, measurements.length, - honestMeasurements.length, - fraudAssessments + measurementsToReward.length, + evaluationOutcomes ) const fraudDetectionDuration = Date.now() - started @@ -138,11 +140,11 @@ export const evaluate = async ({ point.intField('total_participants', Object.keys(participants).length) point.intField('total_measurements', measurements.length) point.intField('total_nodes', countUniqueNodes(measurements)) - point.intField('honest_measurements', honestMeasurements.length) + point.intField('honest_measurements', measurementsToReward.length) point.intField('set_scores_duration_ms', setScoresDuration) point.intField('fraud_detection_duration_ms', fraudDetectionDuration) - for (const [type, count] of Object.entries(fraudAssessments)) { + for (const [type, count] of Object.entries(evaluationOutcomes)) { point.intField(`measurements_${type}`, count) } }) @@ -152,7 +154,9 @@ export const evaluate = async ({ try { recordTelemetry('retrieval_stats_honest', (point) => { point.intField('round_index', roundIndex) - buildRetrievalStats(honestMeasurements, point) + // FIXME: Include non-majority measurements in these stats + // See https://github.com/filecoin-station/spark-evaluate/issues/446 + buildRetrievalStats(measurementsToReward, point) }) } catch (err) { console.error('Cannot record retrieval stats (honest).', err) @@ -177,7 +181,7 @@ export const evaluate = async ({ point.intField('committees_all', committees.length) point.intField('committees_too_small', committees - .filter(c => c.evaluation?.retrievalResult === 'COMMITTEE_TOO_SMALL') + .filter(c => c.decision?.retrievalResult === 'COMMITTEE_TOO_SMALL') .length ) recordCommitteeSizes(committees, point) @@ -271,8 +275,6 @@ export const runFraudDetection = async ({ // or missing some of the required fields like `inet_group` // for (const m of measurements) { - if (m.fraudAssessment) continue - // sanity checks to get nicer errors if we forget to set required fields in unit tests assert(typeof m.inet_group === 'string', 'missing inet_group') assert(typeof m.finished_at === 'number', 'missing finished_at') @@ -281,7 +283,7 @@ export const runFraudDetection = async ({ t => t.cid === m.cid && t.minerId === m.minerId ) if (!isValidTask) { - m.fraudAssessment = 'TASK_NOT_IN_ROUND' + m.taskingEvaluation = 'TASK_NOT_IN_ROUND' continue } @@ -289,7 +291,7 @@ export const runFraudDetection = async ({ t => t.cid === m.cid && t.minerId === m.minerId ) if (!isValidTaskForNode) { - m.fraudAssessment = 'TASK_WRONG_NODE' + m.taskingEvaluation = 'TASK_WRONG_NODE' } } @@ -299,7 +301,7 @@ export const runFraudDetection = async ({ /** @type {Map} */ const inetGroups = new Map() for (const m of measurements) { - if (m.fraudAssessment) continue + if (m.taskingEvaluation) continue const key = m.inet_group let group = inetGroups.get(key) @@ -347,18 +349,18 @@ export const runFraudDetection = async ({ if (tasksSeen.has(taskId)) { debug(' pa: %s h: %s task: %s - task was already rewarded', m.participantAddress, h, taskId) - m.fraudAssessment = 'DUP_INET_GROUP' + m.taskingEvaluation = 'DUP_INET_GROUP' continue } if (tasksSeen.size >= sparkRoundDetails.maxTasksPerNode) { debug(' pa: %s h: %s task: %s - already rewarded max tasks', m.participantAddress, h, taskId) - m.fraudAssessment = 'TOO_MANY_TASKS' + m.taskingEvaluation = 'TOO_MANY_TASKS' continue } tasksSeen.add(taskId) - m.fraudAssessment = 'OK' + m.taskingEvaluation = 'OK' debug(' pa: %s h: %s task: %s - REWARD', m.participantAddress, h, taskId) } } @@ -372,7 +374,7 @@ export const runFraudDetection = async ({ // needs is to iterate over the accepted measurements once. const iterateAcceptedMeasurements = function * () { for (const m of measurements) { - if (m.fraudAssessment !== 'OK') continue + if (m.taskingEvaluation !== 'OK') continue yield m } } diff --git a/lib/platform-stats.js b/lib/platform-stats.js index ebe88b22..d6ee5c65 100644 --- a/lib/platform-stats.js +++ b/lib/platform-stats.js @@ -67,7 +67,7 @@ export const updateStationsAndParticipants = async ( } stationStats.total++ - if (m.fraudAssessment === 'OK') stationStats.accepted++ + if (m.taskingEvaluation === 'OK' && m.consensusEvaluation === 'MAJORITY_RESULT') stationStats.accepted++ let subnetsSet = subnets.get(participantId) if (!subnetsSet) { diff --git a/lib/preprocess.js b/lib/preprocess.js index d3d8e5d1..736b61e7 100644 --- a/lib/preprocess.js +++ b/lib/preprocess.js @@ -22,8 +22,10 @@ export class Measurement { // Note: providerId is recorded by spark-publish but we don't use it for evaluations yet this.providerId = pointerize(m.provider_id) this.spark_version = pointerize(m.spark_version) - /** @type {import('./typings.js').FraudAssesment} */ - this.fraudAssessment = null + /** @type {import('./typings.js').TaskingEvaluation} */ + this.taskingEvaluation = null + /** @type {import('./typings.js').ConsensusEvaluation} */ + this.consensusEvaluation = null this.inet_group = pointerize(m.inet_group) this.finished_at = parseDateTime(m.finished_at) this.provider_address = pointerize(m.provider_address) diff --git a/lib/public-stats.js b/lib/public-stats.js index 9eaef3b1..cf817a51 100644 --- a/lib/public-stats.js +++ b/lib/public-stats.js @@ -71,10 +71,10 @@ const updateIndexerQueryStats = async (pgClient, committees) => { for (const c of committees) { const dealId = getTaskId(c.retrievalTask) - const evaluation = c.evaluation - if (!evaluation) continue - if (evaluation.indexerResult) dealsWithIndexerResults.add(dealId) - if (evaluation.indexerResult === 'OK') dealsWithHttpAdvertisement.add(dealId) + const decision = c.decision + if (!decision) continue + if (decision.indexerResult) dealsWithIndexerResults.add(dealId) + if (decision.indexerResult === 'OK') dealsWithHttpAdvertisement.add(dealId) } const tested = dealsWithIndexerResults.size @@ -145,26 +145,26 @@ const updateDailyDealsStats = async (pgClient, committees, findDealClients) => { stats.tested++ - const evaluation = c.evaluation - if (!evaluation) continue + const decision = c.decision + if (!decision) continue - if (evaluation.hasIndexMajority) { + if (decision.indexMajorityFound) { stats.index_majority_found++ } - if (evaluation.indexerResult === 'OK' || evaluation.indexerResult === 'HTTP_NOT_ADVERTISED') { + if (decision.indexerResult === 'OK' || decision.indexerResult === 'HTTP_NOT_ADVERTISED') { stats.indexed++ } - if (evaluation.indexerResult === 'OK') { + if (decision.indexerResult === 'OK') { stats.indexed_http++ } - if (evaluation.hasRetrievalMajority) { + if (decision.retrievalMajorityFound) { stats.retrieval_majority_found++ } - if (evaluation.retrievalResult === 'OK') { + if (decision.retrievalResult === 'OK') { stats.retrievable++ } } @@ -237,13 +237,14 @@ const updateRetrievalTimings = async (pgClient, committees) => { /** @type {Map} */ const retrievalTimings = new Map() for (const c of committees) { - if (!c.evaluation || !c.evaluation.hasRetrievalMajority || c.evaluation.retrievalResult !== 'OK') continue + if (!c.decision || !c.decision.retrievalMajorityFound || c.decision.retrievalResult !== 'OK') continue const { minerId } = c.retrievalTask const ttfbMeasurments = [] for (const m of c.measurements) { + if (m.taskingEvaluation !== 'OK' || m.consensusEvaluation !== 'MAJORITY_RESULT') continue // FIXME: assert first_byte_at and start_at during preprocessing // See https://github.com/filecoin-station/spark-evaluate/issues/447 - if (m.fraudAssessment !== 'OK' || !m.first_byte_at || !m.start_at || m.start_at > m.first_byte_at) continue + if (!m.first_byte_at || !m.start_at || m.start_at > m.first_byte_at) continue const ttfbMeasurment = m.first_byte_at - m.start_at if (isNaN(ttfbMeasurment)) continue ttfbMeasurments.push(ttfbMeasurment) @@ -261,7 +262,7 @@ const updateRetrievalTimings = async (pgClient, committees) => { const rows = Array.from(retrievalTimings.entries()).flatMap(([miner_id, ttfb_p50]) => ({ miner_id, ttfb_p50 })) await pgClient.query(` - INSERT INTO retrieval_timings (day, miner_id, ttfb_p50) + INSERT INTO retrieval_timings (day, miner_id, ttfb_p50) SELECT now(), miner_id, ttfb_p50 FROM jsonb_to_recordset($1::jsonb) AS t (miner_id text, ttfb_p50 int[]) ON CONFLICT (day, miner_id) DO UPDATE SET diff --git a/lib/retrieval-stats.js b/lib/retrieval-stats.js index eb2d90c8..41fd7641 100644 --- a/lib/retrieval-stats.js +++ b/lib/retrieval-stats.js @@ -81,8 +81,9 @@ export const buildRetrievalStats = (measurements, telemetryPoint) => { const endAt = m.end_at const ttfb = startAt && firstByteAt && (firstByteAt - startAt) const duration = startAt && endAt && (endAt - startAt) + const isAccepted = m.taskingEvaluation === 'OK' && m.consensusEvaluation === 'MAJORITY_RESULT' - debug('size=%s ttfb=%s duration=%s status=%s valid? %s', byteLength, ttfb, duration, m.status_code, m.fraudAssessment === 'OK') + debug('size=%s ttfb=%s duration=%s status=%s accepted? %s', byteLength, ttfb, duration, m.status_code, isAccepted) if (byteLength !== undefined && m.status_code === 200) { downloadBandwidth += byteLength sizeValues.push(byteLength) @@ -93,7 +94,7 @@ export const buildRetrievalStats = (measurements, telemetryPoint) => { const node = `${m.inet_group}::${m.participantAddress}` tasksPerNode.set(node, (tasksPerNode.get(node) ?? 0) + 1) - if (m.fraudAssessment === 'OK') { + if (isAccepted) { acceptedMeasurementsPerInetGroup.set(m.inet_group, (acceptedMeasurementsPerInetGroup.get(m.inet_group) ?? 0) + 1) } @@ -222,7 +223,7 @@ export const recordCommitteeSizes = (committees, point) => { data.nodes.add(`${m.inet_group}::${m.participantAddress}`) data.measurements++ - if (m.fraudAssessment === 'OK') { + if (m.taskingEvaluation === 'OK' && m.consensusEvaluation === 'MAJORITY_RESULT') { data.majoritySize = (data.majoritySize ?? 0) + 1 } } diff --git a/lib/round.js b/lib/round.js index da24f075..01007aac 100644 --- a/lib/round.js +++ b/lib/round.js @@ -1,3 +1,5 @@ +/** @import { Measurement } from './preprocess.js' */ + export class RoundData { /** @type {Map} */ #knownStrings @@ -7,8 +9,9 @@ export class RoundData { */ constructor (index) { this.index = index - /** @type {import('./preprocess.js').Measurement[]} */ + /** @type {string[]} */ this.measurementBatches = [] + /** @type {Measurement[]} */ this.measurements = [] this.details = null this.#knownStrings = new Map() diff --git a/lib/typings.d.ts b/lib/typings.d.ts index 7255a34c..d4e217d1 100644 --- a/lib/typings.d.ts +++ b/lib/typings.d.ts @@ -29,20 +29,22 @@ export type RecordTelemetryFn = ( fn: (point: Point) => void ) => void -export type CommitteeCheckError = +export type ConsensusNotFoundReason = | 'COMMITTEE_TOO_SMALL' | 'MAJORITY_NOT_FOUND' -| 'MINORITY_RESULT' + +export type ConsensusEvaluation = + | 'MAJORITY_RESULT' + | 'MINORITY_RESULT' + | ConsensusNotFoundReason // When adding a new enum value, remember to update the summary initializer inside `evaluate()` -export type FraudAssesment = +export type TaskingEvaluation = | 'OK' | 'TASK_NOT_IN_ROUND' | 'TASK_WRONG_NODE' | 'DUP_INET_GROUP' | 'TOO_MANY_TASKS' - | CommitteeCheckError - // When adding a new enum value, remember to update the summary initializer inside `reportRetrievalStats()` export type RetrievalResult = @@ -63,7 +65,7 @@ export type RetrievalResult = | `HTTP_${number}` | `LASSIE_${number}` | 'UNKNOWN_ERROR' - | CommitteeCheckError + | ConsensusNotFoundReason // Data coming from spark-api and spark-publish diff --git a/test/committee.test.js b/test/committee.test.js index bd21cae8..1f7bc7ab 100644 --- a/test/committee.test.js +++ b/test/committee.test.js @@ -7,13 +7,13 @@ import { Committee } from '../lib/committee.js' /** @type {Measurement} */ const VALID_MEASUREMENT = { ...VALID_MEASUREMENT_BEFORE_ASSESSMENT, - fraudAssessment: 'OK' + taskingEvaluation: 'OK' } Object.freeze(VALID_MEASUREMENT) describe('Committee', () => { describe('evaluate', () => { - it('produces OK result when the absolute majority agrees', () => { + it('produces MAJORITY_RESULT result when the absolute majority agrees', () => { const c = new Committee(VALID_TASK) c.addMeasurement({ ...VALID_MEASUREMENT, retrievalResult: 'OK' }) c.addMeasurement({ ...VALID_MEASUREMENT, retrievalResult: 'OK' }) @@ -22,15 +22,15 @@ describe('Committee', () => { c.evaluate({ requiredCommitteeSize: 2 }) - assert.deepStrictEqual(c.evaluation, { - hasIndexMajority: true, + assert.deepStrictEqual(c.decision, { + indexMajorityFound: true, indexerResult: 'OK', - hasRetrievalMajority: true, + retrievalMajorityFound: true, retrievalResult: 'OK' }) - assert.deepStrictEqual(c.measurements.map(m => m.fraudAssessment), [ - 'OK', - 'OK', + assert.deepStrictEqual(c.measurements.map(m => m.consensusEvaluation), [ + 'MAJORITY_RESULT', + 'MAJORITY_RESULT', 'MINORITY_RESULT' ]) }) @@ -39,13 +39,13 @@ describe('Committee', () => { const c = new Committee(VALID_TASK) c.addMeasurement({ ...VALID_MEASUREMENT }) c.evaluate({ requiredCommitteeSize: 10 }) - assert.deepStrictEqual(c.evaluation, { - hasIndexMajority: false, + assert.deepStrictEqual(c.decision, { + indexMajorityFound: false, indexerResult: 'COMMITTEE_TOO_SMALL', - hasRetrievalMajority: false, + retrievalMajorityFound: false, retrievalResult: 'COMMITTEE_TOO_SMALL' }) - assert.strictEqual(c.measurements[0].fraudAssessment, 'COMMITTEE_TOO_SMALL') + assert.strictEqual(c.measurements[0].consensusEvaluation, 'COMMITTEE_TOO_SMALL') }) it('rejects committees without absolute majority for providerId', () => { @@ -56,13 +56,13 @@ describe('Committee', () => { c.evaluate({ requiredCommitteeSize: 2 }) - assert.deepStrictEqual(c.evaluation, { - hasIndexMajority: false, + assert.deepStrictEqual(c.decision, { + indexMajorityFound: false, indexerResult: 'MAJORITY_NOT_FOUND', - hasRetrievalMajority: false, + retrievalMajorityFound: false, retrievalResult: 'MAJORITY_NOT_FOUND' }) - assert.deepStrictEqual(c.measurements.map(m => m.fraudAssessment), [ + assert.deepStrictEqual(c.measurements.map(m => m.consensusEvaluation), [ 'MAJORITY_NOT_FOUND', 'MAJORITY_NOT_FOUND', 'MAJORITY_NOT_FOUND' @@ -78,15 +78,15 @@ describe('Committee', () => { c.evaluate({ requiredCommitteeSize: 2 }) - assert.deepStrictEqual(c.evaluation, { - hasIndexMajority: true, + assert.deepStrictEqual(c.decision, { + indexMajorityFound: true, indexerResult: 'OK', - hasRetrievalMajority: true, + retrievalMajorityFound: true, retrievalResult: 'OK' }) - assert.deepStrictEqual(c.measurements.map(m => m.fraudAssessment), [ - 'OK', - 'OK', + assert.deepStrictEqual(c.measurements.map(m => m.consensusEvaluation), [ + 'MAJORITY_RESULT', + 'MAJORITY_RESULT', 'MINORITY_RESULT' ]) }) @@ -99,13 +99,13 @@ describe('Committee', () => { c.evaluate({ requiredCommitteeSize: 2 }) - assert.deepStrictEqual(c.evaluation, { - hasIndexMajority: true, + assert.deepStrictEqual(c.decision, { + indexMajorityFound: true, indexerResult: 'OK', - hasRetrievalMajority: false, + retrievalMajorityFound: false, retrievalResult: 'MAJORITY_NOT_FOUND' }) - assert.deepStrictEqual(c.measurements.map(m => m.fraudAssessment), [ + assert.deepStrictEqual(c.measurements.map(m => m.consensusEvaluation), [ 'MAJORITY_NOT_FOUND', 'MAJORITY_NOT_FOUND', 'MAJORITY_NOT_FOUND' @@ -121,15 +121,15 @@ describe('Committee', () => { c.evaluate({ requiredCommitteeSize: 2 }) - assert.deepStrictEqual(c.evaluation, { - hasIndexMajority: true, + assert.deepStrictEqual(c.decision, { + indexMajorityFound: true, indexerResult: 'OK', - hasRetrievalMajority: true, + retrievalMajorityFound: true, retrievalResult: 'CONTENT_VERIFICATION_FAILED' }) - assert.deepStrictEqual(c.measurements.map(m => m.fraudAssessment), [ - 'OK', - 'OK', + assert.deepStrictEqual(c.measurements.map(m => m.consensusEvaluation), [ + 'MAJORITY_RESULT', + 'MAJORITY_RESULT', 'MINORITY_RESULT' ]) }) @@ -142,13 +142,13 @@ describe('Committee', () => { c.evaluate({ requiredCommitteeSize: 2 }) - assert.deepStrictEqual(c.evaluation, { - hasIndexMajority: false, + assert.deepStrictEqual(c.decision, { + indexMajorityFound: false, indexerResult: 'MAJORITY_NOT_FOUND', - hasRetrievalMajority: false, + retrievalMajorityFound: false, retrievalResult: 'MAJORITY_NOT_FOUND' }) - assert.deepStrictEqual(c.measurements.map(m => m.fraudAssessment), [ + assert.deepStrictEqual(c.measurements.map(m => m.consensusEvaluation), [ 'MAJORITY_NOT_FOUND', 'MAJORITY_NOT_FOUND', 'MAJORITY_NOT_FOUND' @@ -164,15 +164,15 @@ describe('Committee', () => { c.evaluate({ requiredCommitteeSize: 2 }) - assert.deepStrictEqual(c.evaluation, { - hasIndexMajority: true, + assert.deepStrictEqual(c.decision, { + indexMajorityFound: true, indexerResult: 'HTTP_NOT_ADVERTISED', - hasRetrievalMajority: true, + retrievalMajorityFound: true, retrievalResult: 'OK' }) - assert.deepStrictEqual(c.measurements.map(m => m.fraudAssessment), [ - 'OK', - 'OK', + assert.deepStrictEqual(c.measurements.map(m => m.consensusEvaluation), [ + 'MAJORITY_RESULT', + 'MAJORITY_RESULT', 'MINORITY_RESULT' ]) }) @@ -186,13 +186,13 @@ describe('Committee', () => { c.evaluate({ requiredCommitteeSize: 2 }) - assert.deepStrictEqual(c.evaluation, { - hasIndexMajority: true, + assert.deepStrictEqual(c.decision, { + indexMajorityFound: true, indexerResult: 'OK', - hasRetrievalMajority: false, + retrievalMajorityFound: false, retrievalResult: 'MAJORITY_NOT_FOUND' }) - assert.deepStrictEqual(c.measurements.map(m => m.fraudAssessment), [ + assert.deepStrictEqual(c.measurements.map(m => m.consensusEvaluation), [ 'MAJORITY_NOT_FOUND', 'MAJORITY_NOT_FOUND', 'MAJORITY_NOT_FOUND' @@ -208,15 +208,15 @@ describe('Committee', () => { c.evaluate({ requiredCommitteeSize: 2 }) - assert.deepStrictEqual(c.evaluation, { - hasIndexMajority: true, + assert.deepStrictEqual(c.decision, { + indexMajorityFound: true, indexerResult: 'OK', - hasRetrievalMajority: true, + retrievalMajorityFound: true, retrievalResult: 'OK' }) - assert.deepStrictEqual(c.measurements.map(m => m.fraudAssessment), [ - 'OK', - 'OK', + assert.deepStrictEqual(c.measurements.map(m => m.consensusEvaluation), [ + 'MAJORITY_RESULT', + 'MAJORITY_RESULT', 'MINORITY_RESULT' ]) }) @@ -229,13 +229,13 @@ describe('Committee', () => { c.evaluate({ requiredCommitteeSize: 2 }) - assert.deepStrictEqual(c.evaluation, { - hasIndexMajority: true, + assert.deepStrictEqual(c.decision, { + indexMajorityFound: true, indexerResult: 'OK', - hasRetrievalMajority: false, + retrievalMajorityFound: false, retrievalResult: 'MAJORITY_NOT_FOUND' }) - assert.deepStrictEqual(c.measurements.map(m => m.fraudAssessment), [ + assert.deepStrictEqual(c.measurements.map(m => m.consensusEvaluation), [ 'MAJORITY_NOT_FOUND', 'MAJORITY_NOT_FOUND', 'MAJORITY_NOT_FOUND' @@ -251,15 +251,15 @@ describe('Committee', () => { c.evaluate({ requiredCommitteeSize: 2 }) - assert.deepStrictEqual(c.evaluation, { - hasIndexMajority: true, + assert.deepStrictEqual(c.decision, { + indexMajorityFound: true, indexerResult: 'OK', - hasRetrievalMajority: true, + retrievalMajorityFound: true, retrievalResult: 'OK' }) - assert.deepStrictEqual(c.measurements.map(m => m.fraudAssessment), [ - 'OK', - 'OK', + assert.deepStrictEqual(c.measurements.map(m => m.consensusEvaluation), [ + 'MAJORITY_RESULT', + 'MAJORITY_RESULT', 'MINORITY_RESULT' ]) }) diff --git a/test/evaluate.js b/test/evaluate.js index c8b3fbf8..175447ef 100644 --- a/test/evaluate.js +++ b/test/evaluate.js @@ -241,6 +241,43 @@ describe('evaluate', async function () { assertPointFieldValue(point, 'total_nodes', '3i') }) + it('rewards majority measurements only', async () => { + const round = new RoundData(0n) + // Majority + round.measurements.push({ ...VALID_MEASUREMENT, participantAddress: '0x10' }) + round.measurements.push({ ...VALID_MEASUREMENT, participantAddress: '0x20', inet_group: 'group2' }) + // Minority + round.measurements.push({ ...VALID_MEASUREMENT, participantAddress: '0x30', inet_group: 'group3', retrievalResult: 'TIMEOUT' }) + + const setScoresCalls = [] + const setScores = async (participantAddresses, scores) => { + setScoresCalls.push({ participantAddresses, scores }) + } + const ieContract = { + async getAddress () { + return '0x811765AccE724cD5582984cb35f5dE02d587CA12' + } + } + /** @returns {Promise} */ + const fetchRoundDetails = async () => ({ ...SPARK_ROUND_DETAILS, retrievalTasks: [VALID_TASK] }) + await evaluate({ + round, + roundIndex: 0n, + requiredCommitteeSize: 1, + ieContract, + setScores, + recordTelemetry, + fetchRoundDetails, + createPgClient, + logger, + prepareProviderRetrievalResultStats: async () => {} + }) + + assert.strictEqual(setScoresCalls.length, 1) + assert.deepStrictEqual(setScoresCalls[0].participantAddresses.sort(), ['0x10', '0x20']) + assert.deepStrictEqual(setScoresCalls[0].scores, [MAX_SCORE / 2n, MAX_SCORE / 2n]) + }) + it('adds a dummy entry to ensure scores add up exactly to MAX_SCORE', async () => { const round = new RoundData(0n) round.measurements.push({ ...VALID_MEASUREMENT, participantAddress: '0x123', inet_group: 'ig1' }) @@ -419,7 +456,7 @@ describe('fraud detection', function () { logger }) assert.deepStrictEqual( - measurements.map(m => m.fraudAssessment), + measurements.map(m => m.taskingEvaluation), ['OK', 'TASK_NOT_IN_ROUND', 'TASK_NOT_IN_ROUND'] ) }) @@ -440,7 +477,7 @@ describe('fraud detection', function () { logger }) assert.deepStrictEqual( - measurements.map(m => m.fraudAssessment), + measurements.map(m => m.taskingEvaluation), ['OK', 'DUP_INET_GROUP'] ) }) @@ -492,7 +529,7 @@ describe('fraud detection', function () { logger }) assert.deepStrictEqual( - measurements.map(m => `${m.participantAddress}::${m.fraudAssessment}`), + measurements.map(m => `${m.participantAddress}::${m.taskingEvaluation}`), [ 'pa1::OK', 'pa1::DUP_INET_GROUP', @@ -548,7 +585,7 @@ describe('fraud detection', function () { }) assert.strictEqual( - measurements.filter(m => m.fraudAssessment === 'OK').length, + measurements.filter(m => m.taskingEvaluation === 'OK').length, 2 // maxTasksPerNode ) }) @@ -617,7 +654,7 @@ describe('fraud detection', function () { }) assert.deepStrictEqual( - measurements.map(m => `${m.participantAddress}::${m.fraudAssessment}`), + measurements.map(m => `${m.participantAddress}::${m.taskingEvaluation}`), [ 'pa1::OK', 'pa1::OK', @@ -686,7 +723,7 @@ describe('fraud detection', function () { }) assert.deepStrictEqual( - measurements.map(m => m.fraudAssessment), + measurements.map(m => m.taskingEvaluation), [ 'OK', 'TOO_MANY_TASKS', @@ -735,7 +772,7 @@ describe('fraud detection', function () { logger }) - assert.deepStrictEqual(measurements.map(m => `${m.cid}::${m.minerId}::${m.fraudAssessment}`), [ + assert.deepStrictEqual(measurements.map(m => `${m.cid}::${m.minerId}::${m.taskingEvaluation}`), [ 'bafyone::f010::TASK_WRONG_NODE', 'bafyone::f020::OK', 'bafyone::f030::OK', diff --git a/test/helpers/test-data.js b/test/helpers/test-data.js index 2be7ff5e..3b196a6c 100644 --- a/test/helpers/test-data.js +++ b/test/helpers/test-data.js @@ -40,7 +40,8 @@ export const VALID_MEASUREMENT = { carTooLarge: false, retrievalResult: 'OK', indexerResult: 'OK', - fraudAssessment: null + taskingEvaluation: null, + consensusEvaluation: null } // Fraud detection is mutating the measurements parsed from JSON @@ -71,13 +72,17 @@ export const today = () => { * @param {Iterable} acceptedMeasurements */ export const buildEvaluatedCommitteesFromMeasurements = (acceptedMeasurements) => { - for (const m of acceptedMeasurements) m.fraudAssessment = 'OK' + for (const m of acceptedMeasurements) { + m.taskingEvaluation = 'OK' + if (!m.consensusEvaluation) m.consensusEvaluation = 'MAJORITY_RESULT' + } + const committees = [...groupMeasurementsToCommittees(acceptedMeasurements).values()] for (const c of committees) { - c.evaluation = { - hasIndexMajority: true, + c.decision = { + indexMajorityFound: true, indexerResult: c.measurements[0].indexerResult, - hasRetrievalMajority: true, + retrievalMajorityFound: true, retrievalResult: c.measurements[0].retrievalResult } } diff --git a/test/platform-stats.test.js b/test/platform-stats.test.js index b93183c9..3102ab8a 100644 --- a/test/platform-stats.test.js +++ b/test/platform-stats.test.js @@ -102,10 +102,10 @@ describe('platform-stats', () => { /** @type {Measurement[]} */ const allMeasurements = [ - { ...VALID_MEASUREMENT, stationId: 'station1', participantAddress: '0x10', inet_group: 'subnet1', fraudAssessment: 'OK' }, - { ...VALID_MEASUREMENT, stationId: 'station1', participantAddress: '0x10', inet_group: 'subnet2', fraudAssessment: 'OK' }, - { ...VALID_MEASUREMENT, stationId: 'station2', participantAddress: '0x20', inet_group: 'subnet3', fraudAssessment: 'OK' }, - { ...VALID_MEASUREMENT, stationId: 'station1', participantAddress: '0x10', inet_group: 'subnet1', fraudAssessment: 'TASK_NOT_IN_ROUND' } + { ...VALID_MEASUREMENT, stationId: 'station1', participantAddress: '0x10', inet_group: 'subnet1', taskingEvaluation: 'OK', consensusEvaluation: 'MAJORITY_RESULT' }, + { ...VALID_MEASUREMENT, stationId: 'station1', participantAddress: '0x10', inet_group: 'subnet2', taskingEvaluation: 'OK', consensusEvaluation: 'MAJORITY_RESULT' }, + { ...VALID_MEASUREMENT, stationId: 'station2', participantAddress: '0x20', inet_group: 'subnet3', taskingEvaluation: 'OK', consensusEvaluation: 'MAJORITY_RESULT' }, + { ...VALID_MEASUREMENT, stationId: 'station1', participantAddress: '0x10', inet_group: 'subnet1', taskingEvaluation: 'TASK_NOT_IN_ROUND' } ] await updateStationsAndParticipants(pgClient, allMeasurements, participantsMap, { day: today }) @@ -166,6 +166,37 @@ describe('platform-stats', () => { ]) }) + it('counts only majority measurements as accepted', async () => { + const participantsMap = await mapParticipantsToIds(pgClient, new Set(['0x10'])) + + /** @type {Measurement[]} */ + const allMeasurements = [ + { ...VALID_MEASUREMENT, participantAddress: '0x10', taskingEvaluation: 'OK', consensusEvaluation: 'MAJORITY_RESULT' }, + { ...VALID_MEASUREMENT, participantAddress: '0x10', taskingEvaluation: 'OK', consensusEvaluation: 'MINORITY_RESULT' } + ] + + await updateStationsAndParticipants(pgClient, allMeasurements, participantsMap, { day: today }) + + const { rows: stationDetails } = await pgClient.query(` + SELECT + day::TEXT, + participant_id, + accepted_measurement_count, + total_measurement_count + FROM recent_station_details + WHERE day = $1::DATE + `, [today]) + + assert.deepStrictEqual(stationDetails, [ + { + day: today, + participant_id: 1, + accepted_measurement_count: 1, + total_measurement_count: 2 + } + ]) + }) + it('updates top measurements participants yesterday materialized view', async () => { const validStationId3 = VALID_STATION_ID.slice(0, -1) + '2' const yesterday = await getYesterdayDate() @@ -174,10 +205,10 @@ describe('platform-stats', () => { /** @type {Measurement[]} */ const allMeasurements = [ - { ...VALID_MEASUREMENT, stationId: VALID_STATION_ID, participantAddress: '0x10', fraudAssessment: 'OK' }, - { ...VALID_MEASUREMENT, stationId: VALID_STATION_ID, participantAddress: '0x10', fraudAssessment: 'OK' }, - { ...VALID_MEASUREMENT, stationId: VALID_STATION_ID_2, participantAddress: '0x10', fraudAssessment: 'OK' }, - { ...VALID_MEASUREMENT, stationId: validStationId3, participantAddress: '0x20', fraudAssessment: 'OK' } + { ...VALID_MEASUREMENT, stationId: VALID_STATION_ID, participantAddress: '0x10', taskingEvaluation: 'OK', consensusEvaluation: 'MAJORITY_RESULT' }, + { ...VALID_MEASUREMENT, stationId: VALID_STATION_ID, participantAddress: '0x10', taskingEvaluation: 'OK', consensusEvaluation: 'MAJORITY_RESULT' }, + { ...VALID_MEASUREMENT, stationId: VALID_STATION_ID_2, participantAddress: '0x10', taskingEvaluation: 'OK', consensusEvaluation: 'MAJORITY_RESULT' }, + { ...VALID_MEASUREMENT, stationId: validStationId3, participantAddress: '0x20', taskingEvaluation: 'OK', consensusEvaluation: 'MAJORITY_RESULT' } ] await updateStationsAndParticipants(pgClient, allMeasurements, participantsMap, { day: yesterday }) @@ -352,7 +383,7 @@ describe('platform-stats', () => { /** @type {Measurement[]} */ const allMeasurements = [ ...honestMeasurements, - { ...VALID_MEASUREMENT, participantAddress: '0x30', fraudAssessment: 'TASK_NOT_IN_ROUND' } + { ...VALID_MEASUREMENT, participantAddress: '0x30', taskingEvaluation: 'TASK_NOT_IN_ROUND' } ] await updatePlatformStats(pgClient, allMeasurements) diff --git a/test/public-stats.test.js b/test/public-stats.test.js index a3a45897..21070322 100644 --- a/test/public-stats.test.js +++ b/test/public-stats.test.js @@ -181,14 +181,19 @@ describe('public-stats', () => { { ...VALID_MEASUREMENT, retrievalResult: 'OK' }, { ...VALID_MEASUREMENT, retrievalResult: 'TIMEOUT' } ] - for (const m of honestMeasurements) m.fraudAssessment = 'OK' + for (const m of honestMeasurements) m.taskingEvaluation = 'OK' const allMeasurements = honestMeasurements const committees = [...groupMeasurementsToCommittees(honestMeasurements).values()] assert.strictEqual(committees.length, 1) committees[0].evaluate({ requiredCommitteeSize: 3 }) - assert.deepStrictEqual(allMeasurements.map(m => m.fraudAssessment), [ + assert.deepStrictEqual(allMeasurements.map(m => m.taskingEvaluation), [ 'OK', 'OK', + 'OK' + ]) + assert.deepStrictEqual(allMeasurements.map(m => m.consensusEvaluation), [ + 'MAJORITY_RESULT', + 'MAJORITY_RESULT', 'MINORITY_RESULT' ]) // The last measurement is rejected because it's a minority result @@ -397,7 +402,7 @@ describe('public-stats', () => { { ...VALID_MEASUREMENT, cid: 'bafy5', indexerResult: 'NO_VALID_ADVERTISEMENT' }, { ...VALID_MEASUREMENT, cid: 'bafy5', indexerResult: 'ERROR_404' } ] - honestMeasurements.forEach(m => { m.fraudAssessment = 'OK' }) + honestMeasurements.forEach(m => { m.taskingEvaluation = 'OK' }) const allMeasurements = honestMeasurements const committees = [...groupMeasurementsToCommittees(honestMeasurements).values()] committees.forEach(c => c.evaluate({ requiredCommitteeSize: 3 })) @@ -435,8 +440,8 @@ describe('public-stats', () => { ] const allMeasurements = honestMeasurements const committees = buildEvaluatedCommitteesFromMeasurements(honestMeasurements) - Object.assign(committees.find(c => c.retrievalTask.cid === 'bafy4').evaluation, { - hasIndexMajority: false, + Object.assign(committees.find(c => c.retrievalTask.cid === 'bafy4').decision, { + indexMajorityFound: false, indexerResult: 'COMMITTEE_TOO_SMALL' }) @@ -481,7 +486,7 @@ describe('public-stats', () => { { ...VALID_MEASUREMENT, cid: 'bafy5', retrievalResult: 'HTTP_404' }, { ...VALID_MEASUREMENT, cid: 'bafy5', retrievalResult: 'HTTP_502' } ] - honestMeasurements.forEach(m => { m.fraudAssessment = 'OK' }) + honestMeasurements.forEach(m => { m.taskingEvaluation = 'OK' }) const allMeasurements = honestMeasurements const committees = [...groupMeasurementsToCommittees(honestMeasurements).values()] committees.forEach(c => c.evaluate({ requiredCommitteeSize: 3 })) @@ -506,18 +511,18 @@ describe('public-stats', () => { /** @type {Measurement[]} */ const honestMeasurements = [ // a majority is found, retrievalResult = OK - { ...VALID_MEASUREMENT, retrievalResult: 'OK' }, + { ...VALID_MEASUREMENT, retrievalResult: 'OK', consensusEvaluation: 'MAJORITY_RESULT' }, // a majority is found, retrievalResult = ERROR_404 - { ...VALID_MEASUREMENT, cid: 'bafy3', retrievalResult: 'HTTP_404' }, + { ...VALID_MEASUREMENT, cid: 'bafy3', retrievalResult: 'HTTP_404', consensusEvaluation: 'MAJORITY_RESULT' }, // committee is too small - { ...VALID_MEASUREMENT, cid: 'bafy4', retrievalResult: 'OK' } + { ...VALID_MEASUREMENT, cid: 'bafy4', retrievalResult: 'OK', consensusEvaluation: 'COMMITTEE_TOO_SMALL' } ] const allMeasurements = honestMeasurements const committees = buildEvaluatedCommitteesFromMeasurements(honestMeasurements) - Object.assign(committees.find(c => c.retrievalTask.cid === 'bafy4').evaluation, { - hasRetrievalMajority: false, + Object.assign(committees.find(c => c.retrievalTask.cid === 'bafy4').decision, { + retrievalMajorityFound: false, retrievalResult: 'COMMITTEE_TOO_SMALL' }) diff --git a/test/retrieval-stats.test.js b/test/retrieval-stats.test.js index 813ed856..373d92a4 100644 --- a/test/retrieval-stats.test.js +++ b/test/retrieval-stats.test.js @@ -161,44 +161,54 @@ describe('retrieval statistics', () => { // inet group 1 - score=2 { ...VALID_MEASUREMENT, - fraudAssessment: 'OK' + taskingEvaluation: 'OK' }, { ...VALID_MEASUREMENT, cid: 'bafyanother', retrievalResult: 'TIMEOUT', - fraudAssessment: 'OK' + taskingEvaluation: 'OK' }, { ...VALID_MEASUREMENT, - fraudAssessment: 'DUP_INET_GROUP' + taskingEvaluation: 'DUP_INET_GROUP' }, // inet group 2 - score=3 { ...VALID_MEASUREMENT, inet_group: 'ig2', - fraudAssessment: 'OK' + taskingEvaluation: 'OK' }, { ...VALID_MEASUREMENT, inet_group: 'ig2', cid: 'bafyanother', - fraudAssessment: 'OK' + taskingEvaluation: 'OK' }, { ...VALID_MEASUREMENT, inet_group: 'ig2', cid: 'bafythree', retrievalResult: 'TIMEOUT', - fraudAssessment: 'OK' + taskingEvaluation: 'OK' }, // inet group 3 - score=1 { ...VALID_MEASUREMENT, inet_group: 'ig3', - fraudAssessment: 'OK' + taskingEvaluation: 'OK' + }, + // measurements not in majority are excluded + { + ...VALID_MEASUREMENT, + inet_group: 'ig3', + taskingEvaluation: 'OK', + consensusEvaluation: 'MINORITY_RESULT' } ] + for (const m of measurements) { + if (!m.consensusEvaluation && m.taskingEvaluation === 'OK') m.consensusEvaluation = 'MAJORITY_RESULT' + } const point = new Point('stats') buildRetrievalStats(measurements, point) @@ -283,7 +293,7 @@ describe('recordCommitteeSizes', () => { cid: 'bafyanother' } ] - measurements.forEach(m => { m.fraudAssessment = 'OK' }) + measurements.forEach(m => { m.taskingEvaluation = 'OK' }) const point = new Point('committees') const committees = groupMeasurementsToCommittees(measurements).values() @@ -324,7 +334,7 @@ describe('recordCommitteeSizes', () => { cid: 'bafyanother' } ] - measurements.forEach(m => { m.fraudAssessment = 'OK' }) + measurements.forEach(m => { m.taskingEvaluation = 'OK' }) const point = new Point('committees') const committees = groupMeasurementsToCommittees(measurements).values() @@ -367,7 +377,7 @@ describe('recordCommitteeSizes', () => { cid: 'bafyanother' } ] - measurements.forEach(m => { m.fraudAssessment = 'OK' }) + measurements.forEach(m => { m.taskingEvaluation = 'OK' }) const point = new Point('committees') const committees = groupMeasurementsToCommittees(measurements).values() @@ -401,11 +411,14 @@ describe('recordCommitteeSizes', () => { cid: 'bafyanother' } ] - measurements.forEach(m => { m.fraudAssessment = 'OK' }) + for (const m of measurements) { + m.taskingEvaluation = 'OK' + m.consensusEvaluation = 'MAJORITY_RESULT' + } const point = new Point('committees') const committees = groupMeasurementsToCommittees(measurements).values() - measurements[0].fraudAssessment = 'MINORITY_RESULT' + measurements[0].consensusEvaluation = 'MINORITY_RESULT' recordCommitteeSizes(committees, point) debug(getPointName(point), point.fields)