From cfa4b556a76bce0e40e4c4a6f32022076f8bc3b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Tue, 10 Sep 2024 13:34:51 +0200 Subject: [PATCH 1/5] Update docs --- packages/signed-api/config/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/signed-api/config/configuration.md b/packages/signed-api/config/configuration.md index cf3dd0b7..6f123d7a 100644 --- a/packages/signed-api/config/configuration.md +++ b/packages/signed-api/config/configuration.md @@ -51,8 +51,8 @@ Default: `json`. ### `LOG_API_DATA` _(optional)_ Enables or disables logging of the API data at the `info` level. When set to `true`, received valid signed data will be -logged with the fields `airnode`, `encodedValue`, `templateId`, and `timestamp`. The `signature` field is intentionally -excluded for security reasons. Options: +logged with the fields `airnode`, `encodedValue`, `templateId`, `timestamp` and `signature`. The logging of this data is +delayed to make sure people with access to the logs won't be able to misuse the beacon data. Options: - `true` - Enables logging of the API data. - `false` - Disables logging of the API data. From 9eadb1b5b0aafff4f176bb6d54dc73feec651496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Tue, 10 Sep 2024 13:39:45 +0200 Subject: [PATCH 2/5] Implement delayed logging --- packages/signed-api/src/handlers.ts | 17 +++++++++++------ packages/signed-api/src/schema.ts | 1 + 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/signed-api/src/handlers.ts b/packages/signed-api/src/handlers.ts index f152b6a3..9d9c800f 100644 --- a/packages/signed-api/src/handlers.ts +++ b/packages/signed-api/src/handlers.ts @@ -19,7 +19,8 @@ import type { } from './types'; import { extractBearerToken, generateErrorResponse, isBatchUnique } from './utils'; -const env = loadEnv(); +const LOG_API_DATA_DELAY_MS = 5 * 60 * 1000; + // Accepts a batch of signed data that is first validated for consistency and data integrity errors. If there is any // issue during this step, the whole batch is rejected. // @@ -106,12 +107,16 @@ export const batchInsertData = async ( return generateErrorResponse(400, message, detail ? { detail, signedData } : { signedData }); } + const env = loadEnv(); if (env.LOG_API_DATA) { - // Log only the required fields to use less space, do not log the signature for security reasons. - const sanitizedData = batchSignedData.map((data) => - pick(data, ['airnode', 'encodedValue', 'templateId', 'timestamp']) - ); - logger.info('Received valid signed data.', { data: sanitizedData }); + // The logging of the data is delayed for security reasons - so people with access to the logs can't misuse the + // signed data. + setTimeout(() => { + const sanitizedData = batchSignedData.map((data) => + pick(data, ['airnode', 'encodedValue', 'templateId', 'timestamp', 'signature']) + ); + logger.info('Received valid signed data.', { data: sanitizedData }); + }, LOG_API_DATA_DELAY_MS); } const newSignedData: InternalSignedData[] = []; diff --git a/packages/signed-api/src/schema.ts b/packages/signed-api/src/schema.ts index be91e568..38141230 100644 --- a/packages/signed-api/src/schema.ts +++ b/packages/signed-api/src/schema.ts @@ -70,6 +70,7 @@ export const configSchema = z.strictObject({ export type Config = z.infer; +// TODO: Move to types export type InternalSignedData = { airnode: string; templateId: string; From ce6f3d4c13b78d98a977881ac90f2beb128e7c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Tue, 10 Sep 2024 13:41:49 +0200 Subject: [PATCH 3/5] Move type from schema to types --- packages/signed-api/src/handlers.ts | 3 ++- packages/signed-api/src/in-memory-cache.ts | 2 +- packages/signed-api/src/schema.ts | 11 ----------- packages/signed-api/src/signed-data-verifier.ts | 2 +- packages/signed-api/src/transform-payload.ts | 2 +- packages/signed-api/src/types.ts | 12 ++++++++++-- packages/signed-api/test/utils.ts | 2 +- 7 files changed, 16 insertions(+), 18 deletions(-) diff --git a/packages/signed-api/src/handlers.ts b/packages/signed-api/src/handlers.ts index 9d9c800f..90644705 100644 --- a/packages/signed-api/src/handlers.ts +++ b/packages/signed-api/src/handlers.ts @@ -7,7 +7,7 @@ import { loadEnv } from './env'; import { createResponseHeaders } from './headers'; import { get, getAll, getAllAirnodeAddresses, prune, putAll } from './in-memory-cache'; import { logger } from './logger'; -import { type Endpoint, evmAddressSchema, type InternalSignedData } from './schema'; +import { type Endpoint, evmAddressSchema } from './schema'; import { getVerifier } from './signed-data-verifier-pool'; import { transformAirnodeFeedPayload } from './transform-payload'; import type { @@ -15,6 +15,7 @@ import type { GetListAirnodesResponseSchema, GetSignedDataResponseSchema, GetUnsignedDataResponseSchema, + InternalSignedData, PostSignedDataResponseSchema, } from './types'; import { extractBearerToken, generateErrorResponse, isBatchUnique } from './utils'; diff --git a/packages/signed-api/src/in-memory-cache.ts b/packages/signed-api/src/in-memory-cache.ts index d8936fc0..805f8a0e 100644 --- a/packages/signed-api/src/in-memory-cache.ts +++ b/packages/signed-api/src/in-memory-cache.ts @@ -1,7 +1,7 @@ import { last, uniqBy } from 'lodash'; import { logger } from './logger'; -import type { InternalSignedData } from './schema'; +import { type InternalSignedData } from './types'; import { isIgnored } from './utils'; type SignedDataCache = Record< diff --git a/packages/signed-api/src/schema.ts b/packages/signed-api/src/schema.ts index 38141230..12ea6a2b 100644 --- a/packages/signed-api/src/schema.ts +++ b/packages/signed-api/src/schema.ts @@ -70,17 +70,6 @@ export const configSchema = z.strictObject({ export type Config = z.infer; -// TODO: Move to types -export type InternalSignedData = { - airnode: string; - templateId: string; - beaconId: string; - timestamp: string; - encodedValue: string; - signature: string; - isOevBeacon: boolean; -}; - export const envBooleanSchema = z.union([z.literal('true'), z.literal('false')]).transform((val) => val === 'true'); // We apply default values to make it convenient to omit certain environment variables. The default values should be diff --git a/packages/signed-api/src/signed-data-verifier.ts b/packages/signed-api/src/signed-data-verifier.ts index c24bfbbf..0e714e7f 100644 --- a/packages/signed-api/src/signed-data-verifier.ts +++ b/packages/signed-api/src/signed-data-verifier.ts @@ -1,7 +1,7 @@ import { goSync } from '@api3/promise-utils'; import workerpool from 'workerpool'; -import type { InternalSignedData } from './schema'; +import { type InternalSignedData } from './types'; import { deriveBeaconId, recoverSignerAddress } from './utils'; interface VerificationError { diff --git a/packages/signed-api/src/transform-payload.ts b/packages/signed-api/src/transform-payload.ts index 09a312e5..21d1cc3e 100644 --- a/packages/signed-api/src/transform-payload.ts +++ b/packages/signed-api/src/transform-payload.ts @@ -2,7 +2,7 @@ import { deriveOevTemplateId, type SignedApiBatchPayloadV1, type SignedApiBatchP import { deriveBeaconId, type Hex } from '@api3/commons'; import { getCache, setCache } from './in-memory-cache'; -import { type InternalSignedData } from './schema'; +import { type InternalSignedData } from './types'; export const getOevTemplateId = (templateId: string) => { const cache = getCache(); diff --git a/packages/signed-api/src/types.ts b/packages/signed-api/src/types.ts index 042d53c3..5f2d04a2 100644 --- a/packages/signed-api/src/types.ts +++ b/packages/signed-api/src/types.ts @@ -1,5 +1,3 @@ -import { type InternalSignedData } from './schema'; - export interface ApiResponse { statusCode: number; headers: Record; @@ -22,3 +20,13 @@ export type PostSignedDataResponseSchema = { }; export type GetListAirnodesResponseSchema = { count: number; 'available-airnodes': string[] }; + +export type InternalSignedData = { + airnode: string; + templateId: string; + beaconId: string; + timestamp: string; + encodedValue: string; + signature: string; + isOevBeacon: boolean; +}; diff --git a/packages/signed-api/test/utils.ts b/packages/signed-api/test/utils.ts index c6e35c42..391d47e3 100644 --- a/packages/signed-api/test/utils.ts +++ b/packages/signed-api/test/utils.ts @@ -2,7 +2,7 @@ import { deriveOevTemplateId, type SignedApiPayloadV1, type SignedApiPayloadV2 } import { ethers } from 'ethers'; import { omit } from 'lodash'; -import type { InternalSignedData } from '../src/schema'; +import type { InternalSignedData } from '../src/types'; import { deriveBeaconId } from '../src/utils'; export const deriveTemplateId = (endpointId: string, encodedParameters: string) => From 6d5fa01a2d03e24392d3b0a8d66e9eb1bf785657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Tue, 10 Sep 2024 14:00:41 +0200 Subject: [PATCH 4/5] Add test --- packages/signed-api/src/handlers.test.ts | 38 ++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/signed-api/src/handlers.test.ts b/packages/signed-api/src/handlers.test.ts index 074b65f9..51145dea 100644 --- a/packages/signed-api/src/handlers.test.ts +++ b/packages/signed-api/src/handlers.test.ts @@ -13,9 +13,11 @@ import { } from '../test/utils'; import * as configModule from './config/config'; +import * as envModule from './env'; import { batchInsertData, getData, listAirnodeAddresses } from './handlers'; import * as inMemoryCacheModule from './in-memory-cache'; import { logger } from './logger'; +import { type EnvConfig } from './schema'; import { initializeVerifierPool } from './signed-data-verifier-pool'; import { type ApiResponse } from './types'; import { deriveBeaconId } from './utils'; @@ -255,6 +257,42 @@ describe(batchInsertData.name, () => { }, }); }); + + it('logs the data with delay if LOG_API_DATA is enabled', async () => { + const airnodeWallet = generateRandomWallet(); + const batchData = [await createSignedDataV1({ airnodeWallet }), await createSignedDataV1({ airnodeWallet })]; + jest.spyOn(envModule, 'loadEnv').mockReturnValue({ LOG_API_DATA: true } as EnvConfig); + jest.spyOn(logger, 'info'); + jest.useFakeTimers(); + + const result = await batchInsertData(undefined, batchData, airnodeWallet.address); + const { body, statusCode } = parseResponse(result); + + expect(statusCode).toBe(201); + expect(body).toStrictEqual({ + count: 2, + skipped: 0, + }); + expect(inMemoryCacheModule.getCache()).toStrictEqual({ + ...inMemoryCacheModule.getInitialCache(), + signedDataCache: { + [batchData[0]!.airnode]: { + [batchData[0]!.templateId]: [{ ...batchData[0], isOevBeacon: false }], + [batchData[1]!.templateId]: [{ ...batchData[1], isOevBeacon: false }], + }, + }, + }); + + // Advance time just before 5 minutes and verify that the logger was not called. + jest.advanceTimersByTime(5 * 60 * 1000 - 1); + expect(logger.info).not.toHaveBeenCalled(); + + // Advance time pass the delay period and verify that the data was called. + jest.advanceTimersByTime(1); + expect(logger.info).toHaveBeenCalledWith('Received valid signed data.', { + data: batchData.map((d) => omit(d, ['beaconId'])), + }); + }); }); describe(getData.name, () => { From d112e2d332172dd805c7e6d4c4dfab49294d3f6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Wed, 11 Sep 2024 10:09:07 +0200 Subject: [PATCH 5/5] Mention delay explicitly in the README --- packages/signed-api/config/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/signed-api/config/configuration.md b/packages/signed-api/config/configuration.md index 6f123d7a..2b904378 100644 --- a/packages/signed-api/config/configuration.md +++ b/packages/signed-api/config/configuration.md @@ -52,7 +52,7 @@ Default: `json`. Enables or disables logging of the API data at the `info` level. When set to `true`, received valid signed data will be logged with the fields `airnode`, `encodedValue`, `templateId`, `timestamp` and `signature`. The logging of this data is -delayed to make sure people with access to the logs won't be able to misuse the beacon data. Options: +delayed by 5 minutes to make sure people with access to the logs won't be able to misuse the beacon data. Options: - `true` - Enables logging of the API data. - `false` - Disables logging of the API data.