Log full data with delay (#365)

* Update docs

* Implement delayed logging

* Move type from schema to types

* Add test

* Mention delay explicitly in the README

packages/signed-api/config/configuration.md

@@ -51,8 +51,8 @@ Default: `json`.
 ### `LOG_API_DATA` _(optional)_
 
 Enables or disables logging of the API data at the `info` level. When set to `true`, received valid signed data will be
-logged with the fields `airnode`, `encodedValue`, `templateId`, and `timestamp`. The `signature` field is intentionally
-excluded for security reasons. Options:
+logged with the fields `airnode`, `encodedValue`, `templateId`, `timestamp` and `signature`. The logging of this data is
+delayed by 5 minutes to make sure people with access to the logs won't be able to misuse the beacon data. Options:
 
 - `true` - Enables logging of the API data.
 - `false` - Disables logging of the API data.

packages/signed-api/src/handlers.test.ts

@@ -13,9 +13,11 @@ import {
 } from '../test/utils';
 
 import * as configModule from './config/config';
+import * as envModule from './env';
 import { batchInsertData, getData, listAirnodeAddresses } from './handlers';
 import * as inMemoryCacheModule from './in-memory-cache';
 import { logger } from './logger';
+import { type EnvConfig } from './schema';
 import { initializeVerifierPool } from './signed-data-verifier-pool';
 import { type ApiResponse } from './types';
 import { deriveBeaconId } from './utils';
@@ -255,6 +257,42 @@ describe(batchInsertData.name, () => {
       },
     });
   });
+
+  it('logs the data with delay if LOG_API_DATA is enabled', async () => {
+    const airnodeWallet = generateRandomWallet();
+    const batchData = [await createSignedDataV1({ airnodeWallet }), await createSignedDataV1({ airnodeWallet })];
+    jest.spyOn(envModule, 'loadEnv').mockReturnValue({ LOG_API_DATA: true } as EnvConfig);
+    jest.spyOn(logger, 'info');
+    jest.useFakeTimers();
+
+    const result = await batchInsertData(undefined, batchData, airnodeWallet.address);
+    const { body, statusCode } = parseResponse(result);
+
+    expect(statusCode).toBe(201);
+    expect(body).toStrictEqual({
+      count: 2,
+      skipped: 0,
+    });
+    expect(inMemoryCacheModule.getCache()).toStrictEqual({
+      ...inMemoryCacheModule.getInitialCache(),
+      signedDataCache: {
+        [batchData[0]!.airnode]: {
+          [batchData[0]!.templateId]: [{ ...batchData[0], isOevBeacon: false }],
+          [batchData[1]!.templateId]: [{ ...batchData[1], isOevBeacon: false }],
+        },
+      },
+    });
+
+    // Advance time just before 5 minutes and verify that the logger was not called.
+    jest.advanceTimersByTime(5 * 60 * 1000 - 1);
+    expect(logger.info).not.toHaveBeenCalled();
+
+    // Advance time pass the delay period and verify that the data was called.
+    jest.advanceTimersByTime(1);
+    expect(logger.info).toHaveBeenCalledWith('Received valid signed data.', {
+      data: batchData.map((d) => omit(d, ['beaconId'])),
+    });
+  });
 });
 
 describe(getData.name, () => {

packages/signed-api/src/handlers.ts

@@ -7,19 +7,21 @@ import { loadEnv } from './env';
 import { createResponseHeaders } from './headers';
 import { get, getAll, getAllAirnodeAddresses, prune, putAll } from './in-memory-cache';
 import { logger } from './logger';
-import { type Endpoint, evmAddressSchema, type InternalSignedData } from './schema';
+import { type Endpoint, evmAddressSchema } from './schema';
 import { getVerifier } from './signed-data-verifier-pool';
 import { transformAirnodeFeedPayload } from './transform-payload';
 import type {
   ApiResponse,
   GetListAirnodesResponseSchema,
   GetSignedDataResponseSchema,
   GetUnsignedDataResponseSchema,
+  InternalSignedData,
   PostSignedDataResponseSchema,
 } from './types';
 import { extractBearerToken, generateErrorResponse, isBatchUnique } from './utils';
 
-const env = loadEnv();
+const LOG_API_DATA_DELAY_MS = 5 * 60 * 1000;
+
 // Accepts a batch of signed data that is first validated for consistency and data integrity errors. If there is any
 // issue during this step, the whole batch is rejected.
 //
@@ -106,12 +108,16 @@ export const batchInsertData = async (
     return generateErrorResponse(400, message, detail ? { detail, signedData } : { signedData });
   }
 
+  const env = loadEnv();
   if (env.LOG_API_DATA) {
-    // Log only the required fields to use less space, do not log the signature for security reasons.
-    const sanitizedData = batchSignedData.map((data) =>
-      pick(data, ['airnode', 'encodedValue', 'templateId', 'timestamp'])
-    );
-    logger.info('Received valid signed data.', { data: sanitizedData });
+    // The logging of the data is delayed for security reasons - so people with access to the logs can't misuse the
+    // signed data.
+    setTimeout(() => {
+      const sanitizedData = batchSignedData.map((data) =>
+        pick(data, ['airnode', 'encodedValue', 'templateId', 'timestamp', 'signature'])
+      );
+      logger.info('Received valid signed data.', { data: sanitizedData });
+    }, LOG_API_DATA_DELAY_MS);
   }
 
   const newSignedData: InternalSignedData[] = [];

packages/signed-api/src/in-memory-cache.ts

@@ -1,7 +1,7 @@
 import { last, uniqBy } from 'lodash';
 
 import { logger } from './logger';
-import type { InternalSignedData } from './schema';
+import { type InternalSignedData } from './types';
 import { isIgnored } from './utils';
 
 type SignedDataCache = Record<

packages/signed-api/src/schema.ts

@@ -70,16 +70,6 @@ export const configSchema = z.strictObject({
 
 export type Config = z.infer<typeof configSchema>;
 
-export type InternalSignedData = {
-  airnode: string;
-  templateId: string;
-  beaconId: string;
-  timestamp: string;
-  encodedValue: string;
-  signature: string;
-  isOevBeacon: boolean;
-};
-
 export const envBooleanSchema = z.union([z.literal('true'), z.literal('false')]).transform((val) => val === 'true');
 
 // We apply default values to make it convenient to omit certain environment variables. The default values should be

packages/signed-api/src/signed-data-verifier.ts

@@ -1,7 +1,7 @@
 import { goSync } from '@api3/promise-utils';
 import workerpool from 'workerpool';
 
-import type { InternalSignedData } from './schema';
+import { type InternalSignedData } from './types';
 import { deriveBeaconId, recoverSignerAddress } from './utils';
 
 interface VerificationError {

packages/signed-api/src/transform-payload.ts

@@ -2,7 +2,7 @@ import { deriveOevTemplateId, type SignedApiBatchPayloadV1, type SignedApiBatchP
 import { deriveBeaconId, type Hex } from '@api3/commons';
 
 import { getCache, setCache } from './in-memory-cache';
-import { type InternalSignedData } from './schema';
+import { type InternalSignedData } from './types';
 
 export const getOevTemplateId = (templateId: string) => {
   const cache = getCache();

packages/signed-api/src/types.ts

@@ -1,5 +1,3 @@
-import { type InternalSignedData } from './schema';
-
 export interface ApiResponse {
   statusCode: number;
   headers: Record<string, string>;
@@ -22,3 +20,13 @@ export type PostSignedDataResponseSchema = {
 };
 
 export type GetListAirnodesResponseSchema = { count: number; 'available-airnodes': string[] };
+
+export type InternalSignedData = {
+  airnode: string;
+  templateId: string;
+  beaconId: string;
+  timestamp: string;
+  encodedValue: string;
+  signature: string;
+  isOevBeacon: boolean;
+};

packages/signed-api/test/utils.ts

@@ -2,7 +2,7 @@ import { deriveOevTemplateId, type SignedApiPayloadV1, type SignedApiPayloadV2 }
 import { ethers } from 'ethers';
 import { omit } from 'lodash';
 
-import type { InternalSignedData } from '../src/schema';
+import type { InternalSignedData } from '../src/types';
 import { deriveBeaconId } from '../src/utils';
 
 export const deriveTemplateId = (endpointId: string, encodedParameters: string) =>