address cmts

indexer/packages/base/src/constants.ts

@@ -13,3 +13,4 @@ export const FIVE_MINUTES_IN_MILLISECONDS: number = 5 * ONE_MINUTE_IN_MILLISECON
 export const TEN_MINUTES_IN_MILLISECONDS: number = 10 * ONE_MINUTE_IN_MILLISECONDS;
 export const ONE_HOUR_IN_MILLISECONDS: number = 60 * ONE_MINUTE_IN_MILLISECONDS;
 export const FOUR_HOURS_IN_MILLISECONDS: number = 4 * ONE_HOUR_IN_MILLISECONDS;
+export const ONE_DAY_IN_MILLISECONDS: number = 24 * ONE_HOUR_IN_MILLISECONDS;

indexer/services/roundtable/__tests__/tasks/take-fast-sync-snapshot.test.ts

@@ -2,7 +2,6 @@ import config from '../../src/config';
 import { asMock } from '@dydxprotocol-indexer/dev';
 import {
   createDBSnapshot,
-  deleteOldFastSyncSnapshots,
   getMostRecentDBSnapshotIdentifier,
 } from '../../src/helpers/aws';
 import takeFastSyncSnapshotTask from '../../src/tasks/take-fast-sync-snapshot';
@@ -29,7 +28,6 @@ describe('fast-sync-export-db-snapshot', () => {
     await takeFastSyncSnapshotTask();
 
     expect(createDBSnapshot).toHaveBeenCalled();
-    expect(deleteOldFastSyncSnapshots).toHaveBeenCalled();
   });
 
   it('Last snapshot was taken less than interval ago', async () => {
@@ -41,7 +39,6 @@ describe('fast-sync-export-db-snapshot', () => {
     await takeFastSyncSnapshotTask();
 
     expect(createDBSnapshot).not.toHaveBeenCalled();
-    expect(deleteOldFastSyncSnapshots).not.toHaveBeenCalled();
   });
 
   it('No existing snapshot', async () => {
@@ -52,6 +49,5 @@ describe('fast-sync-export-db-snapshot', () => {
     await takeFastSyncSnapshotTask();
 
     expect(createDBSnapshot).toHaveBeenCalled();
-    expect(deleteOldFastSyncSnapshots).toHaveBeenCalled();
   });
 });

indexer/services/roundtable/src/config.ts

@@ -16,6 +16,7 @@ import {
   ONE_SECOND_IN_MILLISECONDS,
   TEN_SECONDS_IN_MILLISECONDS,
   FOUR_HOURS_IN_MILLISECONDS,
+  ONE_DAY_IN_MILLISECONDS,
 } from '@dydxprotocol-indexer/base';
 import {
   kafkaConfigSchema,
@@ -42,6 +43,7 @@ export const configSchema = {
   LOOPS_CANCEL_STALE_ORDERS: parseBoolean({ default: true }),
   LOOPS_ENABLED_UPDATE_RESEARCH_ENVIRONMENT: parseBoolean({ default: false }),
   LOOPS_ENABLED_TAKE_FAST_SYNC_SNAPSHOTS: parseBoolean({ default: true }),
+  LOOPS_ENABLED_DELETE_OLD_FAST_SYNC_SNAPSHOTS: parseBoolean({ default: true }),
   LOOPS_ENABLED_TRACK_LAG: parseBoolean({ default: false }),
   LOOPS_ENABLED_REMOVE_OLD_ORDER_UPDATES: parseBoolean({ default: true }),
   LOOPS_ENABLED_AGGREGATE_TRADING_REWARDS: parseBoolean({ default: true }),
@@ -71,6 +73,9 @@ export const configSchema = {
   LOOPS_INTERVAL_MS_TAKE_FAST_SYNC_SNAPSHOTS: parseInteger({
     default: FOUR_HOURS_IN_MILLISECONDS,
   }),
+  LOOPS_INTERVAL_MS_DELETE_OLD_FAST_SYNC_SNAPSHOTS: parseInteger({
+    default: ONE_DAY_IN_MILLISECONDS,
+  }),
   LOOPS_INTERVAL_MS_UPDATE_COMPLIANCE_DATA: parseInteger({
     default: FIVE_MINUTES_IN_MILLISECONDS,
   }),

indexer/services/roundtable/src/index.ts

@@ -11,6 +11,7 @@ import {
 import aggregateTradingRewardsTasks from './tasks/aggregate-trading-rewards';
 import cancelStaleOrdersTask from './tasks/cancel-stale-orders';
 import createPnlTicksTask from './tasks/create-pnl-ticks';
+import deleteOldFastSyncSnapshots from './tasks/delete-old-fast-sync-snapshots';
 import deleteZeroPriceLevelsTask from './tasks/delete-zero-price-levels';
 import marketUpdaterTask from './tasks/market-updater';
 import orderbookInstrumentationTask from './tasks/orderbook-instrumentation';
@@ -109,6 +110,14 @@ async function start(): Promise<void> {
     );
   }
 
+  if (config.LOOPS_ENABLED_DELETE_OLD_FAST_SYNC_SNAPSHOTS) {
+    startLoop(
+      deleteOldFastSyncSnapshots,
+      'delete_old_fast_sync_snapshots',
+      config.LOOPS_INTERVAL_MS_DELETE_OLD_FAST_SYNC_SNAPSHOTS,
+    );
+  }
+
   startLoop(
     () => updateComplianceDataTask(complianceProvider),
     'update_compliance_data',

indexer/services/roundtable/src/tasks/delete-old-fast-sync-snapshots.ts

@@ -0,0 +1,19 @@
+import { logger, stats } from '@dydxprotocol-indexer/base';
+import RDS from 'aws-sdk/clients/rds';
+
+import config from '../config';
+import { deleteOldFastSyncSnapshots } from '../helpers/aws';
+
+const statStart: string = `${config.SERVICE_NAME}.delete_old_fast_sync_snapshots`;
+
+export default async function runTask(): Promise<void> {
+  const at: string = 'delete-old-fast-sync-snapshots#runTask';
+  logger.info({ at, message: 'Starting task.' });
+
+  const rds: RDS = new RDS();
+
+  const startDeleteOldSnapshot: number = Date.now();
+  // Delete old snapshots.
+  await deleteOldFastSyncSnapshots(rds);
+  stats.timing(`${statStart}.deleteOldSnapshots`, Date.now() - startDeleteOldSnapshot);
+}

indexer/services/roundtable/src/tasks/take-fast-sync-snapshot.ts

@@ -5,40 +5,11 @@ import { DateTime } from 'luxon';
 import config from '../config';
 import {
   createDBSnapshot,
-  deleteOldFastSyncSnapshots,
   getMostRecentDBSnapshotIdentifier,
 } from '../helpers/aws';
 
 const statStart: string = `${config.SERVICE_NAME}.fast_sync_export_db_snapshot`;
 
-/**
- * Checks if the difference between two dates is less than a given interval.
- *
- * @param startDate
- * @param endDate
- * @param intervalMs
- */
-function isDifferenceLessThanInterval(
-  startDate: string,
-  endDate: string,
-  intervalMs: number,
-): boolean {
-  const parseDateString = (dateStr: string): Date => {
-    const [year, month, day, hour, minute] = dateStr.split('-').map(Number);
-    return new Date(year, month, day, hour, minute);
-  };
-
-  // Parse the date strings
-  const parsedDate1 = parseDateString(startDate);
-  const parsedDate2 = parseDateString(endDate);
-
-  // Calculate the difference in milliseconds
-  const differenceInMilliseconds = Math.abs(parsedDate1.getTime() - parsedDate2.getTime());
-
-  // Compare with the interval
-  return differenceInMilliseconds < intervalMs;
-}
-
 export default async function runTask(): Promise<void> {
   const at: string = 'fast-sync-export-db-snapshot#runTask';
   logger.info({ at, message: 'Starting task.' });
@@ -66,6 +37,8 @@ export default async function runTask(): Promise<void> {
         at,
         message: 'Last fast sync db snapshot was taken less than the interval ago',
         interval: config.LOOPS_INTERVAL_MS_TAKE_FAST_SYNC_SNAPSHOTS,
+        currentDate: dateString,
+        lastSnapshotDate: s3Date,
       });
       return;
     }
@@ -76,8 +49,32 @@ export default async function runTask(): Promise<void> {
   createDBSnapshot(rds, snapshotIdentifier, config.RDS_INSTANCE_NAME);
   logger.info({ at, message: 'Created DB snapshot.', snapshotIdentifier: createdSnapshotIdentifier });
   stats.timing(`${statStart}.createDbSnapshot`, Date.now() - startSnapshot);
-  const startDeleteOldSnapshot: number = Date.now();
-  // Delete old snapshots.
-  await deleteOldFastSyncSnapshots(rds);
-  stats.timing(`${statStart}.deleteOldSnapshots`, Date.now() - startDeleteOldSnapshot);
+}
+
+/**
+ * Checks if the difference between two dates is less than a given interval.
+ *
+ * @param startDate
+ * @param endDate
+ * @param intervalMs
+ */
+function isDifferenceLessThanInterval(
+  startDate: string,
+  endDate: string,
+  intervalMs: number,
+): boolean {
+  const parseDateString = (dateStr: string): Date => {
+    const [year, month, day, hour, minute] = dateStr.split('-').map(Number);
+    return new Date(year, month, day, hour, minute);
+  };
+
+  // Parse the date strings
+  const parsedDate1 = parseDateString(startDate);
+  const parsedDate2 = parseDateString(endDate);
+
+  // Calculate the difference in milliseconds
+  const differenceInMilliseconds = Math.abs(parsedDate1.getTime() - parsedDate2.getTime());
+
+  // Compare with the interval
+  return differenceInMilliseconds < intervalMs;
 }