Save lastTaskUUID in db & Slack notifications for errors

.env.example

@@ -23,3 +23,5 @@ CLIENT_CERT=
 CLIENT_KEY=
 
 ENABLE_GAS_CHECK=1
+
+SLACK_WEBHOOK_URL=

.env.test

@@ -26,3 +26,5 @@ CLIENT_CERT=test
 CLIENT_KEY=test
 
 ENABLE_GAS_CHECK=1
+
+SLACK_WEBHOOK_URL=

apps/axelar-event-processor/src/approvals-processor/approvals.processor.module.ts

@@ -1,4 +1,4 @@
-import { Module } from '@nestjs/common';
+import { forwardRef, Module } from '@nestjs/common';
 import { ApiConfigModule, DatabaseModule, DynamicModuleUtils } from '@mvx-monorepo/common';
 import { ApprovalsProcessorService } from './approvals.processor.service';
 import { ApiModule } from '@mvx-monorepo/common/api/api.module';
@@ -11,7 +11,7 @@ import { HelpersModule } from '@mvx-monorepo/common/helpers/helpers.module';
     ScheduleModule.forRoot(),
     ApiConfigModule,
     DynamicModuleUtils.getRedisCacheModule(),
-    ApiModule,
+    forwardRef(() => ApiModule),
     ContractsModule,
     DatabaseModule,
     HelpersModule,

apps/axelar-event-processor/src/approvals-processor/approvals.processor.service.ts

@@ -14,13 +14,18 @@ import { MessageApprovedRepository } from '@mvx-monorepo/common/database/reposit
 import { MessageApprovedStatus } from '@prisma/client';
 import { ApiNetworkProvider } from '@multiversx/sdk-network-providers/out';
 import BigNumber from 'bignumber.js';
+import { GasError } from '@mvx-monorepo/common/contracts/entities/gas.error';
+import { GasInfo } from '@mvx-monorepo/common/utils/gas.info';
+import { RedisHelper } from '@mvx-monorepo/common/helpers/redis.helper';
+import {
+  LAST_PROCESSED_DATA_TYPE,
+  LastProcessedDataRepository,
+} from '@mvx-monorepo/common/database/repository/last-processed-data.repository';
+import { SlackApi } from '@mvx-monorepo/common/api/slack.api';
 import TaskItem = Components.Schemas.TaskItem;
 import GatewayTransactionTask = Components.Schemas.GatewayTransactionTask;
 import ExecuteTask = Components.Schemas.ExecuteTask;
 import RefundTask = Components.Schemas.RefundTask;
-import { GasError } from '@mvx-monorepo/common/contracts/entities/gas.error';
-import { GasInfo } from '@mvx-monorepo/common/utils/gas.info';
-import { RedisHelper } from '@mvx-monorepo/common/helpers/redis.helper';
 
 const MAX_NUMBER_OF_RETRIES = 3;
 
@@ -35,13 +40,15 @@ export class ApprovalsProcessorService {
     private readonly transactionsHelper: TransactionsHelper,
     private readonly gatewayContract: GatewayContract,
     private readonly messageApprovedRepository: MessageApprovedRepository,
+    private readonly lastProcessedDataRepository: LastProcessedDataRepository,
     private readonly gasServiceContract: GasServiceContract,
     private readonly api: ApiNetworkProvider,
+    private readonly slackApi: SlackApi,
   ) {
     this.logger = new Logger(ApprovalsProcessorService.name);
   }
 
-  @Cron('0/20 * * * * *')
+  @Cron('1/15 * * * * *')
   async handleNewTasks() {
     await Locker.lock('handleNewTasks', this.handleNewTasksRaw.bind(this));
   }
@@ -52,7 +59,7 @@ export class ApprovalsProcessorService {
   }
 
   async handleNewTasksRaw() {
-    let lastTaskUUID = (await this.redisHelper.get<string>(CacheInfo.LastTaskUUID().key)) || undefined;
+    let lastTaskUUID = await this.lastProcessedDataRepository.get(LAST_PROCESSED_DATA_TYPE.LAST_TASK_ID);
 
     this.logger.debug(`Trying to process tasks for multiversx starting from id: ${lastTaskUUID}`);
 
@@ -76,9 +83,10 @@ export class ApprovalsProcessorService {
 
             lastTaskUUID = task.id;
 
-            await this.redisHelper.set(CacheInfo.LastTaskUUID().key, lastTaskUUID, CacheInfo.LastTaskUUID().ttl);
+            await this.lastProcessedDataRepository.update(LAST_PROCESSED_DATA_TYPE.LAST_TASK_ID, lastTaskUUID);
           } catch (e) {
             this.logger.error(`Could not process task ${task.id}`, task, e);
+            await this.slackApi.sendError('Task processing error', `Could not process task ${task.id}`);
 
             // Stop processing in case of an error and retry from the sam task
             return;
@@ -88,6 +96,10 @@ export class ApprovalsProcessorService {
         this.logger.debug(`Successfully processed ${tasks.length} tasks`);
       } catch (e) {
         this.logger.error('Error retrieving tasks...', e);
+        await this.slackApi.sendError(
+          'Task processing error',
+          `Error retrieving tasks... Last task UUID retrieved: ${lastTaskUUID}`,
+        );
 
         return;
       }
@@ -116,15 +128,22 @@ export class ApprovalsProcessorService {
 
       if (retry === MAX_NUMBER_OF_RETRIES) {
         this.logger.error(`Could not execute Gateway execute transaction with hash ${txHash} after ${retry} retries`);
+        await this.slackApi.sendError(
+          `Gateway transaction error`,
+          `Could not execute Gateway execute transaction with hash ${txHash} after ${retry} retries`,
+        );
 
         continue;
       }
 
       try {
         await this.processGatewayTxTask(externalData, retry);
       } catch (e) {
-        this.logger.error('Error while trying to retry transaction...');
-        this.logger.error(e);
+        this.logger.error('Error while trying to retry transaction...', e);
+        await this.slackApi.sendError(
+          `Gateway transaction retry error`,
+          'Error while trying to retry transaction... Transaction could not be sent to chain. Will be retried',
+        );
 
         // Set value back in cache to be retried again (with same retry number if it failed to even be sent to the chain)
         await this.redisHelper.set<PendingTransaction>(
@@ -192,6 +211,10 @@ export class ApprovalsProcessorService {
       // for transparency
       if (e instanceof GasError) {
         this.logger.warn('Could not estimate gas for Gateway transaction...', e);
+        await this.slackApi.sendWarn(
+          'Gas estimation error',
+          `Could not estimate gas for Gateway transaction... ${transaction.getHash()}`,
+        );
 
         transaction.setGasLimit(GasInfo.GatewayDefault.value);
       } else {
@@ -258,6 +281,11 @@ export class ApprovalsProcessorService {
           ` token ${response.remainingGasBalance.tokenID}, amount ${response.remainingGasBalance.amount}`,
         e,
       );
+      await this.slackApi.sendError(
+        `Refund task error`,
+        `Could not process refund for ${response.message.messageID} for account ${response.refundRecipientAddress},` +
+          ` token ${response.remainingGasBalance.tokenID}, amount ${response.remainingGasBalance.amount}`,
+      );
 
       return;
     }

apps/axelar-event-processor/src/approvals-processor/approvals.processor.spec.ts

@@ -20,6 +20,9 @@ import GatewayTransactionTask = Components.Schemas.GatewayTransactionTask;
 import TaskItem = Components.Schemas.TaskItem;
 import RefundTask = Components.Schemas.RefundTask;
 import ExecuteTask = Components.Schemas.ExecuteTask;
+import { LastProcessedDataRepository } from '@mvx-monorepo/common/database/repository/last-processed-data.repository';
+import { SlackApi } from '@mvx-monorepo/common/api/slack.api';
+import { TransactionHash } from '@multiversx/sdk-core/out/transaction';
 
 const mockExternalData = BinaryUtils.base64Encode('approveMessages@61726731@61726732');
 
@@ -30,8 +33,10 @@ describe('ApprovalsProcessorService', () => {
   let transactionsHelper: DeepMocked<TransactionsHelper>;
   let gatewayContract: DeepMocked<GatewayContract>;
   let messageApprovedRepository: DeepMocked<MessageApprovedRepository>;
+  let lastProcessedDataRepository: DeepMocked<LastProcessedDataRepository>;
   let gasServiceContract: DeepMocked<GasServiceContract>;
   let api: DeepMocked<ApiNetworkProvider>;
+  let slackApi: DeepMocked<SlackApi>;
 
   let service: ApprovalsProcessorService;
 
@@ -42,8 +47,10 @@ describe('ApprovalsProcessorService', () => {
     transactionsHelper = createMock();
     gatewayContract = createMock();
     messageApprovedRepository = createMock();
+    lastProcessedDataRepository = createMock();
     gasServiceContract = createMock();
     api = createMock();
+    slackApi = createMock();
 
     const moduleRef = await Test.createTestingModule({
       providers: [ApprovalsProcessorService],
@@ -73,6 +80,10 @@ describe('ApprovalsProcessorService', () => {
           return messageApprovedRepository;
         }
 
+        if (token === LastProcessedDataRepository) {
+          return lastProcessedDataRepository;
+        }
+
         if (token === GasServiceContract) {
           return gasServiceContract;
         }
@@ -81,11 +92,15 @@ describe('ApprovalsProcessorService', () => {
           return api;
         }
 
+        if (token === SlackApi) {
+          return slackApi;
+        }
+
         return null;
       })
       .compile();
 
-    redisHelper.get.mockImplementation(() => {
+    lastProcessedDataRepository.get.mockImplementation(() => {
       return Promise.resolve(undefined);
     });
 
@@ -140,11 +155,7 @@ describe('ApprovalsProcessorService', () => {
 
       expect(axelarGmpApi.getTasks).toHaveBeenCalledTimes(2);
       expect(axelarGmpApi.getTasks).toHaveBeenCalledWith('multiversx', undefined);
-      expect(redisHelper.set).toHaveBeenCalledWith(
-        CacheInfo.LastTaskUUID().key,
-        'lastUUID1',
-        CacheInfo.LastTaskUUID().ttl,
-      );
+      expect(lastProcessedDataRepository.update).toHaveBeenCalledWith('lastTaskUUID', 'lastUUID1');
     });
 
     it('Should handle gateway tx task', async () => {
@@ -178,7 +189,7 @@ describe('ApprovalsProcessorService', () => {
 
       await service.handleNewTasksRaw();
 
-      expect(redisHelper.get).toHaveBeenCalledTimes(1);
+      expect(lastProcessedDataRepository.get).toHaveBeenCalledTimes(1);
       expect(axelarGmpApi.getTasks).toHaveBeenCalledTimes(2);
       expect(axelarGmpApi.getTasks).toHaveBeenCalledWith('multiversx', undefined);
       expect(axelarGmpApi.getTasks).toHaveBeenCalledWith('multiversx', 'UUID');
@@ -196,7 +207,8 @@ describe('ApprovalsProcessorService', () => {
       expect(transactionsHelper.signAndSendTransaction).toHaveBeenCalledTimes(1);
       expect(transactionsHelper.signAndSendTransaction).toHaveBeenCalledWith(transaction, walletSigner);
 
-      expect(redisHelper.set).toHaveBeenCalledTimes(2);
+      expect(lastProcessedDataRepository.update).toHaveBeenCalledTimes(1);
+      expect(redisHelper.set).toHaveBeenCalledTimes(1);
       expect(redisHelper.set).toHaveBeenCalledWith(
         CacheInfo.PendingTransaction('txHash').key,
         {
@@ -207,7 +219,7 @@ describe('ApprovalsProcessorService', () => {
         CacheInfo.PendingTransaction('txHash').ttl,
       );
 
-      expect(redisHelper.set).toHaveBeenCalledWith(CacheInfo.LastTaskUUID().key, 'UUID', CacheInfo.LastTaskUUID().ttl);
+      expect(lastProcessedDataRepository.update).toHaveBeenCalledWith('lastTaskUUID', 'UUID');
     });
 
     it('Should handle gateway tx task error', async () => {
@@ -234,14 +246,15 @@ describe('ApprovalsProcessorService', () => {
       walletSigner.getAddress.mockReturnValueOnce(userAddress);
 
       const transaction: DeepMocked<Transaction> = createMock();
+      transaction.getHash.mockReturnValue(new TransactionHash('txHash'));
       gatewayContract.buildTransactionExternalFunction.mockReturnValueOnce(transaction);
 
       transactionsHelper.getTransactionGas.mockRejectedValue(new GasError());
       transactionsHelper.signAndSendTransaction.mockReturnValueOnce(Promise.resolve('txHash'));
 
       await service.handleNewTasksRaw();
 
-      expect(redisHelper.get).toHaveBeenCalledTimes(1);
+      expect(lastProcessedDataRepository.get).toHaveBeenCalledTimes(1);
       expect(axelarGmpApi.getTasks).toHaveBeenCalledTimes(2);
       expect(axelarGmpApi.getTasks).toHaveBeenCalledWith('multiversx', undefined);
       expect(axelarGmpApi.getTasks).toHaveBeenCalledWith('multiversx', 'UUID');
@@ -259,7 +272,8 @@ describe('ApprovalsProcessorService', () => {
       expect(transactionsHelper.signAndSendTransaction).toHaveBeenCalledTimes(1);
       expect(transactionsHelper.signAndSendTransaction).toHaveBeenCalledWith(transaction, walletSigner);
 
-      expect(redisHelper.set).toHaveBeenCalledTimes(2);
+      expect(lastProcessedDataRepository.update).toHaveBeenCalledTimes(1);
+      expect(redisHelper.set).toHaveBeenCalledTimes(1);
       expect(redisHelper.set).toHaveBeenCalledWith(
         CacheInfo.PendingTransaction('txHash').key,
         {
@@ -270,7 +284,7 @@ describe('ApprovalsProcessorService', () => {
         CacheInfo.PendingTransaction('txHash').ttl,
       );
 
-      expect(redisHelper.set).toHaveBeenCalledWith(CacheInfo.LastTaskUUID().key, 'UUID', CacheInfo.LastTaskUUID().ttl);
+      expect(lastProcessedDataRepository.update).toHaveBeenCalledWith('lastTaskUUID', 'UUID');
     });
 
     it('Should handle execute task', async () => {
@@ -318,7 +332,7 @@ describe('ApprovalsProcessorService', () => {
         taskItemId: 'UUID',
         availableGasBalance: '100',
       });
-      expect(redisHelper.set).toHaveBeenCalledTimes(1);
+      expect(lastProcessedDataRepository.update).toHaveBeenCalledTimes(1);
     });
 
     it('Should handle execute task invalid gas token', async () => {
@@ -367,7 +381,7 @@ describe('ApprovalsProcessorService', () => {
         taskItemId: 'UUID',
         availableGasBalance: '0',
       });
-      expect(redisHelper.set).toHaveBeenCalledTimes(1);
+      expect(lastProcessedDataRepository.update).toHaveBeenCalledTimes(1);
     });
 
     it('Should not save last task uuid if error', async () => {
@@ -404,14 +418,14 @@ describe('ApprovalsProcessorService', () => {
       expect(redisHelper.set).not.toHaveBeenCalled();
 
       // Mock lastUUID
-      redisHelper.get.mockImplementation(() => {
+      lastProcessedDataRepository.get.mockImplementation(() => {
         return Promise.resolve('lastUUID1');
       });
 
       // Will start processing tasks from lastUUID1
       await service.handleNewTasksRaw();
 
-      expect(redisHelper.get).toHaveBeenCalledTimes(2);
+      expect(lastProcessedDataRepository.get).toHaveBeenCalledTimes(2);
       expect(axelarGmpApi.getTasks).toHaveBeenCalledTimes(2);
       expect(axelarGmpApi.getTasks).toHaveBeenCalledWith('multiversx', 'lastUUID1');
     });
@@ -573,7 +587,7 @@ describe('ApprovalsProcessorService', () => {
 
   describe('processRefundTask', () => {
     function assertRefundSuccess(userAddress: UserAddress, transaction: Transaction, token: string) {
-      expect(redisHelper.set).toHaveBeenCalledTimes(1);
+      expect(lastProcessedDataRepository.update).toHaveBeenCalledTimes(1);
       expect(axelarGmpApi.getTasks).toHaveBeenCalledTimes(2);
       expect(axelarGmpApi.getTasks).toHaveBeenCalledWith('multiversx', undefined);
       expect(axelarGmpApi.getTasks).toHaveBeenCalledWith('multiversx', 'UUID');
@@ -738,7 +752,7 @@ describe('ApprovalsProcessorService', () => {
 
       expect(api.getAccount).toHaveBeenCalledTimes(1);
 
-      expect(redisHelper.set).toHaveBeenCalledTimes(1);
+      expect(lastProcessedDataRepository.update).toHaveBeenCalledTimes(1);
       expect(axelarGmpApi.getTasks).toHaveBeenCalledTimes(2);
 
       expect(gasServiceContract.refund).not.toHaveBeenCalled();
@@ -790,7 +804,7 @@ describe('ApprovalsProcessorService', () => {
 
       expect(api.getFungibleTokenOfAccount).toHaveBeenCalledTimes(1);
 
-      expect(redisHelper.set).toHaveBeenCalledTimes(1);
+      expect(lastProcessedDataRepository.update).toHaveBeenCalledTimes(1);
       expect(axelarGmpApi.getTasks).toHaveBeenCalledTimes(2);
 
       expect(gasServiceContract.refund).not.toHaveBeenCalled();

apps/mvx-event-processor/src/cross-chain-transaction-processor/cross-chain-transaction.processor.module.ts

@@ -1,4 +1,4 @@
-import { Module } from '@nestjs/common';
+import { forwardRef, Module } from '@nestjs/common';
 import { ScheduleModule } from '@nestjs/schedule';
 import { ApiConfigModule, ApiModule, ContractsModule, DatabaseModule } from '@mvx-monorepo/common';
 import { CrossChainTransactionProcessorService } from './cross-chain-transaction.processor.service';
@@ -9,7 +9,7 @@ import { ProcessorsModule } from './processors';
   imports: [
     ScheduleModule.forRoot(),
     DatabaseModule,
-    ApiModule,
+    forwardRef(() => ApiModule),
     HelpersModule,
     ContractsModule,
     ApiConfigModule,