feat: voiding and removing transaction on vertex_removed event

packages/daemon/__tests__/integration/balances.test.ts

@@ -13,6 +13,7 @@ import { cleanDatabase, fetchAddressBalances, validateBalances } from './utils';
 import unvoidedScenarioBalances from './scenario_configs/unvoided_transactions.balances';
 import reorgScenarioBalances from './scenario_configs/reorg.balances';
 import singleChainBlocksAndTransactionsBalances from './scenario_configs/single_chain_blocks_and_transactions.balances';
+import invalidMempoolBalances from './scenario_configs/invalid_mempool_transaction.balances';
 import {
   DB_NAME,
   DB_USER,
@@ -252,13 +253,14 @@ describe('invalid mempool transactions scenario', () => {
 
     await new Promise<void>((resolve) => {
       machine.onTransition(async (state) => {
+        const addressBalances = await fetchAddressBalances(mysql);
         if (state.matches('CONNECTED.idle')) {
           // @ts-ignore
           const lastSyncedEvent = await getLastSyncedEvent(mysql);
+          console.log(lastSyncedEvent);
           if (lastSyncedEvent?.last_event_id === INVALID_MEMPOOL_TRANSACTION_LAST_EVENT) {
-            const addressBalances = await fetchAddressBalances(mysql);
             // @ts-ignore
-            expect(validateBalances(addressBalances, singleChainBlocksAndTransactionsBalances));
+            expect(validateBalances(addressBalances, invalidMempoolBalances));
 
             machine.stop();
 

packages/daemon/__tests__/integration/scenario_configs/invalid_mempool_transaction.balances.ts

@@ -1,4 +1,5 @@
 export default {
+  "HVayMofEDh4XGsaQJeRJKhutYxYodYNop6": 100000000000,
   "HFtz2f59Lms4p3Jfgtsr73s97MbJHsRENh": 6400,
   "HJQbEERnD5Ak3f2dsi8zAmsZrCWTT8FZns": 6400,
   "HRSYchTEsFFpZAkgSTMsohNGQ6eLPyhXvJ": 6400,
@@ -7,10 +8,8 @@ export default {
   "HPNvtPZaDF44i6CL91u4BvZPu6z2xPNt26": 6400,
   "HQijr325t63VJFdc4vYkaTyd87oeBLpSed": 6400,
   "H8fCNrYGkj4B6VzKgtRiHBgoSxM31d65JR": 6400,
-  "HAqrADnn7GyyT68fSX8zmtsRNFyabPzoRQ": 6400,
-  "HRTH6uGo7zn3LWrosBYn7eXkwAeHAHTRh8": 6400,
-  "H9hHteu9QdAS5p6X743Mpfue6G19rV9GeY": 6400,
-  "HNqTfEASfdx7H4vMUGzfD2HyD3GeKuxjTJ": 5400,
-  "HRQe4CXj8AZXzSmuNztU8iQR74QTQMbnTs": 1000,
-  "HVayMofEDh4XGsaQJeRJKhutYxYodYNop6": 100000000000
+  "HRQe4CXj8AZXzSmuNztU8iQR74QTQMbnTs": 6400,
+  "HNqTfEASfdx7H4vMUGzfD2HyD3GeKuxjTJ": 0,
+  "HAqrADnn7GyyT68fSX8zmtsRNFyabPzoRQ": 0,
+  "HRTH6uGo7zn3LWrosBYn7eXkwAeHAHTRh8": 0
 }

packages/daemon/__tests__/integration/scenario_configs/unvoided_transactions.balances.ts

@@ -13,4 +13,4 @@ export default {
   "HNqTfEASfdx7H4vMUGzfD2HyD3GeKuxjTJ": 5400,
   "HRQe4CXj8AZXzSmuNztU8iQR74QTQMbnTs": 1000,
   "HVayMofEDh4XGsaQJeRJKhutYxYodYNop6": 100000000000
-}
+};

packages/daemon/__tests__/integration/scripts/docker-compose.yml

@@ -37,7 +37,7 @@ services:
     ports:
       - "8083:8080"
   invalid_mempool_transaction:
-    image: hathornetwork/hathor-core:v0.62.0-rc.1
+    image: hathornetwork/hathor-core:stable
     command: [
       "events_simulator",
       "--scenario", "INVALID_MEMPOOL_TRANSACTION",

packages/daemon/__tests__/integration/utils/index.ts

@@ -70,7 +70,7 @@ export const validateBalances = async (
     if (totalBalanceA !== balanceB) {
       console.log(totalBalanceA);
       console.log(balanceB);
-      throw new Error(`Balances are not equal for address: ${address}`);
+      throw new Error(`Balances are not equal for address: ${address}, expected: ${balanceB}, received: ${totalBalanceA}`);
     }
   }
 };

packages/daemon/package.json

@@ -14,8 +14,8 @@
     "build": "tsc -b",
     "start": "node dist/index.js",
     "watch": "tsc -w",
-    "test_images_up": "docker-compose -f ./__tests__/integration/scripts/docker-compose.yml up -d",
-    "test_images_down": "docker-compose -f ./__tests__/integration/scripts/docker-compose.yml down",
+    "test_images_up": "docker compose -f ./__tests__/integration/scripts/docker-compose.yml up -d",
+    "test_images_down": "docker compose -f ./__tests__/integration/scripts/docker-compose.yml down",
     "test_images_integration": "jest --config ./jest_integration.config.js --runInBand --forceExit",
     "test_images_migrate": "NODE_ENV=test DB_NAME=hathor DB_PORT=3380 DB_PASS=hathor DB_USER=hathor yarn run sequelize-cli --migrations-path ../../db/migrations --config ./__tests__/integration/scripts/sequelize-db-config.js db:migrate",
     "test_images_wait_for_db": "yarn dlx ts-node ./__tests__/integration/scripts/wait-for-db-up.ts",

packages/daemon/src/actions/index.ts

@@ -6,7 +6,7 @@
  */
 
 import { assign, AssignAction, raise, sendTo } from 'xstate';
-import { CommonEventData, Context, Event, EventTypes } from '../types';
+import { Context, Event, EventTypes } from '../types';
 import { get } from 'lodash';
 import logger from '../logger';
 import { hashTxData } from '../utils';
@@ -168,7 +168,7 @@ export const updateCache = (context: Context) => {
   if (!fullNodeEvent) {
     return;
   }
-  const { metadata, hash }  = fullNodeEvent.event.data as CommonEventData;
+  const { metadata, hash }  = fullNodeEvent.event.data;
   const hashedTxData = hashTxData(metadata);
 
   context.txCache.set(hash, hashedTxData);

packages/daemon/src/guards/index.ts

@@ -5,7 +5,7 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import { CommonEventData, Context, Event, EventTypes, FullNodeEventTypes } from '../types';
+import { Context, Event, EventTypes, FullNodeEventTypes } from '../types';
 import { hashTxData } from '../utils';
 import { METADATA_DIFF_EVENT_TYPES } from '../services';
 import getConfig from '../config';
@@ -173,8 +173,8 @@ export const websocketDisconnected = (_context: Context, event: Event) => {
 
 /*
  * This guard is used in the `idle` state to detect if the transaction in the
- * received event is voided, this can serve many functions, one of them is to
- * ignore transactions that we don't have on our database but are already voided
+ * received event is a vertex removed event, indicating that we should remove
+ * the transaction from our database
  */
 export const vertexRemoved = (_context: Context, event: Event) => {
   if (event.type !== EventTypes.FULLNODE_EVENT) {
@@ -186,8 +186,8 @@ export const vertexRemoved = (_context: Context, event: Event) => {
 
 /*
  * This guard is used in the `idle` state to detect if the transaction in the
- * received event is a vertex removed event, indicating that we should remove
- * the transaction from our database
+ * received event is voided, this can serve many functions, one of them is to
+ * ignore transactions that we don't have on our database but are already voided
  */
 export const voided = (_context: Context, event: Event) => {
   if (event.type !== EventTypes.FULLNODE_EVENT) {
@@ -200,7 +200,7 @@ export const voided = (_context: Context, event: Event) => {
   }
 
   const fullNodeEvent = event.event.event;
-  const { metadata: { voided_by } } = fullNodeEvent.data as CommonEventData;
+  const { metadata: { voided_by } } = fullNodeEvent.data;
 
   return voided_by.length > 0;
 };
@@ -227,13 +227,13 @@ export const unchanged = (context: Context, event: Event) => {
   const { data } = event.event.event;
 
   const txCache = context.txCache;
-  const txHashFromCache = txCache.get((data as CommonEventData).hash);
+  const txHashFromCache = txCache.get(data.hash);
   // Not on the cache, it's not unchanged.
   if (!txHashFromCache) {
     return false;
   }
 
-  const txHashFromEvent = hashTxData((data as CommonEventData).metadata);
+  const txHashFromEvent = hashTxData(data.metadata);
 
   return txHashFromCache === txHashFromEvent;
 };

packages/daemon/src/machines/SyncMachine.ts

@@ -18,6 +18,7 @@ import {
 } from '../types';
 import {
   handleVertexAccepted,
+  handleVertexRemoved,
   metadataDiff,
   handleVoidedTx,
   handleTxFirstBlock,
@@ -223,7 +224,7 @@ const SyncMachine = Machine<Context, any, Event>({
             data: (_context: Context, event: Event) => event,
             onDone: {
               target: 'idle',
-              actions: ['sendAck', 'storeEvent', 'updateCache'],
+              actions: ['sendAck', 'storeEvent'],
             },
             onError: `#${SYNC_MACHINE_STATES.ERROR}`,
           },
@@ -317,6 +318,7 @@ const SyncMachine = Machine<Context, any, Event>({
     handleVoidedTx,
     handleUnvoidedTx,
     handleVertexAccepted,
+    handleVertexRemoved,
     handleTxFirstBlock,
     metadataDiff,
     updateLastSyncedEvent,

packages/daemon/src/services/index.ts

@@ -7,6 +7,7 @@
 
 // @ts-ignore
 import hathorLib from '@hathor/wallet-lib';
+import { Connection as MysqlConnection } from 'mysql2/promise';
 import axios from 'axios';
 import { get } from 'lodash';
 import { NftUtils } from '@wallet-service/common';
@@ -19,7 +20,8 @@ import {
   Event,
   Context,
   FullNodeEvent,
-  FullNodeEventTypes,
+  EventTxInput,
+  EventTxOutput,
 } from '../types';
 import {
   TxInput,
@@ -80,7 +82,7 @@ export const metadataDiff = async (_context: Context, event: Event) => {
   const mysql = await getDbConnection();
 
   try {
-    const fullNodeEvent = event.event as FullNodeEvent<FullNodeEventTypes.VERTEX_METADATA_CHANGED>;
+    const fullNodeEvent = event.event as FullNodeEvent;
     const {
       hash,
       metadata: { voided_by, first_block },
@@ -162,7 +164,7 @@ export const handleVertexAccepted = async (context: Context, _event: Event) => {
   await mysql.beginTransaction();
 
   try {
-    const fullNodeEvent = context.event as FullNodeEvent<FullNodeEventTypes.NEW_VERTEX_ACCEPTED>;
+    const fullNodeEvent = context.event as FullNodeEvent;
     const now = getUnixTimestamp();
     const { NEW_TX_SQS, PUSH_NOTIFICATION_ENABLED } = getConfig();
     const blockRewardLock = context.rewardMinBlocks;
@@ -385,19 +387,34 @@ export const handleVertexRemoved = async (context: Context, _event: Event) => {
   await mysql.beginTransaction();
 
   try {
-    const fullNodeEvent = context.event as FullNodeEvent<FullNodeEventTypes.VERTEX_REMOVED>;
-    const now = getUnixTimestamp();
+    const fullNodeEvent = context.event as FullNodeEvent;
 
-    const { vertex_id: hash } = fullNodeEvent.event.data;
+    const {
+      hash,
+      outputs,
+      inputs,
+      tokens,
+    } = fullNodeEvent.event.data;
 
     const dbTx: DbTransaction | null = await getTransactionById(mysql, hash);
 
     if (!dbTx) {
       throw new Error(`VERTEX_REMOVED event received, but transaction ${hash} was not in the database.`);
     }
 
-    const dbTxOutputs: DbTxOutput[] = await getTxOutputsFromTx(mysql, hash);
+    logger.info(`[VertexRemoved] Voiding tx: ${hash}`);
+    await voidTx(
+      mysql,
+      hash,
+      inputs,
+      outputs,
+      tokens,
+    );
 
+    logger.info(`[VertexRemoved] Removing tx from database: ${hash}`);
+    await cleanupVoidedTx(mysql, hash);
+    await dbUpdateLastSyncedEvent(mysql, fullNodeEvent.event.id);
+    await mysql.commit();
   } catch (e) {
     logger.debug(e);
     await mysql.rollback();
@@ -408,12 +425,44 @@ export const handleVertexRemoved = async (context: Context, _event: Event) => {
   }
 };
 
+export const voidTx = async (
+  mysql: MysqlConnection,
+  hash: string,
+  inputs: EventTxInput[],
+  outputs: EventTxOutput[],
+  tokens: string[],
+) => {
+  const dbTxOutputs: DbTxOutput[] = await getTxOutputsFromTx(mysql, hash);
+  const txOutputs: TxOutputWithIndex[] = prepareOutputs(outputs, tokens);
+  const txInputs: TxInput[] = prepareInputs(inputs, tokens);
+
+  const txOutputsWithLocked = txOutputs.map((output) => {
+    const dbTxOutput = dbTxOutputs.find((_output) => _output.index === output.index);
+
+    if (!dbTxOutput) {
+      throw new Error('Transaction output different from database output!');
+    }
+
+    return {
+      ...output,
+      locked: dbTxOutput.locked,
+    };
+  });
+
+  const addressBalanceMap: StringMap<TokenBalanceMap> = getAddressBalanceMap(txInputs, txOutputsWithLocked);
+  await voidTransaction(mysql, hash, addressBalanceMap);
+  await markUtxosAsVoided(mysql, dbTxOutputs);
+
+  const addresses = Object.keys(addressBalanceMap);
+  await validateAddressBalances(mysql, addresses);
+};
+
 export const handleVoidedTx = async (context: Context) => {
   const mysql = await getDbConnection();
   await mysql.beginTransaction();
 
   try {
-    const fullNodeEvent = context.event as FullNodeEvent<FullNodeEventTypes.VERTEX_METADATA_CHANGED>;
+    const fullNodeEvent = context.event as FullNodeEvent;
 
     const {
       hash,
@@ -423,36 +472,17 @@ export const handleVoidedTx = async (context: Context) => {
     } = fullNodeEvent.event.data;
 
     logger.debug(`Will handle voided tx for ${hash}`);
-
-    const dbTxOutputs: DbTxOutput[] = await getTxOutputsFromTx(mysql, hash);
-    const txOutputs: TxOutputWithIndex[] = prepareOutputs(outputs, tokens);
-    const txInputs: TxInput[] = prepareInputs(inputs, tokens);
-
-    const txOutputsWithLocked = txOutputs.map((output) => {
-      const dbTxOutput = dbTxOutputs.find((_output) => _output.index === output.index);
-
-      if (!dbTxOutput) {
-        throw new Error('Transaction output different from database output!');
-      }
-
-      return {
-        ...output,
-        locked: dbTxOutput.locked,
-      };
-    });
-
-    const addressBalanceMap: StringMap<TokenBalanceMap> = getAddressBalanceMap(txInputs, txOutputsWithLocked);
-    await voidTransaction(mysql, hash, addressBalanceMap);
-    await markUtxosAsVoided(mysql, dbTxOutputs);
-
-    const addresses = Object.keys(addressBalanceMap);
-    await validateAddressBalances(mysql, addresses);
-
-    await dbUpdateLastSyncedEvent(mysql, fullNodeEvent.event.id);
-
+    await voidTx(
+      mysql,
+      hash,
+      inputs,
+      outputs,
+      tokens
+    );
     logger.debug(`Voided tx ${hash}`);
 
     await mysql.commit();
+    await dbUpdateLastSyncedEvent(mysql, fullNodeEvent.event.id);
   } catch (e) {
     logger.debug(e);
     await mysql.rollback();
@@ -468,7 +498,7 @@ export const handleUnvoidedTx = async (context: Context) => {
   await mysql.beginTransaction();
 
   try {
-    const fullNodeEvent = context.event as FullNodeEvent<FullNodeEventTypes.VERTEX_METADATA_CHANGED>;
+    const fullNodeEvent = context.event as FullNodeEvent;
 
     const { hash } = fullNodeEvent.event.data;
 
@@ -494,7 +524,7 @@ export const handleTxFirstBlock = async (context: Context) => {
   await mysql.beginTransaction();
 
   try {
-    const fullNodeEvent = context.event as FullNodeEvent<FullNodeEventTypes.VERTEX_METADATA_CHANGED>;
+    const fullNodeEvent = context.event as FullNodeEvent;
 
     const {
       hash,