parametrize sql timeouts (#95)

b310-digital · Sep 24, 2022 · 720fd44 · 720fd44
1 parent c0cdd0a
commit 720fd44
Show file tree

Hide file tree

Showing 11 changed files with 71 additions and 15 deletions.
diff --git a/.env.default b/.env.default
@@ -28,4 +28,7 @@ POSTGRES_PROD_USER=
 # PROD Requires SSL Connection Support. Use DEV if not available.
 PROD_MODE=PROD
 
+POSTGRES_QUERY_TIMEOUT=100000
+POSTGRES_STATEMENT_TIMEOUT=100000
+
 DELETE_AFTER_DAYS=30
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -79,3 +79,5 @@ jobs:
                   POSTGRES_TEST_PORT: ${{ job.services.postgres.ports[5432] }}
                   POSTGRES_TEST_USER: "teammapper-user"
                   POSTGRES_USER: "teammapper-user"
+                  POSTGRES_QUERY_TIMEOUT: 10000
+                  POSTGRES_STATEMENT_TIMEOUT: 10000
diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml
@@ -16,6 +16,8 @@ services:
       POSTGRES_SSL: ${POSTGRES_PROD_SSL:-true}
       POSTGRES_SSL_REJECT_UNAUTHORIZED: ${POSTGRES_PROD_SSL_REJECT_UNAUTHORIZED:-false}
       POSTGRES_USER: ${POSTGRES_PROD_USER:-teammapper-user}
+      POSTGRES_QUERY_TIMEOUT: ${POSTGRES_QUERY_TIMEOUT:-100000}
+      POSTGRES_STATEMENT_TIMEOUT: ${POSTGRES_STATEMENT_TIMEOUT:-100000}
       DELETE_AFTER_DAYS: ${DELETE_AFTER_DAYS:-30}
     ports:
       - "${APP_PROD_PORT:-80}:3000"

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -17,6 +17,8 @@ services:
       POSTGRES_PORT: ${POSTGRES_PORT:-5432}
       POSTGRES_USER: ${POSTGRES_USER:-teammapper-user}
       POSTGRES_SSL: ${POSTGRES_SSL:-false}
+      POSTGRES_QUERY_TIMEOUT: ${POSTGRES_QUERY_TIMEOUT:-100000}
+      POSTGRES_STATEMENT_TIMEOUT: ${POSTGRES_STATEMENT_TIMEOUT:-100000}
       POSTGRES_TEST_DATABASE: ${DOCKER_COMPOSE_APP_ENV_POSTGRES_TEST_DATABASE:-teammapper-backend-test}
       POSTGRES_TEST_HOST: ${DOCKER_COMPOSE_APP_ENV_POSTGRES_HOST:-postgres}
       POSTGRES_TEST_PASSWORD: ${DOCKER_COMPOSE_APP_ENV_POSTGRES_PASSWORD:-teammapper-password}

diff --git a/teammapper-backend/src/config.service.ts b/teammapper-backend/src/config.service.ts
@@ -62,8 +62,8 @@ class ConfigService {
       },
 
       extra: {
-        query_timeout: 20000,
-        statement_timeout: 20000,
+        query_timeout: this.getValue('POSTGRES_QUERY_TIMEOUT') || 100000,
+        statement_timeout: this.getValue('POSTGRES_STATEMENT_TIMEOUT') || 100000,
       },
 
       synchronize: !this.isProduction(),

diff --git a/teammapper-backend/src/jobs/seedMapData.job.ts b/teammapper-backend/src/jobs/seedMapData.job.ts
@@ -51,13 +51,17 @@ async function bootstrap() {
 
   logger.log('--- Creating maps ... ---');
 
-  for (let i = 0; i < 5000; i++) {
+  for (let i = 0; i < 100000; i++) {
     const rootNode: IMmpClientNode = createNode(true, '', 0, 0);
     const childNode: IMmpClientNode = createNode(false, rootNode.id, 150, 150);
     const secondChildNode: IMmpClientNode = createNode(false, childNode.id, 250, 250);
-    const mapData: IMmpClientMap = createMap([rootNode, childNode, secondChildNode]);
+    const thirdChildNode: IMmpClientNode = createNode(false, childNode.id, 350, 350);
+    const fourthChildNode: IMmpClientNode = createNode(false, childNode.id, 450, 450);
+    const fifthChildNode: IMmpClientNode = createNode(false, childNode.id, 550, 550);
+    const mapData: IMmpClientMap = createMap([rootNode, childNode, secondChildNode, thirdChildNode, fourthChildNode, fifthChildNode]);
     await mapsService.createMap(mapData);
-    logger.log(`--- Map created with id ${mapData.uuid} ---`);
+
+    if (i % 500 == 0) logger.log(`--- Map created with id ${mapData.uuid} ---`);
   }
 
   logger.log('--- Finished creating maps ---');

diff --git a/teammapper-backend/src/map/controllers/maps.gateway.ts b/teammapper-backend/src/map/controllers/maps.gateway.ts
@@ -140,7 +140,7 @@ export class MapsGateway implements OnGatewayDisconnect {
     @ConnectedSocket() client: Socket,
       @MessageBody() request: IMmpClientNodeRequest,
   ): Promise<MmpNode | undefined> {
-    const nodeRemoveStatus: MmpNode | undefined = await this.mapsService.removeNode(
+    const removedNodes: MmpNode | undefined = await this.mapsService.removeNode(
       request.node,
       request.mapId,
     );
@@ -149,7 +149,7 @@ export class MapsGateway implements OnGatewayDisconnect {
       .to(request.mapId)
       .emit('nodeRemoved', { clientId: client.id, nodeId: request?.node?.id });
 
-    return nodeRemoveStatus;
+    return removedNodes;
   }
 
   @SubscribeMessage('updateNodeSelection')

diff --git a/teammapper-backend/src/map/services/maps.service.spec.ts b/teammapper-backend/src/map/services/maps.service.spec.ts
@@ -8,6 +8,7 @@ import { Repository } from 'typeorm';
 import { ConfigModule } from '@nestjs/config';
 import AppModule from '../../app.module';
 import { createTestConfiguration } from '../../../test/db';
+import { mapMmpNodeToClient } from '../utils/clientServerMapping';
 
 describe('MapsController', () => {
   let mapsService: MapsService;
@@ -67,4 +68,26 @@ describe('MapsController', () => {
       expect(mapsService.getDeletedAt(new Date('2022-01-01'), 5)).toEqual(new Date('2022-01-06'));
     });
   });
+
+  describe('removeNode', () => {
+    it('remove all nodes connected together', async () => {
+      const map: MmpMap = await mapsRepo.save({});
+
+      const node: MmpNode = await nodesRepo.save({
+        nodeMapId: map.id,
+        coordinatesX: 3,
+        coordinatesY: 1,
+      });
+
+      const nodeTwo: MmpNode = await nodesRepo.save({
+        nodeMapId: map.id,
+        nodeParent: node,
+        coordinatesX: 3,
+        coordinatesY: 1,
+      });
+
+      await mapsService.removeNode(mapMmpNodeToClient(node), map.id);
+      expect(await nodesRepo.findOne({ where: { id: nodeTwo.id } })).toEqual(undefined);
+    });
+  });
 });
diff --git a/teammapper-backend/src/map/services/maps.service.ts b/teammapper-backend/src/map/services/maps.service.ts
@@ -114,4 +114,20 @@ export class MapsService {
   deleteMap(uuid: string) {
     this.mapsRepository.delete({ id: uuid });
   }
+
+  // In case cascade is too slow, this is a WIP
+  // async recursiveFindAllNodeChildren(node: MmpNode): Promise<MmpNode[]> {
+  //   const children = await this.nodesRepository.find({
+  //     where: {
+  //         nodeParentId: node.id,
+  //         nodeMapId: node.nodeMapId,
+  //     }
+  //   });
+
+  //   if (children === undefined || children.length === 0) return [];
+
+  //   return (await Promise.all(children.map(async (node: MmpNode): Promise<MmpNode[]> => {
+  //     return [node, ...await this.recursiveFindAllNodeChildren(node)];
+  //   }, []))).flat()
+  // }
 }
diff --git a/teammapper-backend/src/migrations/1663927754319-AddIndexForNodesParents.ts b/teammapper-backend/src/migrations/1663927754319-AddIndexForNodesParents.ts
@@ -1,13 +1,13 @@
-import {MigrationInterface, QueryRunner} from "typeorm";
+import { MigrationInterface, QueryRunner } from 'typeorm';
 
 export class AddIndexForNodesParents1663927754319 implements MigrationInterface {
-    name = 'AddIndexForNodesParents1663927754319'
+  name = 'AddIndexForNodesParents1663927754319';
 
-    public async up(queryRunner: QueryRunner): Promise<void> {
-        await queryRunner.query(`CREATE INDEX "IDX_336300b82c56a05f0317f22942" ON "mmp_node" ("nodeMapId", "nodeParentId") `);
-    }
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query('CREATE INDEX "IDX_336300b82c56a05f0317f22942" ON "mmp_node" ("nodeMapId", "nodeParentId") ');
+  }
 
-    public async down(queryRunner: QueryRunner): Promise<void> {
-        await queryRunner.query(`DROP INDEX "public"."IDX_336300b82c56a05f0317f22942"`);
-    }
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query('DROP INDEX "public"."IDX_336300b82c56a05f0317f22942"');
+  }
 }
diff --git a/teammapper-backend/test/db.ts b/teammapper-backend/test/db.ts
@@ -13,4 +13,8 @@ export const createTestConfiguration = (): TypeOrmModuleOptions => ({
   autoLoadEntities: true,
   dropSchema: true,
   keepConnectionAlive: true,
+  extra: {
+    query_timeout: 1000,
+    statement_timeout: 1000,
+  },
 });