From 720fd443b4fef7dd3837cb0ff0455dc0824366d5 Mon Sep 17 00:00:00 2001 From: JannikStreek Date: Sat, 24 Sep 2022 15:16:00 +0200 Subject: [PATCH] parametrize sql timeouts (#95) --- .env.default | 3 +++ .github/workflows/ci.yml | 2 ++ docker-compose-prod.yml | 2 ++ docker-compose.yml | 2 ++ teammapper-backend/src/config.service.ts | 4 ++-- .../src/jobs/seedMapData.job.ts | 10 +++++--- .../src/map/controllers/maps.gateway.ts | 4 ++-- .../src/map/services/maps.service.spec.ts | 23 +++++++++++++++++++ .../src/map/services/maps.service.ts | 16 +++++++++++++ .../1663927754319-AddIndexForNodesParents.ts | 16 ++++++------- teammapper-backend/test/db.ts | 4 ++++ 11 files changed, 71 insertions(+), 15 deletions(-) diff --git a/.env.default b/.env.default index c5ad47d8..7df097ec 100644 --- a/.env.default +++ b/.env.default @@ -28,4 +28,7 @@ POSTGRES_PROD_USER= # PROD Requires SSL Connection Support. Use DEV if not available. PROD_MODE=PROD +POSTGRES_QUERY_TIMEOUT=100000 +POSTGRES_STATEMENT_TIMEOUT=100000 + DELETE_AFTER_DAYS=30 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01198885..563f3987 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,3 +79,5 @@ jobs: POSTGRES_TEST_PORT: ${{ job.services.postgres.ports[5432] }} POSTGRES_TEST_USER: "teammapper-user" POSTGRES_USER: "teammapper-user" + POSTGRES_QUERY_TIMEOUT: 10000 + POSTGRES_STATEMENT_TIMEOUT: 10000 diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index aca479e8..9942bbac 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -16,6 +16,8 @@ services: POSTGRES_SSL: ${POSTGRES_PROD_SSL:-true} POSTGRES_SSL_REJECT_UNAUTHORIZED: ${POSTGRES_PROD_SSL_REJECT_UNAUTHORIZED:-false} POSTGRES_USER: ${POSTGRES_PROD_USER:-teammapper-user} + POSTGRES_QUERY_TIMEOUT: ${POSTGRES_QUERY_TIMEOUT:-100000} + POSTGRES_STATEMENT_TIMEOUT: ${POSTGRES_STATEMENT_TIMEOUT:-100000} DELETE_AFTER_DAYS: ${DELETE_AFTER_DAYS:-30} ports: - "${APP_PROD_PORT:-80}:3000" diff --git a/docker-compose.yml b/docker-compose.yml index a9488758..06854459 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,8 @@ services: POSTGRES_PORT: ${POSTGRES_PORT:-5432} POSTGRES_USER: ${POSTGRES_USER:-teammapper-user} POSTGRES_SSL: ${POSTGRES_SSL:-false} + POSTGRES_QUERY_TIMEOUT: ${POSTGRES_QUERY_TIMEOUT:-100000} + POSTGRES_STATEMENT_TIMEOUT: ${POSTGRES_STATEMENT_TIMEOUT:-100000} POSTGRES_TEST_DATABASE: ${DOCKER_COMPOSE_APP_ENV_POSTGRES_TEST_DATABASE:-teammapper-backend-test} POSTGRES_TEST_HOST: ${DOCKER_COMPOSE_APP_ENV_POSTGRES_HOST:-postgres} POSTGRES_TEST_PASSWORD: ${DOCKER_COMPOSE_APP_ENV_POSTGRES_PASSWORD:-teammapper-password} diff --git a/teammapper-backend/src/config.service.ts b/teammapper-backend/src/config.service.ts index 69524979..397eb3fb 100644 --- a/teammapper-backend/src/config.service.ts +++ b/teammapper-backend/src/config.service.ts @@ -62,8 +62,8 @@ class ConfigService { }, extra: { - query_timeout: 20000, - statement_timeout: 20000, + query_timeout: this.getValue('POSTGRES_QUERY_TIMEOUT') || 100000, + statement_timeout: this.getValue('POSTGRES_STATEMENT_TIMEOUT') || 100000, }, synchronize: !this.isProduction(), diff --git a/teammapper-backend/src/jobs/seedMapData.job.ts b/teammapper-backend/src/jobs/seedMapData.job.ts index 9447b58b..3de172df 100644 --- a/teammapper-backend/src/jobs/seedMapData.job.ts +++ b/teammapper-backend/src/jobs/seedMapData.job.ts @@ -51,13 +51,17 @@ async function bootstrap() { logger.log('--- Creating maps ... ---'); - for (let i = 0; i < 5000; i++) { + for (let i = 0; i < 100000; i++) { const rootNode: IMmpClientNode = createNode(true, '', 0, 0); const childNode: IMmpClientNode = createNode(false, rootNode.id, 150, 150); const secondChildNode: IMmpClientNode = createNode(false, childNode.id, 250, 250); - const mapData: IMmpClientMap = createMap([rootNode, childNode, secondChildNode]); + const thirdChildNode: IMmpClientNode = createNode(false, childNode.id, 350, 350); + const fourthChildNode: IMmpClientNode = createNode(false, childNode.id, 450, 450); + const fifthChildNode: IMmpClientNode = createNode(false, childNode.id, 550, 550); + const mapData: IMmpClientMap = createMap([rootNode, childNode, secondChildNode, thirdChildNode, fourthChildNode, fifthChildNode]); await mapsService.createMap(mapData); - logger.log(`--- Map created with id ${mapData.uuid} ---`); + + if (i % 500 == 0) logger.log(`--- Map created with id ${mapData.uuid} ---`); } logger.log('--- Finished creating maps ---'); diff --git a/teammapper-backend/src/map/controllers/maps.gateway.ts b/teammapper-backend/src/map/controllers/maps.gateway.ts index d64fa048..b0a47ffe 100644 --- a/teammapper-backend/src/map/controllers/maps.gateway.ts +++ b/teammapper-backend/src/map/controllers/maps.gateway.ts @@ -140,7 +140,7 @@ export class MapsGateway implements OnGatewayDisconnect { @ConnectedSocket() client: Socket, @MessageBody() request: IMmpClientNodeRequest, ): Promise { - const nodeRemoveStatus: MmpNode | undefined = await this.mapsService.removeNode( + const removedNodes: MmpNode | undefined = await this.mapsService.removeNode( request.node, request.mapId, ); @@ -149,7 +149,7 @@ export class MapsGateway implements OnGatewayDisconnect { .to(request.mapId) .emit('nodeRemoved', { clientId: client.id, nodeId: request?.node?.id }); - return nodeRemoveStatus; + return removedNodes; } @SubscribeMessage('updateNodeSelection') diff --git a/teammapper-backend/src/map/services/maps.service.spec.ts b/teammapper-backend/src/map/services/maps.service.spec.ts index c4b7ec92..36886436 100644 --- a/teammapper-backend/src/map/services/maps.service.spec.ts +++ b/teammapper-backend/src/map/services/maps.service.spec.ts @@ -8,6 +8,7 @@ import { Repository } from 'typeorm'; import { ConfigModule } from '@nestjs/config'; import AppModule from '../../app.module'; import { createTestConfiguration } from '../../../test/db'; +import { mapMmpNodeToClient } from '../utils/clientServerMapping'; describe('MapsController', () => { let mapsService: MapsService; @@ -67,4 +68,26 @@ describe('MapsController', () => { expect(mapsService.getDeletedAt(new Date('2022-01-01'), 5)).toEqual(new Date('2022-01-06')); }); }); + + describe('removeNode', () => { + it('remove all nodes connected together', async () => { + const map: MmpMap = await mapsRepo.save({}); + + const node: MmpNode = await nodesRepo.save({ + nodeMapId: map.id, + coordinatesX: 3, + coordinatesY: 1, + }); + + const nodeTwo: MmpNode = await nodesRepo.save({ + nodeMapId: map.id, + nodeParent: node, + coordinatesX: 3, + coordinatesY: 1, + }); + + await mapsService.removeNode(mapMmpNodeToClient(node), map.id); + expect(await nodesRepo.findOne({ where: { id: nodeTwo.id } })).toEqual(undefined); + }); + }); }); \ No newline at end of file diff --git a/teammapper-backend/src/map/services/maps.service.ts b/teammapper-backend/src/map/services/maps.service.ts index 0a36be9a..6891dab4 100644 --- a/teammapper-backend/src/map/services/maps.service.ts +++ b/teammapper-backend/src/map/services/maps.service.ts @@ -114,4 +114,20 @@ export class MapsService { deleteMap(uuid: string) { this.mapsRepository.delete({ id: uuid }); } + + // In case cascade is too slow, this is a WIP + // async recursiveFindAllNodeChildren(node: MmpNode): Promise { + // const children = await this.nodesRepository.find({ + // where: { + // nodeParentId: node.id, + // nodeMapId: node.nodeMapId, + // } + // }); + + // if (children === undefined || children.length === 0) return []; + + // return (await Promise.all(children.map(async (node: MmpNode): Promise => { + // return [node, ...await this.recursiveFindAllNodeChildren(node)]; + // }, []))).flat() + // } } diff --git a/teammapper-backend/src/migrations/1663927754319-AddIndexForNodesParents.ts b/teammapper-backend/src/migrations/1663927754319-AddIndexForNodesParents.ts index 94642990..7e845710 100644 --- a/teammapper-backend/src/migrations/1663927754319-AddIndexForNodesParents.ts +++ b/teammapper-backend/src/migrations/1663927754319-AddIndexForNodesParents.ts @@ -1,13 +1,13 @@ -import {MigrationInterface, QueryRunner} from "typeorm"; +import { MigrationInterface, QueryRunner } from 'typeorm'; export class AddIndexForNodesParents1663927754319 implements MigrationInterface { - name = 'AddIndexForNodesParents1663927754319' + name = 'AddIndexForNodesParents1663927754319'; - public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(`CREATE INDEX "IDX_336300b82c56a05f0317f22942" ON "mmp_node" ("nodeMapId", "nodeParentId") `); - } + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query('CREATE INDEX "IDX_336300b82c56a05f0317f22942" ON "mmp_node" ("nodeMapId", "nodeParentId") '); + } - public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`DROP INDEX "public"."IDX_336300b82c56a05f0317f22942"`); - } + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query('DROP INDEX "public"."IDX_336300b82c56a05f0317f22942"'); + } } diff --git a/teammapper-backend/test/db.ts b/teammapper-backend/test/db.ts index dd6c6605..83844984 100644 --- a/teammapper-backend/test/db.ts +++ b/teammapper-backend/test/db.ts @@ -13,4 +13,8 @@ export const createTestConfiguration = (): TypeOrmModuleOptions => ({ autoLoadEntities: true, dropSchema: true, keepConnectionAlive: true, + extra: { + query_timeout: 1000, + statement_timeout: 1000, + }, });