From c0cdd0a23998547141a7c4436928defbd209ced4 Mon Sep 17 00:00:00 2001 From: JannikStreek Date: Fri, 23 Sep 2022 12:59:00 +0200 Subject: [PATCH] add job delete maps (#92) --- README.md | 15 ++++ entrypoint.prod.sh | 2 +- teammapper-backend/README.md | 2 +- teammapper-backend/package.json | 7 +- teammapper-backend/src/config.service.ts | 2 +- teammapper-backend/src/jobs/deleteMaps.job.ts | 25 +++++++ .../src/jobs/seedMapData.job.ts | 69 +++++++++++++++++++ .../src/map/entities/mmpNode.entity.ts | 1 + .../src/map/services/maps.service.ts | 2 +- .../src/map/services/tasks.service.ts | 2 +- ...39669273-AddIndexToForeignKeysOnMmpNode.ts | 16 ++--- .../1663927754319-AddIndexForNodesParents.ts | 13 ++++ 12 files changed, 141 insertions(+), 15 deletions(-) create mode 100644 teammapper-backend/src/jobs/deleteMaps.job.ts create mode 100644 teammapper-backend/src/jobs/seedMapData.job.ts create mode 100644 teammapper-backend/src/migrations/1663927754319-AddIndexForNodesParents.ts diff --git a/README.md b/README.md index 0e44d523..916b3207 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,21 @@ TeamMapper is based on mindmapp (https://github.com/cedoor/mindmapp , discontinu ```bash docker-compose exec app_prod npm -prefix teammapper-backend run typeorm:prod:migrate ``` +#### Running jobs + +Trigger delete job (also executed daily with cron task scheduler): + +``` +docker-compose --file docker-compose-prod.yml --env-file .env.prod exec app_prod npm --prefix teammapper-backend run prod:data:maps:cleanup +``` + +#### Running further queries + +Example of running sql via typeorm: + +``` +npx typeorm query "select * from mmp_node" --config dist/ormconfig.js +``` ### Further details diff --git a/entrypoint.prod.sh b/entrypoint.prod.sh index 219c937d..d3eeb62a 100755 --- a/entrypoint.prod.sh +++ b/entrypoint.prod.sh @@ -1,2 +1,2 @@ -npm --prefix teammapper-backend run typeorm:prod:migrate +npm --prefix teammapper-backend run prod:typeorm:migrate npm run start:prod \ No newline at end of file diff --git a/teammapper-backend/README.md b/teammapper-backend/README.md index 3a8e6995..8e1a11a2 100644 --- a/teammapper-backend/README.md +++ b/teammapper-backend/README.md @@ -69,7 +69,7 @@ npm run typeorm schema:sync For pruction environments, run migrations, see https://github.com/typeorm/typeorm/blob/master/docs/migrations.md ```bash -npm run typeorm:prod:migrate +npm run prod:typeorm:migrate ``` or run migrations on dev: diff --git a/teammapper-backend/package.json b/teammapper-backend/package.json index e70d292e..fa4ac891 100644 --- a/teammapper-backend/package.json +++ b/teammapper-backend/package.json @@ -25,8 +25,11 @@ "test:cov": "jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", "test:e2e": "jest --config ./test/jest-e2e.json", - "typeorm": "node --require ts-node/register ./node_modules/typeorm/cli.js --config src/ormconfig.ts", - "typeorm:prod:migrate": "typeorm migration:run --config dist/ormconfig.js" + "dev:typeorm": "node --require ts-node/register ./node_modules/typeorm/cli.js --config src/ormconfig.ts", + "dev:data:maps:cleanup": "ts-node ./src/jobs/deleteMaps.job.ts", + "dev:data:maps:seed": "ts-node ./src/jobs/seedMapData.job.ts", + "prod:data:maps:cleanup": "node ./dist/jobs/deleteMaps.job.js", + "prod:typeorm:migrate": "typeorm migration:run --config dist/ormconfig.js" }, "dependencies": { "@nestjs/common": "^8.0.0", diff --git a/teammapper-backend/src/config.service.ts b/teammapper-backend/src/config.service.ts index 187c7ff8..69524979 100644 --- a/teammapper-backend/src/config.service.ts +++ b/teammapper-backend/src/config.service.ts @@ -63,7 +63,7 @@ class ConfigService { extra: { query_timeout: 20000, - statement_timeout: 20000 + statement_timeout: 20000, }, synchronize: !this.isProduction(), diff --git a/teammapper-backend/src/jobs/deleteMaps.job.ts b/teammapper-backend/src/jobs/deleteMaps.job.ts new file mode 100644 index 00000000..bfe102cc --- /dev/null +++ b/teammapper-backend/src/jobs/deleteMaps.job.ts @@ -0,0 +1,25 @@ +import { NestFactory } from '@nestjs/core'; +import { MapsService } from '../map/services/maps.service'; +import AppModule from '../app.module'; +import { Logger } from '@nestjs/common'; +import configService from '../config.service'; +import { DeleteResult } from 'typeorm'; + +async function bootstrap() { + const application = await NestFactory.createApplicationContext( + AppModule, + ); + + const logger = new Logger('TaskRunner'); + const mapsService = application.get(MapsService); + + logger.log('--- Deleting old maps ... ---'); + const result: DeleteResult = await mapsService.deleteOutdatedMaps(configService.deleteAfterDays()); + logger.log('Deleted rows: ' + result.affected); + logger.log('--- Finished deleting maps ---'); + + await application.close(); + process.exit(0); +} + +bootstrap(); \ No newline at end of file diff --git a/teammapper-backend/src/jobs/seedMapData.job.ts b/teammapper-backend/src/jobs/seedMapData.job.ts new file mode 100644 index 00000000..9447b58b --- /dev/null +++ b/teammapper-backend/src/jobs/seedMapData.job.ts @@ -0,0 +1,69 @@ +import { NestFactory } from '@nestjs/core'; +import { MapsService } from '../map/services/maps.service'; +import AppModule from '../app.module'; +import { Logger } from '@nestjs/common'; +import { IMmpClientMap, IMmpClientNode } from 'src/map/types'; +import * as crypto from 'crypto'; + +const createNode: any = (isRoot: boolean, parentId: string, x: number, y: number) => { + return { + colors: { + name: '#000000', + background: '#FF0000', + branch: '#000000', + }, + coordinates: { x: x, y: y }, + font: { + style: '', + size: 10, + weight: 'bold', + }, + id: crypto.randomUUID(), + image: { + src: '', + size: 0, + }, + k: 0, + locked: false, + name: 'Seed Data', + parent: parentId, + isRoot: isRoot, + }; +}; + +const createMap: any = (nodes: IMmpClientNode[]) => { + return { + uuid: crypto.randomUUID(), + lastModified: new Date(), + deleteAfterDays: 30, + data: nodes, + deletedAt: undefined, + }; +}; + +async function bootstrap() { + const application = await NestFactory.createApplicationContext( + AppModule, + ); + + const logger = new Logger('TaskRunner'); + const mapsService = application.get(MapsService); + + logger.log('--- Creating maps ... ---'); + + for (let i = 0; i < 5000; i++) { + const rootNode: IMmpClientNode = createNode(true, '', 0, 0); + const childNode: IMmpClientNode = createNode(false, rootNode.id, 150, 150); + const secondChildNode: IMmpClientNode = createNode(false, childNode.id, 250, 250); + const mapData: IMmpClientMap = createMap([rootNode, childNode, secondChildNode]); + await mapsService.createMap(mapData); + logger.log(`--- Map created with id ${mapData.uuid} ---`); + } + + logger.log('--- Finished creating maps ---'); + + await application.close(); + process.exit(0); +} + +bootstrap(); \ No newline at end of file diff --git a/teammapper-backend/src/map/entities/mmpNode.entity.ts b/teammapper-backend/src/map/entities/mmpNode.entity.ts index fb69f32d..94594a85 100644 --- a/teammapper-backend/src/map/entities/mmpNode.entity.ts +++ b/teammapper-backend/src/map/entities/mmpNode.entity.ts @@ -35,6 +35,7 @@ export class MmpNode { { name: 'nodeMapId', referencedColumnName: 'nodeMapId' }, { name: 'nodeParentId', referencedColumnName: 'id' }, ]) + @Index() nodeParent: MmpNode; /* eslint-enable @typescript-eslint/no-unused-vars */ diff --git a/teammapper-backend/src/map/services/maps.service.ts b/teammapper-backend/src/map/services/maps.service.ts index 2c9970b9..0a36be9a 100644 --- a/teammapper-backend/src/map/services/maps.service.ts +++ b/teammapper-backend/src/map/services/maps.service.ts @@ -103,7 +103,7 @@ export class MapsService { } async deleteOutdatedMaps(afterDays: number = 30): Promise { - return await this.mapsRepository + return this.mapsRepository .createQueryBuilder() .where("(lastModified + (INTERVAL '1 day' * :afterDays)) < :today", { afterDays: afterDays, today: new Date() }) .delete() diff --git a/teammapper-backend/src/map/services/tasks.service.ts b/teammapper-backend/src/map/services/tasks.service.ts index 3fca581c..99d8906c 100644 --- a/teammapper-backend/src/map/services/tasks.service.ts +++ b/teammapper-backend/src/map/services/tasks.service.ts @@ -20,7 +20,7 @@ export class TasksService { // every day midnight @Cron('0 0 * * *') - async handleCron() { + async deleteOldMapsInInterval() { this.logger.log('--- Deleting old maps ... ---'); const result: DeleteResult = await this.mapsService.deleteOutdatedMaps(configService.deleteAfterDays()); this.logger.log('Deleted rows: ' + result.affected); diff --git a/teammapper-backend/src/migrations/1663839669273-AddIndexToForeignKeysOnMmpNode.ts b/teammapper-backend/src/migrations/1663839669273-AddIndexToForeignKeysOnMmpNode.ts index b671aef5..0200199e 100644 --- a/teammapper-backend/src/migrations/1663839669273-AddIndexToForeignKeysOnMmpNode.ts +++ b/teammapper-backend/src/migrations/1663839669273-AddIndexToForeignKeysOnMmpNode.ts @@ -1,14 +1,14 @@ -import {MigrationInterface, QueryRunner} from "typeorm"; +import { MigrationInterface, QueryRunner } from 'typeorm'; export class AddIndexToForeignKeysOnMmpNode1663839669273 implements MigrationInterface { - name = 'AddIndexToForeignKeysOnMmpNode1663839669273' + name = 'AddIndexToForeignKeysOnMmpNode1663839669273'; - public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(`CREATE INDEX "IDX_19c5208da416d32ea491315716" ON "mmp_node" ("nodeMapId") `); - } + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query('CREATE INDEX "IDX_19c5208da416d32ea491315716" ON "mmp_node" ("nodeMapId") '); + } - public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`DROP INDEX "public"."IDX_19c5208da416d32ea491315716"`); - } + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query('DROP INDEX "public"."IDX_19c5208da416d32ea491315716"'); + } } diff --git a/teammapper-backend/src/migrations/1663927754319-AddIndexForNodesParents.ts b/teammapper-backend/src/migrations/1663927754319-AddIndexForNodesParents.ts new file mode 100644 index 00000000..94642990 --- /dev/null +++ b/teammapper-backend/src/migrations/1663927754319-AddIndexForNodesParents.ts @@ -0,0 +1,13 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; + +export class AddIndexForNodesParents1663927754319 implements MigrationInterface { + name = 'AddIndexForNodesParents1663927754319' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE INDEX "IDX_336300b82c56a05f0317f22942" ON "mmp_node" ("nodeMapId", "nodeParentId") `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "public"."IDX_336300b82c56a05f0317f22942"`); + } +}