From 822b226f53b16bf167d7c1741725d505783581ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Magrin?= Date: Wed, 22 Jan 2025 17:09:31 +0100 Subject: [PATCH] fix: wip dataloader --- .../dataloaders/dataloader.interface.ts | 18 +++- .../engine/dataloaders/dataloader.module.ts | 3 +- .../engine/dataloaders/dataloader.service.ts | 54 ++++++++++- .../field-metadata/field-metadata.module.ts | 10 ++- .../field-metadata/field-metadata.resolver.ts | 59 ++++++------ .../interfaces/field-metadata.interface.ts | 6 ++ .../field-metadata-relation.service.ts | 90 +++++++++++++++++++ 7 files changed, 201 insertions(+), 39 deletions(-) create mode 100644 packages/twenty-server/src/engine/metadata-modules/field-metadata/relation/field-metadata-relation.service.ts diff --git a/packages/twenty-server/src/engine/dataloaders/dataloader.interface.ts b/packages/twenty-server/src/engine/dataloaders/dataloader.interface.ts index 90b4c9ef4d2a0..07a3ee58e84f2 100644 --- a/packages/twenty-server/src/engine/dataloaders/dataloader.interface.ts +++ b/packages/twenty-server/src/engine/dataloaders/dataloader.interface.ts @@ -1,6 +1,12 @@ import DataLoader from 'dataloader'; -import { RelationMetadataLoaderPayload } from 'src/engine/dataloaders/dataloader.service'; +import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; +import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface'; + +import { + RelationLoaderPayload, + RelationMetadataLoaderPayload, +} from 'src/engine/dataloaders/dataloader.service'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; export interface IDataloaders { @@ -8,4 +14,14 @@ export interface IDataloaders { RelationMetadataLoaderPayload, RelationMetadataEntity >; + + relationLoader: DataLoader< + RelationLoaderPayload, + { + sourceObjectMetadata: ObjectMetadataInterface; + targetObjectMetadata: ObjectMetadataInterface; + sourceFieldMetadata: FieldMetadataInterface; + targetFieldMetadata: FieldMetadataInterface; + } + >; } diff --git a/packages/twenty-server/src/engine/dataloaders/dataloader.module.ts b/packages/twenty-server/src/engine/dataloaders/dataloader.module.ts index a669e0e0501a1..3e5c71a033cc8 100644 --- a/packages/twenty-server/src/engine/dataloaders/dataloader.module.ts +++ b/packages/twenty-server/src/engine/dataloaders/dataloader.module.ts @@ -1,11 +1,12 @@ import { Module } from '@nestjs/common'; import { DataloaderService } from 'src/engine/dataloaders/dataloader.service'; +import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module'; import { RelationMetadataModule } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.module'; @Module({ + imports: [RelationMetadataModule, FieldMetadataModule], providers: [DataloaderService], - imports: [RelationMetadataModule], exports: [DataloaderService], }) export class DataloaderModule {} diff --git a/packages/twenty-server/src/engine/dataloaders/dataloader.service.ts b/packages/twenty-server/src/engine/dataloaders/dataloader.service.ts index e87f236db0c55..0f446c1935857 100644 --- a/packages/twenty-server/src/engine/dataloaders/dataloader.service.ts +++ b/packages/twenty-server/src/engine/dataloaders/dataloader.service.ts @@ -3,8 +3,10 @@ import { Injectable } from '@nestjs/common'; import DataLoader from 'dataloader'; import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; +import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface'; import { IDataloaders } from 'src/engine/dataloaders/dataloader.interface'; +import { FieldMetadataRelationService } from 'src/engine/metadata-modules/field-metadata/relation/field-metadata-relation.service'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; import { RelationMetadataService } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.service'; @@ -16,14 +18,37 @@ export type RelationMetadataLoaderPayload = { >; }; +export type RelationLoaderPayload = { + workspaceId: string; + fieldMetadata: Pick< + FieldMetadataInterface, + | 'type' + | 'id' + | 'objectMetadataId' + | 'relationTargetFieldMetadataId' + | 'relationTargetObjectMetadataId' + >; +}; + @Injectable() export class DataloaderService { constructor( private readonly relationMetadataService: RelationMetadataService, + private readonly fieldMetadataRelationService: FieldMetadataRelationService, ) {} createLoaders(): IDataloaders { - const relationMetadataLoader = new DataLoader< + const relationMetadataLoader = this._createRelationMetadataLoader(); + const relationLoader = this._createRelationLoader(); + + return { + relationMetadataLoader, + relationLoader, + }; + } + + private _createRelationMetadataLoader() { + return new DataLoader< RelationMetadataLoaderPayload, RelationMetadataEntity >(async (dataLoaderParams: RelationMetadataLoaderPayload[]) => { @@ -40,9 +65,30 @@ export class DataloaderService { return relationsMetadataCollection; }); + } - return { - relationMetadataLoader, - }; + private _createRelationLoader() { + return new DataLoader< + RelationLoaderPayload, + { + sourceObjectMetadata: ObjectMetadataInterface; + targetObjectMetadata: ObjectMetadataInterface; + sourceFieldMetadata: FieldMetadataInterface; + targetFieldMetadata: FieldMetadataInterface; + } + >(async (dataLoaderParams: RelationLoaderPayload[]) => { + const workspaceId = dataLoaderParams[0].workspaceId; + const fieldMetadataItems = dataLoaderParams.map( + (dataLoaderParam) => dataLoaderParam.fieldMetadata, + ); + + const fieldMetadataRelationCollection = + await this.fieldMetadataRelationService.findCachedFieldMetadataRelation( + fieldMetadataItems, + workspaceId, + ); + + return fieldMetadataRelationCollection; + }); } } diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts index 9c6c2cfee8e75..5c179eebbdf4d 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts @@ -15,6 +15,7 @@ import { FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dto import { FieldMetadataValidationService } from 'src/engine/metadata-modules/field-metadata/field-metadata-validation.service'; import { FieldMetadataResolver } from 'src/engine/metadata-modules/field-metadata/field-metadata.resolver'; import { FieldMetadataGraphqlApiExceptionInterceptor } from 'src/engine/metadata-modules/field-metadata/interceptors/field-metadata-graphql-api-exception.interceptor'; +import { FieldMetadataRelationService } from 'src/engine/metadata-modules/field-metadata/relation/field-metadata-relation.service'; import { FieldMetadataRelatedRecordsService } from 'src/engine/metadata-modules/field-metadata/services/field-metadata-related-records.service'; import { IsFieldMetadataDefaultValue } from 'src/engine/metadata-modules/field-metadata/validators/is-field-metadata-default-value.validator'; import { IsFieldMetadataOptions } from 'src/engine/metadata-modules/field-metadata/validators/is-field-metadata-options.validator'; @@ -22,6 +23,7 @@ import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadat import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module'; import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; +import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; import { ViewModule } from 'src/modules/view/view.module'; @@ -42,6 +44,7 @@ import { UpdateFieldInput } from './dtos/update-field.input'; WorkspaceMigrationModule, WorkspaceMigrationRunnerModule, WorkspaceMetadataVersionModule, + WorkspaceCacheStorageModule, ObjectMetadataModule, DataSourceModule, TypeORMModule, @@ -86,9 +89,14 @@ import { UpdateFieldInput } from './dtos/update-field.input'; IsFieldMetadataDefaultValue, IsFieldMetadataOptions, FieldMetadataService, + FieldMetadataRelationService, FieldMetadataRelatedRecordsService, FieldMetadataResolver, ], - exports: [FieldMetadataService, FieldMetadataRelatedRecordsService], + exports: [ + FieldMetadataService, + FieldMetadataRelationService, + FieldMetadataRelatedRecordsService, + ], }) export class FieldMetadataModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.resolver.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.resolver.ts index 13c88759d62d9..0d121b41f753b 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.resolver.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.resolver.ts @@ -27,6 +27,7 @@ import { UpdateOneFieldMetadataInput } from 'src/engine/metadata-modules/field-m import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/field-metadata.service'; import { fieldMetadataGraphqlApiExceptionHandler } from 'src/engine/metadata-modules/field-metadata/utils/field-metadata-graphql-api-exception-handler.util'; +import { ObjectMetadataDTO } from 'src/engine/metadata-modules/object-metadata/dtos/object-metadata.dto'; import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util'; @UseGuards(WorkspaceAuthGuard) @@ -135,45 +136,39 @@ export class FieldMetadataResolver { async relation( @AuthWorkspace() workspace: Workspace, @Parent() fieldMetadata: FieldMetadataEntity, + @Context() context: { loaders: IDataloaders }, ): Promise { if (!isRelationFieldMetadataType(fieldMetadata.type)) { return null; } - if (!fieldMetadata.targetFieldMetadataId) { - throw new BadRequestException('Target field metadata id is required'); - } - - const sourceFieldMetadata = - await this.fieldMetadataService.findOneWithinWorkspace(workspace.id, { - where: { - id: fieldMetadata.id, - }, - relations: ['object'], - }); - - const targetFieldMetadata = - await this.fieldMetadataService.findOneWithinWorkspace(workspace.id, { - where: { - id: fieldMetadata.targetFieldMetadataId, - }, - relations: ['object'], + try { + const { + sourceObjectMetadata, + targetObjectMetadata, + sourceFieldMetadata, + targetFieldMetadata, + } = await context.loaders.relationLoader.load({ + fieldMetadata, + workspaceId: workspace.id, }); - if (!targetFieldMetadata) { - throw new BadRequestException('Target field metadata not found'); - } - - if (!fieldMetadata.settings) { - throw new BadRequestException('Relation settings are required'); + if (!fieldMetadata.settings) { + throw new BadRequestException('Relation settings are required'); + } + + return { + type: fieldMetadata.settings.relationType, + // TODO: Fix the typing issues + sourceObjectMetadata: + sourceObjectMetadata as unknown as ObjectMetadataDTO, + targetObjectMetadata: + targetObjectMetadata as unknown as ObjectMetadataDTO, + sourceFieldMetadata: sourceFieldMetadata as unknown as FieldMetadataDTO, + targetFieldMetadata: targetFieldMetadata as unknown as FieldMetadataDTO, + }; + } catch (error) { + fieldMetadataGraphqlApiExceptionHandler(error); } - - return { - type: fieldMetadata.settings.relationType, - sourceObjectMetadata: sourceFieldMetadata.object, - targetObjectMetadata: targetFieldMetadata.object, - sourceFieldMetadata: fieldMetadata, - targetFieldMetadata: targetFieldMetadata, - }; } } diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface.ts index f5ba634d3e7ec..055d8de81c153 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface.ts @@ -4,6 +4,8 @@ import { FieldMetadataDefaultValue } from 'src/engine/metadata-modules/field-met import { FieldMetadataOptions } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-options.interface'; import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-settings.interface'; +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; export interface FieldMetadataInterface< @@ -23,6 +25,10 @@ export interface FieldMetadataInterface< isUnique?: boolean; fromRelationMetadata?: RelationMetadataEntity; toRelationMetadata?: RelationMetadataEntity; + relationTargetFieldMetadataId?: string; + relationTargetFieldMetadata?: FieldMetadataEntity; + relationTargetObjectMetadataId?: string; + relationTargetObjectMetadata?: ObjectMetadataEntity; isCustom?: boolean; generatedType?: 'STORED' | 'VIRTUAL'; asExpression?: string; diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/relation/field-metadata-relation.service.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/relation/field-metadata-relation.service.ts new file mode 100644 index 0000000000000..ba92b54c65576 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/relation/field-metadata-relation.service.ts @@ -0,0 +1,90 @@ +import { Injectable, NotFoundException } from '@nestjs/common'; + +import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; + +import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; + +@Injectable() +export class FieldMetadataRelationService { + constructor( + private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, + ) {} + + async findCachedFieldMetadataRelation( + fieldMetadataItems: Array< + Pick< + FieldMetadataInterface, + | 'id' + | 'type' + | 'objectMetadataId' + | 'relationTargetFieldMetadataId' + | 'relationTargetObjectMetadataId' + > + >, + workspaceId: string, + ) { + const metadataVersion = + await this.workspaceCacheStorageService.getMetadataVersion(workspaceId); + + if (!metadataVersion) { + throw new NotFoundException( + `Metadata version not found for workspace ${workspaceId}`, + ); + } + + const objectMetadataMaps = + await this.workspaceCacheStorageService.getObjectMetadataMaps( + workspaceId, + metadataVersion, + ); + + if (!objectMetadataMaps) { + throw new NotFoundException( + `Object metadata map not found for workspace ${workspaceId} and metadata version ${metadataVersion}`, + ); + } + + const fieldMetadataRelationCollection = fieldMetadataItems.map( + (fieldMetadataItem) => { + const sourceObjectMetadata = + objectMetadataMaps.byId[fieldMetadataItem.objectMetadataId]; + + if (!fieldMetadataItem.relationTargetObjectMetadataId) { + throw new NotFoundException( + `Relation target object metadata id not found for field metadata ${fieldMetadataItem.id}`, + ); + } + + const targetObjectMetadata = + objectMetadataMaps.byId[ + fieldMetadataItem.relationTargetObjectMetadataId + ]; + + if (!fieldMetadataItem.relationTargetFieldMetadataId) { + throw new NotFoundException( + `Relation target field metadata id not found for field metadata ${fieldMetadataItem.id}`, + ); + } + + const sourceFieldMetadata = + sourceObjectMetadata.fieldsById[ + fieldMetadataItem.relationTargetFieldMetadataId + ]; + + const targetFieldMetadata = + targetObjectMetadata.fieldsById[ + fieldMetadataItem.relationTargetFieldMetadataId + ]; + + return { + sourceObjectMetadata: sourceObjectMetadata, + sourceFieldMetadata: sourceFieldMetadata, + targetObjectMetadata: targetObjectMetadata, + targetFieldMetadata: targetFieldMetadata, + }; + }, + ); + + return fieldMetadataRelationCollection; + } +}