From 33376df8b81a7bbe3792ab55feb7afcc24bd9fb8 Mon Sep 17 00:00:00 2001 From: Nick Zelei <2420177+nickzelei@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:34:41 -0800 Subject: [PATCH] NEOS-1618: Fixes IP Address Form, FK Col Rendering, JM Table Perf (#2960) --- .../components/DataGenConnectionCard.tsx | 92 ++- .../components/DataSyncConnectionCard.tsx | 154 +++- .../components/useOnApplyDefaultClick.tsx | 76 ++ .../source/components/useOnImportMappings.tsx | 2 +- .../jobs/[id]/source/components/util.ts | 121 +++ .../new/job/generate/single/schema/page.tsx | 94 ++- .../(mgmt)/[account]/new/job/schema/page.tsx | 135 ++- .../GenerateIpAddressForm.tsx | 1 - .../jobs/JobMappingTable/AttributesCell.tsx | 99 +++ .../jobs/JobMappingTable/Columns.tsx | 409 +++++++++ .../jobs/JobMappingTable/ConstraintsCell.tsx | 85 ++ .../jobs/JobMappingTable/DataTypeCell.tsx | 35 + .../JobMappingTable/IndeterminateCheckbox.tsx | 24 + .../jobs/JobMappingTable/JobMappingTable.tsx | 320 ++++++++ .../jobs/NosqlTable/AddNewNosqlRecord.tsx | 274 +++++++ .../jobs/NosqlTable/EditCollection.tsx | 72 ++ .../jobs/NosqlTable/EditDocumentKey.tsx | 65 ++ .../components/jobs/NosqlTable/NosqlTable.tsx | 775 ++---------------- .../jobs/SchemaTable/AiSchemaColumns.tsx | 2 +- .../components/jobs/SchemaTable/RowAlert.tsx | 51 -- .../jobs/SchemaTable/SchemaColumnHeader.tsx | 16 +- .../jobs/SchemaTable/SchemaColumns.tsx | 591 ------------- .../jobs/SchemaTable/SchemaPageTable.tsx | 203 ----- .../jobs/SchemaTable/SchemaTable.tsx | 176 +++- .../jobs/SchemaTable/SchemaTableToolBar.tsx | 241 +----- .../jobs/SchemaTable/TransformerSelect.tsx | 14 +- .../SchemaTable/schema-constraint-handler.ts | 3 +- .../jobs/SchemaTable/transformer-handler.ts | 12 +- .../jobs/SchemaTable/useOnExportMappings.tsx | 13 +- .../web/components/jobs/SchemaTable/util.ts | 88 ++ .../web/components/labels/LearnMoreLink.tsx | 11 +- frontend/apps/web/util/util.ts | 11 + .../transformer-validations.ts | 3 +- 33 files changed, 2407 insertions(+), 1861 deletions(-) create mode 100644 frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/useOnApplyDefaultClick.tsx create mode 100644 frontend/apps/web/components/jobs/JobMappingTable/AttributesCell.tsx create mode 100644 frontend/apps/web/components/jobs/JobMappingTable/Columns.tsx create mode 100644 frontend/apps/web/components/jobs/JobMappingTable/ConstraintsCell.tsx create mode 100644 frontend/apps/web/components/jobs/JobMappingTable/DataTypeCell.tsx create mode 100644 frontend/apps/web/components/jobs/JobMappingTable/IndeterminateCheckbox.tsx create mode 100644 frontend/apps/web/components/jobs/JobMappingTable/JobMappingTable.tsx create mode 100644 frontend/apps/web/components/jobs/NosqlTable/AddNewNosqlRecord.tsx create mode 100644 frontend/apps/web/components/jobs/NosqlTable/EditCollection.tsx create mode 100644 frontend/apps/web/components/jobs/NosqlTable/EditDocumentKey.tsx delete mode 100644 frontend/apps/web/components/jobs/SchemaTable/RowAlert.tsx delete mode 100644 frontend/apps/web/components/jobs/SchemaTable/SchemaColumns.tsx delete mode 100644 frontend/apps/web/components/jobs/SchemaTable/SchemaPageTable.tsx create mode 100644 frontend/apps/web/components/jobs/SchemaTable/util.ts diff --git a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/DataGenConnectionCard.tsx b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/DataGenConnectionCard.tsx index 2233375b5f..5da58a8417 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/DataGenConnectionCard.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/DataGenConnectionCard.tsx @@ -5,6 +5,8 @@ import { getAllFormErrors, } from '@/components/jobs/SchemaTable/SchemaTable'; import { getSchemaConstraintHandler } from '@/components/jobs/SchemaTable/schema-constraint-handler'; +import { TransformerResult } from '@/components/jobs/SchemaTable/transformer-handler'; +import { getTransformerFilter } from '@/components/jobs/SchemaTable/util'; import { useAccount } from '@/components/providers/account-provider'; import { Alert, AlertTitle } from '@/components/ui/alert'; import { Button } from '@/components/ui/button'; @@ -25,8 +27,10 @@ import { SelectTrigger, SelectValue, } from '@/components/ui/select'; -import { getErrorMessage } from '@/util/util'; +import { useGetTransformersHandler } from '@/libs/hooks/useGetTransformersHandler'; +import { getErrorMessage, getTransformerFromField } from '@/util/util'; import { + JobMappingTransformerForm, convertJobMappingTransformerFormToJobMappingTransformer, convertJobMappingTransformerToForm, } from '@/yup-validations/jobs'; @@ -64,8 +68,12 @@ import { validateJobMapping, } from '../../../util'; import SchemaPageSkeleton from './SchemaPageSkeleton'; +import { useOnApplyDefaultClick } from './useOnApplyDefaultClick'; import { useOnImportMappings } from './useOnImportMappings'; -import { getOnSelectedTableToggle } from './util'; +import { + getFilteredTransformersForBulkSet, + getOnSelectedTableToggle, +} from './util'; interface Props { jobId: string; @@ -138,7 +146,7 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement { [isSchemaMapValidating, isTableConstraintsValidating] ); const [selectedTables, setSelectedTables] = useState>(new Set()); - const { append, remove, fields } = + const { append, remove, fields, update } = useFieldArray({ control: form.control, name: 'mappings', @@ -220,6 +228,22 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement { const { mutateAsync: validateJobMappingsAsync } = useMutation(validateJobMappings); + const { handler, isLoading: isGetTransformersLoading } = + useGetTransformersHandler(account?.id ?? ''); + + function onTransformerUpdate( + index: number, + transformer: JobMappingTransformerForm + ): void { + const val = form.getValues(`mappings.${index}`); + update(index, { + schema: val.schema, + table: val.table, + column: val.column, + transformer, + }); + } + const { onClick: onImportMappingsClick } = useOnImportMappings({ setMappings(mappings) { form.setValue('mappings', mappings, { @@ -247,7 +271,45 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement { setSelectedTables: setSelectedTables, }); - if (isJobLoading || isSchemaDataMapLoading) { + function getAvailableTransformers(idx: number): TransformerResult { + const row = fields[idx]; + return handler.getFilteredTransformers( + getTransformerFilter( + schemaConstraintHandler, + { + schema: row.schema, + table: row.table, + column: row.column, + }, + 'sync' + ) + ); + } + + const { onClick: onApplyDefaultClick } = useOnApplyDefaultClick({ + getMappings() { + return form.getValues('mappings'); + }, + setTransformer: onTransformerUpdate, + constraintHandler: schemaConstraintHandler, + triggerUpdate() { + form.trigger('mappings'); + }, + }); + + function onTransformerBulkUpdate( + indices: number[], + config: JobMappingTransformerForm + ): void { + indices.forEach((idx) => { + onTransformerUpdate(idx, config); + }); + setTimeout(() => { + form.trigger('mappings'); + }, 0); + } + + if (isJobLoading || isSchemaDataMapLoading || isGetTransformersLoading) { return ; } @@ -466,6 +528,28 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement { isJobMappingsValidating={isValidatingMappings} onValidate={validateMappings} onImportMappingsClick={onImportMappingsClick} + onTransformerUpdate={(idx, cfg) => { + onTransformerUpdate(idx, cfg); + }} + getAvailableTransformers={getAvailableTransformers} + getTransformerFromField={(idx) => { + const row = fields[idx]; + return getTransformerFromField(handler, row.transformer); + }} + getAvailableTransformersForBulk={(rows) => { + return getFilteredTransformersForBulkSet( + rows, + handler, + schemaConstraintHandler, + 'generate', + 'relational' + ); + }} + getTransformerFromFieldValue={(fvalue) => { + return getTransformerFromField(handler, fvalue); + }} + onApplyDefaultClick={onApplyDefaultClick} + onTransformerBulkUpdate={onTransformerBulkUpdate} /> {form.formState.errors.mappings && ( diff --git a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/DataSyncConnectionCard.tsx b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/DataSyncConnectionCard.tsx index b6985aa331..99d0f6544e 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/DataSyncConnectionCard.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/DataSyncConnectionCard.tsx @@ -8,6 +8,11 @@ import { getAllFormErrors, } from '@/components/jobs/SchemaTable/SchemaTable'; import { getSchemaConstraintHandler } from '@/components/jobs/SchemaTable/schema-constraint-handler'; +import { TransformerResult } from '@/components/jobs/SchemaTable/transformer-handler'; +import { + getTransformerFilter, + splitCollection, +} from '@/components/jobs/SchemaTable/util'; import { useAccount } from '@/components/providers/account-provider'; import { Button } from '@/components/ui/button'; import { @@ -25,12 +30,14 @@ import { SelectTrigger, SelectValue, } from '@/components/ui/select'; +import { useGetTransformersHandler } from '@/libs/hooks/useGetTransformersHandler'; import { splitConnections } from '@/libs/utils'; -import { getErrorMessage } from '@/util/util'; +import { getErrorMessage, getTransformerFromField } from '@/util/util'; import { DataSyncSourceFormValues, EditDestinationOptionsFormValues, JobMappingFormValues, + JobMappingTransformerForm, VirtualForeignConstraintFormValues, convertJobMappingTransformerFormToJobMappingTransformer, convertJobMappingTransformerToForm, @@ -87,11 +94,13 @@ import { validateJobMapping, } from '../../../util'; import SchemaPageSkeleton from './SchemaPageSkeleton'; +import { useOnApplyDefaultClick } from './useOnApplyDefaultClick'; import { useOnImportMappings } from './useOnImportMappings'; import { getConnectionIdFromSource, getDestinationDetailsRecord, getDynamoDbDestinations, + getFilteredTransformersForBulkSet, getOnSelectedTableToggle, isDynamoDBConnection, isNosqlSource, @@ -466,6 +475,22 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement { [] ); + const { handler, isLoading: isGetTransformersLoading } = + useGetTransformersHandler(account?.id ?? ''); + + function onTransformerUpdate( + index: number, + transformer: JobMappingTransformerForm + ): void { + const val = form.getValues(`mappings.${index}`); + update(index, { + schema: val.schema, + table: val.table, + column: val.column, + transformer, + }); + } + const { onClick: onImportMappingsClick } = useOnImportMappings({ setMappings(mappings) { form.setValue('mappings', mappings, { @@ -480,13 +505,7 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement { appendNewMappings(mappings) { append(mappings); }, - setTransformer(idx, transformer) { - form.setValue(`mappings.${idx}.transformer`, transformer, { - shouldDirty: true, - shouldTouch: true, - shouldValidate: false, - }); - }, + setTransformer: onTransformerUpdate, async triggerUpdate() { await form.trigger('mappings'); setTimeout(() => { @@ -497,7 +516,23 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement { setSelectedTables: setSelectedTables, }); - if (isConnectionsLoading || isSchemaDataMapLoading || isJobDataLoading) { + const { onClick: onApplyDefaultClick } = useOnApplyDefaultClick({ + getMappings() { + return form.getValues('mappings'); + }, + setTransformer: onTransformerUpdate, + constraintHandler: schemaConstraintHandler, + triggerUpdate() { + form.trigger('mappings'); + }, + }); + + if ( + isConnectionsLoading || + isSchemaDataMapLoading || + isJobDataLoading || + isGetTransformersLoading + ) { return ; } @@ -518,6 +553,34 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement { c.connectionConfig?.config.case !== 'gcpCloudstorageConfig' ) ); + + function getAvailableTransformers(idx: number): TransformerResult { + const row = formMappings[idx]; + return handler.getFilteredTransformers( + getTransformerFilter( + schemaConstraintHandler, + { + schema: row.schema, + table: row.table, + column: row.column, + }, + 'sync' + ) + ); + } + + function onTransformerBulkUpdate( + indices: number[], + config: JobMappingTransformerForm + ): void { + indices.forEach((idx) => { + onTransformerUpdate(idx, config); + }); + setTimeout(() => { + form.trigger('mappings'); + }, 0); + } + return (
@@ -588,34 +651,20 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement { validateMappingsResponse )} onValidate={validateMappings} - constraintHandler={schemaConstraintHandler} - onRemoveMappings={(values) => { - const valueSet = new Set( - values.map((v) => `${v.schema}.${v.table}.${v.column}`) - ); - const toRemove: number[] = []; - formMappings.forEach((mapping, idx) => { - if ( - valueSet.has( - `${mapping.schema}.${mapping.table}.${mapping.column}` - ) - ) { - toRemove.push(idx); - } - }); - if (toRemove.length > 0) { - remove(toRemove); + onRemoveMappings={(indices) => { + const indexSet = new Set(indices); + const remainingTables = formMappings + .filter((_, idx) => !indexSet.has(idx)) + .map((fm) => fm.table); + + if (indices.length > 0) { + remove(indices); } if (!source || isDynamoDBConnection(source)) { return; } - const toRemoveSet = new Set(toRemove); - const remainingTables = formMappings - .filter((_, idx) => !toRemoveSet.has(idx)) - .map((fm) => fm.table); - // Check and update destinationOptions if needed const destOpts = form.getValues('destinationOptions'); const updatedDestOpts = destOpts @@ -653,7 +702,7 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement { onAddMappings={(values) => { append( values.map((v) => { - const [schema, table] = v.collection.split('.'); + const [schema, table] = splitCollection(v.collection); return { schema, table, @@ -734,6 +783,25 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement { dynamoDBDestinations.length > 0 )} onImportMappingsClick={onImportMappingsClick} + getAvailableTransformers={getAvailableTransformers} + getTransformerFromField={(idx) => { + const row = formMappings[idx]; + return getTransformerFromField(handler, row.transformer); + }} + getAvailableTransformersForBulk={(rows) => { + return getFilteredTransformersForBulkSet( + rows, + handler, + schemaConstraintHandler, + 'sync', + 'nosql' + ); + }} + getTransformerFromFieldValue={(fvalue) => { + return getTransformerFromField(handler, fvalue); + }} + onApplyDefaultClick={onApplyDefaultClick} + onTransformerBulkUpdate={onTransformerBulkUpdate} /> )} @@ -757,6 +825,28 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement { addVirtualForeignKey={addVirtualForeignKey} removeVirtualForeignKey={removeVirtualForeignKey} onImportMappingsClick={onImportMappingsClick} + onTransformerUpdate={(idx, cfg) => { + onTransformerUpdate(idx, cfg); + }} + getAvailableTransformers={getAvailableTransformers} + getTransformerFromField={(idx) => { + const row = formMappings[idx]; + return getTransformerFromField(handler, row.transformer); + }} + getAvailableTransformersForBulk={(rows) => { + return getFilteredTransformersForBulkSet( + rows, + handler, + schemaConstraintHandler, + 'sync', + 'relational' + ); + }} + getTransformerFromFieldValue={(fvalue) => { + return getTransformerFromField(handler, fvalue); + }} + onApplyDefaultClick={onApplyDefaultClick} + onTransformerBulkUpdate={onTransformerBulkUpdate} /> )}
diff --git a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/useOnApplyDefaultClick.tsx b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/useOnApplyDefaultClick.tsx new file mode 100644 index 0000000000..f4ee996af7 --- /dev/null +++ b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/useOnApplyDefaultClick.tsx @@ -0,0 +1,76 @@ +import { SchemaConstraintHandler } from '@/components/jobs/SchemaTable/schema-constraint-handler'; +import { + convertJobMappingTransformerToForm, + JobMappingFormValues, + JobMappingTransformerForm, +} from '@/yup-validations/jobs'; +import { + GenerateDefault, + JobMappingTransformer, + Passthrough, + TransformerConfig, +} from '@neosync/sdk'; + +interface Props { + getMappings(): JobMappingFormValues[]; + setTransformer(idx: number, transformer: JobMappingTransformerForm): void; + triggerUpdate(): void; + constraintHandler: SchemaConstraintHandler; +} + +interface UseOnApplyDefaultClickResponse { + onClick(override: boolean): void; +} + +// Hook that provides an onClick handler that will handle setting job mappings to a sensible default transformer +export function useOnApplyDefaultClick( + props: Props +): UseOnApplyDefaultClickResponse { + const { getMappings, setTransformer, triggerUpdate, constraintHandler } = + props; + + return { + onClick(override) { + const formMappings = getMappings(); + formMappings.forEach((fm, idx) => { + // skips setting the default transformer if the user has already set the transformer + if (fm.transformer.config.case && !override) { + return; + } else { + const colkey = { + schema: fm.schema, + table: fm.table, + column: fm.column, + }; + const isGenerated = constraintHandler.getIsGenerated(colkey); + const identityType = constraintHandler.getIdentityType(colkey); + const newJm = getDefaultJMTransformer(isGenerated && !identityType); + setTransformer(idx, convertJobMappingTransformerToForm(newJm)); + } + }); + setTimeout(() => { + triggerUpdate(); + }, 0); + }, + }; +} + +function getDefaultJMTransformer(useDefault: boolean): JobMappingTransformer { + return useDefault + ? new JobMappingTransformer({ + config: new TransformerConfig({ + config: { + case: 'generateDefaultConfig', + value: new GenerateDefault(), + }, + }), + }) + : new JobMappingTransformer({ + config: new TransformerConfig({ + config: { + case: 'passthroughConfig', + value: new Passthrough(), + }, + }), + }); +} diff --git a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/useOnImportMappings.tsx b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/useOnImportMappings.tsx index 86c4edf243..5f1f69c0d5 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/useOnImportMappings.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/useOnImportMappings.tsx @@ -24,7 +24,7 @@ interface UseOnImportMappingsResponse { ): void; } -// Hook that provides an onClick handler that will importing job mappings. +// Hook that provides an onClick handler that will import job mappings. export function useOnImportMappings(props: Props): UseOnImportMappingsResponse { const { setMappings, diff --git a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/util.ts b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/util.ts index 4bfd4ed25a..a2c6af60e9 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/util.ts +++ b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/util.ts @@ -1,5 +1,23 @@ import { Action } from '@/components/DualListBox/DualListBox'; +import { + JobMappingRow, + NosqlJobMappingRow, +} from '@/components/jobs/JobMappingTable/Columns'; import { DestinationDetails } from '@/components/jobs/NosqlTable/TableMappings/Columns'; +import { + JobType, + SchemaConstraintHandler, +} from '@/components/jobs/SchemaTable/schema-constraint-handler'; +import { + TransformerConfigCase, + TransformerHandler, + TransformerResult, +} from '@/components/jobs/SchemaTable/transformer-handler'; +import { + fromNosqlRowDataToColKey, + fromRowDataToColKey, + getTransformerFilter, +} from '@/components/jobs/SchemaTable/util'; import { isValidSubsetType } from '@/components/jobs/subsets/utils'; import { JobMappingFormValues, @@ -12,7 +30,10 @@ import { JobDestination, JobMappingTransformer, JobSource, + SystemTransformer, + UserDefinedTransformer, } from '@neosync/sdk'; +import { Row } from '@tanstack/react-table'; export function getConnectionIdFromSource( js: JobSource | undefined @@ -176,3 +197,103 @@ export function getDynamoDbDestinations( (d) => d.options?.config.case === 'dynamodbOptions' ); } + +export function getFilteredTransformersForBulkSet( + rows: Row[] | Row[], + transformerHandler: TransformerHandler, + constraintHandler: SchemaConstraintHandler, + jobType: JobType, + sqlType: 'relational' | 'nosql' +): TransformerResult { + const systemArrays: SystemTransformer[][] = []; + const userDefinedArrays: UserDefinedTransformer[][] = []; + + rows.forEach((row) => { + const colkey = + sqlType === 'nosql' + ? fromNosqlRowDataToColKey(row as Row) + : fromRowDataToColKey(row as Row); + const { system, userDefined } = transformerHandler.getFilteredTransformers( + getTransformerFilter(constraintHandler, colkey, jobType) + ); + systemArrays.push(system); + userDefinedArrays.push(userDefined); + }); + + const uniqueSystemConfigCases = findCommonSystemConfigCases(systemArrays); + const uniqueSystem = uniqueSystemConfigCases + .map((configCase) => + transformerHandler.getSystemTransformerByConfigCase(configCase) + ) + .filter((x): x is SystemTransformer => !!x); + + const uniqueIds = findCommonUserDefinedIds(userDefinedArrays); + const uniqueUserDef = uniqueIds + .map((id) => transformerHandler.getUserDefinedTransformerById(id)) + .filter((x): x is UserDefinedTransformer => !!x); + + return { + system: uniqueSystem, + userDefined: uniqueUserDef, + }; +} + +function findCommonSystemConfigCases( + arrays: SystemTransformer[][] +): TransformerConfigCase[] { + const elementCount: Record = {} as Record< + TransformerConfigCase, + number + >; + const subArrayCount = arrays.length; + const commonElements: TransformerConfigCase[] = []; + + arrays.forEach((subArray) => { + // Use a Set to ensure each element in a sub-array is counted only once + new Set(subArray).forEach((element) => { + if (!element.config?.config.case) { + return; + } + if (!elementCount[element.config.config.case]) { + elementCount[element.config.config.case] = 1; + } else { + elementCount[element.config.config.case]++; + } + }); + }); + + for (const [element, count] of Object.entries(elementCount)) { + if (count === subArrayCount) { + commonElements.push(element as TransformerConfigCase); + } + } + + return commonElements; +} + +function findCommonUserDefinedIds( + arrays: UserDefinedTransformer[][] +): string[] { + const elementCount: Record = {}; + const subArrayCount = arrays.length; + const commonElements: string[] = []; + + arrays.forEach((subArray) => { + // Use a Set to ensure each element in a sub-array is counted only once + new Set(subArray).forEach((element) => { + if (!elementCount[element.id]) { + elementCount[element.id] = 1; + } else { + elementCount[element.id]++; + } + }); + }); + + for (const [element, count] of Object.entries(elementCount)) { + if (count === subArrayCount) { + commonElements.push(element); + } + } + + return commonElements; +} diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/generate/single/schema/page.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/generate/single/schema/page.tsx index 57643e7a4c..479c01046c 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/generate/single/schema/page.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/generate/single/schema/page.tsx @@ -1,8 +1,12 @@ 'use client'; import FormPersist from '@/app/(mgmt)/FormPersist'; +import { useOnApplyDefaultClick } from '@/app/(mgmt)/[account]/jobs/[id]/source/components/useOnApplyDefaultClick'; import { useOnImportMappings } from '@/app/(mgmt)/[account]/jobs/[id]/source/components/useOnImportMappings'; -import { getOnSelectedTableToggle } from '@/app/(mgmt)/[account]/jobs/[id]/source/components/util'; +import { + getFilteredTransformersForBulkSet, + getOnSelectedTableToggle, +} from '@/app/(mgmt)/[account]/jobs/[id]/source/components/util'; import { clearNewJobSession, getCreateNewSingleTableGenerateJobRequest, @@ -16,6 +20,8 @@ import { SchemaTable, } from '@/components/jobs/SchemaTable/SchemaTable'; import { getSchemaConstraintHandler } from '@/components/jobs/SchemaTable/schema-constraint-handler'; +import { TransformerResult } from '@/components/jobs/SchemaTable/transformer-handler'; +import { getTransformerFilter } from '@/components/jobs/SchemaTable/util'; import { useAccount } from '@/components/providers/account-provider'; import { PageProps } from '@/components/types'; import { Alert, AlertTitle } from '@/components/ui/alert'; @@ -30,8 +36,10 @@ import { FormMessage, } from '@/components/ui/form'; import { Input } from '@/components/ui/input'; +import { useGetTransformersHandler } from '@/libs/hooks/useGetTransformersHandler'; import { getSingleOrUndefined } from '@/libs/utils'; -import { getErrorMessage } from '@/util/util'; +import { getErrorMessage, getTransformerFromField } from '@/util/util'; +import { JobMappingTransformerForm } from '@/yup-validations/jobs'; import { useMutation, useQuery } from '@connectrpc/connect-query'; import { yupResolver } from '@hookform/resolvers/yup'; import { ValidateJobMappingsResponse } from '@neosync/sdk'; @@ -217,12 +225,11 @@ export default function Page({ searchParams }: PageProps): ReactElement { [isSchemaMapValidating, isTableConstraintsValidating] ); const [selectedTables, setSelectedTables] = useState>(new Set()); - const { append, remove, fields } = useFieldArray( - { + const { append, remove, fields, update } = + useFieldArray({ control: form.control, name: 'mappings', - } - ); + }); const onSelectedTableToggle = getOnSelectedTableToggle( connectionSchemaDataMap?.schemaMap ?? {}, @@ -255,6 +262,21 @@ export default function Page({ searchParams }: PageProps): ReactElement { ); }, [isSchemaMapLoading]); + const { handler } = useGetTransformersHandler(account?.id ?? ''); + + function onTransformerUpdate( + index: number, + transformer: JobMappingTransformerForm + ): void { + const val = form.getValues(`mappings.${index}`); + update(index, { + schema: val.schema, + table: val.table, + column: val.column, + transformer, + }); + } + const { onClick: onImportMappingsClick } = useOnImportMappings({ setMappings(mappings) { form.setValue('mappings', mappings, { @@ -282,6 +304,44 @@ export default function Page({ searchParams }: PageProps): ReactElement { setSelectedTables: setSelectedTables, }); + function getAvailableTransformers(idx: number): TransformerResult { + const row = formMappings[idx]; + return handler.getFilteredTransformers( + getTransformerFilter( + schemaConstraintHandler, + { + schema: row.schema, + table: row.table, + column: row.column, + }, + 'sync' + ) + ); + } + + const { onClick: onApplyDefaultClick } = useOnApplyDefaultClick({ + getMappings() { + return form.getValues('mappings'); + }, + setTransformer: onTransformerUpdate, + constraintHandler: schemaConstraintHandler, + triggerUpdate() { + form.trigger('mappings'); + }, + }); + + function onTransformerBulkUpdate( + indices: number[], + config: JobMappingTransformerForm + ): void { + indices.forEach((idx) => { + onTransformerUpdate(idx, config); + }); + setTimeout(() => { + form.trigger('mappings'); + }, 0); + } + return (
@@ -346,6 +406,28 @@ export default function Page({ searchParams }: PageProps): ReactElement { onValidate={validateMappings} isJobMappingsValidating={isValidatingMappings} onImportMappingsClick={onImportMappingsClick} + onTransformerUpdate={(idx, cfg) => { + onTransformerUpdate(idx, cfg); + }} + getAvailableTransformers={getAvailableTransformers} + getTransformerFromField={(idx) => { + const row = formMappings[idx]; + return getTransformerFromField(handler, row.transformer); + }} + getAvailableTransformersForBulk={(rows) => { + return getFilteredTransformersForBulkSet( + rows, + handler, + schemaConstraintHandler, + 'sync', + 'relational' + ); + }} + getTransformerFromFieldValue={(fvalue) => { + return getTransformerFromField(handler, fvalue); + }} + onApplyDefaultClick={onApplyDefaultClick} + onTransformerBulkUpdate={onTransformerBulkUpdate} /> )} {form.formState.errors.root && ( diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/schema/page.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/schema/page.tsx index 5c1cc5ea8e..d8eb1f0783 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/schema/page.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/schema/page.tsx @@ -10,14 +10,18 @@ import { SchemaTable, } from '@/components/jobs/SchemaTable/SchemaTable'; import { getSchemaConstraintHandler } from '@/components/jobs/SchemaTable/schema-constraint-handler'; +import { TransformerResult } from '@/components/jobs/SchemaTable/transformer-handler'; +import { getTransformerFilter } from '@/components/jobs/SchemaTable/util'; import { useAccount } from '@/components/providers/account-provider'; import SkeletonForm from '@/components/skeleton/SkeletonForm'; import { PageProps } from '@/components/types'; import { Button } from '@/components/ui/button'; import { Form } from '@/components/ui/form'; +import { useGetTransformersHandler } from '@/libs/hooks/useGetTransformersHandler'; import { getSingleOrUndefined } from '@/libs/utils'; -import { getErrorMessage } from '@/util/util'; +import { getErrorMessage, getTransformerFromField } from '@/util/util'; import { + JobMappingTransformerForm, SchemaFormValues, SchemaFormValuesDestinationOptions, VirtualForeignConstraintFormValues, @@ -51,9 +55,11 @@ import { ReactElement, useCallback, useEffect, useMemo, useState } from 'react'; import { useFieldArray, useForm } from 'react-hook-form'; import { toast } from 'sonner'; import { useSessionStorage } from 'usehooks-ts'; +import { useOnApplyDefaultClick } from '../../../jobs/[id]/source/components/useOnApplyDefaultClick'; import { useOnImportMappings } from '../../../jobs/[id]/source/components/useOnImportMappings'; import { getDestinationDetailsRecord, + getFilteredTransformersForBulkSet, getOnSelectedTableToggle, isConnectionSubsettable, isDynamoDBConnection, @@ -406,7 +412,21 @@ export default function Page({ searchParams }: PageProps): ReactElement { setSelectedTables: setSelectedTables, }); - if (isConnectionLoading || isSchemaMapLoading) { + const { handler, isLoading: isGetTransformersLoading } = + useGetTransformersHandler(account?.id ?? ''); + + const { onClick: onApplyDefaultClick } = useOnApplyDefaultClick({ + getMappings() { + return form.getValues('mappings'); + }, + setTransformer: onTransformerUpdate, + constraintHandler: schemaConstraintHandler, + triggerUpdate() { + form.trigger('mappings'); + }, + }); + + if (isConnectionLoading || isSchemaMapLoading || isGetTransformersLoading) { return ; } @@ -427,6 +447,46 @@ export default function Page({ searchParams }: PageProps): ReactElement { })) : []; + function getAvailableTransformers(idx: number): TransformerResult { + const row = formMappings[idx]; + return handler.getFilteredTransformers( + getTransformerFilter( + schemaConstraintHandler, + { + schema: row.schema, + table: row.table, + column: row.column, + }, + 'sync' + ) + ); + } + + function onTransformerUpdate( + index: number, + transformer: JobMappingTransformerForm + ): void { + const val = form.getValues(`mappings.${index}`); + update(index, { + schema: val.schema, + table: val.table, + column: val.column, + transformer, + }); + } + + function onTransformerBulkUpdate( + indices: number[], + config: JobMappingTransformerForm + ): void { + indices.forEach((idx) => { + onTransformerUpdate(idx, config); + }); + setTimeout(() => { + form.trigger('mappings'); + }, 0); + } + return (
@@ -463,34 +523,20 @@ export default function Page({ searchParams }: PageProps): ReactElement { validateMappingsResponse )} onValidate={validateMappings} - constraintHandler={schemaConstraintHandler} - onRemoveMappings={(values) => { - const valueSet = new Set( - values.map((v) => `${v.schema}.${v.table}.${v.column}`) - ); - const toRemove: number[] = []; - formMappings.forEach((mapping, idx) => { - if ( - valueSet.has( - `${mapping.schema}.${mapping.table}.${mapping.column}` - ) - ) { - toRemove.push(idx); - } - }); - if (toRemove.length > 0) { - remove(toRemove); + onRemoveMappings={(indices) => { + const indexSet = new Set(indices); + const remainingTables = formMappings + .filter((_, idx) => !indexSet.has(idx)) + .map((fm) => fm.table); + + if (indices.length > 0) { + remove(indices); } if (!source || isDynamoDBConnection(source)) { return; } - const toRemoveSet = new Set(toRemove); - const remainingTables = formMappings - .filter((_, idx) => !toRemoveSet.has(idx)) - .map((fm) => fm.table); - // Check and update destinationOptions if needed const destOpts = form.getValues('destinationOptions'); const updatedDestOpts = destOpts @@ -609,6 +655,25 @@ export default function Page({ searchParams }: PageProps): ReactElement { )} destinationOptions={form.watch('destinationOptions')} onImportMappingsClick={onImportMappingsClick} + getAvailableTransformers={getAvailableTransformers} + getTransformerFromField={(idx) => { + const row = formMappings[idx]; + return getTransformerFromField(handler, row.transformer); + }} + getAvailableTransformersForBulk={(rows) => { + return getFilteredTransformersForBulkSet( + rows, + handler, + schemaConstraintHandler, + 'sync', + 'nosql' + ); + }} + getTransformerFromFieldValue={(fvalue) => { + return getTransformerFromField(handler, fvalue); + }} + onApplyDefaultClick={onApplyDefaultClick} + onTransformerBulkUpdate={onTransformerBulkUpdate} /> )} @@ -632,6 +697,28 @@ export default function Page({ searchParams }: PageProps): ReactElement { addVirtualForeignKey={addVirtualForeignKey} removeVirtualForeignKey={removeVirtualForeignKey} onImportMappingsClick={onImportMappingsClick} + onTransformerUpdate={(idx, cfg) => { + onTransformerUpdate(idx, cfg); + }} + getAvailableTransformers={getAvailableTransformers} + getTransformerFromField={(idx) => { + const row = formMappings[idx]; + return getTransformerFromField(handler, row.transformer); + }} + getAvailableTransformersForBulk={(rows) => { + return getFilteredTransformersForBulkSet( + rows, + handler, + schemaConstraintHandler, + 'sync', + 'relational' + ); + }} + getTransformerFromFieldValue={(fvalue) => { + return getTransformerFromField(handler, fvalue); + }} + onApplyDefaultClick={onApplyDefaultClick} + onTransformerBulkUpdate={onTransformerBulkUpdate} /> )}
diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/transformer/TransformerForms/GenerateIpAddressForm.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/transformer/TransformerForms/GenerateIpAddressForm.tsx index ed459c8df0..d9cd6d6dd3 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/transformer/TransformerForms/GenerateIpAddressForm.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/transformer/TransformerForms/GenerateIpAddressForm.tsx @@ -58,7 +58,6 @@ export default function GenerateIpAddressForm(props: Props): ReactElement { GenerateIpAddressType.V4_PRIVATE_A, GenerateIpAddressType.V4_PRIVATE_B, GenerateIpAddressType.V4_PRIVATE_C, - GenerateIpAddressType.V4_PRIVATE_C, GenerateIpAddressType.V4_PUBLIC, GenerateIpAddressType.V6, ].map((version) => ( diff --git a/frontend/apps/web/components/jobs/JobMappingTable/AttributesCell.tsx b/frontend/apps/web/components/jobs/JobMappingTable/AttributesCell.tsx new file mode 100644 index 0000000000..fd241609f1 --- /dev/null +++ b/frontend/apps/web/components/jobs/JobMappingTable/AttributesCell.tsx @@ -0,0 +1,99 @@ +import { Badge } from '@/components/ui/badge'; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from '@/components/ui/tooltip'; +import { ReactElement } from 'react'; + +interface Props { + generatedType: string | undefined; + identityType: string | undefined; + value: string; +} + +export default function AttributesCell(props: Props): ReactElement { + const { generatedType, identityType, value } = props; + + return ( + +
+ {generatedType && ( +
+ + + + + Generated + + + + {getGeneratedStatement(generatedType)} + + + +
+ )} + {!generatedType && identityType && ( + // the API treats generatedType and identityType as mutually exclusive +
+ + + + + Generated + + + {value} + + +
+ )} + {identityType && ( +
+ + + + + Identity + + + {value} + + +
+ )} +
+
+ ); +} + +export function getIdentityStatement(identityType: string): string { + if (identityType === 'a') { + return 'GENERATED ALWAYS AS IDENTITY'; + } else if (identityType === 'd') { + return 'GENERATED BY DEFAULT AS IDENTITY'; + } else if (identityType === 'auto_increment') { + return 'AUTO_INCREMENT'; + } + return identityType; +} + +export function getGeneratedStatement(generatedType: string): string { + if (generatedType === 's' || generatedType === 'STORED GENERATED') { + return 'GENERATED ALWAYS AS STORED'; + } else if (generatedType === 'v' || generatedType === 'VIRTUAL GENERATED') { + return 'GENERATED ALWAYS AS VIRTUAL'; + } + return generatedType; +} diff --git a/frontend/apps/web/components/jobs/JobMappingTable/Columns.tsx b/frontend/apps/web/components/jobs/JobMappingTable/Columns.tsx new file mode 100644 index 0000000000..19b01da1fa --- /dev/null +++ b/frontend/apps/web/components/jobs/JobMappingTable/Columns.tsx @@ -0,0 +1,409 @@ +import EditTransformerOptions from '@/app/(mgmt)/[account]/transformers/EditTransformerOptions'; +import TruncatedText from '@/components/TruncatedText'; +import { Badge } from '@/components/ui/badge'; +import { + getTransformerSelectButtonText, + isInvalidTransformer, +} from '@/util/util'; +import { JobMappingTransformerForm } from '@/yup-validations/jobs'; +import { SystemTransformer } from '@neosync/sdk'; +import { ColumnDef, createColumnHelper } from '@tanstack/react-table'; +import { DataTableRowActions } from '../NosqlTable/data-table-row-actions'; +import EditCollection from '../NosqlTable/EditCollection'; +import EditDocumentKey from '../NosqlTable/EditDocumentKey'; +import { SchemaColumnHeader } from '../SchemaTable/SchemaColumnHeader'; +import TransformerSelect from '../SchemaTable/TransformerSelect'; +import AttributesCell from './AttributesCell'; +import ConstraintsCell from './ConstraintsCell'; +import DataTypeCell from './DataTypeCell'; +import IndeterminateCheckbox from './IndeterminateCheckbox'; + +export interface JobMappingRow { + schema: string; + table: string; + column: string; + constraints: RowConstraint; + dataType: string; + isNullable: boolean; + attributes: RowAttribute; + transformer: JobMappingTransformerForm; +} + +interface RowAttribute { + value: string; // accessor fn value for search + + generatedType: string | undefined; + identityType: string | undefined; +} +interface RowConstraint { + value: string; // accessor fn value for search + isPrimaryKey: boolean; + foreignKey: [boolean, string[]]; + virtualForeignKey: [boolean, string[]]; + isUnique: boolean; +} + +export interface NosqlJobMappingRow { + collection: string; // combined schema.table + column: string; + transformer: JobMappingTransformerForm; +} + +function getJobMappingColumns(): ColumnDef[] { + const columnHelper = createColumnHelper(); + + const checkboxColumn = columnHelper.display({ + id: 'isSelected', + header({ table }) { + return ( + + ); + }, + cell({ row }) { + return ( +
+ +
+ ); + }, + maxSize: 30, + }); + + const schemaColumn = columnHelper.accessor('schema', { + header({ column }) { + return ; + }, + }); + + const tableColumn = columnHelper.accessor((row) => row.table, { + id: 'table', + header({ column }) { + return ; + }, + }); + + const columnColumn = columnHelper.accessor('column', { + header({ column }) { + return ; + }, + cell({ getValue }) { + return ; + }, + }); + + const dataTypeColumn = columnHelper.accessor('dataType', { + header({ column }) { + return ; + }, + cell({ getValue }) { + return ; + }, + }); + + const isNullableColumn = columnHelper.accessor( + (row) => (row.isNullable ? 'Yes' : 'No') as string, + { + id: 'isNullable', + header({ column }) { + return ; + }, + cell({ getValue }) { + return ( + + {getValue()} + + ); + }, + } + ); + + const constraintColumn = columnHelper.accessor( + (row) => row.constraints.value, + { + id: 'constraints', + header({ column }) { + return ; + }, + cell({ row }) { + const constraints = row.original.constraints; + return ( + + ); + }, + } + ); + + const attributeColumn = columnHelper.accessor((row) => row.attributes.value, { + id: 'attributeValues', + header({ column }) { + return ; + }, + cell({ row }) { + const val = row.original.attributes; + return ( + + ); + }, + }); + + const transformerColumn = columnHelper.accessor( + (row) => { + if (row.transformer.config.case) { + return row.transformer.config.case.toLowerCase(); + } + return 'select transformer'; + }, + { + id: 'transformer', + header({ column }) { + return ; + }, + cell({ table, row }) { + const transformer = + table.options.meta?.getTransformerFromField(row.index) ?? + new SystemTransformer(); + const transformerForm = row.original.transformer; + return ( +
+
+ + table.options.meta?.getAvailableTransformers(row.index) ?? { + system: [], + userDefined: [], + } + } + buttonText={getTransformerSelectButtonText(transformer)} + buttonClassName="w-[175px]" + value={transformerForm} + onSelect={(updatedValue) => + table.options.meta?.onTransformerUpdate( + row.index, + updatedValue + ) + } + disabled={false} + /> +
+
+ { + table.options.meta?.onTransformerUpdate( + row.index, + updatedValue + ); + }} + disabled={isInvalidTransformer(transformer)} + /> +
+
+ ); + }, + } + ); + + return [ + checkboxColumn, + schemaColumn, + tableColumn, + columnColumn, + dataTypeColumn, + isNullableColumn, + constraintColumn, + attributeColumn, + transformerColumn, + ]; +} + +function getNosqlJobMappingColumns(): ColumnDef[] { + const columnHelper = createColumnHelper(); + + const checkboxColumn = columnHelper.display({ + id: 'isSelected', + header({ table }) { + return ( + + ); + }, + cell({ row }) { + return ( +
+ +
+ ); + }, + }); + + const collectionColumn = columnHelper.accessor('collection', { + header({ column }) { + return ; + }, + cell({ getValue, table, row }) { + return ( + { + if (table.options.meta?.onRowUpdate) { + table.options.meta.onRowUpdate(row.index, { + ...row.original, + collection: updatedValue.collection, + }); + } + }} + /> + ); + }, + }); + + const columnColumn = columnHelper.accessor('column', { + header({ column }) { + return ; + }, + cell({ getValue, table, row }) { + return ( + { + return ( + newValue !== currValue && + (table.options.meta?.canRenameColumn(row.index, newValue) ?? + false) + ); + }} + onEdit={(updatedValue) => { + if (table.options.meta?.onRowUpdate) { + table.options.meta.onRowUpdate(row.index, { + ...row.original, + column: updatedValue.column, + }); + } + }} + /> + ); + }, + }); + + const transformerColumn = columnHelper.accessor( + (row) => { + if (row.transformer.config.case) { + return row.transformer.config.case.toLowerCase(); + } + return 'select transformer'; + }, + { + id: 'transformer', + header({ column }) { + return ; + }, + cell({ table, row }) { + const transformer = + table.options.meta?.getTransformerFromField(row.index) ?? + new SystemTransformer(); + const transformerForm = row.original.transformer; + return ( +
+
+ + table.options.meta?.getAvailableTransformers(row.index) ?? { + system: [], + userDefined: [], + } + } + buttonText={getTransformerSelectButtonText(transformer)} + buttonClassName="w-[175px]" + value={transformerForm} + onSelect={(updatedValue) => + table.options.meta?.onTransformerUpdate( + row.index, + updatedValue + ) + } + disabled={false} + /> +
+
+ { + table.options.meta?.onTransformerUpdate( + row.index, + updatedValue + ); + }} + disabled={isInvalidTransformer(transformer)} + /> +
+
+ ); + }, + } + ); + + const actionsColumn = columnHelper.display({ + id: 'actions', + header({}) { + return

Actions

; + }, + cell({ row, table }) { + return ( + table.options.meta?.onDuplicateRow(row.index)} + onDelete={() => table.options.meta?.onDeleteRow(row.index)} + /> + ); + }, + }); + + return [ + checkboxColumn, + collectionColumn, + columnColumn, + transformerColumn, + actionsColumn, + ]; +} + +export const SQL_COLUMNS = getJobMappingColumns(); + +export const NOSQL_COLUMNS = getNosqlJobMappingColumns(); diff --git a/frontend/apps/web/components/jobs/JobMappingTable/ConstraintsCell.tsx b/frontend/apps/web/components/jobs/JobMappingTable/ConstraintsCell.tsx new file mode 100644 index 0000000000..b0e3f38993 --- /dev/null +++ b/frontend/apps/web/components/jobs/JobMappingTable/ConstraintsCell.tsx @@ -0,0 +1,85 @@ +import { Badge } from '@/components/ui/badge'; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from '@/components/ui/tooltip'; +import { ReactElement } from 'react'; + +interface Props { + isPrimaryKey: boolean; + foreignKey: [boolean, string[]]; + virtualForeignKey: [boolean, string[]]; + isUnique: boolean; +} + +export default function ConstraintsCell(props: Props): ReactElement { + const { isPrimaryKey, foreignKey, virtualForeignKey, isUnique } = props; + const [isForeignKey, fkCols] = foreignKey; + const [isVirtualForeignKey, vfkCols] = virtualForeignKey; + return ( + +
+ {isPrimaryKey && ( +
+ + Primary Key + +
+ )} + {isForeignKey && ( +
+ + + + + Foreign Key + + + + {fkCols.map((col) => `Primary Key: ${col}`).join('\n')} + + + +
+ )} + {isVirtualForeignKey && ( +
+ + + + + Virtual Foreign Key + + + + {vfkCols.map((col) => `Primary Key: ${col}`).join('\n')} + + + +
+ )} + {isUnique && ( +
+ + Unique + +
+ )} +
+
+ ); +} diff --git a/frontend/apps/web/components/jobs/JobMappingTable/DataTypeCell.tsx b/frontend/apps/web/components/jobs/JobMappingTable/DataTypeCell.tsx new file mode 100644 index 0000000000..fb43f660cc --- /dev/null +++ b/frontend/apps/web/components/jobs/JobMappingTable/DataTypeCell.tsx @@ -0,0 +1,35 @@ +import { Badge } from '@/components/ui/badge'; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from '@/components/ui/tooltip'; +import { ReactElement } from 'react'; +import { handleDataTypeBadge } from '../SchemaTable/util'; + +interface Props { + value: string; +} + +export default function DataTypeCell(props: Props): ReactElement { + const { value } = props; + return ( + + + +
+ + + {handleDataTypeBadge(value)} + + +
+
+ +

{value}

+
+
+
+ ); +} diff --git a/frontend/apps/web/components/jobs/JobMappingTable/IndeterminateCheckbox.tsx b/frontend/apps/web/components/jobs/JobMappingTable/IndeterminateCheckbox.tsx new file mode 100644 index 0000000000..3abb027b9a --- /dev/null +++ b/frontend/apps/web/components/jobs/JobMappingTable/IndeterminateCheckbox.tsx @@ -0,0 +1,24 @@ +import { HTMLProps, useEffect, useRef } from 'react'; + +export default function IndeterminateCheckbox({ + indeterminate, + className = 'w-4 h-4 flex', + ...rest +}: { indeterminate?: boolean } & HTMLProps) { + const ref = useRef(null!); + + useEffect(() => { + if (typeof indeterminate === 'boolean') { + ref.current.indeterminate = !rest.checked && indeterminate; + } + }, [ref, indeterminate, rest.checked]); + + return ( + + ); +} diff --git a/frontend/apps/web/components/jobs/JobMappingTable/JobMappingTable.tsx b/frontend/apps/web/components/jobs/JobMappingTable/JobMappingTable.tsx new file mode 100644 index 0000000000..3e889222b6 --- /dev/null +++ b/frontend/apps/web/components/jobs/JobMappingTable/JobMappingTable.tsx @@ -0,0 +1,320 @@ +import { CardDescription, CardTitle } from '@/components/ui/card'; +import { + StickyHeaderTable, + TableBody, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table'; +import { cn } from '@/libs/utils'; +import { Transformer } from '@/shared/transformers'; +import { JobMappingTransformerForm } from '@/yup-validations/jobs'; +import { JobMapping } from '@neosync/sdk'; +import { + Cell, + ColumnDef, + flexRender, + getCoreRowModel, + getFilteredRowModel, + getSortedRowModel, + Row, + RowData, + useReactTable, +} from '@tanstack/react-table'; +import { useVirtualizer, VirtualItem } from '@tanstack/react-virtual'; +import { memo, ReactElement, useRef } from 'react'; +import { GoWorkflow } from 'react-icons/go'; +import { ImportMappingsConfig } from '../SchemaTable/ImportJobMappingsButton'; +import { SchemaTableToolbar } from '../SchemaTable/SchemaTableToolBar'; +import { TransformerResult } from '../SchemaTable/transformer-handler'; + +interface Props { + data: TData[]; + columns: ColumnDef[]; + onTransformerUpdate(index: number, config: JobMappingTransformerForm): void; + getAvailableTransformers(index: number): TransformerResult; + getTransformerFromField(index: number): Transformer; + + onTransformerBulkUpdate( + indices: number[], + config: JobMappingTransformerForm + ): void; + getAvalableTransformersForBulk(rows: Row[]): TransformerResult; + getTransformerFromFieldValue(value: JobMappingTransformerForm): Transformer; + + isApplyDefaultTransformerButtonDisabled: boolean; + onApplyDefaultClick(override: boolean): void; + + onExportMappingsClick(selected: Row[], shouldFormat: boolean): void; + onImportMappingsClick( + jobmappings: JobMapping[], + config: ImportMappingsConfig + ): void; + + onDuplicateRow(index: number): void; + onDeleteRow(index: number): void; + canRenameColumn(index: number, newColumn: string): boolean; + onRowUpdate(index: number, newValue: TData): void; + getAvailableCollectionsByRow(index: number): string[]; +} + +declare module '@tanstack/react-table' { + interface TableMeta { + onTransformerUpdate( + rowIndex: number, + transformer: JobMappingTransformerForm + ): void; + getAvailableTransformers(rowIndex: number): TransformerResult; + getTransformerFromField(index: number): Transformer; + + onDuplicateRow(rowIndex: number): void; + onDeleteRow(rowIndex: number): void; + canRenameColumn(rowIndex: number, newColumn: string): boolean; + onRowUpdate(rowIndex: number, newValue: TData): void; + // Returns the available schema.table list + getAvailableCollectionsByRow(rowIndex: number): string[]; + } +} + +export default function JobMappingTable( + props: Props +): ReactElement { + const { + data, + columns, + onTransformerUpdate, + getAvailableTransformers, + getTransformerFromField, + onExportMappingsClick, + onImportMappingsClick, + getAvalableTransformersForBulk, + getTransformerFromFieldValue, + isApplyDefaultTransformerButtonDisabled, + onApplyDefaultClick, + onTransformerBulkUpdate, + onDeleteRow, + onDuplicateRow, + canRenameColumn, + onRowUpdate, + getAvailableCollectionsByRow, + } = props; + + const table = useReactTable({ + data, + columns, + getCoreRowModel: getCoreRowModel(), + getSortedRowModel: getSortedRowModel(), + getFilteredRowModel: getFilteredRowModel(), + meta: { + onTransformerUpdate: onTransformerUpdate, + getAvailableTransformers: getAvailableTransformers, + getTransformerFromField: getTransformerFromField, + onDeleteRow, + onDuplicateRow, + canRenameColumn, + onRowUpdate, + getAvailableCollectionsByRow, + }, + }); + + const { rows } = table.getRowModel(); + + const tableContainerRef = useRef(null); + + const rowVirtualizer = useVirtualizer({ + count: rows.length, + estimateSize() { + return 53; + }, + getScrollElement() { + return tableContainerRef.current; + }, + overscan: 50, + }); + + return ( +
+
+
+ +
+ Transformer Mapping +
+ + Map Transformers to every column below. + +
+ + table={table} + displayApplyDefaultTransformersButton={true} + isApplyDefaultButtonDisabled={isApplyDefaultTransformerButtonDisabled} + getAllowedTransformers={getAvalableTransformersForBulk} + getTransformerFromField={getTransformerFromFieldValue} + onApplyDefaultClick={onApplyDefaultClick} + onBulkUpdate={onTransformerBulkUpdate} + onExportMappingsClick={(shouldFormat) => + onExportMappingsClick( + table.getSelectedRowModel().rows, + shouldFormat + ) + } + onImportMappingsClick={onImportMappingsClick} + /> +
+ +
0 && 'overflow-auto' + )} + ref={tableContainerRef} + > + + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => { + return ( + + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef.header, + header.getContext() + )} + + ); + })} + + ))} + + + {rowVirtualizer.getVirtualItems().map((virtualRow) => { + const row = rows[virtualRow.index]; + return ( + + ); + })} + + +
+
+ Total rows: ({getFormattedCount(data.length)}) Rows visible: ( + {getFormattedCount(table.getRowModel().rows.length)}) +
+
+ ); +} + +const US_NUMBER_FORMAT = new Intl.NumberFormat('en-US'); +function getFormattedCount(count: number): string { + return US_NUMBER_FORMAT.format(count); +} + +const MemoizedRow = memo( + ({ + row, + virtualRow, + }: { + row: Row; + virtualRow: VirtualItem; + selected: boolean; + }) => { + return ( + + {row.getVisibleCells().map((cell) => ( + + } /> + + ))} + + ); + }, + (prev, next) => { + // Compare virtualRow properties + if ( + prev.virtualRow.start !== next.virtualRow.start || + prev.virtualRow.size !== next.virtualRow.size + ) { + return false; + } + + // Compare row.id + if (prev.row.id !== next.row.id) { + return false; + } + + // Check row selection state for "isSelected" + if (prev.selected !== next.selected) { + return false; + } + + // Check if visible cells or their values have changed + const prevCells = prev.row.getVisibleCells(); + const nextCells = next.row.getVisibleCells(); + + if (prevCells.length !== nextCells.length) { + return false; + } + + for (let i = 0; i < prevCells.length; i++) { + if ( + prevCells[i].id !== nextCells[i].id || + prevCells[i].getValue() !== nextCells[i].getValue() + ) { + return false; + } + } + + // If no differences are found, skip re-render + return true; + } +); +MemoizedRow.displayName = 'MemoizedRow'; + +const MemoizedCell = memo( + ({ cell }: { cell: Cell }) => + flexRender(cell.column.columnDef.cell, cell.getContext()), + (prev, next) => { + const prevValue = prev.cell.getValue(); + const nextValue = next.cell.getValue(); + + if (prev.cell.column.id === 'isSelected') { + // Always re-render checkbox cells as getIsSelected() is always the same for both + return false; + } + + // For other columns, just compare the values + return prevValue === nextValue; + } +); +MemoizedCell.displayName = 'MemoizedCell'; diff --git a/frontend/apps/web/components/jobs/NosqlTable/AddNewNosqlRecord.tsx b/frontend/apps/web/components/jobs/NosqlTable/AddNewNosqlRecord.tsx new file mode 100644 index 0000000000..8b55941c6c --- /dev/null +++ b/frontend/apps/web/components/jobs/NosqlTable/AddNewNosqlRecord.tsx @@ -0,0 +1,274 @@ +import EditTransformerOptions from '@/app/(mgmt)/[account]/transformers/EditTransformerOptions'; +import { useAccount } from '@/components/providers/account-provider'; +import { Button } from '@/components/ui/button'; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from '@/components/ui/form'; +import { Input } from '@/components/ui/input'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { cn } from '@/libs/utils'; +import { + getTransformerFromField, + getTransformerSelectButtonText, + isInvalidTransformer, +} from '@/util/util'; +import { + convertJobMappingTransformerToForm, + JobMappingTransformerForm, +} from '@/yup-validations/jobs'; +import { PartialMessage } from '@bufbuild/protobuf'; +import { useMutation } from '@connectrpc/connect-query'; +import { yupResolver } from '@hookform/resolvers/yup'; +import { + ConnectError, + JobMappingTransformer, + Passthrough, + SystemTransformer, + TransformerConfig, + TransformerSource, + ValidateUserJavascriptCodeRequest, + ValidateUserJavascriptCodeResponse, +} from '@neosync/sdk'; +import { validateUserJavascriptCode } from '@neosync/sdk/connectquery'; +import { UseMutateAsyncFunction } from '@tanstack/react-query'; +import { ReactElement } from 'react'; +import { useForm } from 'react-hook-form'; +import * as Yup from 'yup'; +import { TransformerHandler } from '../SchemaTable/transformer-handler'; +import TransformerSelect from '../SchemaTable/TransformerSelect'; + +interface Props { + collections: string[]; + onSubmit(values: AddNewNosqlRecordFormValues): void; + transformerHandler: TransformerHandler; + isDuplicateKey: ( + value: string, + schema: string, + table: string, + currValue?: string + ) => boolean; +} + +const AddNewNosqlRecordFormValues = Yup.object({ + collection: Yup.string().required('The Collection is required.'), + key: Yup.string() + .required('The Key is required.') + .test({ + name: 'uniqueMapping', + message: 'This key already exists in the selected collection.', + test: function (value, context) { + const { collection } = this.parent; + + if (!collection || !value) { + return true; + } + + const lastDotIndex = collection.lastIndexOf('.'); + const schema = collection.substring(0, lastDotIndex); + const table = collection.substring(lastDotIndex + 1); + + return ( + !context?.options?.context?.isDuplicateKey(value, schema, table) || + this.createError({ + message: 'This key already exists in this collection.', + }) + ); + }, + }), + transformer: JobMappingTransformerForm, +}); + +export type AddNewNosqlRecordFormValues = Yup.InferType< + typeof AddNewNosqlRecordFormValues +>; + +interface AddNewNosqlRecordFormContext { + accountId: string; + isUserJavascriptCodeValid: UseMutateAsyncFunction< + ValidateUserJavascriptCodeResponse, + ConnectError, + PartialMessage, + unknown + >; + isDuplicateKey: (value: string, schema: string, table: string) => boolean; +} + +export default function AddNewNosqlRecord(props: Props): ReactElement { + const { collections, onSubmit, transformerHandler, isDuplicateKey } = props; + + const { account } = useAccount(); + const { mutateAsync: validateUserJsCodeAsync } = useMutation( + validateUserJavascriptCode + ); + const form = useForm< + AddNewNosqlRecordFormValues, + AddNewNosqlRecordFormContext + >({ + resolver: yupResolver(AddNewNosqlRecordFormValues), + mode: 'onChange', + defaultValues: { + collection: '', + key: '', + transformer: convertJobMappingTransformerToForm( + new JobMappingTransformer({ + source: TransformerSource.PASSTHROUGH, + config: new TransformerConfig({ + config: { + case: 'passthroughConfig', + value: new Passthrough(), + }, + }), + }) + ), + }, + context: { + accountId: account?.id ?? '', + isUserJavascriptCodeValid: validateUserJsCodeAsync, + isDuplicateKey: isDuplicateKey, + }, + }); + + return ( +
+ + ( + + Collection + + The collection that you want to map. + + + + + + + )} + /> + ( + + Document Key + + Use dot notation to select a key for the mapping. + + + + + + + )} + /> + { + const fv = field.value; + const transformer = getTransformerFromField(transformerHandler, fv); + return ( + + Transformer + Select a transformer to map + +
+
+ + transformerHandler.getTransformers() + } + buttonText={getTransformerSelectButtonText(transformer)} + value={fv} + onSelect={field.onChange} + side={'left'} + disabled={false} + buttonClassName="w-[175px]" + /> +
+ { + field.onChange(newvalue); + }} + disabled={isInvalidTransformer(transformer)} + /> +
+
+ +
+ ); + }} + /> +
+ +
+ +
+ ); +} diff --git a/frontend/apps/web/components/jobs/NosqlTable/EditCollection.tsx b/frontend/apps/web/components/jobs/NosqlTable/EditCollection.tsx new file mode 100644 index 0000000000..d88692b106 --- /dev/null +++ b/frontend/apps/web/components/jobs/NosqlTable/EditCollection.tsx @@ -0,0 +1,72 @@ +import TruncatedText from '@/components/TruncatedText'; +import { Button } from '@/components/ui/button'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { CheckIcon, Pencil1Icon } from '@radix-ui/react-icons'; +import { ReactElement, useState } from 'react'; +import { NosqlJobMappingRow } from '../JobMappingTable/Columns'; + +interface Props { + collections: string[]; + text: string; + onEdit(updatedObject: Pick): void; +} + +export default function EditCollection(props: Props): ReactElement { + const { text, collections, onEdit } = props; + + const [isEditingMapping, setIsEditingMapping] = useState(false); + const [isSelectedCollection, setSelectedCollection] = useState(text); + + const handleSave = () => { + onEdit({ collection: isSelectedCollection }); + setIsEditingMapping(false); + }; + + return ( +
+ {isEditingMapping ? ( + + ) : ( + + )} + +
+ ); +} diff --git a/frontend/apps/web/components/jobs/NosqlTable/EditDocumentKey.tsx b/frontend/apps/web/components/jobs/NosqlTable/EditDocumentKey.tsx new file mode 100644 index 0000000000..eddc8268f0 --- /dev/null +++ b/frontend/apps/web/components/jobs/NosqlTable/EditDocumentKey.tsx @@ -0,0 +1,65 @@ +import TruncatedText from '@/components/TruncatedText'; +import { Button } from '@/components/ui/button'; +import { Input } from '@/components/ui/input'; +import { cn } from '@/libs/utils'; +import { CheckIcon, Pencil1Icon } from '@radix-ui/react-icons'; +import { ReactElement, useState } from 'react'; +import { NosqlJobMappingRow } from '../JobMappingTable/Columns'; + +interface Props { + text: string; + onEdit(updatedObject: Pick): void; + isDuplicate(val: string, currValue?: string): boolean; +} + +export default function EditDocumentKey(props: Props): ReactElement { + const { text, onEdit, isDuplicate } = props; + const [isEditingMapping, setIsEditingMapping] = useState(false); + const [inputValue, setInputValue] = useState(text); + const [duplicateError, setDuplicateError] = useState(false); + + const handleSave = () => { + onEdit({ column: inputValue }); + setIsEditingMapping(false); + }; + + const handleDocumentKeyChange = (val: string) => { + setInputValue(val); + setDuplicateError(isDuplicate(val, text)); + }; + + return ( +
+ {isEditingMapping ? ( + <> + handleDocumentKeyChange(e.target.value)} + className={cn(duplicateError ? 'border border-red-400 ring-' : '')} + /> +
+ {duplicateError && 'Already exists'} +
+ + ) : ( + + )} + +
+ ); +} diff --git a/frontend/apps/web/components/jobs/NosqlTable/NosqlTable.tsx b/frontend/apps/web/components/jobs/NosqlTable/NosqlTable.tsx index b417c6a001..0711ab4d18 100644 --- a/frontend/apps/web/components/jobs/NosqlTable/NosqlTable.tsx +++ b/frontend/apps/web/components/jobs/NosqlTable/NosqlTable.tsx @@ -1,8 +1,5 @@ -import EditTransformerOptions from '@/app/(mgmt)/[account]/transformers/EditTransformerOptions'; import Spinner from '@/components/Spinner'; -import TruncatedText from '@/components/TruncatedText'; import { useAccount } from '@/components/providers/account-provider'; -import { Button } from '@/components/ui/button'; import { Card, CardContent, @@ -10,94 +7,45 @@ import { CardHeader, CardTitle, } from '@/components/ui/card'; -import { - Form, - FormControl, - FormDescription, - FormField, - FormItem, - FormLabel, - FormMessage, -} from '@/components/ui/form'; -import { Input } from '@/components/ui/input'; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from '@/components/ui/select'; import { useGetTransformersHandler } from '@/libs/hooks/useGetTransformersHandler'; -import { cn } from '@/libs/utils'; +import { Transformer } from '@/shared/transformers'; import { - getTransformerFromField, - getTransformerSelectButtonText, - isInvalidTransformer, -} from '@/util/util'; -import { - convertJobMappingTransformerToForm, EditDestinationOptionsFormValues, JobMappingFormValues, JobMappingTransformerForm, } from '@/yup-validations/jobs'; -import { PartialMessage } from '@bufbuild/protobuf'; -import { useMutation } from '@connectrpc/connect-query'; -import { yupResolver } from '@hookform/resolvers/yup'; -import { - ConnectError, - GetConnectionSchemaResponse, - JobMapping, - JobMappingTransformer, - Passthrough, - SystemTransformer, - TransformerConfig, - TransformerSource, - ValidateUserJavascriptCodeRequest, - ValidateUserJavascriptCodeResponse, -} from '@neosync/sdk'; -import { validateUserJavascriptCode } from '@neosync/sdk/connectquery'; -import { CheckIcon, Pencil1Icon, TableIcon } from '@radix-ui/react-icons'; -import { UseMutateAsyncFunction } from '@tanstack/react-query'; -import { ColumnDef } from '@tanstack/react-table'; +import { GetConnectionSchemaResponse, JobMapping } from '@neosync/sdk'; +import { TableIcon } from '@radix-ui/react-icons'; +import { Row } from '@tanstack/react-table'; import { nanoid } from 'nanoid'; -import { - HTMLProps, - ReactElement, - useCallback, - useEffect, - useMemo, - useRef, - useState, -} from 'react'; -import { useForm } from 'react-hook-form'; -import * as Yup from 'yup'; +import { ReactElement, useCallback, useMemo } from 'react'; +import { NOSQL_COLUMNS, NosqlJobMappingRow } from '../JobMappingTable/Columns'; +import JobMappingTable from '../JobMappingTable/JobMappingTable'; import FormErrorsCard, { FormError } from '../SchemaTable/FormErrorsCard'; import { ImportMappingsConfig } from '../SchemaTable/ImportJobMappingsButton'; -import { SchemaColumnHeader } from '../SchemaTable/SchemaColumnHeader'; -import SchemaPageTable, { Row } from '../SchemaTable/SchemaPageTable'; -import TransformerSelect from '../SchemaTable/TransformerSelect'; -import { SchemaConstraintHandler } from '../SchemaTable/schema-constraint-handler'; -import { TransformerHandler } from '../SchemaTable/transformer-handler'; +import { TransformerResult } from '../SchemaTable/transformer-handler'; import { useOnExportMappings } from '../SchemaTable/useOnExportMappings'; +import { splitCollection } from '../SchemaTable/util'; +import AddNewNosqlRecord, { + AddNewNosqlRecordFormValues, +} from './AddNewNosqlRecord'; import { DestinationDetails, OnTableMappingUpdateRequest, } from './TableMappings/Columns'; import TableMappingsCard from './TableMappings/TableMappingsCard'; -import { DataTableRowActions } from './data-table-row-actions'; interface Props { data: JobMappingFormValues[]; schema: Record; isSchemaDataReloading: boolean; - constraintHandler: SchemaConstraintHandler; isJobMappingsValidating?: boolean; onValidate?(): void; formErrors: FormError[]; onAddMappings(values: AddNewNosqlRecordFormValues[]): void; - onRemoveMappings(values: JobMappingFormValues[]): void; + onRemoveMappings(indices: number[]): void; onEditMappings(values: JobMappingFormValues, index: number): void; destinationOptions: EditDestinationOptionsFormValues[]; @@ -108,6 +56,17 @@ interface Props { jobmappings: JobMapping[], importConfig: ImportMappingsConfig ): void; + getAvailableTransformers(index: number): TransformerResult; + getTransformerFromField(index: number): Transformer; + onApplyDefaultClick(override: boolean): void; + getAvailableTransformersForBulk( + rows: Row[] + ): TransformerResult; + getTransformerFromFieldValue(value: JobMappingTransformerForm): Transformer; + onTransformerBulkUpdate( + indices: number[], + config: JobMappingTransformerForm + ): void; } export default function NosqlTable(props: Props): ReactElement { @@ -116,7 +75,6 @@ export default function NosqlTable(props: Props): ReactElement { schema, formErrors, isJobMappingsValidating, - constraintHandler, onValidate, onAddMappings, onRemoveMappings, @@ -126,9 +84,15 @@ export default function NosqlTable(props: Props): ReactElement { onDestinationTableMappingUpdate, showDestinationTableMappings, onImportMappingsClick, + getAvailableTransformers, + getTransformerFromField, + onApplyDefaultClick, + getAvailableTransformersForBulk, + getTransformerFromFieldValue, + onTransformerBulkUpdate, } = props; const { account } = useAccount(); - const { handler, isLoading, isValidating } = useGetTransformersHandler( + const { handler, isValidating } = useGetTransformersHandler( account?.id ?? '' ); @@ -145,8 +109,9 @@ export default function NosqlTable(props: Props): ReactElement { // useCallback ensures that we only re-run the function if the keySet changes const isDuplicateKey = useCallback( - (newValue: string, schema: string, table: string) => { - const key = `${schema}.${table}.${newValue}`; + (index: number, newValue: string) => { + const row = data[index]; + const key = `${row.schema}.${row.table}.${newValue}`; return keySet.has(key); }, [keySet] @@ -154,7 +119,7 @@ export default function NosqlTable(props: Props): ReactElement { // used to calculate the collections that can be updated based on a given key value // for ex. if a collection.key is "a.b.c" and we want to update it to "d.e.c" but "d.e.c" already exists, then we don't want to show the "d.e." collection as an uption for the update - const filteredCollections = useCallback( + const getAvailableCollectionsByRow = useCallback( (index: number) => { const currentColumn = data[index].column; const currentSchemaTable = `${data[index].schema}.${data[index].table}`; @@ -173,35 +138,20 @@ export default function NosqlTable(props: Props): ReactElement { [data, collections] ); - const columns = useMemo( - () => - getColumns({ - onDelete(row) { - onRemoveMappings([row]); - }, - onEdit(row, index) { - onEditMappings(row, index); - }, - onDuplicate(row: Row) { - const newKey = createDuplicateKey(row.column); - onAddMappings([ - { - collection: `${row.schema}.${row.table}`, - key: newKey, - transformer: row.transformer, - }, - ]); - }, - transformerHandler: handler, - filteredCollections, - isDuplicateKey, - }), - [onRemoveMappings, onEditMappings, handler, isLoading] - ); + const tableData = useMemo(() => { + return data.map((d): NosqlJobMappingRow => { + return { + collection: `${d.schema}.${d.table}`, + column: d.column, + transformer: d.transformer, + }; + }); + }, [data.length]); - const { onClick: onExportMappingsClick } = useOnExportMappings({ - jobMappings: data, - }); + const { onClick: onExportMappingsClick } = + useOnExportMappings({ + jobMappings: data, + }); return (
@@ -221,9 +171,11 @@ export default function NosqlTable(props: Props): ReactElement { - { + return keySet.has(`${schema}.${table}.${newVal}`); + }} onSubmit={(values) => { onAddMappings([values]); }} @@ -246,599 +198,52 @@ export default function NosqlTable(props: Props): ReactElement { />
)} - { + const row = data[idx]; + onEditMappings({ ...row, transformer: config }, idx); + }} + getAvailableTransformers={getAvailableTransformers} + getTransformerFromField={getTransformerFromField} onExportMappingsClick={onExportMappingsClick} onImportMappingsClick={onImportMappingsClick} - /> -
- ); -} -interface AddNewRecordProps { - collections: string[]; - onSubmit(values: AddNewNosqlRecordFormValues): void; - transformerHandler: TransformerHandler; - isDuplicateKey: ( - value: string, - schema: string, - table: string, - currValue?: string - ) => boolean; -} - -const AddNewNosqlRecordFormValues = Yup.object({ - collection: Yup.string().required('The Collection is required.'), - key: Yup.string() - .required('The Key is required.') - .test({ - name: 'uniqueMapping', - message: 'This key already exists in the selected collection.', - test: function (value, context) { - const { collection } = this.parent; - - if (!collection || !value) { - return true; - } - - const lastDotIndex = collection.lastIndexOf('.'); - const schema = collection.substring(0, lastDotIndex); - const table = collection.substring(lastDotIndex + 1); - - return ( - !context?.options?.context?.isDuplicateKey(value, schema, table) || - this.createError({ - message: 'This key already exists in this collection.', - }) - ); - }, - }), - transformer: JobMappingTransformerForm, -}); - -type AddNewNosqlRecordFormValues = Yup.InferType< - typeof AddNewNosqlRecordFormValues ->; - -interface AddNewNosqlRecordFormContext { - accountId: string; - isUserJavascriptCodeValid: UseMutateAsyncFunction< - ValidateUserJavascriptCodeResponse, - ConnectError, - PartialMessage, - unknown - >; - isDuplicateKey: (value: string, schema: string, table: string) => boolean; -} - -function AddNewRecord(props: AddNewRecordProps): ReactElement { - const { collections, onSubmit, transformerHandler, isDuplicateKey } = props; - - const { account } = useAccount(); - const { mutateAsync: validateUserJsCodeAsync } = useMutation( - validateUserJavascriptCode - ); - const form = useForm< - AddNewNosqlRecordFormValues, - AddNewNosqlRecordFormContext - >({ - resolver: yupResolver(AddNewNosqlRecordFormValues), - mode: 'onChange', - defaultValues: { - collection: '', - key: '', - transformer: convertJobMappingTransformerToForm( - new JobMappingTransformer({ - source: TransformerSource.PASSTHROUGH, - config: new TransformerConfig({ - config: { - case: 'passthroughConfig', - value: new Passthrough(), + isApplyDefaultTransformerButtonDisabled={data.length === 0} + getAvalableTransformersForBulk={getAvailableTransformersForBulk} + getTransformerFromFieldValue={getTransformerFromFieldValue} + onTransformerBulkUpdate={onTransformerBulkUpdate} + onApplyDefaultClick={onApplyDefaultClick} + onDeleteRow={(idx) => onRemoveMappings([idx])} + onDuplicateRow={(idx) => { + const row = data[idx]; + onAddMappings([ + { + collection: `${row.schema}.${row.table}`, + key: createDuplicateKey(row.column), + transformer: row.transformer, }, - }), - }) - ), - }, - context: { - accountId: account?.id ?? '', - isUserJavascriptCodeValid: validateUserJsCodeAsync, - isDuplicateKey: isDuplicateKey, - }, - }); - - return ( -
-
- ( - - Collection - - The collection that you want to map. - - - - - - - )} - /> - ( - - Document Key - - Use dot notation to select a key for the mapping. - - - - - - - )} - /> - { - const fv = field.value; - const transformer = getTransformerFromField(transformerHandler, fv); - return ( - - Transformer - Select a transformer to map - -
-
- - transformerHandler.getTransformers() - } - buttonText={getTransformerSelectButtonText(transformer)} - value={fv} - onSelect={field.onChange} - side={'left'} - disabled={false} - buttonClassName="w-[175px]" - /> -
- { - field.onChange(newvalue); - }} - disabled={isInvalidTransformer(transformer)} - /> -
-
- -
- ); - }} - /> -
- -
- + ]); + }} + canRenameColumn={isDuplicateKey} + onRowUpdate={(idx, val) => { + const [schema, table] = splitCollection(val.collection); + onEditMappings( + { + ...val, + schema, + table, + }, + idx + ); + }} + getAvailableCollectionsByRow={getAvailableCollectionsByRow} + />
); } -interface GetColumnsProps { - onDelete(row: Row): void; - onDuplicate(row: Row): void; - transformerHandler: TransformerHandler; - onEdit(row: Row, index: number): void; - isDuplicateKey: ( - newValue: string, - schema: string, - table: string, - currValue?: string - ) => boolean; - filteredCollections(ind: number): string[]; -} - -function getColumns(props: GetColumnsProps): ColumnDef[] { - const { - onDelete, - transformerHandler, - onEdit, - onDuplicate, - isDuplicateKey, - filteredCollections, - } = props; - return [ - { - accessorKey: 'isSelected', - header: ({ table }) => ( - - ), - cell: ({ row }) => ( -
- -
- ), - enableSorting: false, - enableHiding: false, - size: 30, - }, - { - accessorKey: 'schema', - header: ({ column }) => ( - - ), - }, - { - accessorKey: 'table', - header: ({ column }) => ( - - ), - }, - { - accessorFn: (row) => { - if (row.schema && row.table) { - return `${row.schema}.${row.table}`; - } - if (row.schema) { - return row.schema; - } - return row.table; - }, - id: 'schemaTable', - footer: (props) => props.column.id, - header: ({ column }) => ( - - ), - cell: ({ getValue, row }) => { - return ( - ()} - collections={filteredCollections(row.index)} - onEdit={(updatedObject) => { - const lastDotIndex = updatedObject.collection.lastIndexOf('.'); - onEdit( - { - schema: updatedObject.collection.substring(0, lastDotIndex), - table: updatedObject.collection.substring(lastDotIndex + 1), - column: row.getValue('column'), - transformer: row.getValue('transformer'), - }, - row.index - ); - }} - /> - ); - }, - maxSize: 500, - size: 300, - }, - { - accessorKey: 'column', - header: ({ column }) => ( - - ), - cell: ({ row }) => { - const text = row.getValue('column'); - return ( - - currValue !== newValue && - isDuplicateKey( - newValue, - row.getValue('schema'), - row.getValue('table'), - currValue - ) - } - onEdit={(updatedObject) => { - onEdit( - { - schema: row.getValue('schema'), - table: row.getValue('table'), - column: updatedObject.column, - transformer: row.getValue('transformer'), - }, - row.index - ); - }} - /> - ); - }, - maxSize: 500, - size: 200, - }, - { - id: 'transformer', - accessorKey: 'transformer', - header: ({ column }) => ( - - ), - cell: ({ row }) => { - // row.getValue doesn't work here due to a tanstack bug where the transformer value is out of sync with getValue - // row.original works here. There must be a caching bug with the transformer prop being an object. - // This may be related: https://github.com/TanStack/table/issues/5363 - const fv = row.original.transformer; - const transformer = getTransformerFromField(transformerHandler, fv); - return ( - -
-
- transformerHandler.getTransformers()} - buttonText={getTransformerSelectButtonText(transformer)} - value={fv} - onSelect={(updatedTransformer) => - onEdit( - { - schema: row.getValue('schema'), - table: row.getValue('table'), - column: row.getValue('column'), - transformer: updatedTransformer, - }, - row.index - ) - } - side={'left'} - disabled={false} - buttonClassName="w-[175px]" - /> -
- { - onEdit( - { - schema: row.getValue('schema'), - table: row.getValue('table'), - column: row.getValue('column'), - transformer: updatedTransformer, - }, - row.index - ); - }} - disabled={isInvalidTransformer(transformer)} - /> -
-
- ); - }, - }, - { - id: 'actions', - header: () =>

Actions

, - cell: ({ row }) => { - return ( - onDuplicate(row.original)} - onDelete={() => - onDelete({ - schema: row.getValue('schema'), - table: row.getValue('table'), - column: row.getValue('column'), - transformer: row.getValue('transformer'), - }) - } - /> - ); - }, - }, - ]; -} - function createDuplicateKey(key: string): string { const uniqueSuffix = nanoid(6); return `${key}_${uniqueSuffix}`; } - -function IndeterminateCheckbox({ - indeterminate, - className = 'w-4 h-4 flex', - ...rest -}: { indeterminate?: boolean } & HTMLProps) { - const ref = useRef(null!); - - useEffect(() => { - if (typeof indeterminate === 'boolean') { - ref.current.indeterminate = !rest.checked && indeterminate; - } - }, [ref, indeterminate, rest.checked]); - - return ( - - ); -} -interface EditDocumentKeyProps { - text: string; - onEdit: (updatedObject: { column: string }) => void; - isDuplicate: (val: string, currValue?: string) => boolean; -} - -function EditDocumentKey(props: EditDocumentKeyProps): ReactElement { - const { text, onEdit, isDuplicate } = props; - const [isEditingMapping, setIsEditingMapping] = useState(false); - const [inputValue, setInputValue] = useState(text); - const [duplicateError, setDuplicateError] = useState(false); - - const handleSave = () => { - onEdit({ column: inputValue }); - setIsEditingMapping(false); - }; - - const handleDocumentKeyChange = (val: string) => { - setInputValue(val); - setDuplicateError(isDuplicate(val, text)); - }; - - return ( -
- {isEditingMapping ? ( - <> - handleDocumentKeyChange(e.target.value)} - className={cn(duplicateError ? 'border border-red-400 ring-' : '')} - /> -
- {duplicateError && 'Already exists'} -
- - ) : ( - - )} - -
- ); -} - -interface EditCollectionProps { - collections: string[]; - text: string; - onEdit: (updatedObject: { collection: string }) => void; -} - -function EditCollection(props: EditCollectionProps): ReactElement { - const { text, collections, onEdit } = props; - - const [isEditingMapping, setIsEditingMapping] = useState(false); - const [isSelectedCollection, setSelectedCollection] = useState(text); - - const handleSave = () => { - onEdit({ collection: isSelectedCollection }); - setIsEditingMapping(false); - }; - - return ( -
- {isEditingMapping ? ( - - ) : ( - - )} - -
- ); -} diff --git a/frontend/apps/web/components/jobs/SchemaTable/AiSchemaColumns.tsx b/frontend/apps/web/components/jobs/SchemaTable/AiSchemaColumns.tsx index ffa7aa14ec..b241036041 100644 --- a/frontend/apps/web/components/jobs/SchemaTable/AiSchemaColumns.tsx +++ b/frontend/apps/web/components/jobs/SchemaTable/AiSchemaColumns.tsx @@ -7,8 +7,8 @@ import { } from '@/components/ui/tooltip'; import { ColumnDef } from '@tanstack/react-table'; import { SchemaColumnHeader } from './SchemaColumnHeader'; -import { handleDataTypeBadge, toColKey } from './SchemaColumns'; import { SchemaConstraintHandler } from './schema-constraint-handler'; +import { handleDataTypeBadge, toColKey } from './util'; interface Props { constraintHandler: SchemaConstraintHandler; diff --git a/frontend/apps/web/components/jobs/SchemaTable/RowAlert.tsx b/frontend/apps/web/components/jobs/SchemaTable/RowAlert.tsx deleted file mode 100644 index ea145db7a7..0000000000 --- a/frontend/apps/web/components/jobs/SchemaTable/RowAlert.tsx +++ /dev/null @@ -1,51 +0,0 @@ -import { - Tooltip, - TooltipContent, - TooltipProvider, - TooltipTrigger, -} from '@/components/ui/tooltip'; -import { ExclamationTriangleIcon } from '@radix-ui/react-icons'; -import { ReactElement } from 'react'; -import { - ColumnKey, - SchemaConstraintHandler, -} from './schema-constraint-handler'; - -interface Props { - rowKey: ColumnKey; - handler: SchemaConstraintHandler; - onRemoveClick(): void; -} - -export default function SchemaRowAlert(props: Props): ReactElement { - const { rowKey, handler, onRemoveClick } = props; - const isInSchema = handler.getIsInSchema(rowKey); - - const messages: string[] = []; - - if (!isInSchema) { - messages.push('This column was not found in the backing source schema'); - } - - if (messages.length === 0) { - return
; - } - - return ( - - - -
- onRemoveClick()} - /> -
-
- -

{messages.join('\n')}

-
-
-
- ); -} diff --git a/frontend/apps/web/components/jobs/SchemaTable/SchemaColumnHeader.tsx b/frontend/apps/web/components/jobs/SchemaTable/SchemaColumnHeader.tsx index 154d97ff19..051909f45a 100644 --- a/frontend/apps/web/components/jobs/SchemaTable/SchemaColumnHeader.tsx +++ b/frontend/apps/web/components/jobs/SchemaTable/SchemaColumnHeader.tsx @@ -8,7 +8,9 @@ import { ArrowUpIcon, CaretSortIcon, } from '@radix-ui/react-icons'; +import { useState } from 'react'; import { FaSearch } from 'react-icons/fa'; +import { useDebounceCallback } from 'usehooks-ts'; interface DataTableColumnHeaderProps extends React.HTMLAttributes { @@ -21,6 +23,13 @@ export function SchemaColumnHeader({ title, className, }: DataTableColumnHeaderProps) { + const [inputValue, setInputValue] = useState( + (column.getFilterValue() ?? '') as string + ); + const onInputChange = useDebounceCallback( + (value) => column.setFilterValue(value), + 300 + ); return (
{column.getCanFilter() && ( @@ -28,8 +37,11 @@ export function SchemaColumnHeader({
column.setFilterValue(e.target.value)} + value={inputValue} + onChange={(e) => { + setInputValue(e.target.value); + onInputChange(e.target.value); + }} placeholder={title} className="border border-gray-300 dark:border-gray-700 bg-white dark:bg-transparent text-xs h-8 pl-8" /> diff --git a/frontend/apps/web/components/jobs/SchemaTable/SchemaColumns.tsx b/frontend/apps/web/components/jobs/SchemaTable/SchemaColumns.tsx deleted file mode 100644 index bb02206535..0000000000 --- a/frontend/apps/web/components/jobs/SchemaTable/SchemaColumns.tsx +++ /dev/null @@ -1,591 +0,0 @@ -'use client'; - -import { SingleTableSchemaFormValues } from '@/app/(mgmt)/[account]/new/job/job-form-validations'; -import EditTransformerOptions from '@/app/(mgmt)/[account]/transformers/EditTransformerOptions'; -import TruncatedText from '@/components/TruncatedText'; -import { Badge } from '@/components/ui/badge'; -import { FormControl, FormField, FormItem } from '@/components/ui/form'; -import { - Tooltip, - TooltipContent, - TooltipProvider, - TooltipTrigger, -} from '@/components/ui/tooltip'; -import { - getTransformerFromField, - getTransformerSelectButtonText, - isInvalidTransformer, -} from '@/util/util'; -import { - JobMappingTransformerForm, - SchemaFormValues, -} from '@/yup-validations/jobs'; -import { ExclamationTriangleIcon } from '@radix-ui/react-icons'; -import { ColumnDef, Row } from '@tanstack/react-table'; -import { HTMLProps, useEffect, useRef } from 'react'; -import { useFieldArray, useFormContext } from 'react-hook-form'; -import SchemaRowAlert from './RowAlert'; -import { SchemaColumnHeader } from './SchemaColumnHeader'; -import { Row as RowData } from './SchemaPageTable'; -import TransformerSelect from './TransformerSelect'; -import { JobType, SchemaConstraintHandler } from './schema-constraint-handler'; -import { - TransformerFilters, - TransformerHandler, - toSupportedJobtype, -} from './transformer-handler'; - -interface ColumnKey { - schema: string; - table: string; - column: string; -} - -export function fromRowDataToColKey(row: Row): ColumnKey { - return { - schema: row.getValue('schema'), - table: row.getValue('table'), - column: row.getValue('column'), - }; -} -export function toColKey( - schema: string, - table: string, - column: string -): ColumnKey { - return { - schema, - table, - column, - }; -} - -interface Props { - transformerHandler: TransformerHandler; - constraintHandler: SchemaConstraintHandler; - jobType: JobType; -} - -export function getSchemaColumns(props: Props): ColumnDef[] { - const { transformerHandler, constraintHandler, jobType } = props; - return [ - { - accessorKey: 'isSelected', - header: ({ table }) => ( - - ), - cell: ({ row }) => ( -
- -
- ), - enableSorting: false, - enableHiding: false, - maxSize: 30, - }, - { - accessorKey: 'schema', - header: ({ column }) => ( - - ), - }, - { - accessorKey: 'table', - header: ({ column }) => ( - - ), - }, - { - accessorFn: (row) => `${row.schema}.${row.table}`, - id: 'schemaTable', - footer: (props) => props.column.id, - header: ({ column }) => ( - - ), - cell: ({ row, getValue }) => { - // eslint-disable-next-line react-hooks/rules-of-hooks - const form = useFormContext< - SchemaFormValues | SingleTableSchemaFormValues - >(); - // eslint-disable-next-line react-hooks/rules-of-hooks - const { remove } = useFieldArray< - SchemaFormValues | SingleTableSchemaFormValues - >({ - control: form.control, - name: 'mappings', - }); - const columnKey: ColumnKey = { - schema: row.getValue('schema'), - table: row.getValue('table'), - column: row.getValue('column'), - }; - return ( -
- remove(row.index)} - /> - - {getValue()} - -
- ); - }, - maxSize: 500, - }, - { - accessorKey: 'column', - header: ({ column }) => ( - - ), - cell: ({ row }) => { - return ('column')} />; - }, - maxSize: 500, - }, - { - id: 'constraints', - accessorKey: 'constraints', - header: ({ column }) => ( - - ), - accessorFn: (row) => { - const key = toColKey(row.schema, row.table, row.column); - const isPrimaryKey = constraintHandler.getIsPrimaryKey(key); - const [isForeignKey, fkCols] = constraintHandler.getIsForeignKey(key); - const isUnique = constraintHandler.getIsUniqueConstraint(key); - - const pieces: string[] = []; - if (isPrimaryKey) { - pieces.push('Primary Key'); - } - if (isForeignKey) { - fkCols.forEach((col) => pieces.push(`Foreign Key: ${col}`)); - } - if (isUnique) { - pieces.push('Unique'); - } - return pieces.join('\n'); - }, - cell: ({ row }) => { - const key: ColumnKey = { - schema: row.getValue('schema'), - table: row.getValue('table'), - column: row.getValue('column'), - }; - const isPrimaryKey = constraintHandler.getIsPrimaryKey(key); - const [isForeignKey, fkCols] = constraintHandler.getIsForeignKey(key); - const isUniqueConstraint = constraintHandler.getIsUniqueConstraint(key); - const [isVirtualForeignKey, vfkCols] = - constraintHandler.getIsVirtualForeignKey(key); - return ( - -
- {isPrimaryKey && ( -
- - Primary Key - -
- )} - {isForeignKey && ( -
- - - - - Foreign Key - - - - {fkCols.map((col) => `Primary Key: ${col}`).join('\n')} - - - -
- )} - {isVirtualForeignKey && ( -
- - - - - Virtual Foreign Key - - - - {vfkCols.map((col) => `Primary Key: ${col}`).join('\n')} - - - -
- )} - {isUniqueConstraint && ( -
- - Unique - -
- )} -
-
- ); - }, - }, - { - accessorKey: 'isNullable', - accessorFn: (row) => { - const key = toColKey(row.schema, row.table, row.column); - return constraintHandler.getIsNullable(key) ? 'Yes' : 'No'; - }, - header: ({ column }) => ( - - ), - cell: ({ row }) => { - const key: ColumnKey = { - schema: row.getValue('schema'), - table: row.getValue('table'), - column: row.getValue('column'), - }; - const isNullable = constraintHandler.getIsNullable(key); - const text = isNullable ? 'Yes' : 'No'; - return ( - - {text} - - ); - }, - }, - { - accessorKey: 'attributes', - accessorFn: (row) => { - const key = toColKey(row.schema, row.table, row.column); - const generatedType = constraintHandler.getGeneratedType(key); - const identityType = constraintHandler.getIdentityType(key); - - const pieces: string[] = []; - if (generatedType) { - pieces.push(getGeneratedStatement(generatedType)); - } else if (identityType) { - pieces.push(getIdentityStatement(identityType)); - } - return pieces.join('\n'); - }, - header: ({ column }) => ( - - ), - cell: ({ row }) => { - const key = toColKey( - row.getValue('schema'), - row.getValue('table'), - row.getValue('column') - ); - const generatedType = constraintHandler.getGeneratedType(key); - const identityType = constraintHandler.getIdentityType(key); - return ( - -
- {generatedType && ( -
- - - - - Generated - - - - {getGeneratedStatement(generatedType)} - - - -
- )} - {!generatedType && identityType && ( - // the API treats generatedType and identityType as mutually exclusive -
- - - - - Generated - - - - {getIdentityStatement(identityType)} - - - -
- )} - {identityType && ( -
- - - - - Identity - - - - {getIdentityStatement(identityType)} - - - -
- )} -
-
- ); - }, - }, - { - accessorKey: 'dataType', - accessorFn: (row) => { - const key = toColKey(row.schema, row.table, row.column); - return handleDataTypeBadge(constraintHandler.getDataType(key)); - }, - header: ({ column }) => ( - - ), - cell: ({ row }) => { - const key: ColumnKey = { - schema: row.getValue('schema'), - table: row.getValue('table'), - column: row.getValue('column'), - }; - const datatype = constraintHandler.getDataType(key); - return ( - - - -
- - - {handleDataTypeBadge(datatype)} - - -
-
- -

{datatype}

-
-
-
- ); - }, - }, - - { - accessorKey: 'transformer', - - id: 'transformer', - header: ({ column }) => ( - - ), - filterFn: (row, _id, value) => { - // row.getValue doesn't work here due to a tanstack bug where the transformer value is out of sync with getValue - // row.original works here. There must be a caching bug with the transformer prop being an object. - // This may be related: https://github.com/TanStack/table/issues/5363 - const rowVal = row.original.transformer; - const tsource = transformerHandler.getSystemTransformerByConfigCase( - rowVal.config.case - ); - const sourceName = tsource?.name.toLowerCase() ?? 'select transformer'; - return sourceName.includes((value as string)?.toLowerCase()); - }, - cell: (info) => { - // eslint-disable-next-line react-hooks/rules-of-hooks - const fctx = useFormContext< - SchemaFormValues | SingleTableSchemaFormValues - >(); - return ( -
- - name={`mappings.${info.row.index}.transformer`} - control={fctx.control} - render={({ field, fieldState, formState }) => { - const fv = field.value as JobMappingTransformerForm; - const colkey = fromRowDataToColKey(info.row); - const filtered = transformerHandler.getFilteredTransformers( - getTransformerFilter(constraintHandler, colkey, jobType) - ); - - const filteredTransformerHandler = new TransformerHandler( - filtered.system, - filtered.userDefined - ); - - const transformer = getTransformerFromField( - filteredTransformerHandler, - fv - ); - return ( - - -
- {formState.errors.mappings && ( -
- {fieldState.error ? ( -
-
{fieldState.error.message}
- -
- ) : ( -
- )} -
- )} -
- filtered} - buttonText={getTransformerSelectButtonText( - transformer - )} - value={fv} - onSelect={field.onChange} - side={'left'} - disabled={false} - buttonClassName="w-[175px]" - /> -
- { - field.onChange(newvalue); - }} - disabled={isInvalidTransformer(transformer)} - /> -
-
-
- ); - }} - /> -
- ); - }, - }, - ]; -} - -function IndeterminateCheckbox({ - indeterminate, - className = 'w-4 h-4 flex', - ...rest -}: { indeterminate?: boolean } & HTMLProps) { - const ref = useRef(null!); - - useEffect(() => { - if (typeof indeterminate === 'boolean') { - ref.current.indeterminate = !rest.checked && indeterminate; - } - }, [ref, indeterminate, rest.checked]); - - return ( - - ); -} - -// cleans up the data type values since some are too long , can add on more here -export function handleDataTypeBadge(dataType: string): string { - // Check for "timezone" and replace accordingly without entering the switch - if (dataType.includes('timezone')) { - return dataType - .replace('timestamp with time zone', 'timestamp(tz)') - .replace('timestamp without time zone', 'timestamp'); - } - - const splitDt = dataType.split('('); - switch (splitDt[0]) { - case 'character varying': - // The condition inside the if statement seemed reversed. It should return 'varchar' directly if splitDt[1] is undefined. - return splitDt[1] !== undefined ? `varchar(${splitDt[1]}` : 'varchar'; - default: - return dataType; - } -} - -export function getTransformerFilter( - constraintHandler: SchemaConstraintHandler, - colkey: ColumnKey, - jobType: JobType -): TransformerFilters { - const [isForeignKey] = constraintHandler.getIsForeignKey(colkey); - const [isVirtualForeignKey] = - constraintHandler.getIsVirtualForeignKey(colkey); - const isNullable = constraintHandler.getIsNullable(colkey); - const convertedDataType = constraintHandler.getConvertedDataType(colkey); - const hasDefault = constraintHandler.getHasDefault(colkey); - const isGenerated = constraintHandler.getIsGenerated(colkey); - return { - dataType: convertedDataType, - hasDefault, - isForeignKey, - isVirtualForeignKey, - isNullable, - jobType: toSupportedJobtype(jobType), - isGenerated, - identityType: constraintHandler.getIdentityType(colkey), - }; -} - -function getIdentityStatement(identityType: string): string { - if (identityType === 'a') { - return 'GENERATED ALWAYS AS IDENTITY'; - } else if (identityType === 'd') { - return 'GENERATED BY DEFAULT AS IDENTITY'; - } else if (identityType === 'auto_increment') { - return 'AUTO_INCREMENT'; - } - return identityType; -} - -function getGeneratedStatement(generatedType: string): string { - if (generatedType === 's' || generatedType === 'STORED GENERATED') { - return 'GENERATED ALWAYS AS STORED'; - } else if (generatedType === 'v' || generatedType === 'VIRTUAL GENERATED') { - return 'GENERATED ALWAYS AS VIRTUAL'; - } - return generatedType; -} diff --git a/frontend/apps/web/components/jobs/SchemaTable/SchemaPageTable.tsx b/frontend/apps/web/components/jobs/SchemaTable/SchemaPageTable.tsx deleted file mode 100644 index 997c5f5292..0000000000 --- a/frontend/apps/web/components/jobs/SchemaTable/SchemaPageTable.tsx +++ /dev/null @@ -1,203 +0,0 @@ -'use client'; -import React, { ReactElement } from 'react'; - -import { - ColumnDef, - flexRender, - getCoreRowModel, - getFacetedMinMaxValues, - getFacetedRowModel, - getFacetedUniqueValues, - getFilteredRowModel, - getSortedRowModel, - Row as TanStackRow, - useReactTable, -} from '@tanstack/react-table'; - -import { useVirtualizer } from '@tanstack/react-virtual'; - -import { CardDescription, CardTitle } from '@/components/ui/card'; -import { - StickyHeaderTable, - TableBody, - TableHead, - TableHeader, - TableRow, -} from '@/components/ui/table'; -import { cn } from '@/libs/utils'; -import { JobMappingFormValues } from '@/yup-validations/jobs'; -import { JobMapping } from '@neosync/sdk'; -import { GoWorkflow } from 'react-icons/go'; -import { ImportMappingsConfig } from './ImportJobMappingsButton'; -import { SchemaTableToolbar } from './SchemaTableToolBar'; -import { JobType, SchemaConstraintHandler } from './schema-constraint-handler'; -import { TransformerHandler } from './transformer-handler'; - -export type Row = JobMappingFormValues; - -interface DataTableProps { - columns: ColumnDef[]; - data: TData[]; - transformerHandler: TransformerHandler; - constraintHandler: SchemaConstraintHandler; - jobType: JobType; - onExportMappingsClick( - selected: TanStackRow[], - shouldFormat: boolean - ): void; - onImportMappingsClick( - jobmappings: JobMapping[], - config: ImportMappingsConfig - ): void; -} - -export default function SchemaPageTable({ - columns, - data, - transformerHandler, - constraintHandler, - jobType, - onExportMappingsClick, - onImportMappingsClick, -}: DataTableProps): ReactElement { - const table = useReactTable({ - data, - columns, - initialState: { - columnVisibility: { - schema: false, - table: false, - }, - }, - getCoreRowModel: getCoreRowModel(), - getSortedRowModel: getSortedRowModel(), - getFilteredRowModel: getFilteredRowModel(), - getFacetedRowModel: getFacetedRowModel(), - getFacetedUniqueValues: getFacetedUniqueValues(), - getFacetedMinMaxValues: getFacetedMinMaxValues(), - }); - - const { rows } = table.getRowModel(); - - const tableContainerRef = React.useRef(null); - - const rowVirtualizer = useVirtualizer({ - count: rows.length, - estimateSize: () => 33, - getScrollElement: () => tableContainerRef.current, - overscan: 5, - }); - return ( -
-
-
- -
- Transformer Mapping -
- - Map Transformers to every column below. - -
- - onExportMappingsClick( - table.getSelectedRowModel().rows, - shouldFormat - ) - } - onImportMappingsClick={onImportMappingsClick} - /> -
-
0 && 'overflow-auto' - )} - ref={tableContainerRef} - > - - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => { - return ( - - {header.isPlaceholder - ? null - : flexRender( - header.column.columnDef.header, - header.getContext() - )} - - ); - })} - - ))} - - - {rows.length === 0 && ( - - No Schema(s) or Table(s) selected. - - )} - {rowVirtualizer.getVirtualItems().map((virtualRow) => { - const row = rows[virtualRow.index]; - return ( - rowVirtualizer.measureElement(node)} //measure dynamic row height - key={row.id} - style={{ - transform: `translateY(${virtualRow.start}px)`, - }} - className="items-center flex absolute w-full justify-between px-2" - > - {row.getVisibleCells().map((cell) => { - return ( - - {flexRender( - cell.column.columnDef.cell, - cell.getContext() - )} - - ); - })} - - ); - })} - - -
-
- Total rows: ({new Intl.NumberFormat('en-US').format(data.length)}) Rows - visible: ( - {new Intl.NumberFormat('en-US').format(table.getRowModel().rows.length)} - ) -
-
- ); -} diff --git a/frontend/apps/web/components/jobs/SchemaTable/SchemaTable.tsx b/frontend/apps/web/components/jobs/SchemaTable/SchemaTable.tsx index f35397bdaf..e0cbe7b526 100644 --- a/frontend/apps/web/components/jobs/SchemaTable/SchemaTable.tsx +++ b/frontend/apps/web/components/jobs/SchemaTable/SchemaTable.tsx @@ -4,8 +4,6 @@ import DualListBox, { Action, Option, } from '@/components/DualListBox/DualListBox'; -import Spinner from '@/components/Spinner'; -import { useAccount } from '@/components/providers/account-provider'; import SkeletonTable from '@/components/skeleton/SkeletonTable'; import { Card, @@ -15,9 +13,10 @@ import { CardTitle, } from '@/components/ui/card'; import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; -import { useGetTransformersHandler } from '@/libs/hooks/useGetTransformersHandler'; +import { Transformer } from '@/shared/transformers'; import { JobMappingFormValues, + JobMappingTransformerForm, SchemaFormValues, VirtualForeignConstraintFormValues, } from '@/yup-validations/jobs'; @@ -27,17 +26,24 @@ import { ValidateJobMappingsResponse, } from '@neosync/sdk'; import { TableIcon } from '@radix-ui/react-icons'; +import { Row } from '@tanstack/react-table'; import { ReactElement, useMemo } from 'react'; import { FieldErrors } from 'react-hook-form'; +import { + getGeneratedStatement, + getIdentityStatement, +} from '../JobMappingTable/AttributesCell'; +import { JobMappingRow, SQL_COLUMNS } from '../JobMappingTable/Columns'; +import JobMappingTable from '../JobMappingTable/JobMappingTable'; import FormErrorsCard, { FormError } from './FormErrorsCard'; import { ImportMappingsConfig } from './ImportJobMappingsButton'; -import { getSchemaColumns } from './SchemaColumns'; -import SchemaPageTable from './SchemaPageTable'; import { getVirtualForeignKeysColumns } from './VirtualFkColumns'; import VirtualFkPageTable from './VirtualFkPageTable'; import { VirtualForeignKeyForm } from './VirtualForeignKeyForm'; import { JobType, SchemaConstraintHandler } from './schema-constraint-handler'; +import { TransformerResult } from './transformer-handler'; import { useOnExportMappings } from './useOnExportMappings'; +import { handleDataTypeBadge } from './util'; interface Props { data: JobMappingFormValues[]; @@ -59,6 +65,18 @@ interface Props { jobmappings: JobMapping[], importConfig: ImportMappingsConfig ): void; + onTransformerUpdate(index: number, config: JobMappingTransformerForm): void; + getAvailableTransformers(index: number): TransformerResult; + getTransformerFromField(index: number): Transformer; + onTransformerBulkUpdate( + indices: number[], + config: JobMappingTransformerForm + ): void; + getAvailableTransformersForBulk( + rows: Row[] + ): TransformerResult; + getTransformerFromFieldValue(value: JobMappingTransformerForm): Transformer; + onApplyDefaultClick(override: boolean): void; } export function SchemaTable(props: Props): ReactElement { @@ -76,19 +94,80 @@ export function SchemaTable(props: Props): ReactElement { isJobMappingsValidating, onValidate, onImportMappingsClick, + onTransformerUpdate, + getAvailableTransformers, + getTransformerFromField, + getAvailableTransformersForBulk, + getTransformerFromFieldValue, + onApplyDefaultClick, + onTransformerBulkUpdate, } = props; - const { account } = useAccount(); - const { handler, isLoading, isValidating } = useGetTransformersHandler( - account?.id ?? '' - ); + const tableData = useMemo((): JobMappingRow[] => { + return data.map((d): JobMappingRow => { + const colKey = { + schema: d.schema, + table: d.table, + column: d.column, + }; + const isPrimaryKey = constraintHandler.getIsPrimaryKey(colKey); + const [isForeignKey, fkCols] = constraintHandler.getIsForeignKey(colKey); + const [isVirtualForeignKey, vfkCols] = + constraintHandler.getIsVirtualForeignKey(colKey); + const isUnique = constraintHandler.getIsUniqueConstraint(colKey); + + const constraintPieces: string[] = []; + if (isPrimaryKey) { + constraintPieces.push('Primary Key'); + } + if (isForeignKey) { + fkCols.forEach((col) => constraintPieces.push(`Foreign Key: ${col}`)); + } + if (isVirtualForeignKey) { + vfkCols.forEach((col) => + constraintPieces.push(`Virtual Foreign Key: ${col}`) + ); + } + if (isUnique) { + constraintPieces.push('Unique'); + } + const constraints = constraintPieces.join('\n'); + + const generatedType = constraintHandler.getGeneratedType(colKey); + const identityType = constraintHandler.getIdentityType(colKey); + + const attributePieces: string[] = []; + if (generatedType) { + attributePieces.push(getGeneratedStatement(generatedType)); + } else if (identityType) { + attributePieces.push(getIdentityStatement(identityType)); + } + attributePieces.push( + constraintHandler.getIsNullable(colKey) ? 'Is Nullable' : 'Not Nullable' + ); + const attributes = attributePieces.join('\n'); - const columns = useMemo(() => { - return getSchemaColumns({ - transformerHandler: handler, - constraintHandler, - jobType, + return { + schema: d.schema, + table: d.table, + column: d.column, + dataType: handleDataTypeBadge(constraintHandler.getDataType(colKey)), + attributes: { + value: attributes, + generatedType: generatedType, + identityType: identityType, + }, + constraints: { + value: constraints, + foreignKey: [isForeignKey, fkCols], + virtualForeignKey: [isVirtualForeignKey, vfkCols], + isPrimaryKey: isPrimaryKey, + isUnique: isUnique, + }, + isNullable: constraintHandler.getIsNullable(colKey), + transformer: d.transformer, + }; }); - }, [handler, constraintHandler, jobType]); + }, [data]); const virtualForeignKeyColumns = useMemo(() => { return getVirtualForeignKeysColumns({ removeVirtualForeignKey }); @@ -100,11 +179,13 @@ export function SchemaTable(props: Props): ReactElement { [schema, data] ); - const { onClick: onExportMappingsClick } = useOnExportMappings({ - jobMappings: data, - }); + const { onClick: onExportMappingsClick } = useOnExportMappings( + { + jobMappings: data, + } + ); - if (isLoading || !data) { + if (!data) { return ; } @@ -118,7 +199,6 @@ export function SchemaTable(props: Props): ReactElement {
Table Selection -
{isValidating ? : null}
Select the tables that you want to transform and move them from @@ -150,14 +230,31 @@ export function SchemaTable(props: Props): ReactElement { - + console.warn('on delete row is not implemented') + } + onDuplicateRow={() => + console.warn('on duplicate row is not implemented') + } + canRenameColumn={() => false} + onRowUpdate={() => console.warn('onRowUpdate is not implemented')} + getAvailableCollectionsByRow={() => { + console.warn('getAvailableCollections is not implemented'); + return []; + }} /> @@ -176,14 +273,29 @@ export function SchemaTable(props: Props): ReactElement { ) : ( - console.warn('on delete row is not implemented')} + onDuplicateRow={() => + console.warn('on duplicate row is not implemented') + } + canRenameColumn={() => false} + onRowUpdate={() => console.warn('onRowUpdate is not implemented')} + getAvailableCollectionsByRow={() => { + console.warn('getAvailableCollections is not implemented'); + return []; + }} /> )}
diff --git a/frontend/apps/web/components/jobs/SchemaTable/SchemaTableToolBar.tsx b/frontend/apps/web/components/jobs/SchemaTable/SchemaTableToolBar.tsx index c276e9271c..7795622d61 100644 --- a/frontend/apps/web/components/jobs/SchemaTable/SchemaTableToolBar.tsx +++ b/frontend/apps/web/components/jobs/SchemaTable/SchemaTableToolBar.tsx @@ -2,7 +2,6 @@ import { Row, Table } from '@tanstack/react-table'; -import { SingleTableSchemaFormValues } from '@/app/(mgmt)/[account]/new/job/job-form-validations'; import EditTransformerOptions from '@/app/(mgmt)/[account]/transformers/EditTransformerOptions'; import ButtonText from '@/components/ButtonText'; import FormErrorMessage from '@/components/FormErrorMessage'; @@ -10,64 +9,61 @@ import { Button } from '@/components/ui/button'; import { cn } from '@/libs/utils'; import { isSystemTransformer, Transformer } from '@/shared/transformers'; import { - getTransformerFromField, - getTransformerSelectButtonText, isInvalidTransformer, + useTransformerSelectButtonText, } from '@/util/util'; import { convertJobMappingTransformerToForm, JobMappingTransformerForm, - SchemaFormValues, } from '@/yup-validations/jobs'; import { - GenerateDefault, JobMapping, JobMappingTransformer, - Passthrough, SystemTransformer, - TransformerConfig, UserDefinedTransformer, } from '@neosync/sdk'; import { CheckIcon, Cross2Icon } from '@radix-ui/react-icons'; import { useState } from 'react'; -import { useFormContext } from 'react-hook-form'; import ApplyDefaultTransformersButton from './ApplyDefaultTransformersButton'; import ExportJobMappingsButton from './ExportJobMappingsButton'; import ImportJobMappingsButton, { ImportMappingsConfig, } from './ImportJobMappingsButton'; -import { fromRowDataToColKey, getTransformerFilter } from './SchemaColumns'; -import { Row as RowData } from './SchemaPageTable'; import { SchemaTableViewOptions } from './SchemaTableViewOptions'; import TransformerSelect from './TransformerSelect'; -import { JobType, SchemaConstraintHandler } from './schema-constraint-handler'; -import { - TransformerConfigCase, - TransformerHandler, -} from './transformer-handler'; +import { TransformerResult } from './transformer-handler'; interface DataTableToolbarProps { table: Table; - transformerHandler: TransformerHandler; - constraintHandler: SchemaConstraintHandler; - jobType: JobType; + getAllowedTransformers(rows: Row[]): TransformerResult; + getTransformerFromField(selected: JobMappingTransformerForm): Transformer; + onBulkUpdate(indices: number[], value: JobMappingTransformerForm): void; onExportMappingsClick(shouldFormat: boolean): void; onImportMappingsClick( jobmappings: JobMapping[], config: ImportMappingsConfig ): void; + displayApplyDefaultTransformersButton: boolean; + isApplyDefaultButtonDisabled: boolean; + onApplyDefaultClick(override: boolean): void; } +const DEFAULT_TRANSFORMER_BUTTON_TEXT = 'Bulk set transformers'; + export function SchemaTableToolbar({ table, - transformerHandler, - constraintHandler, - jobType, onExportMappingsClick, onImportMappingsClick, + getAllowedTransformers, + getTransformerFromField, + onBulkUpdate, + displayApplyDefaultTransformersButton, + isApplyDefaultButtonDisabled, + onApplyDefaultClick, }: DataTableToolbarProps) { - const isFiltered = table.getState().columnFilters.length > 0; - const hasSelectedRows = Object.values(table.getState().rowSelection).some( + const tableState = table.getState(); + const isFiltered = tableState.columnFilters.length > 0; + const hasSelectedRows = Object.values(tableState.rowSelection).some( (value) => value ); @@ -76,25 +72,18 @@ export function SchemaTableToolbar({ convertJobMappingTransformerToForm(new JobMappingTransformer()) ); - const form = useFormContext(); - - const transformer = getTransformerFromField( - transformerHandler, - bulkTransformer + const transformer = getTransformerFromField(bulkTransformer); + const allowedTransformers = getAllowedTransformers( + table.getSelectedRowModel().rows ); - // conditionally computed the allowed transformers only if there are selected rows - const allowedTransformers = hasSelectedRows - ? getFilteredTransformersForBulkSet( - table.getSelectedRowModel().rows, - transformerHandler, - constraintHandler, - jobType - ) - : { system: [], userDefined: [] }; const isBulkApplyDisabled = !bulkTransformer || !hasSelectedRows || !isTransformerAllowed(allowedTransformers, transformer); + const transformerSelectButtontext = useTransformerSelectButtonText( + transformer, + DEFAULT_TRANSFORMER_BUTTON_TEXT + ); return (
@@ -107,10 +96,7 @@ export function SchemaTableToolbar({ onSelect={(value) => { setBulkTransformer(value); }} - buttonText={getTransformerSelectButtonText( - transformer, - 'Bulk set transformers' - )} + buttonText={transformerSelectButtontext} disabled={!hasSelectedRows} buttonClassName="md:max-w-[275px]" notFoundText="No transformers found for the given selection." @@ -127,21 +113,16 @@ export function SchemaTableToolbar({ variant="outline" className={cn(isBulkApplyDisabled ? undefined : 'border-blue-600')} onClick={() => { - table.getSelectedRowModel().rows.forEach((r) => { - form.setValue( - `mappings.${r.index}.transformer`, - bulkTransformer, - { - shouldDirty: true, - shouldTouch: true, - shouldValidate: false, // this is really expensive, see the trigger call below - } - ); - }); + const rowIndices = table + .getSelectedRowModel() + .rows.map((r) => r.index); + if (rowIndices.length === 0) { + return; + } + onBulkUpdate(rowIndices, bulkTransformer); setBulkTransformer( convertJobMappingTransformerToForm(new JobMappingTransformer()) ); - form.trigger('mappings'); // trigger validation after bulk updating the selected form options table.resetRowSelection(true); }} > @@ -174,56 +155,10 @@ export function SchemaTableToolbar({ /> )} - {jobType === 'sync' && ( + {displayApplyDefaultTransformersButton && ( { - const formMappings = form.getValues('mappings'); - formMappings.forEach((fm, idx) => { - // skips setting the default transformer if the user has already set the transformer - if (fm.transformer.config.case && !override) { - return; - } else { - const colkey = { - schema: fm.schema, - table: fm.table, - column: fm.column, - }; - const isGenerated = - constraintHandler.getIsGenerated(colkey); - const identityType = - constraintHandler.getIdentityType(colkey); - const newJm = - isGenerated && !identityType - ? new JobMappingTransformer({ - config: new TransformerConfig({ - config: { - case: 'generateDefaultConfig', - value: new GenerateDefault(), - }, - }), - }) - : new JobMappingTransformer({ - config: new TransformerConfig({ - config: { - case: 'passthroughConfig', - value: new Passthrough(), - }, - }), - }); - form.setValue( - `mappings.${idx}.transformer`, - convertJobMappingTransformerToForm(newJm), - { - shouldDirty: true, - shouldTouch: true, - shouldValidate: false, - } - ); - } - }); - form.trigger('mappings'); // trigger validation after bulk updating the selected form options - }} + isDisabled={isApplyDefaultButtonDisabled} + onClick={onApplyDefaultClick} /> )} @@ -257,105 +192,3 @@ function isTransformerAllowed( return userDefined.some((t) => t.id === selected.id); } } - -function getFilteredTransformersForBulkSet( - rows: Row[], - transformerHandler: TransformerHandler, - constraintHandler: SchemaConstraintHandler, - jobType: JobType -): { - system: SystemTransformer[]; - userDefined: UserDefinedTransformer[]; -} { - const systemArrays: SystemTransformer[][] = []; - const userDefinedArrays: UserDefinedTransformer[][] = []; - - rows.forEach((row) => { - const { system, userDefined } = transformerHandler.getFilteredTransformers( - getTransformerFilter( - constraintHandler, - fromRowDataToColKey(row as unknown as Row), // this will bite us at some point - jobType - ) - ); - systemArrays.push(system); - userDefinedArrays.push(userDefined); - }); - - const uniqueSystemConfigCases = findCommonSystemConfigCases(systemArrays); - const uniqueSystem = uniqueSystemConfigCases - .map((configCase) => - transformerHandler.getSystemTransformerByConfigCase(configCase) - ) - .filter((x): x is SystemTransformer => !!x); - - const uniqueIds = findCommonUserDefinedIds(userDefinedArrays); - const uniqueUserDef = uniqueIds - .map((id) => transformerHandler.getUserDefinedTransformerById(id)) - .filter((x): x is UserDefinedTransformer => !!x); - - return { - system: uniqueSystem, - userDefined: uniqueUserDef, - }; -} - -function findCommonSystemConfigCases( - arrays: SystemTransformer[][] -): TransformerConfigCase[] { - const elementCount: Record = {} as Record< - TransformerConfigCase, - number - >; - const subArrayCount = arrays.length; - const commonElements: TransformerConfigCase[] = []; - - arrays.forEach((subArray) => { - // Use a Set to ensure each element in a sub-array is counted only once - new Set(subArray).forEach((element) => { - if (!element.config?.config.case) { - return; - } - if (!elementCount[element.config.config.case]) { - elementCount[element.config.config.case] = 1; - } else { - elementCount[element.config.config.case]++; - } - }); - }); - - for (const [element, count] of Object.entries(elementCount)) { - if (count === subArrayCount) { - commonElements.push(element as TransformerConfigCase); - } - } - - return commonElements; -} - -function findCommonUserDefinedIds( - arrays: UserDefinedTransformer[][] -): string[] { - const elementCount: Record = {}; - const subArrayCount = arrays.length; - const commonElements: string[] = []; - - arrays.forEach((subArray) => { - // Use a Set to ensure each element in a sub-array is counted only once - new Set(subArray).forEach((element) => { - if (!elementCount[element.id]) { - elementCount[element.id] = 1; - } else { - elementCount[element.id]++; - } - }); - }); - - for (const [element, count] of Object.entries(elementCount)) { - if (count === subArrayCount) { - commonElements.push(element); - } - } - - return commonElements; -} diff --git a/frontend/apps/web/components/jobs/SchemaTable/TransformerSelect.tsx b/frontend/apps/web/components/jobs/SchemaTable/TransformerSelect.tsx index 34a80e1d0f..9f6f011aec 100644 --- a/frontend/apps/web/components/jobs/SchemaTable/TransformerSelect.tsx +++ b/frontend/apps/web/components/jobs/SchemaTable/TransformerSelect.tsx @@ -26,7 +26,8 @@ import { UserDefinedTransformerConfig, } from '@neosync/sdk'; import { CaretSortIcon, CheckIcon } from '@radix-ui/react-icons'; -import { ReactElement, useState } from 'react'; +import { ReactElement, useEffect, useState } from 'react'; +import { TransformerResult } from './transformer-handler'; type Side = (typeof SIDE_OPTIONS)[number]; @@ -59,9 +60,14 @@ export default function TransformerSelect(props: Props): ReactElement { } = props; const [open, setOpen] = useState(false); - const { system, userDefined } = open - ? getTransformers() - : { system: [], userDefined: [] }; + const [{ system, userDefined }, setTransformerResult] = + useState({ system: [], userDefined: [] }); + + useEffect(() => { + if (open) { + setTransformerResult(getTransformers()); + } + }, [open]); return ( diff --git a/frontend/apps/web/components/jobs/SchemaTable/schema-constraint-handler.ts b/frontend/apps/web/components/jobs/SchemaTable/schema-constraint-handler.ts index 4ed7bebeb1..b88f7a1821 100644 --- a/frontend/apps/web/components/jobs/SchemaTable/schema-constraint-handler.ts +++ b/frontend/apps/web/components/jobs/SchemaTable/schema-constraint-handler.ts @@ -73,7 +73,6 @@ export function getSchemaConstraintHandler( uniqueConstraints, vfkMap ); - return { getDataType(key) { return colmap[fromColKey(key)]?.dataType ?? ''; @@ -251,7 +250,7 @@ function buildColDetailsMap( if (constraint.foreignKey) { fkconstraintsMap[col] = new ForeignKey({ table: constraint.foreignKey?.table, - column: constraint.foreignKey?.columns[idx], + columns: [constraint.foreignKey?.columns[idx]], }); } else { fkconstraintsMap[col] = new ForeignKey(); diff --git a/frontend/apps/web/components/jobs/SchemaTable/transformer-handler.ts b/frontend/apps/web/components/jobs/SchemaTable/transformer-handler.ts index 4fa4df67e1..f67e7efbcf 100644 --- a/frontend/apps/web/components/jobs/SchemaTable/transformer-handler.ts +++ b/frontend/apps/web/components/jobs/SchemaTable/transformer-handler.ts @@ -16,6 +16,11 @@ export type TransformerConfigCase = NonNullable< ExtractCase >; +export interface TransformerResult { + system: SystemTransformer[]; + userDefined: UserDefinedTransformer[]; +} + export class TransformerHandler { private readonly systemTransformers: SystemTransformer[]; private readonly userDefinedTransformers: UserDefinedTransformer[]; @@ -55,10 +60,9 @@ export class TransformerHandler { }; } - public getFilteredTransformers(filters: TransformerFilters): { - system: SystemTransformer[]; - userDefined: UserDefinedTransformer[]; - } { + public getFilteredTransformers( + filters: TransformerFilters + ): TransformerResult { const systemMap = new Map( this.systemTransformers.map((t) => [ t.source, diff --git a/frontend/apps/web/components/jobs/SchemaTable/useOnExportMappings.tsx b/frontend/apps/web/components/jobs/SchemaTable/useOnExportMappings.tsx index b47b2d8751..657cdc16f4 100644 --- a/frontend/apps/web/components/jobs/SchemaTable/useOnExportMappings.tsx +++ b/frontend/apps/web/components/jobs/SchemaTable/useOnExportMappings.tsx @@ -11,21 +11,20 @@ interface Props { jobMappings: JobMappingFormValues[]; } -interface UseOnExportMappingsResponse { - onClick( - selectedRows: Row[], - shouldFormat: boolean - ): Promise; +interface UseOnExportMappingsResponse { + onClick(selectedRows: Row[], shouldFormat: boolean): Promise; } // Hook that provides an onClick handler that will download job mappings to disk -export function useOnExportMappings(props: Props): UseOnExportMappingsResponse { +export function useOnExportMappings( + props: Props +): UseOnExportMappingsResponse { const { jobMappings } = props; const { downloadFile } = useJsonFileDownload(); return { onClick: async function ( - selectedRows: Row[], + selectedRows: Row[], shouldFormat: boolean ): Promise { // Using the raw jobMappings instead of the row due to tanstack sometimes not giving the most up to date values. diff --git a/frontend/apps/web/components/jobs/SchemaTable/util.ts b/frontend/apps/web/components/jobs/SchemaTable/util.ts new file mode 100644 index 0000000000..cac3a07e77 --- /dev/null +++ b/frontend/apps/web/components/jobs/SchemaTable/util.ts @@ -0,0 +1,88 @@ +import { Row } from '@tanstack/react-table'; +import { JobMappingRow, NosqlJobMappingRow } from '../JobMappingTable/Columns'; +import { + ColumnKey, + JobType, + SchemaConstraintHandler, +} from './schema-constraint-handler'; +import { toSupportedJobtype, TransformerFilters } from './transformer-handler'; + +export function fromRowDataToColKey(row: Row): ColumnKey { + return { + schema: row.getValue('schema'), + table: row.getValue('table'), + column: row.getValue('column'), + }; +} +export function fromNosqlRowDataToColKey( + row: Row +): ColumnKey { + const [schema, table] = splitCollection(row.getValue('collection')); + return { + schema, + table, + column: row.getValue('column'), + }; +} + +export function toColKey( + schema: string, + table: string, + column: string +): ColumnKey { + return { + schema, + table, + column, + }; +} + +// cleans up the data type values since some are too long , can add on more here +export function handleDataTypeBadge(dataType: string): string { + // Check for "timezone" and replace accordingly without entering the switch + if (dataType.includes('timezone')) { + return dataType + .replace('timestamp with time zone', 'timestamp(tz)') + .replace('timestamp without time zone', 'timestamp'); + } + + const splitDt = dataType.split('('); + switch (splitDt[0]) { + case 'character varying': + // The condition inside the if statement seemed reversed. It should return 'varchar' directly if splitDt[1] is undefined. + return splitDt[1] !== undefined ? `varchar(${splitDt[1]}` : 'varchar'; + default: + return dataType; + } +} + +export function getTransformerFilter( + constraintHandler: SchemaConstraintHandler, + colkey: ColumnKey, + jobType: JobType +): TransformerFilters { + const [isForeignKey] = constraintHandler.getIsForeignKey(colkey); + const [isVirtualForeignKey] = + constraintHandler.getIsVirtualForeignKey(colkey); + const isNullable = constraintHandler.getIsNullable(colkey); + const convertedDataType = constraintHandler.getConvertedDataType(colkey); + const hasDefault = constraintHandler.getHasDefault(colkey); + const isGenerated = constraintHandler.getIsGenerated(colkey); + return { + dataType: convertedDataType, + hasDefault, + isForeignKey, + isVirtualForeignKey, + isNullable, + jobType: toSupportedJobtype(jobType), + isGenerated, + identityType: constraintHandler.getIdentityType(colkey), + }; +} + +export function splitCollection(collection: string): [string, string] { + const lastDotIndex = collection.lastIndexOf('.'); + const schema = collection.substring(0, lastDotIndex); + const table = collection.substring(lastDotIndex + 1); + return [schema, table]; +} diff --git a/frontend/apps/web/components/labels/LearnMoreLink.tsx b/frontend/apps/web/components/labels/LearnMoreLink.tsx index 1a9fc9bbc0..fedffdf9f1 100644 --- a/frontend/apps/web/components/labels/LearnMoreLink.tsx +++ b/frontend/apps/web/components/labels/LearnMoreLink.tsx @@ -15,12 +15,13 @@ export default function LearnMoreLink(props: Props): ReactElement { -
-
Learn more
- -
+ Learn more + ); } diff --git a/frontend/apps/web/util/util.ts b/frontend/apps/web/util/util.ts index efc9adf602..16749cad03 100644 --- a/frontend/apps/web/util/util.ts +++ b/frontend/apps/web/util/util.ts @@ -13,6 +13,7 @@ import { UserDefinedTransformer, } from '@neosync/sdk'; import { format } from 'date-fns'; +import { useMemo } from 'react'; export function formatDateTime( dateStr?: string | Date | number, @@ -168,6 +169,16 @@ export function isInvalidTransformer(transformer: Transformer): boolean { return transformer.config == null; } +export function useTransformerSelectButtonText( + transformer: Transformer, + defaultText: string = 'Select Transformer' +): string { + return useMemo( + () => getTransformerSelectButtonText(transformer, defaultText), + [transformer.name, defaultText, transformer.config] + ); +} + export function getTransformerSelectButtonText( transformer: Transformer, defaultText: string = 'Select Transformer' diff --git a/frontend/apps/web/yup-validations/transformer-validations.ts b/frontend/apps/web/yup-validations/transformer-validations.ts index b35f20c724..39a2bd836b 100644 --- a/frontend/apps/web/yup-validations/transformer-validations.ts +++ b/frontend/apps/web/yup-validations/transformer-validations.ts @@ -371,8 +371,7 @@ const transformPiiTextConfig = Yup.object({ }); const generateIpAddressConfig = Yup.object({ - version: Yup.string().default('GENERATE_IP_ADDRESS_VERSION_V4'), - class: Yup.string().default('GENERATE_IP_ADDRESS_CLASS_PUBLIC'), + ipType: Yup.string().default('GENERATE_IP_ADDRESS_TYPE_V4_PUBLIC'), }); type ConfigType = TransformerConfig['config'];