From 8f0797329446ee35e4d3534cd193892a2766dc09 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 14 Apr 2024 14:26:09 +0300 Subject: [PATCH] Index spi impl (#522) * init * tests and types * Refactor index retrieval and add index service * lint fixes * Update API endpoints for index operations * Update API endpoint for creating indexes * Commented out unused database configurations * Add index provider test support and generate random domain and SPI indexes * Update import statement for IIndexProvider * Add logger to IndexProvider constructor * Fix index creation and error handling * Add IndexProvider to connection provider * Refactor test description to be more descriptive * Update supported operations for external databases * Refactor index creation and removal in MySQL index provider * Refactor index creation and removal in PostgresIndexProvider * Refactor index API response structure * Refactor index creation in PostgresIndexProvider * Add support for indexes in capabilities response * Add extractIndexFromIndexQueryForCollection function to postgres_utils * Fix index query for collection name * Refactor query for retrieving active index creation queries --------- Co-authored-by: Ido Kahlon --- .../test/drivers/index_api_rest_matchers.ts | 49 ++++++ .../drivers/index_api_rest_test_support.ts | 30 ++++ .../velo-external-db/test/e2e/app.e2e.spec.ts | 14 +- .../test/e2e/app_index.e2e.spec.ts | 107 ++++++++++++ apps/velo-external-db/test/gen.ts | 45 +++++- .../velo-external-db/test/utils/eventually.ts | 24 +++ .../src/supported_operations.ts | 1 + .../src/supported_operations.ts | 3 +- .../src/supported_operations.ts | 5 +- .../src/supported_operations.ts | 7 +- .../src/connection_provider.ts | 4 +- .../src/mysql_index_provider.ts | 152 ++++++++++++++++++ .../src/connection_provider.ts | 3 + .../src/postgres_index_provider.ts | 131 +++++++++++++++ .../src/postgres_utils.spec.ts | 47 +++++- .../src/postgres_utils.ts | 16 ++ .../src/supported_operations.ts | 7 +- .../src/libs/db_connector.ts | 6 +- .../src/libs/errors.ts | 13 ++ libs/velo-external-db-core/src/index.ts | 8 +- libs/velo-external-db-core/src/router.ts | 52 +++++- .../src/service/indexing.spec.ts | 81 ++++++++++ .../src/service/indexing.ts | 45 ++++++ .../src/spi-model/indexing.ts | 74 +++++++++ .../drivers/index_provider_test_support.ts | 28 ++++ libs/velo-external-db-core/test/gen.ts | 22 +++ .../src/collection_types.ts | 1 + libs/velo-external-db-types/src/index.ts | 6 +- .../src/indexing_types.ts | 22 +++ package.json | 1 + 30 files changed, 976 insertions(+), 28 deletions(-) create mode 100644 apps/velo-external-db/test/drivers/index_api_rest_matchers.ts create mode 100644 apps/velo-external-db/test/drivers/index_api_rest_test_support.ts create mode 100644 apps/velo-external-db/test/e2e/app_index.e2e.spec.ts create mode 100644 apps/velo-external-db/test/utils/eventually.ts create mode 100644 libs/external-db-mysql/src/mysql_index_provider.ts create mode 100644 libs/external-db-postgres/src/postgres_index_provider.ts create mode 100644 libs/velo-external-db-core/src/service/indexing.spec.ts create mode 100644 libs/velo-external-db-core/src/service/indexing.ts create mode 100644 libs/velo-external-db-core/src/spi-model/indexing.ts create mode 100644 libs/velo-external-db-core/test/drivers/index_provider_test_support.ts create mode 100644 libs/velo-external-db-types/src/indexing_types.ts diff --git a/apps/velo-external-db/test/drivers/index_api_rest_matchers.ts b/apps/velo-external-db/test/drivers/index_api_rest_matchers.ts new file mode 100644 index 000000000..64a423d38 --- /dev/null +++ b/apps/velo-external-db/test/drivers/index_api_rest_matchers.ts @@ -0,0 +1,49 @@ +import { indexSpi } from '@wix-velo/velo-external-db-core' +const { IndexFieldOrder, IndexStatus } = indexSpi + +const indexWith = (index: indexSpi.Index, extraProps: Partial) => ({ + ...index, + fields: index.fields.map(field => ({ + ...field, + order: expect.toBeOneOf([IndexFieldOrder.ASC, IndexFieldOrder.DESC]), + })), + caseInsensitive: expect.any(Boolean), // TODO: remove this when we support case insensitive indexes + ...extraProps +}) + +export const failedIndexCreationResponse = (index: indexSpi.Index) => ({ + index: indexWith(index, { status: IndexStatus.FAILED }) +}) + + +export const listIndexResponseWithDefaultIndex = () => ({ + indexes: expect.arrayContaining([toHaveDefaultIndex()]) +}) + +export const listIndexResponseWith = (indexes: indexSpi.Index[]) => ({ + indexes: expect.arrayContaining( + [...indexes.map((index: indexSpi.Index) => indexWith(index, { status: IndexStatus.ACTIVE }))] + ) +}) + +export const toHaveDefaultIndex = () => ({ + name: expect.any(String), + fields: expect.arrayContaining([ + expect.objectContaining({ + path: '_id', + order: expect.toBeOneOf([IndexFieldOrder.ASC, IndexFieldOrder.DESC]) + }) + ]), + caseInsensitive: expect.any(Boolean), + status: IndexStatus.ACTIVE, + unique: true +}) + + +export const createIndexResponseWith = (index: indexSpi.Index) => ({ index: indexWith(index, { status: IndexStatus.BUILDING }) }) + +export const removeIndexResponse = () => (({})) + +export const listIndexResponseWithFailedIndex = (index: indexSpi.Index) => { + return expect.arrayContaining([indexWith(index, { status: IndexStatus.FAILED })]) +} diff --git a/apps/velo-external-db/test/drivers/index_api_rest_test_support.ts b/apps/velo-external-db/test/drivers/index_api_rest_test_support.ts new file mode 100644 index 000000000..2e67a0b08 --- /dev/null +++ b/apps/velo-external-db/test/drivers/index_api_rest_test_support.ts @@ -0,0 +1,30 @@ +import axios from 'axios' +import waitUntil from 'async-wait-until' +import { indexSpi } from '@wix-velo/velo-external-db-core' + +const axiosInstance = axios.create({ + baseURL: 'http://localhost:8080/v3' +}) + +export const givenIndexes = async(collectionName: string, indexes: indexSpi.Index[], auth: any) => { + for (const index of indexes) { + await axiosInstance.post('/indexes/create', { collectionId: collectionName, index } as indexSpi.CreateIndexRequest, auth) + } + await Promise.all(indexes.map(index => indexCreated(collectionName, index.name, auth))) +} + +const indexCreated = async(collectionName: string, indexName: string, auth: any) => { + await waitUntil(async() => { + const { indexes } = await retrieveIndexesFor(collectionName, auth) as indexSpi.ListIndexesResponse + return indexes.some(index => index.name === indexName) + }) +} + +export const createIndexFor = async(collectionName: string, index: indexSpi.Index, auth: any) => await + axiosInstance.post('/indexes/create', { collectionId: collectionName, index } as indexSpi.CreateIndexRequest, auth).then(res => res.data) + +export const removeIndexFor = async(collectionName: string, indexName: string, auth: any) => await + axiosInstance.post('/indexes/remove', { collectionId: collectionName, indexName } as indexSpi.RemoveIndexRequest, auth).then(res => res.data) + +export const retrieveIndexesFor = async(collectionName: string, auth: any) => await +axiosInstance.post('/indexes/list', { collectionId: collectionName } as indexSpi.ListIndexesRequest, { transformRequest: auth.transformRequest }).then(res => res.data) diff --git a/apps/velo-external-db/test/e2e/app.e2e.spec.ts b/apps/velo-external-db/test/e2e/app.e2e.spec.ts index ec6c877ed..26b3d9cda 100644 --- a/apps/velo-external-db/test/e2e/app.e2e.spec.ts +++ b/apps/velo-external-db/test/e2e/app.e2e.spec.ts @@ -1,5 +1,6 @@ import { authOwner } from '@wix-velo/external-db-testkit' -import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName, env } from '../resources/e2e_resources' +import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName, env, supportedOperations } from '../resources/e2e_resources' +import { SchemaOperations } from '@wix-velo/velo-external-db-types' import { givenHideAppInfoEnvIsTrue } from '../drivers/app_info_config_test_support' const axios = require('axios').create({ @@ -36,11 +37,12 @@ describe(`Velo External DB: ${currentDbImplementationName()}`, () => { expect((await axios.post('/v3/capabilities/get', { }, authOwner)).data).toEqual(expect.objectContaining({ supportsCollectionModifications: true, supportedFieldTypes: expect.toBeArray(), - supportsCollectionDisplayName: false, - supportsCollectionDisplayField: false, - supportsCollectionPermissions: false, - supportsCollectionFieldDisplayName: false, - supportsCollectionFieldDescription: false, + indexptions: { + supportsIndexes: supportedOperations.includes(SchemaOperations.Indexing), + maxNumberOfRegularIndexesPerCollection: 10, + maxNumberOfUniqueIndexesPerCollection: 10, + maxNumberOfIndexesPerCollection: 20, + }, })) }) diff --git a/apps/velo-external-db/test/e2e/app_index.e2e.spec.ts b/apps/velo-external-db/test/e2e/app_index.e2e.spec.ts new file mode 100644 index 000000000..5a072e134 --- /dev/null +++ b/apps/velo-external-db/test/e2e/app_index.e2e.spec.ts @@ -0,0 +1,107 @@ +import Chance = require('chance') +import { Uninitialized, testIfSupportedOperationsIncludes } from '@wix-velo/test-commons' +import { authOwner } from '@wix-velo/external-db-testkit' +import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName, supportedOperations } from '../resources/e2e_resources' +import * as schema from '../drivers/schema_api_rest_test_support' +import * as matchers from '../drivers/index_api_rest_matchers' +import * as index from '../drivers/index_api_rest_test_support' +import * as gen from '../gen' +import axios from 'axios' +const chance = new Chance() +import { eventually } from '../utils/eventually' +import { InputField, SchemaOperations } from '@wix-velo/velo-external-db-types' +import { indexSpi } from '@wix-velo/velo-external-db-core' + +const { Indexing } = SchemaOperations + +const axiosServer = axios.create({ + baseURL: 'http://localhost:8080/v3' +}) + + +describe(`Velo External DB Index API: ${currentDbImplementationName()}`, () => { + beforeAll(async() => { + await setupDb() + await initApp() + }) + + afterAll(async() => { + await dbTeardown() + }, 20000) + + testIfSupportedOperationsIncludes(supportedOperations, [Indexing])('list', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + + await expect(index.retrieveIndexesFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.listIndexResponseWithDefaultIndex()) + }) + + testIfSupportedOperationsIncludes(supportedOperations, [Indexing])('list with multiple indexes', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await index.givenIndexes(ctx.collectionName, [ctx.index], authOwner) + + await expect(index.retrieveIndexesFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.listIndexResponseWith([ctx.index])) + }) + + testIfSupportedOperationsIncludes(supportedOperations, [Indexing])('create', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + + // in-progress + await expect(index.createIndexFor(ctx.collectionName, ctx.index, authOwner)).resolves.toEqual(matchers.createIndexResponseWith(ctx.index)) + + // active + await eventually(async() => + await expect(index.retrieveIndexesFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.listIndexResponseWith([ctx.index])) + ) + }) + + testIfSupportedOperationsIncludes(supportedOperations, [Indexing])('create an index on a column that already has an existing index, should return the index with status failed', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await index.givenIndexes(ctx.collectionName, [ctx.index], authOwner) + + await eventually(async() => await expect(axiosServer.post('/indexes/create', { + dataCollectionId: ctx.collectionName, + index: ctx.index + }, authOwner).then(res => res.data)).resolves.toEqual(matchers.failedIndexCreationResponse(ctx.index) + )) + }) + + testIfSupportedOperationsIncludes(supportedOperations, [Indexing])('creation of index with invalid column should return the index with status failed', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + + await eventually(async() => await expect(index.createIndexFor(ctx.collectionName, ctx.invalidIndex, authOwner)).resolves.toEqual(matchers.failedIndexCreationResponse(ctx.invalidIndex))) + }) + + testIfSupportedOperationsIncludes(supportedOperations, [Indexing])('remove', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await index.givenIndexes(ctx.collectionName, [ctx.index], authOwner) + + await expect( index.removeIndexFor(ctx.collectionName, ctx.index.name, authOwner)).resolves.toEqual(matchers.removeIndexResponse()) + + await expect(index.retrieveIndexesFor(ctx.collectionName, authOwner)).resolves.not.toEqual(matchers.listIndexResponseWith([ctx.index])) + }) + + afterAll(async() => { + await teardownApp() + }) + + interface Ctx { + collectionName: string + column: InputField + index: indexSpi.Index + invalidIndex: indexSpi.Index + } + + const ctx: Ctx = { + collectionName: Uninitialized, + column: Uninitialized, + index: Uninitialized, + invalidIndex: Uninitialized, + } + + beforeEach(() => { + ctx.collectionName = chance.word() + ctx.column = gen.randomColumn() + ctx.index = gen.spiIndexFor(ctx.collectionName, [ctx.column.name]) + ctx.invalidIndex = gen.spiIndexFor(ctx.collectionName, ['wrongColumn']) + }) +}) diff --git a/apps/velo-external-db/test/gen.ts b/apps/velo-external-db/test/gen.ts index d34b312cb..85ed3d61a 100644 --- a/apps/velo-external-db/test/gen.ts +++ b/apps/velo-external-db/test/gen.ts @@ -1,8 +1,12 @@ import { SystemFields } from '@wix-velo/velo-external-db-commons' -import { collectionSpi, schemaUtils } from '@wix-velo/velo-external-db-core' -import { InputField } from '@wix-velo/velo-external-db-types' +import { collectionSpi, schemaUtils, indexSpi } from '@wix-velo/velo-external-db-core' +import { + InputField, + } from '@wix-velo/velo-external-db-types' +import { DomainIndex, DomainIndexStatus } from '@wix-velo/velo-external-db-types' import * as Chance from 'chance' +const { IndexFieldOrder } = indexSpi const chance = Chance() @@ -27,7 +31,14 @@ export const randomDbEntity = (columns: string[]) => { return entity } - +export const randomArrayOf= (gen: any): T[] => { + const arr = [] + const num = chance.natural({ min: 2, max: 20 }) + for (let i = 0; i < num; i++) { + arr.push(gen()) + } + return arr +} export const randomDbEntities = (columns: string[]) => { const num = chance.natural({ min: 2, max: 20 }) @@ -114,3 +125,31 @@ export const randomCollection = (): collectionSpi.Collection => { pagingMode: collectionSpi.PagingMode.offset } } + +export const randomDomainIndex = (): DomainIndex => ({ + name: chance.word(), + columns: randomArrayOf(() => chance.word()), + isUnique: chance.bool(), + caseInsensitive: chance.bool(), + order: chance.pickone(['ASC', 'DESC']), + status: DomainIndexStatus.ACTIVE, +}) + +export const randomSpiIndex = (): indexSpi.Index => ({ + name: chance.word(), + fields: randomArrayOf(() => ({ + name: chance.word(), + order: chance.pickone(['ASC', 'DESC']), + })), + unique: chance.bool(), + caseInsensitive: chance.bool(), +}) + +export const spiIndexFor = (_collectionName: string, columns: string[]): indexSpi.Index => { + return { + name: chance.word(), + fields: columns.map((column: string) => ({ path: column, order: chance.pickone([IndexFieldOrder.ASC, IndexFieldOrder.DESC]) })), + unique: chance.bool(), + caseInsensitive: chance.bool(), + } +} diff --git a/apps/velo-external-db/test/utils/eventually.ts b/apps/velo-external-db/test/utils/eventually.ts new file mode 100644 index 000000000..2125a35f0 --- /dev/null +++ b/apps/velo-external-db/test/utils/eventually.ts @@ -0,0 +1,24 @@ +import * as trier from 'trier-promise' + +const defaults = { + timeout: 5000, + interval: 200 +} + +export const eventually = async(fn: any, opts?: { timeout?: number; interval?: number }) => { + return Promise.resolve().then(() => { + let error = null + const action = () => Promise.resolve().then(fn).catch(err => { + error = err + throw err + }) + const options = Object.assign({ action }, defaults, opts) + + return trier(options).catch(() => { + if (error !== null) { + error.message = `Timeout of ${options.timeout} ms with: ` + error.message + } + throw error + }) + }) +} diff --git a/libs/external-db-bigquery/src/supported_operations.ts b/libs/external-db-bigquery/src/supported_operations.ts index 3e2cca112..5af8ca9e2 100644 --- a/libs/external-db-bigquery/src/supported_operations.ts +++ b/libs/external-db-bigquery/src/supported_operations.ts @@ -7,6 +7,7 @@ const notSupportedOperations = [ SchemaOperations.StartWithCaseInsensitive, SchemaOperations.PrimaryKey, SchemaOperations.ChangeColumnType, + SchemaOperations.Indexing, ] export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op)) diff --git a/libs/external-db-dynamodb/src/supported_operations.ts b/libs/external-db-dynamodb/src/supported_operations.ts index 532050b56..19f5f8de3 100644 --- a/libs/external-db-dynamodb/src/supported_operations.ts +++ b/libs/external-db-dynamodb/src/supported_operations.ts @@ -8,7 +8,8 @@ const notSupportedOperations = [ SchemaOperations.FindObject, SchemaOperations.IncludeOperator, SchemaOperations.Matches, - SchemaOperations.NonAtomicBulkInsert + SchemaOperations.NonAtomicBulkInsert, + SchemaOperations.Indexing, ] diff --git a/libs/external-db-mongo/src/supported_operations.ts b/libs/external-db-mongo/src/supported_operations.ts index c266ba31d..cad285b9f 100644 --- a/libs/external-db-mongo/src/supported_operations.ts +++ b/libs/external-db-mongo/src/supported_operations.ts @@ -1,5 +1,8 @@ import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons' import { SchemaOperations } from '@wix-velo/velo-external-db-types' -const notSupportedOperations = [SchemaOperations.AtomicBulkInsert] +const notSupportedOperations = [ + SchemaOperations.AtomicBulkInsert, + SchemaOperations.Indexing, +] export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op)) diff --git a/libs/external-db-mssql/src/supported_operations.ts b/libs/external-db-mssql/src/supported_operations.ts index b1c9e223b..faada90c3 100644 --- a/libs/external-db-mssql/src/supported_operations.ts +++ b/libs/external-db-mssql/src/supported_operations.ts @@ -1,5 +1,10 @@ import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons' import { SchemaOperations } from '@wix-velo/velo-external-db-types' -const notSupportedOperations = [SchemaOperations.QueryNestedFields, SchemaOperations.FindObject, SchemaOperations.NonAtomicBulkInsert] +const notSupportedOperations = [ + SchemaOperations.QueryNestedFields, + SchemaOperations.FindObject, + SchemaOperations.NonAtomicBulkInsert, + SchemaOperations.Indexing, +] export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op)) diff --git a/libs/external-db-mysql/src/connection_provider.ts b/libs/external-db-mysql/src/connection_provider.ts index ae2886fe1..8fd883500 100644 --- a/libs/external-db-mysql/src/connection_provider.ts +++ b/libs/external-db-mysql/src/connection_provider.ts @@ -5,6 +5,7 @@ import FilterParser from './sql_filter_transformer' import DatabaseOperations from './mysql_operations' import { MySqlConfig } from './types' import { ILogger } from '@wix-velo/external-db-logger' +import IndexProvider from './mysql_index_provider' export default (cfg: MySqlConfig, _poolOptions: Record, logger?: ILogger) => { @@ -32,6 +33,7 @@ export default (cfg: MySqlConfig, _poolOptions: Record, logger? const filterParser = new FilterParser() const dataProvider = new DataProvider(pool, filterParser, logger) const schemaProvider = new SchemaProvider(pool, logger) + const indexProvider = new IndexProvider(pool, logger) - return { dataProvider, schemaProvider, databaseOperations, connection: pool, cleanup: async() => await pool.end() } + return { dataProvider, schemaProvider, databaseOperations, indexProvider, connection: pool, cleanup: async() => await pool.end() } } diff --git a/libs/external-db-mysql/src/mysql_index_provider.ts b/libs/external-db-mysql/src/mysql_index_provider.ts new file mode 100644 index 000000000..797c713d2 --- /dev/null +++ b/libs/external-db-mysql/src/mysql_index_provider.ts @@ -0,0 +1,152 @@ +import { promisify } from 'util' +import { errors } from '@wix-velo/velo-external-db-commons' +import { DomainIndex, IIndexProvider, DomainIndexStatus } from '@wix-velo/velo-external-db-types' +import { ILogger } from '@wix-velo/external-db-logger' +import { Pool as MySqlPool } from 'mysql' +import { MySqlQuery } from './types' +import { escapeId, escapeTable } from './mysql_utils' + +export default class IndexProvider implements IIndexProvider { + pool: MySqlPool + query: MySqlQuery + logger?: ILogger + constructor(pool: any, logger?: ILogger) { + this.pool = pool + this.logger = logger + this.query = promisify(this.pool.query).bind(this.pool) + } + + async list(collectionName: string): Promise { + const activeIndexes = await this.getActiveIndexesFor(collectionName) + const inProgressIndexes = await this.getInProgressIndexesFor(collectionName) + const indexes = { ...inProgressIndexes, ...activeIndexes } + return Object.values(indexes) + } + + async create(collectionName: string, index: DomainIndex): Promise { + const unique = index.isUnique ? 'UNIQUE' : '' + + const columnsToIndex = await Promise.all(index.columns.map(async(col: string) => { + return { + name: col, + partialString: await this.partialStringFor(col, collectionName) + } + })) + + const sql = `CREATE ${unique} INDEX ${escapeId(index.name)} ON ${escapeTable(collectionName)} (${columnsToIndex.map((col: { name: string, partialString: string }) => `${escapeId(col.name)}${col.partialString}`)})` + this.logger?.debug('mysql-create-index', { sql }) + const createIndexPromise = this.query(sql) + + const status = await this.returnStatusAfterXSeconds(1, createIndexPromise) + + return { ...index, status } + } + + async remove(collectionName: string, indexName: string): Promise { + const sql = `DROP INDEX ${escapeId(indexName)} ON ${escapeTable(collectionName)}` + this.logger?.debug('mysql-remove-index', { sql }) + await this.query(sql) + .catch(e => { throw this.translateErrorCodes(e) }) + } + + + private async getActiveIndexesFor(collectionName: string): Promise<{ [x: string]: DomainIndex }> { + const sql = `SHOW INDEXES FROM ${escapeTable(collectionName)}` + this.logger?.debug('mysql-getActiveIndexesFor', { sql }) + const res = await this.query(sql) + .catch(this.translateErrorCodes) + const indexes: { [x: string]: DomainIndex } = {} + + res.forEach((r: { Key_name: string; Column_name: string; Non_unique: number }) => { + if (!indexes[r.Key_name]) { + indexes[r.Key_name] = { + name: r.Key_name, + columns: [], + isUnique: r.Non_unique === 0, + caseInsensitive: true, // by default true but can be changed by the user - need to check. + order: 'ASC', + status: DomainIndexStatus.ACTIVE + } + } + indexes[r.Key_name].columns.push(r.Column_name) + }) + return indexes + } + + private async getInProgressIndexesFor(collectionName: string): Promise<{ [x: string]: DomainIndex }> { + const databaseName = this.pool.config.connectionConfig.database + const sql = 'SELECT * FROM information_schema.processlist WHERE db = ? AND info LIKE \'CREATE%INDEX%\'' + this.logger?.debug('mysql-getInProgressIndexesFor', { sql, parameters: databaseName }) + const inProgressIndexes = await this.query('SELECT * FROM information_schema.processlist WHERE db = ? AND info LIKE \'CREATE%INDEX%\'', [databaseName]) + const domainIndexesForCollection = inProgressIndexes.map((r: any) => this.extractIndexFromQueryForCollection(collectionName, r.INFO)).filter(Boolean) as DomainIndex[] + return domainIndexesForCollection.reduce((acc, index) => { + acc[index.name] = index + return acc + }, {} as { [x: string]: DomainIndex }) + } + + private extractIndexFromQueryForCollection(collectionName: string, createIndexQuery: string): DomainIndex | undefined { + const regex = /CREATE\s+(UNIQUE)?\s?INDEX\s+`(\w+)`\s+ON\s+`(\w+)`\s+\(([\w\s`,]+)\)/ + const match = createIndexQuery.match(regex) + if (match) { + const [, isUnique, name, collection, columnsString] = match + if (collection === collectionName) { + const columns = columnsString.replace(/`/g, '').split(',').map((column) => column.trim()) + return { + name, + columns, + isUnique: !!isUnique, + caseInsensitive: true, + order: 'ASC', + status: DomainIndexStatus.BUILDING + } + } + } + return + } + + private async returnStatusAfterXSeconds(x: number, promise: Promise): Promise { + return new Promise((resolve, _reject) => { + promise.catch((e: any) => { + this.logger?.error('failed to create index', e) + resolve(DomainIndexStatus.FAILED) + }) + + setTimeout(() => { + resolve(DomainIndexStatus.BUILDING) + }, x * 1000) + }) + } + + private translateErrorCodes(e: any) { + switch (e.code) { + case 'ER_DUP_INDEX': + case 'ER_DUP_KEYNAME': + return new errors.IndexAlreadyExists(`Index already exists: ${e.sqlMessage}`) + case 'ER_DUP_ENTRY': + return new errors.ItemAlreadyExists(`Duplicate entry in unique index: ${e.sqlMessage}`) + case 'ER_CANT_DROP_FIELD_OR_KEY': + return new errors.IndexDoesNotExist(`Index does not exist: ${e.sqlMessage}`) + default: + return new errors.UnrecognizedError(`Error while creating index: ${e.sqlMessage}`) + } + } + + private isTextType(type: string) { + return ['char', 'varchar', 'text', 'tinytext', 'mediumtext', 'longtext'].includes(type) + } + + private async partialStringFor(col: string, collectionName: string) { + const typeResp = await this.query('SELECT DATA_TYPE FROM information_schema.COLUMNS WHERE TABLE_NAME = ? AND COLUMN_NAME = ?', [collectionName, col]).catch(_e => []) + const type = typeResp[0]?.DATA_TYPE + + if (this.isTextType(type)) { + const lengthResp = await this.query('SELECT CHARACTER_MAXIMUM_LENGTH FROM information_schema.COLUMNS WHERE TABLE_NAME = ? AND COLUMN_NAME = ?', [collectionName, col]) + const length = lengthResp[0].CHARACTER_MAXIMUM_LENGTH + if (length) { + return length > 767 ? '(767)' : `(${length})` // 767 is the max length for a text index + } + } + return '' + } +} diff --git a/libs/external-db-postgres/src/connection_provider.ts b/libs/external-db-postgres/src/connection_provider.ts index e1d036b4a..0ad579e67 100644 --- a/libs/external-db-postgres/src/connection_provider.ts +++ b/libs/external-db-postgres/src/connection_provider.ts @@ -6,6 +6,7 @@ import FilterParser from './sql_filter_transformer' import DatabaseOperations from './postgres_operations' import { PostgresConfig, postgresPoolOptions } from './types' import { ILogger } from '@wix-velo/external-db-logger' +import IndexProvider from './postgres_index_provider' types.setTypeParser(builtins.NUMERIC, val => parseFloat(val)) @@ -32,12 +33,14 @@ export default (cfg: PostgresConfig, _poolOptions: postgresPoolOptions, logger?: const databaseOperations = new DatabaseOperations(pool) const dataProvider = new DataProvider(pool, filterParser, logger) const schemaProvider = new SchemaProvider(pool, logger) + const indexProvider = new IndexProvider(pool, logger) return { dataProvider, schemaProvider, databaseOperations, connection: pool, + indexProvider, cleanup: async() => pool.end(() => {}) } } diff --git a/libs/external-db-postgres/src/postgres_index_provider.ts b/libs/external-db-postgres/src/postgres_index_provider.ts new file mode 100644 index 000000000..7e9b005a2 --- /dev/null +++ b/libs/external-db-postgres/src/postgres_index_provider.ts @@ -0,0 +1,131 @@ +import { Pool } from 'pg' +import { escapeIdentifier, extractIndexFromIndexQueryForCollection } from './postgres_utils' +import { errors } from '@wix-velo/velo-external-db-commons' +import { DomainIndex, IIndexProvider, DomainIndexStatus } from '@wix-velo/velo-external-db-types' +import { ILogger } from '@wix-velo/external-db-logger' + + +export default class IndexProvider implements IIndexProvider { + pool: Pool + logger?: ILogger + + constructor(pool: any, logger?: ILogger) { + this.pool = pool + this.logger = logger + } + + async list(collectionName: string): Promise { + const activeIndexes = await this.getActiveIndexesFor(collectionName) + const inProgressIndexes = await this.getInProgressIndexesFor(collectionName) + const indexes = { ...inProgressIndexes, ...activeIndexes } + return Object.values(indexes) + } + + async create(collectionName: string, index: DomainIndex): Promise { + const unique = index.isUnique ? 'UNIQUE' : '' + + const sql = `CREATE ${unique} INDEX ${escapeIdentifier(index.name)} ON ${escapeIdentifier(collectionName)} (${index.columns.map((col: string) => `${escapeIdentifier(col)}`)})` + this.logger?.debug('postgres-create-index', { sql }) + const createIndexPromise = this.pool.query(sql) + + const status = await this.returnStatusAfterXSeconds(1, createIndexPromise, index) + + return { ...index, status } + } + + async remove(collectionName: string, indexName: string): Promise { + const sql = `DROP INDEX ${escapeIdentifier(indexName)}` + this.logger?.debug('postgres-remove-index', { sql }) + await this.pool.query(sql) + .catch(e => { throw this.translateErrorCodes(e) }) + } + + + private async getActiveIndexesFor(collectionName: string): Promise<{ [x: string]: DomainIndex }> { + const sql = 'SELECT * FROM pg_indexes WHERE schemaname = current_schema() AND tablename = $1' + this.logger?.debug('postgres-get-active-indexes', { sql, parameters: [collectionName] }) + const { rows } = await this.pool.query(sql, [collectionName]) + .catch(err => { throw this.translateErrorCodes(err) }) + + + const indexs: { [x: string]: DomainIndex } = {} + + // postgres return the following properties for each index: + type IndexRow = { + // Table name + tablename: string + // Index name + indexname: string + // Index creation command + indexdef: string + } + + rows.forEach((r: IndexRow) => { + if (!indexs[r.indexname]) { + indexs[r.indexname] = { + name: r.indexname, + columns: [], + isUnique: r.indexdef.includes('UNIQUE'), + caseInsensitive: false, + order: 'ASC', + status: DomainIndexStatus.ACTIVE + } + } + // TODO: extract this column extraction to a function + indexs[r.indexname].columns.push(r.indexdef.split('ON')[1].split('(')[1].split(')')[0]) + }) + + return indexs + } + + private async getInProgressIndexesFor(collectionName: string): Promise<{ [x: string]: DomainIndex }> { + const sql = ` + SELECT query + FROM pg_stat_activity + WHERE + (query ILIKE 'CREATE INDEX%' OR query ILIKE 'CREATE UNIQUE INDEX%') + AND (query LIKE '%${escapeIdentifier(collectionName)}(%') + AND state = 'active' + GROUP BY query; + ` + this.logger?.debug('postgres-getInProgressIndexesFor', { sql }) + const { rows } = await this.pool.query(sql) + .catch(err => { throw this.translateErrorCodes(err) }) + const domainIndexesForCollection = rows.map((r: { query: string }) => extractIndexFromIndexQueryForCollection(r.query)) + + return domainIndexesForCollection.reduce((acc, index) => { + acc[index.name] = index + return acc + }, {} as { [x: string]: DomainIndex }) + } + + + + private async returnStatusAfterXSeconds(x: number, promise: Promise, _index: DomainIndex): Promise { + return new Promise((resolve, _reject) => { + promise.catch((e: any) => { + this.logger?.error('failed to create index', this.translateErrorCodes(e)) + resolve(DomainIndexStatus.FAILED) + }) + + setTimeout(() => { + resolve(DomainIndexStatus.BUILDING) + }, x * 1000) + }) + } + + private translateErrorCodes(e: any) { + switch (e.code) { + case '42P07': + return new errors.IndexAlreadyExists(`Index already exists: ${e.sqlMessage}`) + case '42703': + return new errors.FieldDoesNotExist(`Field does not exist: ${e.sqlMessage}`) + default: + console.log(e) + return new errors.UnrecognizedError(`Error while creating index: ${e} ${e.code}`) + } + } + + + +} diff --git a/libs/external-db-postgres/src/postgres_utils.spec.ts b/libs/external-db-postgres/src/postgres_utils.spec.ts index 19df3a5a2..51ed5b34e 100644 --- a/libs/external-db-postgres/src/postgres_utils.spec.ts +++ b/libs/external-db-postgres/src/postgres_utils.spec.ts @@ -1,6 +1,7 @@ -import { prepareStatementVariablesForBulkInsert } from './postgres_utils' +import { DomainIndexStatus } from '@wix-velo/velo-external-db-types' +import { prepareStatementVariablesForBulkInsert, extractIndexFromIndexQueryForCollection } from './postgres_utils' describe('Postgres utils', () => { describe('Prepare statement variables for BulkInsert', () => { @@ -24,4 +25,48 @@ describe('Postgres utils', () => { }) }) + + + describe('Index utils functions', () => { + test('extract DomainIndex from index query on 1 column', () => { + const indexQuery = 'CREATE INDEX idx_table_col1 ON table(col1)' + const result = extractIndexFromIndexQueryForCollection(indexQuery) + const expected = { + name: 'idx_table_col1', + columns: ['col1'], + isUnique: false, + caseInsensitive: true, + order: 'ASC', + status: DomainIndexStatus.BUILDING + } + expect(result).toEqual(expected) + }) + test('extract DomainIndex from unique index query on 1 column', () => { + const indexQuery = 'CREATE UNIQUE INDEX idx_table_col1 ON table(col1)' + const result = extractIndexFromIndexQueryForCollection(indexQuery) + const expected = { + name: 'idx_table_col1', + columns: ['col1'], + isUnique: true, + caseInsensitive: true, + order: 'ASC', + status: DomainIndexStatus.BUILDING + } + expect(result).toEqual(expected) + }) + + test('extract DomainIndex from unique index query on 2 column', () => { + const indexQuery = 'CREATE UNIQUE INDEX idx_table_col1_col2 ON table(col1, col2)' + const result = extractIndexFromIndexQueryForCollection(indexQuery) + const expected = { + name: 'idx_table_col1_col2', + columns: ['col1', 'col2'], + isUnique: true, + caseInsensitive: true, + order: 'ASC', + status: DomainIndexStatus.BUILDING + } + expect(result).toEqual(expected) + }) + }) }) diff --git a/libs/external-db-postgres/src/postgres_utils.ts b/libs/external-db-postgres/src/postgres_utils.ts index a13d96cec..3a5ed2451 100644 --- a/libs/external-db-postgres/src/postgres_utils.ts +++ b/libs/external-db-postgres/src/postgres_utils.ts @@ -1,5 +1,6 @@ // Ported from PostgreSQL 9.2.4 source code in src/interfaces/libpq/fe-exec.c +import { DomainIndex, DomainIndexStatus } from '@wix-velo/velo-external-db-types' export const escapeIdentifier = (str: string) => str === '*' ? '*' : `"${(str || '').replace(/"/g, '""')}"` @@ -20,3 +21,18 @@ export const prepareStatementVariablesForBulkInsert = (rowsCount: number, column } return segments.join(',') } + + +export const extractIndexFromIndexQueryForCollection = (query: string): DomainIndex => { + const isUnique = query.includes('UNIQUE') + const name = query.split('INDEX')[1].split('ON')[0].trim() + const columns = query.split('ON')[1].split('(')[1].split(')')[0].split(',').map(c => c.trim()) + return { + name, + columns, + isUnique, + caseInsensitive: true, + order: 'ASC', + status: DomainIndexStatus.BUILDING + } +} diff --git a/libs/external-db-spanner/src/supported_operations.ts b/libs/external-db-spanner/src/supported_operations.ts index 5d9238231..4fdd5e3a8 100644 --- a/libs/external-db-spanner/src/supported_operations.ts +++ b/libs/external-db-spanner/src/supported_operations.ts @@ -1,6 +1,11 @@ import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons' import { SchemaOperations } from '@wix-velo/velo-external-db-types' //change column types - https://cloud.google.com/spanner/docs/schema-updates#supported_schema_updates -const notSupportedOperations = [SchemaOperations.ChangeColumnType, SchemaOperations.NonAtomicBulkInsert, SchemaOperations.FindObject] +const notSupportedOperations = [ + SchemaOperations.ChangeColumnType, + SchemaOperations.NonAtomicBulkInsert, + SchemaOperations.FindObject, + SchemaOperations.Indexing, +] export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op)) diff --git a/libs/velo-external-db-commons/src/libs/db_connector.ts b/libs/velo-external-db-commons/src/libs/db_connector.ts index ef7a15e36..d5739f8df 100644 --- a/libs/velo-external-db-commons/src/libs/db_connector.ts +++ b/libs/velo-external-db-commons/src/libs/db_connector.ts @@ -1,5 +1,5 @@ import { ILogger } from '@wix-velo/external-db-logger' -import { ConnectionCleanUp, DbProviders, IConfigValidator, IDatabaseOperations, IDataProvider, ISchemaProvider } from '@wix-velo/velo-external-db-types' +import { ConnectionCleanUp, DbProviders, IConfigValidator, IDatabaseOperations, IDataProvider, ISchemaProvider, IIndexProvider } from '@wix-velo/velo-external-db-types' export default class DbConnector { @@ -8,6 +8,7 @@ export default class DbConnector { init: (config: any, option: any, logger?: ILogger) => Promise> | DbProviders dataProvider!: IDataProvider schemaProvider!: ISchemaProvider + indexProvider?: IIndexProvider databaseOperations!: IDatabaseOperations connection: any cleanup!: ConnectionCleanUp @@ -26,7 +27,8 @@ export default class DbConnector { } async initialize(config: any, options: any, logger?: ILogger) { - const { dataProvider, schemaProvider, databaseOperations, connection, cleanup } = await this.init(config, options, logger) + const { dataProvider, schemaProvider, databaseOperations, indexProvider, connection, cleanup } = await this.init(config, options, logger) + this.indexProvider = indexProvider this.dataProvider = dataProvider this.schemaProvider = schemaProvider this.databaseOperations = databaseOperations diff --git a/libs/velo-external-db-commons/src/libs/errors.ts b/libs/velo-external-db-commons/src/libs/errors.ts index b280605bf..ca1ddc66c 100644 --- a/libs/velo-external-db-commons/src/libs/errors.ts +++ b/libs/velo-external-db-commons/src/libs/errors.ts @@ -131,3 +131,16 @@ export class CollectionChangeNotSupportedError extends BaseHttpError { this.fieldName = fieldName ?? '' } } + + +export class IndexAlreadyExists extends BaseHttpError { + constructor(message: string) { + super(message) + } +} + +export class IndexDoesNotExist extends BaseHttpError { + constructor(message: string) { + super(message) + } +} diff --git a/libs/velo-external-db-core/src/index.ts b/libs/velo-external-db-core/src/index.ts index b8d03a8dc..c1b643253 100644 --- a/libs/velo-external-db-core/src/index.ts +++ b/libs/velo-external-db-core/src/index.ts @@ -1,6 +1,7 @@ 'use strict' import DataService from './service/data' +import IndexService from './service/indexing' import CacheableSchemaInformation from './service/schema_information' import SchemaService from './service/schema' import OperationService from './service/operation' @@ -45,6 +46,7 @@ export class ExternalDbRouter { router: Router config: ExternalDbRouterConfig logger?: ILogger + indexService: IndexService constructor({ connector, config, hooks, logger }: ExternalDbRouterConstructorParams) { this.isInitialized(connector) this.connector = connector @@ -60,6 +62,7 @@ export class ExternalDbRouter { this.aggregationTransformer = new AggregationTransformer(this.filterTransformer) this.queryValidator = new QueryValidator() this.dataService = new DataService(connector.dataProvider) + this.indexService = new IndexService(connector.indexProvider) this.itemTransformer = new ItemTransformer() this.schemaAwareDataService = new SchemaAwareDataService(this.dataService, this.queryValidator, this.schemaInformation, this.itemTransformer) this.schemaService = new SchemaService(connector.schemaProvider, this.schemaInformation) @@ -67,12 +70,12 @@ export class ExternalDbRouter { this.roleAuthorizationService = new RoleAuthorizationService(config.authorization?.roleConfig?.collectionPermissions) this.cleanup = connector.cleanup - initServices(this.schemaAwareDataService, this.schemaService, this.operationService, this.configValidator, { ...config, type: connector.type }, this.filterTransformer, this.aggregationTransformer, this.roleAuthorizationService, hooks, logger) + initServices(this.schemaAwareDataService, this.schemaService, this.operationService, this.indexService, this.configValidator, { ...config, type: connector.type }, this.filterTransformer, this.aggregationTransformer, this.roleAuthorizationService, hooks, logger) this.router = createRouter() } reloadHooks(hooks?: Hooks) { - initServices(this.schemaAwareDataService, this.schemaService, this.operationService, this.configValidator, { ...this.config, type: this.connector.type }, this.filterTransformer, this.aggregationTransformer, this.roleAuthorizationService, hooks || {}, this.logger) + initServices(this.schemaAwareDataService, this.schemaService, this.operationService, this.indexService, this.configValidator, { ...this.config, type: this.connector.type }, this.filterTransformer, this.aggregationTransformer, this.roleAuthorizationService, hooks || {}, this.logger) } isInitialized(connector: DbConnector) { @@ -84,6 +87,7 @@ export class ExternalDbRouter { export * as types from './types' export * as dataSpi from './spi-model/data_source' +export * as indexSpi from './spi-model/indexing' export * as collectionSpi from './spi-model/collection' export * as schemaUtils from '../src/utils/schema_utils' export * as dataConvertUtils from './converters/data_utils' diff --git a/libs/velo-external-db-core/src/router.ts b/libs/velo-external-db-core/src/router.ts index a52e04bd1..698a00855 100644 --- a/libs/velo-external-db-core/src/router.ts +++ b/libs/velo-external-db-core/src/router.ts @@ -13,6 +13,7 @@ import { getAppInfoPage } from './utils/router_utils' import { requestContextFor, DataActions, dataPayloadFor, DataHooksForAction } from './data_hooks_utils' import { SchemaActions, SchemaHooksForAction, schemaPayloadFor } from './schema_hooks_utils' import SchemaService from './service/schema' +import IndexService from './service/indexing' import OperationService from './service/operation' import { AnyFixMe, CollectionOperationSPI, DataOperation } from '@wix-velo/velo-external-db-types' import SchemaAwareDataService from './service/schema_aware_data' @@ -26,21 +27,23 @@ import * as schemaSource from './spi-model/collection' import { JWTVerifier } from './web/jwt-verifier' import { JWTVerifierDecoderMiddleware } from './web/jwt-verifier-decoder-middleware' import { ILogger } from '@wix-velo/external-db-logger' +import { CreateIndexRequest, ListIndexesRequest, RemoveIndexRequest } from './spi-model/indexing' const { query: Query, count: Count, aggregate: Aggregate, insert: Insert, update: Update, remove: Remove, truncate: Truncate } = DataOperation const { Get, Create, Update: UpdateSchema, Delete } = CollectionOperationSPI type RouterConfig = { type?: string, vendor?: string, hideAppInfo?: boolean, jwtPublicKey: string, appDefId: string, readOnlySchema?: boolean } -let schemaService: SchemaService, operationService: OperationService, externalDbConfigClient: ConfigValidator, schemaAwareDataService: SchemaAwareDataService, cfg: RouterConfig, filterTransformer: FilterTransformer, aggregationTransformer: AggregationTransformer, dataHooks: DataHooks, schemaHooks: SchemaHooks //roleAuthorizationService: RoleAuthorizationService, +let schemaService: SchemaService, indexService: IndexService, operationService: OperationService, externalDbConfigClient: ConfigValidator, schemaAwareDataService: SchemaAwareDataService, cfg: RouterConfig, filterTransformer: FilterTransformer, aggregationTransformer: AggregationTransformer, dataHooks: DataHooks, schemaHooks: SchemaHooks //roleAuthorizationService: RoleAuthorizationService, let logger: ILogger | undefined export const initServices = (_schemaAwareDataService: SchemaAwareDataService, _schemaService: SchemaService, _operationService: OperationService, - _externalDbConfigClient: ConfigValidator, _cfg: RouterConfig, + _indexService: IndexService, _externalDbConfigClient: ConfigValidator, _cfg: RouterConfig, _filterTransformer: FilterTransformer, _aggregationTransformer: AggregationTransformer, _roleAuthorizationService: RoleAuthorizationService, _hooks: Hooks, _logger: ILogger | undefined) => { schemaService = _schemaService operationService = _operationService + indexService = _indexService externalDbConfigClient = _externalDbConfigClient cfg = _cfg schemaAwareDataService = _schemaAwareDataService @@ -129,11 +132,12 @@ export const createRouter = () => { const capabilitiesResponse = { supportsCollectionModifications: cfg.readOnlySchema ? false : true, supportedFieldTypes: Object.values(schemaSource.FieldType).filter(t => !unsupportedFieldTypes.includes(t)), - supportsCollectionDisplayName: false, - supportsCollectionDisplayField: false, - supportsCollectionPermissions: false, - supportsCollectionFieldDisplayName: false, - supportsCollectionFieldDescription: false, + indexptions: { + supportsIndexes: indexService.storage ? true : false, + maxNumberOfRegularIndexesPerCollection: 10, + maxNumberOfUniqueIndexesPerCollection: 10, + maxNumberOfIndexesPerCollection: 20, + } } res.json(capabilitiesResponse) @@ -330,6 +334,40 @@ export const createRouter = () => { } }) + // *************** Indexes API ********************** + + router.post('/v3/indexes/list', async(req, res, next) => { + try { + const { collectionId } = req.body as ListIndexesRequest + const indexes = await indexService.list(collectionId) + res.json({ indexes }) + } catch (e) { + next(e) + } + }) + + router.post('/v3/indexes/create', async(req, res, next) => { + try { + const { collectionId, index } = req.body as CreateIndexRequest + const createdIndex = await indexService.create(collectionId, index) + res.json({ + index: createdIndex + }) + } catch (e) { + next(e) + } + }) + + router.post('/v3/indexes/remove', async(req, res, next) => { + try { + const { collectionId, indexName } = req.body as RemoveIndexRequest + await indexService.remove(collectionId, indexName) + res.json({}) + } catch (e) { + next(e) + } + }) + // *********************************************** router.use(errorMiddleware(logger)) diff --git a/libs/velo-external-db-core/src/service/indexing.spec.ts b/libs/velo-external-db-core/src/service/indexing.spec.ts new file mode 100644 index 000000000..89420f14d --- /dev/null +++ b/libs/velo-external-db-core/src/service/indexing.spec.ts @@ -0,0 +1,81 @@ +import Chance = require('chance') +import { Uninitialized } from '@wix-velo/test-commons' +import { DomainIndex } from '@wix-velo/velo-external-db-types' +import * as gen from '../../test/gen' +import IndexService from './indexing' +import * as driver from '../../test/drivers/index_provider_test_support' +import { Index as SpiIndex, IndexField, IndexStatus } from '../spi-model/indexing' +const chance = new Chance() + +describe('Index Service', () => { + describe('transformers', () => { + test('domainIndexToSpiIndex', () => { + expect(env.indexService['domainIndexToSpiIndex'](ctx.index)).toEqual({ + name: ctx.index.name, + fields: ctx.index.columns.map(column => ({ path: column, order: ctx.index.order })) as IndexField[], + unique: ctx.index.isUnique, + caseInsensitive: ctx.index.caseInsensitive, + status: IndexStatus[ctx.index.status as keyof typeof IndexStatus], + }) + }) + + test('spiIndexToDomainIndex', () => { + expect(env.indexService['spiIndexToDomainIndex'](ctx.spiIndex)).toEqual({ + name: ctx.spiIndex.name, + columns: ctx.spiIndex.fields.map(field => field.path), + isUnique: ctx.spiIndex.unique, + caseInsensitive: ctx.spiIndex.caseInsensitive, + order: ctx.spiIndex.fields[0].order, + }) + }) + }) + + test('list will issue a call to list and translate data to spi format', () => { + driver.givenListResult(ctx.indexes, ctx.collectionName) + + return expect(env.indexService.list(ctx.collectionName)).resolves.toEqual(ctx.indexes.map(env.indexService['domainIndexToSpiIndex'])) + }) + + test('create will issue a call to create and translate data to spi format', () => { + driver.givenCreateResult(ctx.index, ctx.collectionName) + + return expect(env.indexService.create(ctx.collectionName, env.indexService['domainIndexToSpiIndex'](ctx.index))) + .resolves.toEqual(env.indexService['domainIndexToSpiIndex'](ctx.index)) + }) + + test('remove will issue a call to remove', () => { + driver.givenRemoveResult(ctx.collectionName, ctx.index.name) + + return expect(env.indexService.remove(ctx.collectionName, ctx.index.name)).resolves.toEqual({}) + }) + + + const ctx: { + collectionName: string + indexes: DomainIndex[], + index: DomainIndex, + spiIndex: SpiIndex + } = { + collectionName: Uninitialized, + indexes: Uninitialized, + index: Uninitialized, + spiIndex: Uninitialized, + } + const env: { + indexService: IndexService + } = { + indexService: Uninitialized + } + + beforeAll(() => { + env.indexService = new IndexService(driver.indexProvider) + ctx.collectionName = chance.word() + ctx.indexes = gen.randomArrayOf(gen.randomDomainIndex) + ctx.index = gen.randomDomainIndex() + ctx.spiIndex = gen.randomSpiIndex() + }) + + afterEach(() => { + driver.reset() + }) +}) diff --git a/libs/velo-external-db-core/src/service/indexing.ts b/libs/velo-external-db-core/src/service/indexing.ts new file mode 100644 index 000000000..1b3fb0c61 --- /dev/null +++ b/libs/velo-external-db-core/src/service/indexing.ts @@ -0,0 +1,45 @@ +import { DomainIndex, IIndexProvider } from '@wix-velo/velo-external-db-types' +import { Index as SpiIndex, IndexField, IndexStatus } from '../spi-model/indexing' + +export default class IndexService { + storage: IIndexProvider + constructor(storage: any) { + this.storage = storage + } + + async list(collectionName: string) { + const indexes = await this.storage.list(collectionName) + return indexes.map(this.domainIndexToSpiIndex) + } + + async create(collectionName: string, index: SpiIndex) { + const domainIndex = this.spiIndexToDomainIndex(index) + const createdIndex = await this.storage.create(collectionName, domainIndex) + return this.domainIndexToSpiIndex(createdIndex) + } + + async remove(collectionName: string, indexName: string) { + await this.storage.remove(collectionName, indexName) + return {} + } + + private domainIndexToSpiIndex(domainIndex: DomainIndex): SpiIndex { + return { + name: domainIndex.name, + fields: domainIndex.columns.map(column => ({ path: column, order: domainIndex.order })) as IndexField[], + unique: domainIndex.isUnique, + caseInsensitive: domainIndex.caseInsensitive, + status: IndexStatus[domainIndex.status as keyof typeof IndexStatus], + } + } + + private spiIndexToDomainIndex(spiIndex: SpiIndex): DomainIndex { + return { + name: spiIndex.name, + columns: spiIndex.fields.map(field => field.path), + isUnique: spiIndex.unique, + caseInsensitive: spiIndex.caseInsensitive, + order: spiIndex.fields[0].order, + } + } +} diff --git a/libs/velo-external-db-core/src/spi-model/indexing.ts b/libs/velo-external-db-core/src/spi-model/indexing.ts new file mode 100644 index 000000000..350309591 --- /dev/null +++ b/libs/velo-external-db-core/src/spi-model/indexing.ts @@ -0,0 +1,74 @@ +export interface Index { + // Index name + name: string; + + // Fields over which the index is defined + fields: IndexField[]; + + // Indicates current status of index + status?: IndexStatus; + + // If true index will enforce that values in the field are unique in scope of a collection. Default is false. + unique: boolean; + + // If true index will be case-insensitive. Default is false. + caseInsensitive: boolean; + + // Contains details about failure reason when index is in *FAILED* status + // wix.api.ApplicationError failure = 8 [(wix.api.readOnly) = true]; + failure?: ApplicationError; +} + +export enum IndexStatus { + UNKNOWN = 'UNKNOWN', + BUILDING = 'BUILDING', + ACTIVE = 'ACTIVE', + DROPPING = 'DROPPING', + DROPPED = 'DROPPED', + FAILED = 'FAILED', + INVALID = 'INVALID' +} + +export interface IndexField { + // Order determines how values are ordered in the index. This is important when + // ordering and/or range querying by indexed fields. + order: IndexFieldOrder; + + // The field path to index. + path: string; +} + +export enum IndexFieldOrder { + ASC = 'ASC', + DESC = 'DESC' +} + +interface ApplicationError { + code: string, + errorMessage: string, + data: any +} + +export interface ListIndexesRequest { + collectionId: string; +} + +export type ListIndexesResponse = { + indexes: Index[]; +} + +export interface CreateIndexRequest { + collectionId: string; + index: Index; +} + +export interface CreateIndexResponse { + index: Index; +} + +export interface RemoveIndexRequest { + collectionId: string; + indexName: string; +} + +export interface RemoveIndexResponse {} diff --git a/libs/velo-external-db-core/test/drivers/index_provider_test_support.ts b/libs/velo-external-db-core/test/drivers/index_provider_test_support.ts new file mode 100644 index 000000000..9cbaad021 --- /dev/null +++ b/libs/velo-external-db-core/test/drivers/index_provider_test_support.ts @@ -0,0 +1,28 @@ +import { DomainIndex } from '@wix-velo/velo-external-db-types' +import { when } from 'jest-when' + +export const indexProvider = { + list: jest.fn(), + create: jest.fn(), + remove: jest.fn(), +} + +export const givenListResult = (indexes: DomainIndex[], collectionName: string) => { + when(indexProvider.list).calledWith(collectionName).mockResolvedValue(indexes) +} + +export const givenCreateResult = (index: DomainIndex, collectionName: string) => { + const { status, ...indexWithoutStatus } = index + when(indexProvider.create).calledWith(collectionName, indexWithoutStatus).mockResolvedValue(index) +} + +export const reset = () => { + indexProvider.list.mockReset() + indexProvider.create.mockReset() + indexProvider.remove.mockReset() +} + + +export function givenRemoveResult(collectionName: string, name: string) { + when(indexProvider.remove).calledWith(collectionName, name).mockResolvedValue({}) +} diff --git a/libs/velo-external-db-core/test/gen.ts b/libs/velo-external-db-core/test/gen.ts index cfcf2266a..0617c9e6d 100644 --- a/libs/velo-external-db-core/test/gen.ts +++ b/libs/velo-external-db-core/test/gen.ts @@ -1,6 +1,8 @@ import * as Chance from 'chance' import { AdapterOperators } from '@wix-velo/velo-external-db-commons' import { gen as genCommon } from '@wix-velo/test-commons' +import { DomainIndex, DomainIndexStatus } from '@wix-velo/velo-external-db-types' +import { Index } from '../src/spi-model/indexing' import { CollectionCapabilities, CollectionOperation, @@ -108,3 +110,23 @@ export const randomBodyWith = (obj: any) => ({ ...genCommon.randomObject(), ...obj }) + +export const randomDomainIndex = (): DomainIndex => ({ + name: chance.word(), + columns: randomArrayOf(() => chance.word()), + isUnique: chance.bool(), + caseInsensitive: chance.bool(), + order: chance.pickone(['ASC', 'DESC']), + status: DomainIndexStatus.ACTIVE, +}) + + +export const randomSpiIndex = (): Index => ({ + name: chance.word(), + fields: randomArrayOf(() => ({ + name: chance.word(), + order: chance.pickone(['ASC', 'DESC']), + })), + unique: chance.bool(), + caseInsensitive: chance.bool(), +}) diff --git a/libs/velo-external-db-types/src/collection_types.ts b/libs/velo-external-db-types/src/collection_types.ts index 524311bdc..16a536399 100644 --- a/libs/velo-external-db-types/src/collection_types.ts +++ b/libs/velo-external-db-types/src/collection_types.ts @@ -104,6 +104,7 @@ export enum SchemaOperations { NonAtomicBulkInsert = 'NonAtomicBulkInsert', AtomicBulkInsert = 'AtomicBulkInsert', PrimaryKey = 'PrimaryKey', + Indexing = 'Indexing', } export type InputField = FieldAttributes & { name: string } diff --git a/libs/velo-external-db-types/src/index.ts b/libs/velo-external-db-types/src/index.ts index 40f44bc1a..0f2483497 100644 --- a/libs/velo-external-db-types/src/index.ts +++ b/libs/velo-external-db-types/src/index.ts @@ -1,10 +1,11 @@ import { ResponseField, SchemaOperations, - ISchemaProvider + ISchemaProvider, } from './collection_types' - +import { IIndexProvider } from './indexing_types' export * from './collection_types' +export * from './indexing_types' export enum AdapterOperator { //in velo-external-db-core eq = 'eq', @@ -130,6 +131,7 @@ export type ConnectionCleanUp = () => Promise | void export type DbProviders = { dataProvider: IDataProvider schemaProvider: ISchemaProvider + indexProvider?: IIndexProvider databaseOperations: IDatabaseOperations connection: T cleanup: ConnectionCleanUp diff --git a/libs/velo-external-db-types/src/indexing_types.ts b/libs/velo-external-db-types/src/indexing_types.ts new file mode 100644 index 000000000..1a3acfe0e --- /dev/null +++ b/libs/velo-external-db-types/src/indexing_types.ts @@ -0,0 +1,22 @@ +export interface IIndexProvider { + list(collectionName: string): Promise + create(collectionName: string, index: DomainIndex): Promise + remove(collectionName: string, indexName: string): Promise +} + + +export interface DomainIndex { + name: string + columns: string[] + isUnique: boolean + caseInsensitive: boolean + order : 'ASC' | 'DESC' + status?: DomainIndexStatus + error?: any +} + +export enum DomainIndexStatus { + ACTIVE = 'ACTIVE', + BUILDING = 'BUILDING', + FAILED = 'FAILED' +} diff --git a/package.json b/package.json index 669bf9182..7fcb0c2ed 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "pg-escape": "^0.2.0", "pino": "^8.17.2", "sqlstring": "^2.3.3", + "trier-promise": "^1.0.1", "tslib": "^2.3.0", "tsqlstring": "^1.0.1", "uuid": "^8.3.2"