Index spi impl (#522)

* init * tests and types * Refactor index retrieval and add index service * lint fixes * Update API endpoints for index operations * Update API endpoint for creating indexes * Commented out unused database configurations * Add index provider test support and generate random domain and SPI indexes * Update import statement for IIndexProvider * Add logger to IndexProvider constructor * Fix index creation and error handling * Add IndexProvider to connection provider * Refactor test description to be more descriptive * Update supported operations for external databases * Refactor index creation and removal in MySQL index provider * Refactor index creation and removal in PostgresIndexProvider * Refactor index API response structure * Refactor index creation in PostgresIndexProvider * Add support for indexes in capabilities response * Add extractIndexFromIndexQueryForCollection function to postgres_utils * Fix index query for collection name * Refactor query for retrieving active index creation queries --------- Co-authored-by: Ido Kahlon <[email protected]>
wix · Apr 14, 2024 · 8f07973 · 8f07973
1 parent 0ea951c
commit 8f07973
Show file tree

Hide file tree

Showing 30 changed files with 976 additions and 28 deletions.
diff --git a/apps/velo-external-db/test/drivers/index_api_rest_matchers.ts b/apps/velo-external-db/test/drivers/index_api_rest_matchers.ts
@@ -0,0 +1,49 @@
+import { indexSpi } from '@wix-velo/velo-external-db-core'
+const { IndexFieldOrder, IndexStatus } = indexSpi
+
+const indexWith = (index: indexSpi.Index, extraProps: Partial<indexSpi.Index>) => ({
+    ...index,
+    fields: index.fields.map(field => ({
+        ...field,
+        order: expect.toBeOneOf([IndexFieldOrder.ASC, IndexFieldOrder.DESC]),
+    })),
+    caseInsensitive: expect.any(Boolean), // TODO: remove this when we support case insensitive indexes
+    ...extraProps
+})
+
+export const failedIndexCreationResponse = (index: indexSpi.Index) => ({
+    index: indexWith(index, { status: IndexStatus.FAILED }) 
+})
+
+
+export const listIndexResponseWithDefaultIndex = () => ({
+    indexes: expect.arrayContaining([toHaveDefaultIndex()])
+})
+
+export const listIndexResponseWith = (indexes: indexSpi.Index[]) => ({
+    indexes: expect.arrayContaining(
+        [...indexes.map((index: indexSpi.Index) => indexWith(index, { status: IndexStatus.ACTIVE }))]
+    )
+})
+
+export const toHaveDefaultIndex = () => ({
+    name: expect.any(String),
+    fields: expect.arrayContaining([
+        expect.objectContaining({
+            path: '_id',
+            order: expect.toBeOneOf([IndexFieldOrder.ASC, IndexFieldOrder.DESC])
+        })
+    ]),
+    caseInsensitive: expect.any(Boolean),
+    status: IndexStatus.ACTIVE,
+    unique: true
+})
+
+
+export const createIndexResponseWith = (index: indexSpi.Index) => ({ index: indexWith(index, { status: IndexStatus.BUILDING }) })
+
+export const removeIndexResponse = () => (({}))
+
+export const listIndexResponseWithFailedIndex = (index: indexSpi.Index) => {
+    return expect.arrayContaining([indexWith(index, { status: IndexStatus.FAILED })])
+}
diff --git a/apps/velo-external-db/test/drivers/index_api_rest_test_support.ts b/apps/velo-external-db/test/drivers/index_api_rest_test_support.ts
@@ -0,0 +1,30 @@
+import axios from 'axios'
+import waitUntil from 'async-wait-until'
+import { indexSpi } from '@wix-velo/velo-external-db-core'
+
+const axiosInstance = axios.create({
+    baseURL: 'http://localhost:8080/v3'
+})
+
+export const givenIndexes = async(collectionName: string, indexes: indexSpi.Index[], auth: any) => {    
+    for (const index of indexes) {
+        await axiosInstance.post('/indexes/create', { collectionId: collectionName, index } as indexSpi.CreateIndexRequest, auth)
+    }
+    await Promise.all(indexes.map(index => indexCreated(collectionName, index.name, auth)))
+}
+
+const indexCreated = async(collectionName: string, indexName: string, auth: any) => {
+    await waitUntil(async() => {
+        const { indexes } = await retrieveIndexesFor(collectionName, auth) as indexSpi.ListIndexesResponse
+        return indexes.some(index => index.name === indexName)
+    })
+}
+
+export const createIndexFor = async(collectionName: string, index: indexSpi.Index, auth: any) => await
+    axiosInstance.post('/indexes/create', { collectionId: collectionName, index } as indexSpi.CreateIndexRequest, auth).then(res => res.data)
+
+export const removeIndexFor = async(collectionName: string, indexName: string, auth: any) => await
+    axiosInstance.post('/indexes/remove', { collectionId: collectionName, indexName } as indexSpi.RemoveIndexRequest, auth).then(res => res.data)
+
+export const retrieveIndexesFor = async(collectionName: string, auth: any) => await 
+axiosInstance.post('/indexes/list', { collectionId: collectionName } as indexSpi.ListIndexesRequest, { transformRequest: auth.transformRequest }).then(res => res.data)
diff --git a/apps/velo-external-db/test/e2e/app.e2e.spec.ts b/apps/velo-external-db/test/e2e/app.e2e.spec.ts
@@ -1,5 +1,6 @@
 import { authOwner } from '@wix-velo/external-db-testkit'
-import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName, env } from '../resources/e2e_resources'
+import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName, env, supportedOperations } from '../resources/e2e_resources'
+import { SchemaOperations } from '@wix-velo/velo-external-db-types'
 import { givenHideAppInfoEnvIsTrue } from '../drivers/app_info_config_test_support'
 
 const axios = require('axios').create({
@@ -36,11 +37,12 @@ describe(`Velo External DB: ${currentDbImplementationName()}`,  () => {
         expect((await axios.post('/v3/capabilities/get', { }, authOwner)).data).toEqual(expect.objectContaining({
             supportsCollectionModifications: true,
             supportedFieldTypes: expect.toBeArray(),
-            supportsCollectionDisplayName: false,
-            supportsCollectionDisplayField: false,
-            supportsCollectionPermissions: false,
-            supportsCollectionFieldDisplayName: false,
-            supportsCollectionFieldDescription: false,
+            indexptions: {
+                supportsIndexes: supportedOperations.includes(SchemaOperations.Indexing),
+                maxNumberOfRegularIndexesPerCollection: 10,
+                maxNumberOfUniqueIndexesPerCollection: 10,
+                maxNumberOfIndexesPerCollection: 20,
+            },
          }))
     })
 

diff --git a/apps/velo-external-db/test/e2e/app_index.e2e.spec.ts b/apps/velo-external-db/test/e2e/app_index.e2e.spec.ts
@@ -0,0 +1,107 @@
+import Chance = require('chance')
+import { Uninitialized, testIfSupportedOperationsIncludes } from '@wix-velo/test-commons'
+import { authOwner } from '@wix-velo/external-db-testkit'
+import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName, supportedOperations } from '../resources/e2e_resources'
+import * as schema from '../drivers/schema_api_rest_test_support'
+import * as matchers from '../drivers/index_api_rest_matchers'
+import * as index from '../drivers/index_api_rest_test_support'
+import * as gen from '../gen'
+import axios from 'axios'
+const chance = new Chance()
+import { eventually } from '../utils/eventually'
+import { InputField, SchemaOperations } from '@wix-velo/velo-external-db-types'
+import { indexSpi } from '@wix-velo/velo-external-db-core'
+
+const { Indexing } = SchemaOperations
+
+const axiosServer = axios.create({
+    baseURL: 'http://localhost:8080/v3'
+})
+
+
+describe(`Velo External DB Index API: ${currentDbImplementationName()}`, () => {
+    beforeAll(async() => {
+        await setupDb()
+        await initApp()
+    })
+
+    afterAll(async() => {
+        await dbTeardown()
+    }, 20000)
+
+    testIfSupportedOperationsIncludes(supportedOperations, [Indexing])('list', async() => {
+        await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+
+        await expect(index.retrieveIndexesFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.listIndexResponseWithDefaultIndex())
+    })
+
+    testIfSupportedOperationsIncludes(supportedOperations, [Indexing])('list with multiple indexes', async() => {
+        await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+        await index.givenIndexes(ctx.collectionName, [ctx.index], authOwner)
+
+        await expect(index.retrieveIndexesFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.listIndexResponseWith([ctx.index]))
+    })
+
+    testIfSupportedOperationsIncludes(supportedOperations, [Indexing])('create', async() => {
+        await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+
+        // in-progress
+        await expect(index.createIndexFor(ctx.collectionName, ctx.index, authOwner)).resolves.toEqual(matchers.createIndexResponseWith(ctx.index))
+
+        // active
+        await eventually(async() =>
+            await expect(index.retrieveIndexesFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.listIndexResponseWith([ctx.index]))
+        )
+    })
+
+    testIfSupportedOperationsIncludes(supportedOperations, [Indexing])('create an index on a column that already has an existing index, should return the index with status failed', async() => {
+        await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+        await index.givenIndexes(ctx.collectionName, [ctx.index], authOwner)
+
+        await eventually(async() => await expect(axiosServer.post('/indexes/create', {
+            dataCollectionId: ctx.collectionName,
+            index: ctx.index
+        }, authOwner).then(res => res.data)).resolves.toEqual(matchers.failedIndexCreationResponse(ctx.index)
+    ))
+    })
+
+    testIfSupportedOperationsIncludes(supportedOperations, [Indexing])('creation of index with invalid column should return the index with status failed', async() => {
+        await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+
+        await eventually(async() => await expect(index.createIndexFor(ctx.collectionName, ctx.invalidIndex, authOwner)).resolves.toEqual(matchers.failedIndexCreationResponse(ctx.invalidIndex)))
+    })
+
+    testIfSupportedOperationsIncludes(supportedOperations, [Indexing])('remove', async() => {
+        await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+        await index.givenIndexes(ctx.collectionName, [ctx.index], authOwner)
+
+        await expect( index.removeIndexFor(ctx.collectionName, ctx.index.name, authOwner)).resolves.toEqual(matchers.removeIndexResponse())
+
+        await expect(index.retrieveIndexesFor(ctx.collectionName, authOwner)).resolves.not.toEqual(matchers.listIndexResponseWith([ctx.index]))
+    })
+
+    afterAll(async() => {
+        await teardownApp()
+    })
+
+    interface Ctx {
+        collectionName: string
+        column: InputField
+        index: indexSpi.Index
+        invalidIndex: indexSpi.Index
+    }
+
+    const ctx: Ctx = {
+        collectionName: Uninitialized,
+        column: Uninitialized,
+        index: Uninitialized,
+        invalidIndex: Uninitialized,
+    }
+
+    beforeEach(() => {
+        ctx.collectionName = chance.word()
+        ctx.column = gen.randomColumn()
+        ctx.index = gen.spiIndexFor(ctx.collectionName, [ctx.column.name])
+        ctx.invalidIndex = gen.spiIndexFor(ctx.collectionName, ['wrongColumn'])
+    })
+})
diff --git a/apps/velo-external-db/test/gen.ts b/apps/velo-external-db/test/gen.ts
@@ -1,8 +1,12 @@
 
 import { SystemFields } from '@wix-velo/velo-external-db-commons'
-import { collectionSpi, schemaUtils } from '@wix-velo/velo-external-db-core'
-import { InputField } from '@wix-velo/velo-external-db-types'
+import { collectionSpi, schemaUtils, indexSpi } from '@wix-velo/velo-external-db-core'
+import { 
+    InputField,
+ } from '@wix-velo/velo-external-db-types'
+import { DomainIndex, DomainIndexStatus } from '@wix-velo/velo-external-db-types'
 import * as Chance from 'chance'
+const { IndexFieldOrder } = indexSpi
 
 const chance = Chance()
 
@@ -27,7 +31,14 @@ export const randomDbEntity = (columns: string[]) => {
     return entity
 }
 
-
+export const randomArrayOf= <T>(gen: any): T[] => {
+    const arr = []
+    const num = chance.natural({ min: 2, max: 20 })
+    for (let i = 0; i < num; i++) {
+        arr.push(gen())
+    }
+    return arr
+}
 
 export const randomDbEntities = (columns: string[]) => {
     const num = chance.natural({ min: 2, max: 20 })
@@ -114,3 +125,31 @@ export const randomCollection = (): collectionSpi.Collection => {
         pagingMode: collectionSpi.PagingMode.offset
     }
 }
+
+export const randomDomainIndex = (): DomainIndex => ({
+    name: chance.word(),
+    columns: randomArrayOf(() => chance.word()),
+    isUnique: chance.bool(),
+    caseInsensitive: chance.bool(),
+    order: chance.pickone(['ASC', 'DESC']),
+    status: DomainIndexStatus.ACTIVE,
+})
+
+export const randomSpiIndex = (): indexSpi.Index => ({
+    name: chance.word(),
+    fields: randomArrayOf(() => ({
+        name: chance.word(),
+        order: chance.pickone(['ASC', 'DESC']),
+    })),
+    unique: chance.bool(),
+    caseInsensitive: chance.bool(),
+})
+
+export const spiIndexFor = (_collectionName: string, columns: string[]): indexSpi.Index => { 
+    return {
+        name: chance.word(),
+        fields: columns.map((column: string) => ({ path: column, order: chance.pickone([IndexFieldOrder.ASC, IndexFieldOrder.DESC]) })),
+        unique: chance.bool(),
+        caseInsensitive: chance.bool(),
+    }
+}
diff --git a/apps/velo-external-db/test/utils/eventually.ts b/apps/velo-external-db/test/utils/eventually.ts
@@ -0,0 +1,24 @@
+import * as trier from 'trier-promise'
+
+const defaults = {
+    timeout: 5000,
+    interval: 200
+}
+
+export const eventually = async(fn: any, opts?: { timeout?: number; interval?: number }) => {
+    return Promise.resolve().then(() => {
+        let error = null
+        const action = () => Promise.resolve().then(fn).catch(err => {
+            error = err
+            throw err
+        })
+        const options = Object.assign({ action }, defaults, opts)
+
+        return trier(options).catch(() => {
+            if (error !== null) {
+                error.message = `Timeout of ${options.timeout} ms with: ` + error.message
+            }
+            throw error
+        })
+    })
+}
diff --git a/libs/external-db-bigquery/src/supported_operations.ts b/libs/external-db-bigquery/src/supported_operations.ts
@@ -7,6 +7,7 @@ const notSupportedOperations = [
     SchemaOperations.StartWithCaseInsensitive,
     SchemaOperations.PrimaryKey,
     SchemaOperations.ChangeColumnType,
+    SchemaOperations.Indexing,
 ]
 
 export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op))
diff --git a/libs/external-db-dynamodb/src/supported_operations.ts b/libs/external-db-dynamodb/src/supported_operations.ts
@@ -8,7 +8,8 @@ const notSupportedOperations = [
     SchemaOperations.FindObject,
     SchemaOperations.IncludeOperator,
     SchemaOperations.Matches,
-    SchemaOperations.NonAtomicBulkInsert
+    SchemaOperations.NonAtomicBulkInsert,
+    SchemaOperations.Indexing,
 ]
 
 

diff --git a/libs/external-db-mongo/src/supported_operations.ts b/libs/external-db-mongo/src/supported_operations.ts
@@ -1,5 +1,8 @@
 import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons'
 import { SchemaOperations } from '@wix-velo/velo-external-db-types'
-const notSupportedOperations = [SchemaOperations.AtomicBulkInsert]
+const notSupportedOperations = [
+    SchemaOperations.AtomicBulkInsert,
+    SchemaOperations.Indexing,
+]
 
 export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op))
diff --git a/libs/external-db-mssql/src/supported_operations.ts b/libs/external-db-mssql/src/supported_operations.ts
@@ -1,5 +1,10 @@
 import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons'
 import { SchemaOperations } from '@wix-velo/velo-external-db-types'
-const notSupportedOperations = [SchemaOperations.QueryNestedFields, SchemaOperations.FindObject, SchemaOperations.NonAtomicBulkInsert]
+const notSupportedOperations = [
+    SchemaOperations.QueryNestedFields, 
+    SchemaOperations.FindObject, 
+    SchemaOperations.NonAtomicBulkInsert,
+    SchemaOperations.Indexing,
+]
 
 export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op))
diff --git a/libs/external-db-mysql/src/connection_provider.ts b/libs/external-db-mysql/src/connection_provider.ts
@@ -5,6 +5,7 @@ import FilterParser from './sql_filter_transformer'
 import DatabaseOperations from './mysql_operations'
 import { MySqlConfig } from './types'
 import { ILogger } from '@wix-velo/external-db-logger'
+import IndexProvider from './mysql_index_provider'
 
 
 export default (cfg: MySqlConfig, _poolOptions: Record<string, unknown>, logger?: ILogger)  => {
@@ -32,6 +33,7 @@ export default (cfg: MySqlConfig, _poolOptions: Record<string, unknown>, logger?
     const filterParser = new FilterParser()
     const dataProvider = new DataProvider(pool, filterParser, logger)
     const schemaProvider = new SchemaProvider(pool, logger)
+    const indexProvider = new IndexProvider(pool, logger)
 
-    return { dataProvider, schemaProvider, databaseOperations, connection: pool, cleanup: async() => await pool.end() }
+    return { dataProvider, schemaProvider, databaseOperations, indexProvider, connection: pool, cleanup: async() => await pool.end() }
 }