diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 894ca419e..efcbf47b0 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -79,6 +79,19 @@ export enum PostgresIndexType { BRIN = 'BRIN', } +/** + * Portable ascending/descending indexes that exist on MongoDB and every SQL dialect. + * Map to Mongo 1/-1 and SQL BTREE ASC/DESC. Optional uniqueness lives on index options. + */ +export enum CompatibleIndexType { + Ascending = 'Ascending', + Descending = 'Descending', +} + +export type IndexType = MongoIndexType | PostgresIndexType | CompatibleIndexType; + +export type ModelOptionsIndexTypes = IndexType | readonly IndexType[]; + export type Array = any[]; export interface ConduitStringValidation { @@ -244,7 +257,7 @@ export interface ConduitSchemaOptions { } export interface SchemaFieldIndex { - type?: MongoIndexType | PostgresIndexType; + type?: IndexType; options?: MongoIndexOptions | PostgresIndexOptions; [field: string]: any; @@ -252,8 +265,10 @@ export interface SchemaFieldIndex { export interface ModelOptionsIndexes { fields: string[] | readonly string[]; - types?: MongoIndexType[] | PostgresIndexType; + types?: ModelOptionsIndexTypes; options?: MongoIndexOptions | PostgresIndexOptions; + /** Optional. Generated deterministically when omitted. */ + name?: string; [field: string]: any; } diff --git a/modules/authorization/src/models/ActorIndex.schema.ts b/modules/authorization/src/models/ActorIndex.schema.ts index 2663dee2b..30cce8a83 100644 --- a/modules/authorization/src/models/ActorIndex.schema.ts +++ b/modules/authorization/src/models/ActorIndex.schema.ts @@ -1,8 +1,8 @@ import { + CompatibleIndexType, ConduitModel, ConduitSchemaOptions, DatabaseProvider, - MongoIndexType, TYPE, } from '@conduitplatform/grpc-sdk'; import { ConduitActiveSchema } from '@conduitplatform/module-tools'; @@ -23,7 +23,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, subjectId: { @@ -41,7 +41,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, entityId: { @@ -69,6 +69,7 @@ const schemaOptions: ConduitSchemaOptions = { indexes: [ { fields: ['subject', 'entity'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], }, ], conduit: { diff --git a/modules/authorization/src/models/ObjectIndex.schema.ts b/modules/authorization/src/models/ObjectIndex.schema.ts index 7ef08abd1..0366dd08a 100644 --- a/modules/authorization/src/models/ObjectIndex.schema.ts +++ b/modules/authorization/src/models/ObjectIndex.schema.ts @@ -1,8 +1,8 @@ import { + CompatibleIndexType, ConduitModel, ConduitSchemaOptions, DatabaseProvider, - MongoIndexType, TYPE, } from '@conduitplatform/grpc-sdk'; import { ConduitActiveSchema } from '@conduitplatform/module-tools'; @@ -23,7 +23,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, subjectId: { @@ -47,7 +47,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, entityId: { @@ -75,7 +75,7 @@ const schema: ConduitModel = { type: [TYPE.String], default: [], index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, createdAt: TYPE.Date, @@ -86,12 +86,27 @@ const schemaOptions: ConduitSchemaOptions = { indexes: [ { fields: ['subjectType', 'subjectPermission', 'entity'], + types: [ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ], }, { fields: ['entity', 'subjectType', 'subjectPermission'], + types: [ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ], }, { fields: ['entityType', 'entityId', 'entityPermission'], + types: [ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ], }, ], conduit: { diff --git a/modules/authorization/src/models/Permission.schema.ts b/modules/authorization/src/models/Permission.schema.ts index 76dd2e3a5..ad1502649 100644 --- a/modules/authorization/src/models/Permission.schema.ts +++ b/modules/authorization/src/models/Permission.schema.ts @@ -1,7 +1,7 @@ import { + CompatibleIndexType, ConduitModel, DatabaseProvider, - MongoIndexType, TYPE, } from '@conduitplatform/grpc-sdk'; import { ConduitActiveSchema } from '@conduitplatform/module-tools'; @@ -18,7 +18,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, resourceId: { @@ -37,7 +37,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, subjectId: { @@ -61,7 +61,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, options: { unique: true, }, diff --git a/modules/authorization/src/models/Relationship.schema.ts b/modules/authorization/src/models/Relationship.schema.ts index 4961f5813..88cff5fc7 100644 --- a/modules/authorization/src/models/Relationship.schema.ts +++ b/modules/authorization/src/models/Relationship.schema.ts @@ -1,7 +1,7 @@ import { + CompatibleIndexType, ConduitModel, DatabaseProvider, - MongoIndexType, TYPE, } from '@conduitplatform/grpc-sdk'; import { ConduitActiveSchema } from '@conduitplatform/module-tools'; @@ -17,7 +17,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, resourceId: { @@ -36,7 +36,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, // user:1adasdas @@ -61,7 +61,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, options: { unique: true, }, diff --git a/modules/chat/src/models/ChatRoom.schema.ts b/modules/chat/src/models/ChatRoom.schema.ts index 6a7c10f61..92eae55e7 100644 --- a/modules/chat/src/models/ChatRoom.schema.ts +++ b/modules/chat/src/models/ChatRoom.schema.ts @@ -1,4 +1,5 @@ import { + CompatibleIndexType, ConduitModel, ConduitSchemaOptions, DatabaseProvider, @@ -40,7 +41,13 @@ const schema: ConduitModel = { }; const modelOptions: ConduitSchemaOptions = { timestamps: true, - indexes: [{ fields: ['participants'] }, { fields: ['participants', 'deleted'] }], + indexes: [ + { fields: ['participants'], types: [CompatibleIndexType.Ascending] }, + { + fields: ['participants', 'deleted'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], conduit: { permissions: { extendable: true, diff --git a/modules/chat/src/models/Message.schema.ts b/modules/chat/src/models/Message.schema.ts index 7f5bf5546..752928a49 100644 --- a/modules/chat/src/models/Message.schema.ts +++ b/modules/chat/src/models/Message.schema.ts @@ -1,8 +1,9 @@ import { + CompatibleIndexType, ConduitModel, + ConduitSchemaOptions, DatabaseProvider, TYPE, - ConduitSchemaOptions, } from '@conduitplatform/grpc-sdk'; import { ConduitActiveSchema } from '@conduitplatform/module-tools'; import { ChatRoom } from './ChatRoom.schema.js'; @@ -53,8 +54,18 @@ const schema: ConduitModel = { const modelOptions: ConduitSchemaOptions = { timestamps: true, indexes: [ - { fields: ['room', 'createdAt'] }, - { fields: ['room', 'deleted', 'createdAt'] }, + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + { + fields: ['room', 'deleted', 'createdAt'], + types: [ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ], + }, ], conduit: { permissions: { diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index ed46fa5bd..b6a8e67f9 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -167,26 +167,27 @@ export default class DatabaseModule extends ManagedModule { const isReplica = this.grpcSdk.isAvailable('database'); await this._activeAdapter.registerSystemSchema(models.DeclaredSchema, isReplica); await this._activeAdapter.registerSystemSchema(models.MigratedSchemas, isReplica); - let modelPromises = DATABASE_SYSTEM_SCHEMAS.filter( - model => - model.name !== models.DeclaredSchema.name && - model.name !== models.MigratedSchemas.name, - ).map(model => this._activeAdapter.registerSystemSchema(model, isReplica)); - await Promise.all(modelPromises); + for (const model of DATABASE_SYSTEM_SCHEMAS) { + if ( + model.name === models.DeclaredSchema.name || + model.name === models.MigratedSchemas.name + ) { + continue; + } + await this._activeAdapter.registerSystemSchema(model, isReplica); + } await this._activeAdapter.retrieveForeignSchemas(); await this._activeAdapter.recoverSchemasFromDatabase(); await this._activeAdapter.recoverViewsFromDatabase(); if (!isReplica) { await runMigrations(this._activeAdapter); } - modelPromises = DATABASE_SYSTEM_SCHEMAS.map(model => { - return this._activeAdapter.registerSystemSchema(model, isReplica).then(() => { - if (this._activeAdapter.getDatabaseType() !== 'MongoDB' && !isReplica) { - return this._activeAdapter.syncSchema(model.name); - } - }); - }); - await Promise.all(modelPromises); + for (const model of DATABASE_SYSTEM_SCHEMAS) { + await this._activeAdapter.registerSystemSchema(model, isReplica); + if (this._activeAdapter.getDatabaseType() !== 'MongoDB' && !isReplica) { + await this._activeAdapter.syncSchema(model.name); + } + } this.updateHealth(HealthCheckStatus.SERVING); } diff --git a/modules/database/src/__tests__/indexes/adapters.test.ts b/modules/database/src/__tests__/indexes/adapters.test.ts new file mode 100644 index 000000000..918bfd699 --- /dev/null +++ b/modules/database/src/__tests__/indexes/adapters.test.ts @@ -0,0 +1,810 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { status } from '@grpc/grpc-js'; +import { + CompatibleIndexType, + MongoIndexType, + PostgresIndexType, + TYPE, +} from '@conduitplatform/grpc-sdk'; +import { MongooseAdapter } from '../../adapters/mongoose-adapter/index.js'; +import { SequelizeAdapter } from '../../adapters/sequelize-adapter/index.js'; + +function makeMongooseAdapter(overrides: Record = {}) { + const createIndex = jest.fn().mockResolvedValue('email_1'); + const dropIndex = jest.fn().mockResolvedValue(undefined); + const indexes = jest.fn().mockResolvedValue([{ v: 2, key: { _id: 1 }, name: '_id_' }]); + const findOne = jest + .fn() + .mockResolvedValue({ _id: 'declared-1', modelOptions: { indexes: [] } }); + const findByIdAndUpdate = jest.fn().mockResolvedValue({}); + const publish = jest.fn(); + const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; + adapter.mongoose = { + model: () => ({ collection: { createIndex, dropIndex, indexes } }), + } as MongooseAdapter['mongoose']; + (adapter as unknown as { grpcSdk: { bus: { publish: typeof publish } } }).grpcSdk = { + bus: { publish }, + }; + const originalSchema = { + name: 'User', + ownerModule: 'chat', + collectionName: 'cnd_User', + fields: { + email: { type: 'String' }, + room: { type: 'String' }, + createdAt: { type: 'Date' }, + }, + compiledFields: { + email: { type: 'String' }, + room: { type: 'String' }, + createdAt: { type: 'Date' }, + }, + modelOptions: { indexes: [] as unknown[] }, + ...((overrides.originalSchema as object) ?? {}), + }; + adapter.models = { + User: { originalSchema }, + _DeclaredSchema: { findOne, findByIdAndUpdate }, + } as MongooseAdapter['models']; + findOne.mockImplementation(async () => ({ + _id: 'declared-1', + modelOptions: originalSchema.modelOptions, + })); + return { + adapter, + createIndex, + dropIndex, + indexes, + findOne, + findByIdAndUpdate, + publish, + originalSchema, + }; +} + +class TestSequelizeAdapter extends SequelizeAdapter { + protected async hasLegacyCollections(): Promise { + return false; + } +} + +function makeSequelizeAdapter(dialect = 'postgres') { + const addIndex = jest.fn().mockResolvedValue(undefined); + const removeIndex = jest.fn().mockResolvedValue(undefined); + const showIndex = jest.fn().mockResolvedValue([]); + const findOne = jest + .fn() + .mockResolvedValue({ _id: 'declared-1', modelOptions: { indexes: [] } }); + const findByIdAndUpdate = jest.fn().mockResolvedValue({}); + const publish = jest.fn(); + const sync = jest.fn().mockResolvedValue(undefined); + const adapter = Object.create(TestSequelizeAdapter.prototype) as SequelizeAdapter; + adapter.sequelize = { + getDialect: () => dialect, + getQueryInterface: () => ({ addIndex, removeIndex, showIndex }), + } as SequelizeAdapter['sequelize']; + (adapter as unknown as { grpcSdk: { bus: { publish: typeof publish } } }).grpcSdk = { + bus: { publish }, + }; + const originalSchema = { + name: 'User', + ownerModule: 'database', + collectionName: 'custom_users', + fields: { + email: { type: 'String' }, + room: { type: 'String' }, + createdAt: { type: 'Date' }, + }, + compiledFields: { + email: { type: 'String' }, + room: { type: 'String' }, + createdAt: { type: 'Date' }, + }, + modelOptions: { indexes: [] as unknown[] }, + }; + adapter.models = { + User: { originalSchema, sync }, + _DeclaredSchema: { findOne, findByIdAndUpdate }, + } as SequelizeAdapter['models']; + findOne.mockImplementation(async () => ({ + _id: 'declared-1', + modelOptions: originalSchema.modelOptions, + })); + return { + adapter, + addIndex, + removeIndex, + showIndex, + sync, + findOne, + findByIdAndUpdate, + publish, + originalSchema, + }; +} + +describe('mongoose adapter indexes', () => { + it('creates a single key spec object, not an array of objects', async () => { + const { adapter, createIndex } = makeMongooseAdapter(); + await adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'chat', + ); + expect(createIndex).toHaveBeenCalledTimes(1); + expect(createIndex.mock.calls[0][0]).toEqual({ email: MongoIndexType.Ascending }); + expect(Array.isArray(createIndex.mock.calls[0][0])).toBe(false); + }); + + it('persists created and deleted indexes on _DeclaredSchema', async () => { + const created = makeMongooseAdapter(); + await created.adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'chat', + ); + expect(created.findByIdAndUpdate).toHaveBeenCalledTimes(1); + expect(created.findOne.mock.calls[0][1]).toEqual({ readPreference: 'primary' }); + const update = created.findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { name?: string }[] }; + }; + expect(update.modelOptions.indexes[0].name).toMatch(/email/); + + const deleted = makeMongooseAdapter({ + originalSchema: { + modelOptions: { indexes: [{ fields: ['email'], name: 'cnd_idx_email_asc' }] }, + }, + }); + await deleted.adapter.deleteIndexes('User', ['cnd_idx_email_asc']); + expect(deleted.originalSchema.modelOptions.indexes).toEqual([]); + expect(deleted.findByIdAndUpdate).toHaveBeenCalled(); + }); + + it('awaits dropIndex', async () => { + const { adapter, dropIndex } = makeMongooseAdapter(); + let resolveDrop: () => void = () => undefined; + const dropped = new Promise(resolve => { + resolveDrop = resolve; + }); + dropIndex.mockImplementation(async () => { + resolveDrop(); + }); + await adapter.deleteIndexes('User', ['cnd_idx_email_asc']); + await dropped; + expect(dropIndex).toHaveBeenCalledWith('cnd_idx_email_asc'); + }); + + it('reads live engine indexes', async () => { + const { adapter, indexes } = makeMongooseAdapter(); + indexes.mockResolvedValue([ + { v: 2, key: { _id: 1 }, name: '_id_' }, + { v: 2, key: { email: 1 }, name: 'cnd_idx_email_asc', unique: false }, + ]); + const result = await adapter.getIndexes('User'); + expect(indexes).toHaveBeenCalled(); + expect(result.map(index => index.name)).toEqual(['_id_', 'cnd_idx_email_asc']); + expect(result[1].fields).toEqual(['email']); + }); + + it('throws on Admin-bound invalid types and allows privileged unique', async () => { + const { adapter } = makeMongooseAdapter(); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [PostgresIndexType.GIST] }], + 'database', + { privileged: true }, + ), + ).rejects.toMatchObject({ code: status.INVALID_ARGUMENT }); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], options: { unique: true } }], + 'database', + { privileged: true }, + ), + ).resolves.toBe('Indexes created!'); + }); + + it('adopts a live compound name and skips createIndex', async () => { + const { adapter, createIndex, indexes, findByIdAndUpdate } = makeMongooseAdapter(); + indexes.mockResolvedValue([ + { v: 2, key: { _id: 1 }, name: '_id_' }, + { v: 2, key: { room: 1, createdAt: 1 }, name: 'room_1_createdAt_1', unique: false }, + ]); + await adapter.createIndexes( + 'User', + [ + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + 'chat', + ); + expect(createIndex).not.toHaveBeenCalled(); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { name?: string }[] }; + }; + expect(update.modelOptions.indexes[0].name).toBe('room_1_createdAt_1'); + }); + + it('throws on unique-data collisions and does not persist that index', async () => { + const { adapter, createIndex, findByIdAndUpdate } = makeMongooseAdapter(); + createIndex.mockRejectedValue({ code: 11000, message: 'E11000 duplicate key' }); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], options: { unique: true } }], + 'chat', + ), + ).rejects.toMatchObject({ code: status.INTERNAL }); + expect(findByIdAndUpdate).not.toHaveBeenCalled(); + }); + + it('persists the prefix and publishes after a later unique-data failure', async () => { + const { adapter, createIndex, findByIdAndUpdate, publish } = makeMongooseAdapter(); + createIndex + .mockResolvedValueOnce('cnd_idx_email_asc') + .mockRejectedValueOnce({ code: 11000, message: 'E11000 duplicate key' }); + await expect( + adapter.createIndexes( + 'User', + [ + { fields: ['email'], types: [CompatibleIndexType.Ascending] }, + { fields: ['room'], options: { unique: true } }, + ], + 'chat', + ), + ).rejects.toMatchObject({ code: status.INTERNAL }); + expect(findByIdAndUpdate).toHaveBeenCalledTimes(1); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { name?: string }[] }; + }; + expect(update.modelOptions.indexes.map(index => index.name)).toEqual([ + 'cnd_idx_cnd_User_email_asc', + ]); + expect(publish).toHaveBeenCalledWith('database:create:schema', expect.any(String)); + }); + + it('overlays declared Compatible types onto a live index by fields', async () => { + const { adapter, indexes, originalSchema } = makeMongooseAdapter(); + originalSchema.modelOptions.indexes = [ + { + fields: ['room', 'createdAt'], + name: 'cnd_idx_room_createdAt_asc_asc', + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ]; + indexes.mockResolvedValue([ + { v: 2, key: { _id: 1 }, name: '_id_' }, + { v: 2, key: { room: 1, createdAt: 1 }, name: 'room_1_createdAt_1', unique: false }, + ]); + const result = await adapter.getIndexes('User'); + expect(result[1].name).toBe('room_1_createdAt_1'); + expect(result[1].types).toEqual([ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ]); + }); + + it('publishes the schema after a successful persist', async () => { + const { adapter, publish } = makeMongooseAdapter(); + await adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'chat', + ); + expect(publish).toHaveBeenCalledWith('database:create:schema', expect.any(String)); + }); + + it('throws on a name-already-exists error when the live name indexes different fields', async () => { + const { adapter, createIndex, indexes, findByIdAndUpdate } = makeMongooseAdapter(); + createIndex.mockRejectedValue({ message: 'index user_idx already exists' }); + indexes + .mockResolvedValueOnce([{ v: 2, key: { _id: 1 }, name: '_id_' }]) + .mockResolvedValueOnce([ + { v: 2, key: { _id: 1 }, name: '_id_' }, + { v: 2, key: { room: 1 }, name: 'user_idx', unique: false }, + ]); + await expect( + adapter.createIndexes('User', [{ fields: ['email'], name: 'user_idx' }], 'chat'), + ).rejects.toMatchObject({ code: status.INTERNAL }); + expect(findByIdAndUpdate).not.toHaveBeenCalled(); + }); + + it('rebinds to the live name on Mongo 86 instead of persisting a generated name', async () => { + const { adapter, createIndex, indexes, findByIdAndUpdate } = makeMongooseAdapter(); + createIndex.mockRejectedValue({ code: 86, message: 'IndexKeySpecsConflict' }); + indexes + .mockResolvedValueOnce([{ v: 2, key: { _id: 1 }, name: '_id_' }]) + .mockResolvedValueOnce([ + { v: 2, key: { _id: 1 }, name: '_id_' }, + { v: 2, key: { email: 1 }, name: 'email_1', unique: false }, + ]); + await adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'chat', + ); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { name?: string }[] }; + }; + expect(update.modelOptions.indexes[0].name).toBe('email_1'); + }); + + it('persists only names that were actually dropped', async () => { + const { adapter, dropIndex, findByIdAndUpdate } = makeMongooseAdapter({ + originalSchema: { + modelOptions: { + indexes: [ + { fields: ['email'], name: 'idx_a' }, + { fields: ['room'], name: 'idx_b' }, + ], + }, + }, + }); + dropIndex + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error('missing')); + await expect(adapter.deleteIndexes('User', ['idx_a', 'idx_b'])).rejects.toMatchObject( + { + code: status.INTERNAL, + }, + ); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { name?: string }[] }; + }; + expect(update.modelOptions.indexes.map(index => index.name)).toEqual(['idx_b']); + }); + + it('returns empty indexes when the Mongo namespace is missing', async () => { + const { adapter, indexes } = makeMongooseAdapter(); + indexes.mockRejectedValue({ + code: 26, + codeName: 'NamespaceNotFound', + message: 'ns does not exist: test.cnd_adminapitokens', + }); + await expect(adapter.getIndexes('User')).resolves.toEqual([]); + }); + + it('creates indexes when listLiveIndexes hits a missing namespace', async () => { + const { adapter, createIndex, indexes } = makeMongooseAdapter(); + indexes.mockRejectedValue({ + code: 26, + codeName: 'NamespaceNotFound', + message: 'ns does not exist: test.cnd_adminapitokens', + }); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'chat', + ), + ).resolves.toBe('Indexes created!'); + expect(createIndex).toHaveBeenCalledTimes(1); + }); + + it('rethrows non-namespace Mongo index listing errors', async () => { + const { adapter, indexes } = makeMongooseAdapter(); + indexes.mockRejectedValue({ code: 13, message: 'unauthorized' }); + await expect(adapter.getIndexes('User')).rejects.toMatchObject({ + message: 'unauthorized', + }); + }); + + it('creates both a single-field array index and a compound sharing the leading field', async () => { + const { adapter, createIndex, originalSchema } = makeMongooseAdapter(); + originalSchema.fields.participants = [{ type: TYPE.Relation, model: 'User' }]; + originalSchema.fields.deleted = { type: TYPE.Boolean }; + originalSchema.compiledFields.participants = [{ type: TYPE.Relation, model: 'User' }]; + originalSchema.compiledFields.deleted = { type: TYPE.Boolean }; + await adapter.createIndexes( + 'User', + [ + { fields: ['participants'], types: [CompatibleIndexType.Ascending] }, + { + fields: ['participants', 'deleted'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + 'chat', + ); + expect(createIndex).toHaveBeenCalledTimes(2); + expect(createIndex.mock.calls[0][0]).toEqual({ + participants: MongoIndexType.Ascending, + }); + expect(createIndex.mock.calls[1][0]).toEqual({ + participants: MongoIndexType.Ascending, + deleted: MongoIndexType.Ascending, + }); + }); +}); + +describe('sequelize adapter indexes', () => { + it('keeps getDatabaseType as PostgreSQL', () => { + const { adapter } = makeSequelizeAdapter('postgres'); + expect(adapter.getDatabaseType()).toBe('PostgreSQL'); + }); + + it('uses originalSchema.collectionName instead of a hardcoded cnd_ prefix', async () => { + const { adapter, addIndex, removeIndex, showIndex } = makeSequelizeAdapter(); + await adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'database', + { privileged: true }, + ); + expect(addIndex.mock.calls[0][0]).toBe('custom_users'); + expect(addIndex.mock.calls[0][0]).not.toBe('cnd_User'); + await adapter.getIndexes('User'); + expect(showIndex).toHaveBeenCalledWith('custom_users'); + await adapter.deleteIndexes('User', ['cnd_idx_email_asc']); + expect(removeIndex).toHaveBeenCalledWith('custom_users', 'cnd_idx_email_asc'); + }); + + it('does not rebuild or sync the schema when creating indexes', async () => { + const { adapter, sync } = makeSequelizeAdapter(); + await adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'database', + ); + expect(sync).not.toHaveBeenCalled(); + }); + + it('reads the live engine and overlays declared Compatible types', async () => { + const { adapter, showIndex, originalSchema } = makeSequelizeAdapter(); + originalSchema.modelOptions.indexes = [ + { + fields: ['email'], + name: 'cnd_idx_email_asc', + types: [CompatibleIndexType.Ascending], + }, + ]; + showIndex.mockResolvedValue([ + { + name: 'cnd_idx_email_asc', + unique: false, + fields: [{ attribute: 'email', order: 'ASC' }], + definition: 'CREATE INDEX cnd_idx_email_asc ON custom_users USING btree (email)', + }, + ]); + const result = await adapter.getIndexes('User'); + expect(showIndex).toHaveBeenCalledWith('custom_users'); + expect(result[0].types).toEqual([CompatibleIndexType.Ascending]); + expect(result[0].fields).toEqual(['email']); + }); + + it('awaits removeIndex and persists the deletion', async () => { + const { adapter, removeIndex, findByIdAndUpdate } = makeSequelizeAdapter(); + await adapter.deleteIndexes('User', ['cnd_idx_email_asc']); + expect(removeIndex).toHaveBeenCalledTimes(1); + expect(findByIdAndUpdate).toHaveBeenCalled(); + }); + + it('allows HASH on mysql and rejects it on sqlite', async () => { + const mysql = makeSequelizeAdapter('mysql'); + await expect( + mysql.adapter.createIndexes( + 'User', + [{ fields: ['email'], types: PostgresIndexType.HASH }], + 'database', + { privileged: true }, + ), + ).resolves.toBe('Indexes created!'); + + const sqlite = makeSequelizeAdapter('sqlite'); + await expect( + sqlite.adapter.createIndexes( + 'User', + [{ fields: ['email'], types: PostgresIndexType.HASH }], + 'database', + { privileged: true }, + ), + ).rejects.toMatchObject({ message: expect.stringMatching(/sqlite/i) }); + }); + + it('adopts a live compound name and skips addIndex', async () => { + const { adapter, addIndex, showIndex, findByIdAndUpdate } = makeSequelizeAdapter(); + showIndex.mockResolvedValue([ + { + name: 'room_createdAt', + unique: false, + fields: [{ attribute: 'room' }, { attribute: 'createdAt' }], + }, + ]); + await adapter.createIndexes( + 'User', + [ + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + 'database', + ); + expect(addIndex).not.toHaveBeenCalled(); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { name?: string }[] }; + }; + expect(update.modelOptions.indexes[0].name).toBe('room_createdAt'); + }); + + it('skips and persists on 42P07 when the live name has the same identity', async () => { + const { adapter, addIndex, showIndex, findByIdAndUpdate, publish } = + makeSequelizeAdapter(); + showIndex.mockResolvedValueOnce([]).mockResolvedValueOnce([ + { + name: 'cnd_idx_email_asc', + unique: false, + fields: [{ attribute: 'email', order: 'ASC' }], + }, + ]); + addIndex.mockRejectedValue({ + original: { code: '42P07', message: 'relation "cnd_idx_email_asc" already exists' }, + }); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'database', + ), + ).resolves.toBe('Indexes created!'); + expect(findByIdAndUpdate).toHaveBeenCalled(); + expect(publish).toHaveBeenCalled(); + }); + + it('throws on 42P07 when the live name indexes different fields', async () => { + const { adapter, addIndex, showIndex, findByIdAndUpdate } = makeSequelizeAdapter(); + showIndex.mockResolvedValueOnce([]).mockResolvedValueOnce([ + { + name: 'user_idx', + unique: false, + fields: [{ attribute: 'room' }], + }, + ]); + addIndex.mockRejectedValue({ + original: { code: '42P07', message: 'relation "user_idx" already exists' }, + }); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], name: 'user_idx' }], + 'database', + ), + ).rejects.toMatchObject({ code: status.INTERNAL }); + expect(findByIdAndUpdate).not.toHaveBeenCalled(); + }); + + it('persists the applied prefix when a later 42P07 name has different fields', async () => { + const { adapter, addIndex, showIndex, findByIdAndUpdate, publish } = + makeSequelizeAdapter(); + showIndex.mockResolvedValueOnce([]).mockResolvedValueOnce([ + { + name: 'user_idx', + unique: false, + fields: [{ attribute: 'username' }], + }, + ]); + addIndex.mockResolvedValueOnce(undefined).mockRejectedValueOnce({ + original: { code: '42P07', message: 'relation "user_idx" already exists' }, + }); + await expect( + adapter.createIndexes( + 'User', + [ + { fields: ['email'], types: [CompatibleIndexType.Ascending] }, + { fields: ['room'], name: 'user_idx' }, + ], + 'database', + ), + ).rejects.toMatchObject({ code: status.INTERNAL }); + expect(findByIdAndUpdate).toHaveBeenCalledTimes(1); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { name?: string }[] }; + }; + expect(update.modelOptions.indexes.map(index => index.name)).toEqual([ + 'cnd_idx_custom_users_email_asc', + ]); + expect(publish).toHaveBeenCalled(); + }); + + it('throws on 23505 unique collisions and does not persist that index', async () => { + const { adapter, addIndex, findByIdAndUpdate } = makeSequelizeAdapter(); + addIndex.mockRejectedValue({ + name: 'SequelizeUniqueConstraintError', + original: { code: '23505' }, + }); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], options: { unique: true } }], + 'database', + { privileged: true }, + ), + ).rejects.toMatchObject({ code: status.INTERNAL }); + expect(findByIdAndUpdate).not.toHaveBeenCalled(); + }); + + it('overlays declared Compatible types onto a live SQL index by fields', async () => { + const { adapter, showIndex, originalSchema } = makeSequelizeAdapter(); + originalSchema.modelOptions.indexes = [ + { + fields: ['room', 'createdAt'], + name: 'cnd_idx_room_createdAt_asc_asc', + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ]; + showIndex.mockResolvedValue([ + { + name: 'room_createdAt', + unique: false, + fields: [{ attribute: 'room' }, { attribute: 'createdAt' }], + definition: + 'CREATE INDEX room_createdAt ON custom_users USING btree (room, createdAt)', + }, + ]); + const result = await adapter.getIndexes('User'); + expect(result[0].name).toBe('room_createdAt'); + expect(result[0].types).toEqual([ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ]); + }); + + it('rejects MySQL JSON btree indexes and allows them on postgres', async () => { + const mysql = makeSequelizeAdapter('mysql'); + mysql.originalSchema.fields.inheritanceTree = { type: [TYPE.String] }; + mysql.originalSchema.compiledFields.inheritanceTree = { type: [TYPE.String] }; + await expect( + mysql.adapter.createIndexes( + 'User', + [{ fields: ['inheritanceTree'], types: [CompatibleIndexType.Ascending] }], + 'database', + ), + ).rejects.toMatchObject({ + code: status.INVALID_ARGUMENT, + message: expect.stringMatching(/MySQL JSON field 'inheritanceTree'/), + }); + expect(mysql.addIndex).not.toHaveBeenCalled(); + expect(mysql.findByIdAndUpdate).not.toHaveBeenCalled(); + + const postgres = makeSequelizeAdapter('postgres'); + postgres.originalSchema.fields.payload = { type: TYPE.JSON }; + postgres.originalSchema.compiledFields.payload = { type: TYPE.JSON }; + await expect( + postgres.adapter.createIndexes( + 'User', + [{ fields: ['payload'], types: [CompatibleIndexType.Ascending] }], + 'database', + ), + ).resolves.toBe('Indexes created!'); + expect(postgres.addIndex).toHaveBeenCalled(); + }); + + it('rejects SQL indexes on extracted relation fields', async () => { + const { adapter, addIndex, findByIdAndUpdate } = makeSequelizeAdapter('postgres'); + adapter.models.User.originalSchema.fields.participants = [ + { type: TYPE.Relation, model: 'User' }, + ]; + adapter.models.User.originalSchema.compiledFields.participants = [ + { type: TYPE.Relation, model: 'User' }, + ]; + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['participants'], types: [CompatibleIndexType.Ascending] }], + 'database', + ), + ).rejects.toMatchObject({ + code: status.INVALID_ARGUMENT, + message: expect.stringMatching(/relation join table/), + }); + expect(addIndex).not.toHaveBeenCalled(); + expect(findByIdAndUpdate).not.toHaveBeenCalled(); + }); + + it('creates scalar Relation compounds on roomId while persisting declared room', async () => { + const { adapter, addIndex, findByIdAndUpdate, originalSchema } = + makeSequelizeAdapter('postgres'); + originalSchema.fields.room = { type: TYPE.Relation, model: 'ChatRoom' }; + originalSchema.compiledFields.room = { type: TYPE.Relation, model: 'ChatRoom' }; + await adapter.createIndexes( + 'User', + [ + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + 'database', + ); + expect(addIndex.mock.calls[0][1].fields).toEqual([ + { name: 'roomId', order: 'ASC' }, + { name: 'createdAt', order: 'ASC' }, + ]); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { fields: string[]; name?: string }[] }; + }; + expect(update.modelOptions.indexes[0].fields).toEqual(['room', 'createdAt']); + expect(update.modelOptions.indexes[0].name).toBe( + 'cnd_idx_custom_users_room_createdAt_asc_asc', + ); + }); + + it('adopts a live roomId compound for declared room and skips addIndex', async () => { + const { adapter, addIndex, showIndex, findByIdAndUpdate, originalSchema } = + makeSequelizeAdapter(); + originalSchema.fields.room = { type: TYPE.Relation, model: 'ChatRoom' }; + originalSchema.compiledFields.room = { type: TYPE.Relation, model: 'ChatRoom' }; + showIndex.mockResolvedValue([ + { + name: 'roomId_createdAt', + unique: false, + fields: [{ attribute: 'roomId' }, { attribute: 'createdAt' }], + }, + ]); + await adapter.createIndexes( + 'User', + [ + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + 'database', + ); + expect(addIndex).not.toHaveBeenCalled(); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { fields: string[]; name?: string }[] }; + }; + expect(update.modelOptions.indexes[0].name).toBe('roomId_createdAt'); + expect(update.modelOptions.indexes[0].fields).toEqual(['room', 'createdAt']); + }); + + it('returns declared room names from live roomId indexes', async () => { + const { adapter, showIndex, originalSchema } = makeSequelizeAdapter(); + originalSchema.fields.room = { type: TYPE.Relation, model: 'ChatRoom' }; + originalSchema.compiledFields.room = { type: TYPE.Relation, model: 'ChatRoom' }; + originalSchema.modelOptions.indexes = [ + { + fields: ['room', 'createdAt'], + name: 'cnd_idx_custom_users_room_createdAt_asc_asc', + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ]; + showIndex.mockResolvedValue([ + { + name: 'roomId_createdAt', + unique: false, + fields: [{ attribute: 'roomId' }, { attribute: 'createdAt' }], + definition: + 'CREATE INDEX roomId_createdAt ON custom_users USING btree ("roomId", "createdAt")', + }, + ]); + const result = await adapter.getIndexes('User'); + expect(result[0].fields).toEqual(['room', 'createdAt']); + expect(result[0].name).toBe('roomId_createdAt'); + expect(result[0].types).toEqual([ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ]); + }); + + it('leaves Authz String *Id fields unchanged on SQL create', async () => { + const { adapter, addIndex, originalSchema } = makeSequelizeAdapter(); + originalSchema.fields.resource = { type: TYPE.String }; + originalSchema.fields.resourceId = { type: TYPE.String }; + originalSchema.compiledFields.resource = { type: TYPE.String }; + originalSchema.compiledFields.resourceId = { type: TYPE.String }; + await adapter.createIndexes( + 'User', + [{ fields: ['resource'], types: [CompatibleIndexType.Ascending] }], + 'database', + ); + expect(addIndex.mock.calls[0][1].fields).toEqual([ + { name: 'resource', order: 'ASC' }, + ]); + }); +}); diff --git a/modules/database/src/__tests__/indexes/admin.test.ts b/modules/database/src/__tests__/indexes/admin.test.ts new file mode 100644 index 000000000..7eaf4f5e0 --- /dev/null +++ b/modules/database/src/__tests__/indexes/admin.test.ts @@ -0,0 +1,257 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { status } from '@grpc/grpc-js'; +import { + CompatibleIndexType, + ConduitGrpcSdk, + ParsedRouterRequest, +} from '@conduitplatform/grpc-sdk'; +import { SchemaAdmin } from '../../admin/schema.admin.js'; +import { DatabaseAdapter } from '../../adapters/DatabaseAdapter.js'; +import { MongooseSchema } from '../../adapters/mongoose-adapter/MongooseSchema.js'; +import { SequelizeSchema } from '../../adapters/sequelize-adapter/SequelizeSchema.js'; +import { SchemaController } from '../../controllers/cms/schema.controller.js'; +import { CustomEndpointController } from '../../controllers/customEndpoints/customEndpoint.controller.js'; +import { ADMIN_INDEX_CALLER } from '../../adapters/utils/indexes.js'; +import { validateSchemaInput } from '../../utils/utilities.js'; + +function makeCall(params: Record): ParsedRouterRequest { + return { request: { params } } as unknown as ParsedRouterRequest; +} + +function setup() { + const findOne = jest.fn().mockResolvedValue({ + _id: 'schema-1', + name: 'User', + ownerModule: 'database', + fields: { + email: { type: 'String' }, + room: { type: 'Relation', model: 'ChatRoom' }, + createdAt: { type: 'Date' }, + resource: { type: 'String' }, + resourceId: { type: 'String' }, + }, + compiledFields: { + email: { type: 'String' }, + room: { type: 'Relation', model: 'ChatRoom' }, + createdAt: { type: 'Date' }, + resource: { type: 'String' }, + resourceId: { type: 'String' }, + }, + }); + const findMany = jest.fn().mockResolvedValue([{ name: 'User' }, { name: 'ChatRoom' }]); + const countDocuments = jest.fn().mockResolvedValue(2); + const createIndexes = jest.fn().mockResolvedValue('Indexes created!'); + const getIndexes = jest + .fn() + .mockResolvedValue([{ name: 'cnd_idx_email_asc', fields: ['email'] }]); + const deleteIndexes = jest.fn().mockResolvedValue('Indexes deleted'); + const getSchemaModel = jest.fn().mockReturnValue({ + model: { findOne, findMany, countDocuments }, + }); + const database = { + getSchemaModel, + createIndexes, + getIndexes, + deleteIndexes, + systemSchemas: ['_DeclaredSchema'], + models: { + User: { + originalSchema: { + ownerModule: 'database', + fields: { + email: { type: 'String' }, + room: { type: 'Relation', model: 'ChatRoom' }, + createdAt: { type: 'Date' }, + resource: { type: 'String' }, + resourceId: { type: 'String' }, + }, + compiledFields: { + email: { type: 'String' }, + room: { type: 'Relation', model: 'ChatRoom' }, + createdAt: { type: 'Date' }, + resource: { type: 'String' }, + resourceId: { type: 'String' }, + }, + }, + }, + ChatRoom: { originalSchema: { ownerModule: 'chat' } }, + }, + } as unknown as DatabaseAdapter; + const admin = new SchemaAdmin( + {} as ConduitGrpcSdk, + database, + {} as SchemaController, + {} as CustomEndpointController, + ); + return { admin, createIndexes, getIndexes, findMany, countDocuments, findOne }; +} + +describe('SchemaAdmin indexes', () => { + it('creates indexes as a privileged Admin caller', async () => { + const { admin, createIndexes } = setup(); + await admin.createIndexes( + makeCall({ + id: 'schema-1', + indexes: [{ fields: ['email'], options: { unique: true } }], + }), + ); + expect(createIndexes).toHaveBeenCalledWith( + 'User', + [{ fields: ['email'], options: { unique: true } }], + ADMIN_INDEX_CALLER, + { privileged: true }, + ); + }); + + it('canonicalizes inbound roomId to room when room is a scalar Relation', async () => { + const { admin, createIndexes } = setup(); + await admin.createIndexes( + makeCall({ + id: 'schema-1', + indexes: [{ fields: ['roomId', 'createdAt'] }], + }), + ); + expect(createIndexes).toHaveBeenCalledWith( + 'User', + [{ fields: ['room', 'createdAt'] }], + ADMIN_INDEX_CALLER, + { privileged: true }, + ); + }); + + it('does not rewrite Authz String *Id fields on Admin create', async () => { + const { admin, createIndexes } = setup(); + await admin.createIndexes( + makeCall({ + id: 'schema-1', + indexes: [{ fields: ['resourceId'] }], + }), + ); + expect(createIndexes).toHaveBeenCalledWith( + 'User', + [{ fields: ['resourceId'] }], + ADMIN_INDEX_CALLER, + { privileged: true }, + ); + }); + + it('exports indexes with skip/limit pagination', async () => { + const { admin, findMany, countDocuments, getIndexes } = setup(); + const result = (await admin.exportIndexes(makeCall({ skip: 10, limit: 5 }))) as { + indexes: unknown[]; + count: number; + }; + expect(findMany).toHaveBeenCalledWith( + expect.any(Object), + expect.objectContaining({ skip: 10, limit: 5 }), + ); + expect(countDocuments).toHaveBeenCalled(); + expect(getIndexes).toHaveBeenCalled(); + expect(result.count).toBe(2); + expect(result.indexes.every(index => 'schemaName' in (index as object))).toBe(true); + }); + + it('continues exporting when getIndexes throws for one schema', async () => { + const { admin, getIndexes } = setup(); + getIndexes + .mockRejectedValueOnce(new Error('ns does not exist: test.cnd_adminapitokens')) + .mockResolvedValueOnce([{ name: 'ok', fields: ['email'] }]); + jest.spyOn(ConduitGrpcSdk.Logger, 'warn').mockImplementation(() => undefined); + const result = (await admin.exportIndexes(makeCall({}))) as { + indexes: Array<{ schemaName: string }>; + count: number; + }; + expect(result.count).toBe(2); + expect(result.indexes).toHaveLength(1); + expect(result.indexes[0].schemaName).toBe('ChatRoom'); + }); + + it('skips same-name indexes on import', async () => { + const { admin, createIndexes, getIndexes } = setup(); + getIndexes.mockResolvedValue([{ name: 'keep_me', fields: ['email'] }]); + await admin.importIndexes( + makeCall({ + indexes: [ + { schemaName: 'User', fields: ['email'], name: 'keep_me' }, + { schemaName: 'User', fields: ['name'], name: 'new_name' }, + ], + }), + ); + expect(createIndexes).toHaveBeenCalledTimes(1); + const created = createIndexes.mock.calls[0][1] as { name?: string }[]; + expect(created.map(index => index.name)).toEqual(['new_name']); + }); + + it('imports unique indexes without Admin privilege so owner rules apply', async () => { + const { admin, createIndexes } = setup(); + await admin.importIndexes( + makeCall({ + indexes: [ + { + schemaName: 'ChatRoom', + fields: ['name'], + name: 'chat_name', + options: { unique: true }, + }, + ], + }), + ); + expect(createIndexes).toHaveBeenCalledWith( + 'ChatRoom', + expect.any(Array), + ADMIN_INDEX_CALLER, + { privileged: false }, + ); + }); + + it('canonicalizes inbound roomId on import when room is a scalar Relation', async () => { + const { admin, createIndexes } = setup(); + await admin.importIndexes( + makeCall({ + indexes: [{ schemaName: 'User', fields: ['roomId'], name: 'msg_room' }], + }), + ); + expect(createIndexes).toHaveBeenCalledWith( + 'User', + [expect.objectContaining({ fields: ['room'], name: 'msg_room' })], + ADMIN_INDEX_CALLER, + { privileged: false }, + ); + }); + + it('rejects an empty import payload', async () => { + const { admin } = setup(); + await expect(admin.importIndexes(makeCall({ indexes: [] }))).rejects.toMatchObject({ + code: status.INVALID_ARGUMENT, + }); + }); +}); + +describe('validateModelOptions indexes', () => { + it('accepts modelOptions.indexes and conduit.readPreference together', () => { + expect(() => + validateSchemaInput( + 'User', + { email: 'String' }, + { + timestamps: true, + indexes: [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + conduit: { readPreference: 'secondaryPreferred' }, + }, + ), + ).not.toThrow(); + }); + + it('still rejects unknown conduit keys and unknown model option keys', () => { + expect(() => + validateSchemaInput('User', { email: 'String' }, { unknown: true } as Parameters< + typeof validateSchemaInput + >[2]), + ).toThrow(/indexes/); + expect(() => + validateSchemaInput('User', { email: 'String' }, { + conduit: { notARealKey: true }, + } as Parameters[2]), + ).toThrow(/readPreference/); + }); +}); diff --git a/modules/database/src/__tests__/indexes/converters.test.ts b/modules/database/src/__tests__/indexes/converters.test.ts new file mode 100644 index 000000000..0a803f30f --- /dev/null +++ b/modules/database/src/__tests__/indexes/converters.test.ts @@ -0,0 +1,447 @@ +import { beforeEach, describe, expect, it, jest } from '@jest/globals'; +import { + CompatibleIndexType, + ConduitGrpcSdk, + ConduitSchema, + MongoIndexType, + PostgresIndexType, + TYPE, +} from '@conduitplatform/grpc-sdk'; +import { + convertModelOptionsIndexes, + convertSchemaFieldIndexes, +} from '../../adapters/utils/database-transform-utils.js'; +import { sqlSchemaConverter } from '../../adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.js'; +import { pgSchemaConverter } from '../../adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.js'; +import { schemaConverter } from '../../adapters/mongoose-adapter/SchemaConverter.js'; + +type ConvertedSqlIndex = { + fields: Array; + using?: PostgresIndexType; +}; + +function schemaWithIndexes( + indexes: ConduitSchema['modelOptions']['indexes'], + fields: ConduitSchema['fields'] = { email: { type: TYPE.String } }, +) { + return new ConduitSchema('User', fields, { indexes }); +} + +describe('SQL index converters', () => { + beforeEach(() => { + jest.spyOn(ConduitGrpcSdk.Logger, 'warn').mockImplementation(() => undefined); + }); + + it('maps Compatible types to BTREE plus ASC/DESC on postgres and mysql', () => { + const postgres = convertModelOptionsIndexes( + schemaWithIndexes([{ fields: ['email'], types: [CompatibleIndexType.Descending] }]), + 'postgres', + ); + const pgIndex = postgres.modelOptions.indexes![0] as ConvertedSqlIndex; + expect(pgIndex.using).toBe(PostgresIndexType.BTREE); + expect(pgIndex.fields[0]).toEqual({ name: 'email', order: 'DESC' }); + + const mysql = convertModelOptionsIndexes( + schemaWithIndexes([{ fields: ['email'], types: [CompatibleIndexType.Ascending] }]), + 'mysql', + ); + const mysqlIndex = mysql.modelOptions.indexes![0] as ConvertedSqlIndex; + expect(mysqlIndex.using).toBe(PostgresIndexType.BTREE); + expect(mysqlIndex.fields[0]).toEqual({ name: 'email', order: 'ASC' }); + }); + + it('maps Compatible types to BTREE on sqlite', () => { + const copy = convertModelOptionsIndexes( + schemaWithIndexes([{ fields: ['email'], types: CompatibleIndexType.Ascending }]), + 'sqlite', + ); + expect((copy.modelOptions.indexes![0] as ConvertedSqlIndex).using).toBe( + PostgresIndexType.BTREE, + ); + }); + + it('warns and skips Mongo-only leftovers on SQL', () => { + const warn = ConduitGrpcSdk.Logger.warn as jest.Mock; + const copy = convertModelOptionsIndexes( + schemaWithIndexes( + [ + { fields: ['loc'], types: [MongoIndexType.GeoSpatial2dSphere] }, + { fields: ['email'], types: [CompatibleIndexType.Ascending] }, + ], + { email: { type: TYPE.String }, loc: { type: TYPE.JSON } }, + ), + 'postgres', + ); + expect(copy.modelOptions.indexes).toHaveLength(1); + expect(warn).toHaveBeenCalled(); + }); + + it('skips postgres-only types on mysql, mariadb, and sqlite', () => { + for (const dialect of ['mysql', 'mariadb', 'sqlite'] as const) { + const copy = convertModelOptionsIndexes( + schemaWithIndexes([{ fields: ['email'], types: PostgresIndexType.GIST }]), + dialect, + ); + expect(copy.modelOptions.indexes).toHaveLength(0); + } + }); + + it('converts field-level Compatible indexes into sequelize model indexes', () => { + const copy = convertSchemaFieldIndexes( + new ConduitSchema( + 'Perm', + { + resource: { + type: TYPE.String, + index: { type: CompatibleIndexType.Ascending }, + }, + }, + {}, + ), + 'mysql', + ); + expect(copy.modelOptions.indexes).toHaveLength(1); + expect((copy.fields.resource as { index?: unknown }).index).toBeUndefined(); + }); + + it('keeps HASH on mysql and drops it on sqlite', () => { + const schema = new ConduitSchema( + 'User', + { email: { type: TYPE.String } }, + { + indexes: [ + { fields: ['email'], types: [CompatibleIndexType.Descending] }, + { fields: ['email'], types: PostgresIndexType.HASH }, + ], + }, + ); + const [mysql] = sqlSchemaConverter(schema, 'mysql'); + const [sqlite] = sqlSchemaConverter(schema, 'sqlite'); + expect(mysql.modelOptions.indexes).toHaveLength(2); + expect(sqlite.modelOptions.indexes).toHaveLength(1); + }); + + it('keeps postgres-only types in the pg converter', () => { + const schema = new ConduitSchema( + 'User', + { email: { type: TYPE.String } }, + { + indexes: [{ fields: ['email'], types: PostgresIndexType.GIN }], + }, + ); + const [pg] = pgSchemaConverter(schema); + expect(pg.modelOptions.indexes).toHaveLength(1); + expect((pg.modelOptions.indexes![0] as ConvertedSqlIndex).using).toBe( + PostgresIndexType.GIN, + ); + }); + + it('gives distinct generated names to two schemas with the same fields', () => { + const permission = convertModelOptionsIndexes( + new ConduitSchema( + 'Permission', + { resource: { type: TYPE.String } }, + { indexes: [{ fields: ['resource'], types: [CompatibleIndexType.Ascending] }] }, + ), + 'postgres', + ); + const relationship = convertModelOptionsIndexes( + new ConduitSchema( + 'Relationship', + { resource: { type: TYPE.String } }, + { indexes: [{ fields: ['resource'], types: [CompatibleIndexType.Ascending] }] }, + ), + 'postgres', + ); + expect((permission.modelOptions.indexes![0] as { name?: string }).name).toBe( + 'cnd_idx_Permission_resource_asc', + ); + expect((relationship.modelOptions.indexes![0] as { name?: string }).name).toBe( + 'cnd_idx_Relationship_resource_asc', + ); + }); + + it('skips MySQL JSON field-level indexes and keeps them on postgres', () => { + const mysql = convertSchemaFieldIndexes( + new ConduitSchema( + 'ObjectIndex', + { + inheritanceTree: { + type: [TYPE.String], + index: { type: CompatibleIndexType.Ascending }, + }, + }, + {}, + ), + 'mysql', + ); + const postgres = convertSchemaFieldIndexes( + new ConduitSchema( + 'ObjectIndex', + { + inheritanceTree: { + type: [TYPE.String], + index: { type: CompatibleIndexType.Ascending }, + }, + }, + {}, + ), + 'postgres', + ); + expect(mysql.modelOptions.indexes).toHaveLength(0); + expect(postgres.modelOptions.indexes).toHaveLength(1); + }); + + it('skips extracted array-relation indexes on SQL and keeps scalar relation compounds', () => { + const chatRoom = new ConduitSchema( + 'ChatRoom', + { + participants: [{ type: TYPE.Relation, model: 'User', required: true }], + deleted: { type: TYPE.Boolean }, + }, + { + timestamps: true, + indexes: [ + { fields: ['participants'], types: [CompatibleIndexType.Ascending] }, + { + fields: ['participants', 'deleted'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + }, + ); + const [mysql] = sqlSchemaConverter(chatRoom, 'mysql'); + const [pg] = pgSchemaConverter(chatRoom); + expect(mysql.modelOptions.indexes ?? []).toHaveLength(0); + expect(pg.modelOptions.indexes ?? []).toHaveLength(0); + + const message = new ConduitSchema( + 'Message', + { + room: { type: TYPE.Relation, model: 'ChatRoom', required: true }, + deleted: { type: TYPE.Boolean }, + createdAt: { type: TYPE.Date }, + }, + { + timestamps: true, + indexes: [ + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + }, + ); + const [mysqlMessage] = sqlSchemaConverter(message, 'mysql'); + const [pgMessage] = pgSchemaConverter(message); + const expectedFields = [ + { name: 'roomId', order: 'ASC' }, + { name: 'createdAt', order: 'ASC' }, + ]; + expect(mysqlMessage.modelOptions.indexes).toHaveLength(1); + expect(pgMessage.modelOptions.indexes).toHaveLength(1); + expect((mysqlMessage.modelOptions.indexes![0] as ConvertedSqlIndex).fields).toEqual( + expectedFields, + ); + expect((pgMessage.modelOptions.indexes![0] as ConvertedSqlIndex).fields).toEqual( + expectedFields, + ); + expect((mysqlMessage.modelOptions.indexes![0] as { name?: string }).name).toBe( + 'cnd_idx_Message_room_createdAt_asc_asc', + ); + expect((pgMessage.modelOptions.indexes![0] as { name?: string }).name).toBe( + 'cnd_idx_Message_room_createdAt_asc_asc', + ); + + const scalar = new ConduitSchema( + 'Message', + { + deleted: { type: TYPE.Boolean }, + createdAt: { type: TYPE.Date }, + }, + { + timestamps: true, + indexes: [ + { + fields: ['deleted', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + }, + ); + const [mysqlScalar] = sqlSchemaConverter(scalar, 'mysql'); + const [pgScalar] = pgSchemaConverter(scalar); + expect(mysqlScalar.modelOptions.indexes).toHaveLength(1); + expect(pgScalar.modelOptions.indexes).toHaveLength(1); + }); + + it('lifts field-level scalar Relation indexes before extract and skips field-level arrays', () => { + const message = new ConduitSchema( + 'Message', + { + room: { + type: TYPE.Relation, + model: 'ChatRoom', + required: true, + index: { type: CompatibleIndexType.Ascending }, + }, + createdAt: { type: TYPE.Date }, + }, + { timestamps: true }, + ); + const [pg] = pgSchemaConverter(message); + const [mysql] = sqlSchemaConverter(message, 'mysql'); + expect(pg.modelOptions.indexes).toHaveLength(1); + expect(mysql.modelOptions.indexes).toHaveLength(1); + expect((pg.modelOptions.indexes![0] as { name?: string }).name).toBe( + 'cnd_idx_Message_room_asc', + ); + expect((pg.modelOptions.indexes![0] as ConvertedSqlIndex).fields).toEqual([ + { name: 'roomId', order: 'ASC' }, + ]); + expect(pg.fields.room).toBeUndefined(); + + const chatRoom = new ConduitSchema( + 'ChatRoom', + { + participants: { + type: [{ type: TYPE.Relation, model: 'User', required: true }], + index: { type: CompatibleIndexType.Ascending }, + }, + deleted: { type: TYPE.Boolean }, + }, + { timestamps: true }, + ); + const [pgRoom] = pgSchemaConverter(chatRoom); + const [mysqlRoom] = sqlSchemaConverter(chatRoom, 'mysql'); + expect(pgRoom.modelOptions.indexes ?? []).toHaveLength(0); + expect(mysqlRoom.modelOptions.indexes ?? []).toHaveLength(0); + }); + + it('does not rewrite Authz String *Id fields to a second Id suffix', () => { + const permission = convertModelOptionsIndexes( + new ConduitSchema( + 'Permission', + { + resource: { type: TYPE.String }, + resourceId: { type: TYPE.String }, + }, + { indexes: [{ fields: ['resource'], types: [CompatibleIndexType.Ascending] }] }, + ), + 'postgres', + ); + const index = permission.modelOptions.indexes![0] as ConvertedSqlIndex & { + name?: string; + }; + expect(index.name).toBe('cnd_idx_Permission_resource_asc'); + expect(index.fields[0]).toEqual({ name: 'resource', order: 'ASC' }); + }); +}); + +describe('mongoose SchemaConverter indexes', () => { + beforeEach(() => { + jest.spyOn(ConduitGrpcSdk.Logger, 'warn').mockImplementation(() => undefined); + }); + + it('treats Compatible types as Mongo 1/-1', () => { + const converted = schemaConverter( + new ConduitSchema( + 'User', + { + email: { + type: TYPE.String, + index: { type: CompatibleIndexType.Descending }, + }, + }, + {}, + ), + ); + expect( + (converted.fields.email as { index: { type: MongoIndexType } }).index.type, + ).toBe(MongoIndexType.Descending); + }); + + it('warns and skips postgres leftovers on Mongo', () => { + const warn = ConduitGrpcSdk.Logger.warn as jest.Mock; + const converted = schemaConverter( + new ConduitSchema( + 'User', + { + email: { + type: TYPE.String, + index: { type: PostgresIndexType.GIN }, + }, + }, + {}, + ), + ); + expect((converted.fields.email as { index?: unknown }).index).toBeUndefined(); + expect(warn).toHaveBeenCalled(); + }); + + it('maps the Compatible string Ascending, not a Mongo enum key', () => { + const converted = schemaConverter( + new ConduitSchema( + 'User', + { + email: { + type: TYPE.String, + index: { type: CompatibleIndexType.Ascending }, + }, + }, + {}, + ), + ); + expect( + (converted.fields.email as { index: { type: MongoIndexType } }).index.type, + ).toBe(MongoIndexType.Ascending); + }); + + it('keeps single-field array indexes on modelOptions with compounds', () => { + const converted = schemaConverter( + new ConduitSchema( + 'ChatRoom', + { + participants: [{ type: TYPE.Relation, model: 'User', required: true }], + deleted: { type: TYPE.Boolean }, + }, + { + indexes: [ + { fields: ['participants'], types: [CompatibleIndexType.Ascending] }, + { + fields: ['participants', 'deleted'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + }, + ), + ); + const remaining = converted.modelOptions.indexes ?? []; + expect(remaining.map(index => index.fields)).toEqual([ + ['participants'], + ['participants', 'deleted'], + ]); + }); + + it('lifts field-level array Compatible indexes onto modelOptions', () => { + const converted = schemaConverter( + new ConduitSchema( + 'ObjectIndex', + { + inheritanceTree: { + type: [TYPE.String], + default: [], + index: { type: CompatibleIndexType.Ascending }, + }, + }, + {}, + ), + ); + expect( + (converted.fields.inheritanceTree as { index?: unknown }).index, + ).toBeUndefined(); + expect(converted.modelOptions.indexes?.map(index => index.fields)).toEqual([ + ['inheritanceTree'], + ]); + }); +}); diff --git a/modules/database/src/__tests__/indexes/helpers.test.ts b/modules/database/src/__tests__/indexes/helpers.test.ts new file mode 100644 index 000000000..8544398c7 --- /dev/null +++ b/modules/database/src/__tests__/indexes/helpers.test.ts @@ -0,0 +1,617 @@ +import { describe, expect, it } from '@jest/globals'; +import { status } from '@grpc/grpc-js'; +import { + CompatibleIndexType, + MongoIndexType, + PostgresIndexType, + TYPE, +} from '@conduitplatform/grpc-sdk'; +import { + assertUniqueIndexPrivilege, + bindDeclaredIndexesToLive, + canonicalizeDeclaredIndexFields, + collectExistingIndexNames, + collectSchemaIndexFields, + ensureIndexName, + generateIndexName, + indexIdentity, + isCompatibleIndexType, + isIndexAlreadyExistsError, + isMongoIndexType, + isMongoNamespaceMissingError, + keepDeclaredIndexExtras, + liveNameConflictAllowsReuse, + mapCompatibleToMongo, + mapCompatibleToSqlOrder, + mapIndexFieldsToDeclared, + mapIndexFieldsToSqlEngine, + mergeDeclaredIndexes, + mongoAllowsIndexType, + overlayDeclaredOnLive, + persistDeclaredSchemaIndexes, + removeDeclaredIndexes, + removeIndexFromSchemaFields, + resolveIndexName, + sqlDeclaredIndexFieldName, + sqlDialectAllowsIndexType, + sqlEngineIndexFieldName, + sqlIndexFieldNormalizer, + sqlIndexFields, + sqlIndexUnsupportedReason, + validateIndexFields, +} from '../../adapters/utils/indexes.js'; + +describe('index helpers', () => { + it('keeps CompatibleIndexType as portable strings, not Mongo 1/-1', () => { + expect(CompatibleIndexType.Ascending).toBe('Ascending'); + expect(CompatibleIndexType.Descending).toBe('Descending'); + expect(CompatibleIndexType.Ascending).not.toBe(MongoIndexType.Ascending); + expect(isCompatibleIndexType(CompatibleIndexType.Ascending)).toBe(true); + expect(isCompatibleIndexType(1)).toBe(false); + expect(isMongoIndexType('Ascending')).toBe(false); + }); + + it('generates a deterministic name unique per collection', () => { + const name = generateIndexName( + ['email'], + [CompatibleIndexType.Ascending], + false, + 'cnd_User', + ); + expect(name).toBe('cnd_idx_cnd_User_email_asc'); + const permission = generateIndexName( + ['resource'], + [CompatibleIndexType.Ascending], + false, + 'cnd_Permission', + ); + const relationship = generateIndexName( + ['resource'], + [CompatibleIndexType.Ascending], + false, + 'cnd_Relationship', + ); + expect(permission).toBe('cnd_idx_cnd_Permission_resource_asc'); + expect(relationship).toBe('cnd_idx_cnd_Relationship_resource_asc'); + expect(permission).not.toBe(relationship); + const unique = generateIndexName( + ['room', 'createdAt'], + [CompatibleIndexType.Ascending, CompatibleIndexType.Descending], + true, + 'cnd_User', + ); + expect(unique).toBe( + generateIndexName( + ['room', 'createdAt'], + [CompatibleIndexType.Ascending, CompatibleIndexType.Descending], + true, + 'cnd_User', + ), + ); + expect(unique).toMatch(/^cnd_uidx_/); + expect(unique).toContain('cnd_User'); + }); + + it('hashes long names with collectionName in the identity', () => { + const fields = ['one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight']; + const left = generateIndexName(fields, undefined, false, 'cnd_VeryLongTableNameOne'); + const right = generateIndexName(fields, undefined, false, 'cnd_VeryLongTableNameTwo'); + expect(left.length).toBeLessThanOrEqual(63); + expect(right.length).toBeLessThanOrEqual(63); + expect(left).not.toBe(right); + }); + + it('keeps a provided name on the index and options', () => { + const named = ensureIndexName( + { + fields: ['email'], + name: 'custom_email_idx', + }, + 'cnd_User', + ); + expect(resolveIndexName(named)).toBe('custom_email_idx'); + expect(named.options?.name).toBe('custom_email_idx'); + }); + + it('maps Compatible and Mongo directions to engine types', () => { + expect(mapCompatibleToMongo(CompatibleIndexType.Ascending)).toBe(1); + expect(mapCompatibleToMongo(CompatibleIndexType.Descending)).toBe(-1); + expect(mapCompatibleToMongo(undefined)).toBe(1); + expect(mapCompatibleToSqlOrder(CompatibleIndexType.Ascending)).toBe('ASC'); + expect(mapCompatibleToSqlOrder(CompatibleIndexType.Descending)).toBe('DESC'); + expect(sqlIndexFields({ fields: ['createdAt', 'room'] })).toEqual([ + 'createdAt', + 'room', + ]); + expect( + sqlIndexFields({ + fields: ['createdAt', 'room'], + types: [CompatibleIndexType.Descending, CompatibleIndexType.Ascending], + }), + ).toEqual([ + { name: 'createdAt', order: 'DESC' }, + { name: 'room', order: 'ASC' }, + ]); + expect( + sqlIndexFields({ + fields: ['createdAt'], + types: [MongoIndexType.Descending], + }), + ).toEqual([{ name: 'createdAt', order: 'DESC' }]); + }); + + it('preserves unique when generating a name', () => { + const unique = ensureIndexName( + { + fields: ['email'], + types: [CompatibleIndexType.Ascending], + options: { unique: true }, + }, + 'cnd_User', + ); + expect(unique.options?.unique).toBe(true); + expect(resolveIndexName(unique)).toMatch(/uidx/); + }); + + it('merges declared indexes by name without overwriting the first', () => { + const merged = mergeDeclaredIndexes( + [{ fields: ['a'], name: 'idx_a' }], + [ + { fields: ['a'], name: 'idx_a', options: { unique: true } }, + { fields: ['b'], name: 'idx_b' }, + ], + 'cnd_User', + ); + expect(merged.map(index => index.name)).toEqual(['idx_a', 'idx_b']); + expect(merged[0].options?.unique).toBeUndefined(); + }); + + it('removes declared indexes and field-level index metadata by name', () => { + expect( + removeDeclaredIndexes( + [ + { fields: ['a'], name: 'idx_a' }, + { fields: ['b'], name: 'idx_b' }, + ], + ['idx_a'], + ), + ).toEqual([{ fields: ['b'], name: 'idx_b' }]); + expect( + removeIndexFromSchemaFields({ fields: { a: { index: { name: 'x' } } } }, 'x'), + ).toBe(true); + }); + + it('rejects unknown index fields', () => { + expect(() => + validateIndexFields( + { compiledFields: { email: 'String' }, fields: {} }, + { fields: ['missing'] }, + ), + ).toThrow(/Invalid fields/); + }); + + it('enforces unique-index privilege for owner, Admin, and import', () => { + expect(() => + assertUniqueIndexPrivilege({ + unique: true, + schemaOwner: 'chat', + callerModule: 'database', + privileged: false, + }), + ).toThrow(expect.objectContaining({ code: status.PERMISSION_DENIED })); + expect(() => + assertUniqueIndexPrivilege({ + unique: true, + schemaOwner: 'chat', + callerModule: 'chat', + privileged: false, + }), + ).not.toThrow(); + expect(() => + assertUniqueIndexPrivilege({ + unique: true, + schemaOwner: 'chat', + callerModule: 'database', + privileged: true, + }), + ).not.toThrow(); + expect(() => + assertUniqueIndexPrivilege({ + unique: true, + schemaOwner: 'authorization', + callerModule: 'database', + privileged: false, + }), + ).toThrow(/Not authorized to create unique index/); + }); + + it('collects existing names and detects name/relation already-exists errors only', () => { + expect( + collectExistingIndexNames([{ fields: ['a'], options: { name: 'idx_a' } }]).has( + 'idx_a', + ), + ).toBe(true); + expect( + isIndexAlreadyExistsError({ + code: '42P07', + message: 'relation "x" already exists', + }), + ).toBe(true); + expect( + isIndexAlreadyExistsError({ code: '1061', message: "Duplicate key name 'x'" }), + ).toBe(true); + expect( + isIndexAlreadyExistsError({ + original: { code: '42P07', message: 'relation "idx" already exists' }, + }), + ).toBe(true); + expect(isIndexAlreadyExistsError(new Error('index foo already exists'))).toBe(true); + expect(isIndexAlreadyExistsError(new Error('already exists'))).toBe(false); + expect( + isIndexAlreadyExistsError({ code: 11000, message: 'E11000 duplicate key' }), + ).toBe(false); + expect( + isIndexAlreadyExistsError({ + name: 'SequelizeUniqueConstraintError', + original: { code: '23505' }, + }), + ).toBe(false); + expect( + isIndexAlreadyExistsError({ + code: 85, + message: 'Index with name: x already exists', + }), + ).toBe(false); + expect( + isIndexAlreadyExistsError({ + code: 86, + message: 'Index already exists with different options', + }), + ).toBe(false); + }); + + it('binds unnamed declared indexes to live names by fields+unique', () => { + const bound = bindDeclaredIndexesToLive( + [ + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + [ + { + name: '_id_', + fields: ['_id'], + options: { name: '_id_' }, + }, + { + name: 'room_1_createdAt_1', + fields: ['room', 'createdAt'], + options: { name: 'room_1_createdAt_1', unique: false }, + }, + ], + 'cnd_User', + ); + expect(resolveIndexName(bound[0])).toBe('room_1_createdAt_1'); + expect(indexIdentity(bound[0])).toEqual({ + fields: ['room', 'createdAt'], + unique: false, + }); + }); + + it('treats unique vs non-unique as different identities', () => { + const bound = bindDeclaredIndexesToLive( + [{ fields: ['email'], options: { unique: true } }], + [ + { + name: 'email_1', + fields: ['email'], + options: { name: 'email_1', unique: false }, + }, + ], + 'cnd_User', + ); + expect(resolveIndexName(bound[0])).not.toBe('email_1'); + expect(resolveIndexName(bound[0])).toMatch(/uidx/); + }); + + it('keeps Admin extras and drops stale generated names for the same identity', () => { + const unioned = keepDeclaredIndexExtras( + [ + { + fields: ['room', 'createdAt'], + name: 'room_1_createdAt_1', + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + [ + { + fields: ['room', 'createdAt'], + name: 'cnd_idx_room_createdAt_asc_asc', + }, + { fields: ['email'], name: 'admin_email_idx' }, + ], + 'cnd_User', + ); + expect(unioned.map(index => index.name)).toEqual([ + 'room_1_createdAt_1', + 'admin_email_idx', + ]); + }); + + it('overlays declared Compatible types onto live indexes by identity when names differ', () => { + const overlaid = overlayDeclaredOnLive( + { + name: 'room_1_createdAt_1', + fields: ['room', 'createdAt'], + types: [MongoIndexType.Ascending, MongoIndexType.Ascending], + options: { name: 'room_1_createdAt_1' }, + }, + [ + { + fields: ['room', 'createdAt'], + name: 'cnd_idx_room_createdAt_asc_asc', + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + ); + expect(overlaid.types).toEqual([ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ]); + expect(overlaid.name).toBe('room_1_createdAt_1'); + }); + + it('persists applied indexes against a re-read declared schema list', async () => { + let findOneOptions: unknown; + const findOne = async (_query: Record, options?: unknown) => { + findOneOptions = options; + return { + _id: 'declared-1', + modelOptions: { indexes: [{ fields: ['keep'], name: 'keep_me' }] }, + }; + }; + const findByIdAndUpdate = async () => ({}); + const originalSchema = { + modelOptions: { indexes: [] as { fields: string[]; name?: string }[] }, + }; + const persisted = await persistDeclaredSchemaIndexes({ + declaredSchemaModel: { findOne, findByIdAndUpdate }, + schemaName: 'User', + originalSchema, + applied: [{ fields: ['email'], name: 'cnd_idx_email_asc' }], + }); + expect(persisted).toBe(true); + expect(findOneOptions).toEqual({ readPreference: 'primary' }); + expect(originalSchema.modelOptions.indexes.map(index => index.name)).toEqual([ + 'keep_me', + 'cnd_idx_email_asc', + ]); + }); + + it('reuses a live name-conflict only when identity matches', () => { + const live = [ + { + name: 'user_idx', + fields: ['email'], + options: { name: 'user_idx', unique: false }, + }, + ]; + expect( + liveNameConflictAllowsReuse( + { name: 'user_idx', fields: ['email'], options: { name: 'user_idx' } }, + live, + ), + ).toBe(true); + expect( + liveNameConflictAllowsReuse( + { name: 'user_idx', fields: ['username'], options: { name: 'user_idx' } }, + live, + ), + ).toBe(false); + expect( + liveNameConflictAllowsReuse( + { + name: 'user_idx', + fields: ['email'], + options: { name: 'user_idx', unique: true }, + }, + live, + ), + ).toBe(false); + expect( + liveNameConflictAllowsReuse( + { name: 'user_idx', fields: ['email'], options: { name: 'user_idx' } }, + [], + ), + ).toBe(false); + }); + + it('allows dialect-native types and rejects foreign leftovers', () => { + expect(sqlDialectAllowsIndexType('mysql', CompatibleIndexType.Ascending)).toBe(true); + expect(sqlDialectAllowsIndexType('mariadb', CompatibleIndexType.Descending)).toBe( + true, + ); + expect(sqlDialectAllowsIndexType('sqlite', PostgresIndexType.BTREE)).toBe(true); + expect(sqlDialectAllowsIndexType('sqlite', PostgresIndexType.HASH)).toBe(false); + expect(sqlDialectAllowsIndexType('mysql', PostgresIndexType.GIST)).toBe(false); + expect(sqlDialectAllowsIndexType('postgres', PostgresIndexType.GIN)).toBe(true); + expect(mongoAllowsIndexType(CompatibleIndexType.Ascending)).toBe(true); + expect(mongoAllowsIndexType(PostgresIndexType.BTREE)).toBe(false); + }); + + it('adopts a live old global name and generates a table-qualified name when unmatched', () => { + const adopted = bindDeclaredIndexesToLive( + [{ fields: ['resource'], types: [CompatibleIndexType.Ascending] }], + [ + { + name: 'cnd_idx_resource_asc', + fields: ['resource'], + options: { name: 'cnd_idx_resource_asc', unique: false }, + }, + ], + 'cnd_Permission', + ); + expect(resolveIndexName(adopted[0])).toBe('cnd_idx_resource_asc'); + const generated = bindDeclaredIndexesToLive( + [{ fields: ['resource'], types: [CompatibleIndexType.Ascending] }], + [], + 'cnd_Relationship', + ); + expect(resolveIndexName(generated[0])).toBe('cnd_idx_cnd_Relationship_resource_asc'); + }); + + it('does not treat a missing-ns Mongo error as a live name conflict', () => { + expect( + isMongoNamespaceMissingError({ + code: 26, + codeName: 'NamespaceNotFound', + message: 'ns does not exist: test.cnd_adminapitokens', + }), + ).toBe(true); + expect(isMongoNamespaceMissingError({ message: 'unauthorized' })).toBe(false); + }); + + it('reports unsupported SQL JSON and extracted-relation indexes', () => { + expect( + sqlIndexUnsupportedReason( + 'mysql', + { fields: ['inheritanceTree'] }, + { inheritanceTree: { type: [TYPE.String] } }, + ), + ).toMatch(/MySQL JSON field 'inheritanceTree'/); + expect( + sqlIndexUnsupportedReason( + 'postgres', + { fields: ['inheritanceTree'] }, + { inheritanceTree: { type: [TYPE.String] } }, + ), + ).toBeUndefined(); + expect( + sqlIndexUnsupportedReason( + 'postgres', + { fields: ['participants'] }, + { + participants: [{ type: TYPE.Relation, model: 'User' }], + }, + ), + ).toMatch(/relation join table/); + expect( + sqlIndexUnsupportedReason( + 'postgres', + { fields: ['room', 'createdAt'] }, + { + room: { type: TYPE.Relation, model: 'ChatRoom' }, + createdAt: { type: TYPE.Date }, + }, + { timestamps: true }, + ), + ).toBeUndefined(); + expect( + sqlIndexUnsupportedReason( + 'mysql', + { fields: ['email'] }, + { email: { type: TYPE.String } }, + ), + ).toBeUndefined(); + }); + + it('maps declared scalar relations to engine *Id and never Authz String *Id fields', () => { + const fields = collectSchemaIndexFields({ + fields: { + room: { type: TYPE.Relation, model: 'ChatRoom' }, + createdAt: { type: TYPE.Date }, + resource: { type: TYPE.String }, + resourceId: { type: TYPE.String }, + subject: { type: TYPE.String }, + subjectId: { type: TYPE.String }, + }, + }); + expect(sqlEngineIndexFieldName('room', fields)).toBe('roomId'); + expect(sqlDeclaredIndexFieldName('roomId', fields)).toBe('room'); + expect(sqlEngineIndexFieldName('resource', fields)).toBe('resource'); + expect(sqlEngineIndexFieldName('resourceId', fields)).toBe('resourceId'); + expect(sqlDeclaredIndexFieldName('resourceId', fields)).toBe('resourceId'); + expect(sqlDeclaredIndexFieldName('subjectId', fields)).toBe('subjectId'); + expect(mapIndexFieldsToSqlEngine({ fields: ['room', 'createdAt'] }, fields)).toEqual([ + 'roomId', + 'createdAt', + ]); + expect(mapIndexFieldsToDeclared({ fields: ['roomId', 'createdAt'] }, fields)).toEqual( + ['room', 'createdAt'], + ); + expect( + canonicalizeDeclaredIndexFields({ fields: ['roomId', 'createdAt'] }, fields).fields, + ).toEqual(['room', 'createdAt']); + expect( + canonicalizeDeclaredIndexFields({ fields: ['resourceId'] }, fields).fields, + ).toEqual(['resourceId']); + }); + + it('generates index names from declared fields, not engine *Id columns', () => { + expect( + generateIndexName( + ['room', 'createdAt'], + [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + false, + 'cnd_ChatMessage', + ), + ).toBe('cnd_idx_cnd_ChatMessage_room_createdAt_asc_asc'); + expect( + generateIndexName( + ['roomId', 'createdAt'], + [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + false, + 'cnd_ChatMessage', + ), + ).not.toBe('cnd_idx_cnd_ChatMessage_room_createdAt_asc_asc'); + }); + + it('binds live engine roomId to declared room without renaming Authz String *Id', () => { + const fields = { + room: { type: TYPE.Relation, model: 'ChatRoom' }, + createdAt: { type: TYPE.Date }, + resource: { type: TYPE.String }, + resourceId: { type: TYPE.String }, + }; + const normalize = sqlIndexFieldNormalizer(fields); + const bound = bindDeclaredIndexesToLive( + [ + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + [ + { + name: 'roomId_createdAt', + fields: ['roomId', 'createdAt'], + options: { name: 'roomId_createdAt', unique: false }, + }, + ], + 'cnd_ChatMessage', + normalize, + ); + expect(resolveIndexName(bound[0])).toBe('roomId_createdAt'); + expect(bound[0].fields).toEqual(['room', 'createdAt']); + expect(indexIdentity(bound[0])).toEqual({ + fields: ['room', 'createdAt'], + unique: false, + }); + + const authz = bindDeclaredIndexesToLive( + [{ fields: ['resource'], types: [CompatibleIndexType.Ascending] }], + [ + { + name: 'cnd_idx_cnd_Permission_resource_asc', + fields: ['resource'], + options: { name: 'cnd_idx_cnd_Permission_resource_asc', unique: false }, + }, + ], + 'cnd_Permission', + normalize, + ); + expect(resolveIndexName(authz[0])).toBe('cnd_idx_cnd_Permission_resource_asc'); + expect(authz[0].fields).toEqual(['resource']); + }); +}); diff --git a/modules/database/src/__tests__/indexes/regressions.test.ts b/modules/database/src/__tests__/indexes/regressions.test.ts new file mode 100644 index 000000000..385cbd536 --- /dev/null +++ b/modules/database/src/__tests__/indexes/regressions.test.ts @@ -0,0 +1,82 @@ +import { existsSync, readFileSync } from 'fs'; +import { resolve } from 'path'; +import { describe, expect, it } from '@jest/globals'; + +function repoFile(...parts: string[]) { + const candidates = [ + resolve(process.cwd(), '..', ...parts), + resolve(process.cwd(), ...parts), + resolve(process.cwd(), '../..', ...parts), + ]; + const found = candidates.find(existsSync); + if (!found) throw new Error(`Missing ${parts.join('/')}`); + return found; +} + +describe('platform models use CompatibleIndexType', () => { + it('authz and chat schemas declare Compatible indexes, not Mongo-only types', () => { + const files = [ + repoFile('authorization', 'src', 'models', 'Permission.schema.ts'), + repoFile('authorization', 'src', 'models', 'Relationship.schema.ts'), + repoFile('authorization', 'src', 'models', 'ActorIndex.schema.ts'), + repoFile('authorization', 'src', 'models', 'ObjectIndex.schema.ts'), + repoFile('chat', 'src', 'models', 'ChatRoom.schema.ts'), + repoFile('chat', 'src', 'models', 'Message.schema.ts'), + ]; + for (const file of files) { + const source = readFileSync(file, 'utf8'); + expect(source).toContain('CompatibleIndexType'); + expect(source).not.toContain('MongoIndexType'); + } + }); +}); + +describe('do not port old PR #643 bugs', () => { + it("does not use `case 'mysql' || 'mariadb'`", () => { + const files = [ + resolve(process.cwd(), 'src/adapters/utils/indexes.ts'), + resolve(process.cwd(), 'src/adapters/sequelize-adapter/index.ts'), + resolve(process.cwd(), 'src/adapters/utils/database-transform-utils.ts'), + ]; + for (const file of files) { + const source = readFileSync(file, 'utf8'); + expect(source).not.toMatch(/case ['"]mysql['"]\s*\|\|/); + } + }); + + it('does not rename getDatabaseType PostgreSQL to postgres', () => { + const source = readFileSync( + resolve(process.cwd(), 'src/adapters/sequelize-adapter/index.ts'), + 'utf8', + ); + expect(source).toContain("return 'PostgreSQL'"); + }); + + it('does not use metadata-only getIndexes or createSchemaFromAdapter rebuild for indexes', () => { + const sequelize = readFileSync( + resolve(process.cwd(), 'src/adapters/sequelize-adapter/index.ts'), + 'utf8', + ); + expect(sequelize).toContain('showIndex'); + expect(sequelize).toContain('addIndex'); + expect(sequelize).not.toMatch(/createIndexes[\s\S]*createSchemaFromAdapter/); + expect(sequelize).not.toMatch(/await this\.models\[schemaName\]\.sync\(\)/); + }); + + it('skips index creation on replica instanceSync paths', () => { + const mongoose = readFileSync( + resolve(process.cwd(), 'src/adapters/mongoose-adapter/index.ts'), + 'utf8', + ); + const sequelize = readFileSync( + resolve(process.cwd(), 'src/adapters/sequelize-adapter/index.ts'), + 'utf8', + ); + expect(mongoose).toMatch(/if \(indexes && !isInstanceSync\)/); + expect(mongoose).toMatch( + /if \(!isInstanceSync\) \{\s*await this\.createMongooseFieldIndexes/, + ); + expect(sequelize).toMatch(/isInstanceSync/); + expect(sequelize).toMatch(/noSync/); + }); +}); diff --git a/modules/database/src/__tests__/sequelize-relations.test.ts b/modules/database/src/__tests__/sequelize-relations.test.ts new file mode 100644 index 000000000..eacd60cf7 --- /dev/null +++ b/modules/database/src/__tests__/sequelize-relations.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from '@jest/globals'; +import { ConduitDatabaseSchema } from '../interfaces/index.js'; +import { resolveRelatedSchemas } from '../adapters/sequelize-adapter/utils/schema.js'; + +describe('resolveRelatedSchemas', () => { + it('accepts a related model that exists but is not yet synced', async () => { + const chatRoom = { synced: false }; + const models = { + ChatRoom: chatRoom, + ChatParticipantsLog: { synced: false }, + }; + const started = Date.now(); + const related = await resolveRelatedSchemas( + { name: 'ChatParticipantsLog' } as ConduitDatabaseSchema, + { chatRoom: { type: 'Relation', model: 'ChatRoom' } }, + models, + ); + expect(Date.now() - started).toBeLessThan(400); + expect(related.chatRoom).toBe(chatRoom); + }); + + it('resolves a cyclic peer that appears in models before sync', async () => { + const chatRoom = { synced: false }; + const models: Record = { ChatRoom: chatRoom }; + const pending = resolveRelatedSchemas( + { name: 'ChatRoom' } as ConduitDatabaseSchema, + { + participantsLog: [{ type: 'Relation', model: 'ChatParticipantsLog' }], + }, + models, + ); + const chatParticipantsLog = { synced: false }; + models.ChatParticipantsLog = chatParticipantsLog; + await expect(pending).resolves.toEqual({ + participantsLog: [chatParticipantsLog], + }); + }); +}); diff --git a/modules/database/src/adapters/DatabaseAdapter.ts b/modules/database/src/adapters/DatabaseAdapter.ts index 96d33b4dc..85c7a3314 100644 --- a/modules/database/src/adapters/DatabaseAdapter.ts +++ b/modules/database/src/adapters/DatabaseAdapter.ts @@ -24,6 +24,11 @@ import { Schema, } from '../interfaces/index.js'; import { stitchSchema, validateExtensionFields } from './utils/extensions.js'; +import { + indexNameCollection, + keepDeclaredIndexExtras, + persistDeclaredSchemaIndexes, +} from './utils/indexes.js'; import { status } from '@grpc/grpc-js'; import { isEqual, isNil } from 'lodash-es'; import ObjectHash from 'object-hash'; @@ -204,8 +209,9 @@ export abstract class DatabaseAdapter { abstract createIndexes( schemaName: string, - indexes: ModelOptionsIndexes[], + indexes: readonly ModelOptionsIndexes[], callerModule: string, + options?: { privileged?: boolean }, ): Promise; abstract getIndexes(schemaName: string): Promise; @@ -600,6 +606,31 @@ export abstract class DatabaseAdapter { instanceSync: boolean, ): Promise; + protected snapshotRegisteredSchema(schema: ConduitSchema) { + this.registeredSchemas.set( + schema.name, + Object.freeze(JSON.parse(JSON.stringify(schema))), + ); + } + + protected async persistIndexesAndPublish(args: { + schemaName: string; + originalSchema: ConduitDatabaseSchema; + applied?: ModelOptionsIndexes[]; + droppedNames?: string[]; + }): Promise { + if (!this.models['_DeclaredSchema']) return false; + const persisted = await persistDeclaredSchemaIndexes({ + declaredSchemaModel: this.models['_DeclaredSchema'], + schemaName: args.schemaName, + originalSchema: args.originalSchema, + applied: args.applied, + droppedNames: args.droppedNames, + }); + if (persisted) this.publishSchema(args.originalSchema); + return persisted; + } + protected async saveSchemaToDatabase(schema: ConduitSchema) { if (schema.name === '_DeclaredSchema') return; const model = await this.models['_DeclaredSchema'].findOne( @@ -607,6 +638,11 @@ export abstract class DatabaseAdapter { { readPreference: 'primary' }, ); if (model) { + schema.modelOptions.indexes = keepDeclaredIndexExtras( + schema.modelOptions.indexes ?? [], + model.modelOptions?.indexes, + indexNameCollection(schema), + ); await this.models['_DeclaredSchema'].findByIdAndUpdate(model._id, { name: schema.name, fields: schema.fields, diff --git a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts index fe9dab7ba..e087905c2 100644 --- a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts +++ b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts @@ -1,5 +1,6 @@ import { Schema } from 'mongoose'; import { + ConduitGrpcSdk, ConduitModelField, ConduitSchema, ModelOptionsIndexes, @@ -10,6 +11,13 @@ import { cloneDeep, isArray, isNil, isObject } from 'lodash-es'; import { checkIfMongoOptions } from './utils.js'; import { applyMongoVectorField } from '../utils/vectorMappings.js'; import { isVectorTypeName } from '../utils/vectorField.js'; +import { + isArrayLikeConduitField, + isCompatibleIndexType, + mapCompatibleToMongo, + mongoAllowsIndexType, + normalizeIndexTypes, +} from '../utils/indexes.js'; import * as deepdash from 'deepdash-es/standalone'; @@ -23,10 +31,10 @@ export function schemaConverter(jsonSchema: ConduitSchema) { delete copy.fields['_id']; } copy = convertSchemaFieldIndexes(copy); - deepdash.eachDeep(copy.fields, convert); if (copy.modelOptions.indexes) { copy = convertModelOptionsIndexes(copy); } + deepdash.eachDeep(copy.fields, convert); iterDeep(copy.fields); return copy; } @@ -104,60 +112,94 @@ function convert(value: any, key: any, parentValue: any) { } function convertSchemaFieldIndexes(copy: ConduitSchema) { + const lifted: ModelOptionsIndexes[] = []; for (const field of Object.entries(copy.fields)) { - const index = (field[1] as ConduitModelField).index; + const modelField = field[1] as ConduitModelField; + const index = modelField.index; if (!index) continue; const type = index.type; const options = index.options; - if (type && !Object.values(MongoIndexType).includes(type)) { - throw new Error('Incorrect index type for MongoDB'); + if (type && !mongoAllowsIndexType(type)) { + ConduitGrpcSdk.Logger.warn( + `Invalid index type for MongoDB found in '${copy.name}', ignoring index`, + ); + delete modelField.index; + continue; + } + if (type && isCompatibleIndexType(type)) { + index.type = mapCompatibleToMongo(type); } if (options) { if (!checkIfMongoOptions(options)) { - throw new Error('Incorrect index options for MongoDB'); + ConduitGrpcSdk.Logger.warn( + `Invalid index options for MongoDB found in '${copy.name}', ignoring index`, + ); + delete modelField.index; + continue; } for (const [option, optionValue] of Object.entries(options)) { index[option as keyof SchemaFieldIndex] = optionValue; } delete index.options; } + if (isArrayLikeConduitField(field[1])) { + lifted.push({ + fields: [field[0]], + types: index.type !== undefined ? [index.type] : undefined, + options: { + ...(typeof index.unique === 'boolean' ? { unique: index.unique } : {}), + ...(typeof index.name === 'string' ? { name: index.name } : {}), + }, + name: index.name, + }); + delete modelField.index; + } + } + if (lifted.length) { + copy.modelOptions.indexes = [...(copy.modelOptions.indexes ?? []), ...lifted]; } return copy; } function convertModelOptionsIndexes(copy: ConduitSchema) { if (!copy.modelOptions.indexes?.length) return copy; - const mutIndexes = copy.modelOptions.indexes as ModelOptionsIndexes[]; - for (const index of mutIndexes) { - // compound indexes are maintained in modelOptions in order to be created after schema creation - // single field index => add it to specified schema field - if (index.fields.length !== 1) continue; - const modelField = copy.fields[index.fields[0]] as ConduitModelField; - if (!modelField) { - throw new Error(`Field ${modelField} in index definition doesn't exist`); - } + const remaining: ModelOptionsIndexes[] = []; + for (const index of copy.modelOptions.indexes) { + let mappedTypes: MongoIndexType[] | undefined; if (index.types) { + const types = normalizeIndexTypes(index.types, index.fields.length) ?? []; if ( - !isArray(index.types) || - !Object.values(MongoIndexType).includes(index.types[0]) || - index.fields.length !== index.types.length + types.some(type => !mongoAllowsIndexType(type)) || + (isArray(index.types) && index.fields.length !== index.types.length) ) { - throw new Error('Invalid index type for MongoDB'); + ConduitGrpcSdk.Logger.warn( + `Invalid index type for MongoDB found in '${copy.name}', ignoring index`, + ); + continue; } - const type = index.types[0] as MongoIndexType; - modelField.index = { - type: type, - }; + mappedTypes = types.map(mapCompatibleToMongo); + index.types = mappedTypes; } - if (index.options) { - if (!checkIfMongoOptions(index.options)) { - throw new Error('Incorrect index options for MongoDB'); - } - for (const [option, optionValue] of Object.entries(index.options)) { - modelField.index![option as keyof SchemaFieldIndex] = optionValue; - } + if (index.fields.length !== 1) { + remaining.push(index); + continue; + } + const modelField = copy.fields[index.fields[0]] as ConduitModelField; + if (!modelField || isArrayLikeConduitField(modelField)) { + remaining.push(index); + continue; + } + if (index.options && !checkIfMongoOptions(index.options)) { + ConduitGrpcSdk.Logger.warn( + `Invalid index options for MongoDB found in '${copy.name}', ignoring index`, + ); + continue; } - mutIndexes.splice(mutIndexes.indexOf(index), 1); + modelField.index = { + ...(mappedTypes ? { type: mappedTypes[0] } : {}), + ...index.options, + }; } + copy.modelOptions.indexes = remaining; return copy; } diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 47c3b1a4b..65832aad3 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -30,6 +30,25 @@ import { planMongoVectorIndexCreate, assertMongoVectorSearchIndexDropTarget, } from '../utils/index.js'; +import { + assertUniqueIndexPrivilege, + bindDeclaredIndexesToLive, + ensureIndexName, + findLiveIndex, + indexNameCollection, + isIndexAlreadyExistsError, + isIndexKeySpecsConflictError, + isMongoNamespaceMissingError, + liveIndexFromMongo, + liveNameConflictAllowsReuse, + mapCompatibleToMongo, + mongoAllowsIndexType, + normalizeIndexTypes, + overlayDeclaredOnLive, + removeIndexFromSchemaFields, + toMutableIndexes, + validateIndexFields, +} from '../utils/indexes.js'; import pluralize from '../../utils/pluralize.js'; import { mongoSchemaConverter } from '../../introspection/mongoose/utils.js'; import { status } from '@grpc/grpc-js'; @@ -640,25 +659,84 @@ export class MongooseAdapter extends DatabaseAdapter { schemaName: string, indexes: readonly ModelOptionsIndexes[], callerModule: string, + options?: { privileged?: boolean }, ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - this.checkIndexes(schemaName, indexes, callerModule); + const collectionName = indexNameCollection(this.models[schemaName].originalSchema); + const live = await this.listLiveIndexes(schemaName); + const prepared = bindDeclaredIndexesToLive( + this.checkIndexes(schemaName, indexes, callerModule, options?.privileged), + live, + collectionName, + ); const collection = this.mongoose.model(schemaName).collection; - for (const index of indexes) { - const indexSpecs = []; + const applied: ModelOptionsIndexes[] = []; + let failure: unknown; + for (const index of prepared) { + const existing = findLiveIndex(live, index); + if (existing) { + applied.push(index); + continue; + } + const spec: Record = {}; + const types = normalizeIndexTypes(index.types, index.fields.length); for (let i = 0; i < index.fields.length; i++) { - const spec: any = {}; - spec[index.fields[i]] = index.types ? index.types[i] : 1; - indexSpecs.push(spec); + spec[index.fields[i]] = types + ? mapCompatibleToMongo(types[i]) + : MongoIndexType.Ascending; } - await collection.createIndex(indexSpecs, index.options).catch((e: Error) => { - throw new GrpcError(status.INTERNAL, e.message); + try { + await collection.createIndex(spec, index.options); + applied.push(index); + live.push(index); + } catch (e) { + if (isIndexAlreadyExistsError(e)) { + const relisted = await this.listLiveIndexes(schemaName); + if (liveNameConflictAllowsReuse(index, relisted)) { + applied.push(index); + live.splice(0, live.length, ...relisted); + continue; + } + failure = e; + break; + } + if (isIndexKeySpecsConflictError(e)) { + const relisted = await this.listLiveIndexes(schemaName); + const match = findLiveIndex(relisted, index); + if (match) { + applied.push(bindDeclaredIndexesToLive([index], relisted, collectionName)[0]); + live.splice(0, live.length, ...relisted); + continue; + } + } + failure = e; + break; + } + } + if (!failure || applied.length > 0) { + await this.persistIndexesAndPublish({ + schemaName, + originalSchema: this.models[schemaName].originalSchema, + applied, }); } + if (failure) { + throw new GrpcError(status.INTERNAL, (failure as Error).message); + } return 'Indexes created!'; } + private async listLiveIndexes(schemaName: string): Promise { + try { + const result = await this.mongoose.model(schemaName).collection.indexes(); + return result.map(liveIndexFromMongo); + } catch (e) { + if (isMongoNamespaceMissingError(e)) return []; + throw e; + } + } + private async createMongooseFieldIndexes(schemaName: string): Promise { const model = this.models[schemaName]; if (!model) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); @@ -667,6 +745,7 @@ export class MongooseAdapter extends DatabaseAdapter { const declaredIndexes = model.model.schema.indexes(); if (!declaredIndexes.length) return; + const live = await this.listLiveIndexes(schemaName); const collection = this.mongoose.model(schemaName).collection; for (const [keys, rawOptions] of declaredIndexes) { const indexKeys = keys as IndexSpecification; @@ -675,9 +754,20 @@ export class MongooseAdapter extends DatabaseAdapter { const options = this.sanitizeMongooseIndexOptions( rawOptions as Record, ); - await collection.createIndex(indexKeys, options).catch((e: Error) => { - throw new GrpcError(status.INTERNAL, e.message); - }); + const declared = { + fields: Object.keys((indexKeys as Record) ?? {}), + options: { + unique: options?.unique === true, + name: typeof options?.name === 'string' ? options.name : undefined, + }, + }; + if (findLiveIndex(live, declared)) continue; + try { + await collection.createIndex(indexKeys, options); + } catch (e) { + if (isIndexAlreadyExistsError(e) || isIndexKeySpecsConflictError(e)) continue; + throw new GrpcError(status.INTERNAL, (e as Error).message); + } } } @@ -700,41 +790,68 @@ export class MongooseAdapter extends DatabaseAdapter { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); const collection = this.mongoose.model(schemaName).collection; - const result = await collection.indexes(); - result.filter(index => { - index.options = {}; - for (const indexEntry of Object.entries(index)) { - if (indexEntry[0] === 'key' || indexEntry[0] === 'options') { - continue; - } - if (indexEntry[0] === 'v') { - delete index.v; - continue; - } - index.options[indexEntry[0]] = indexEntry[1]; - delete index[indexEntry[0]]; + let result: Awaited>; + try { + result = await collection.indexes(); + } catch (e) { + if (isMongoNamespaceMissingError(e)) return []; + throw e; + } + const declared = this.models[schemaName].originalSchema.modelOptions.indexes; + return result.map(index => { + const options: Record = {}; + for (const [key, value] of Object.entries(index)) { + if (key === 'key' || key === 'options' || key === 'v') continue; + options[key] = value; } - index.fields = []; - index.types = []; - for (const keyEntry of Object.entries(index.key)) { - index.fields.push(keyEntry[0]); - index.types.push(keyEntry[1]); - //@ts-expect-error - delete index.key; + const fields: string[] = []; + const types: MongoIndexType[] = []; + for (const [field, type] of Object.entries(index.key ?? {})) { + fields.push(field); + types.push(type as MongoIndexType); } + const name = typeof options.name === 'string' ? options.name : index.name; + return overlayDeclaredOnLive( + { + name, + fields, + types, + options: { ...options, name }, + }, + declared, + ); }); - return result as unknown as ModelOptionsIndexes[]; } async deleteIndexes(schemaName: string, indexNames: string[]): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); const collection = this.mongoose.model(schemaName).collection; + const dropped: string[] = []; + let failure: unknown; for (const name of indexNames) { - collection.dropIndex(name).catch(() => { - throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); + try { + await collection.dropIndex(name); + dropped.push(name); + } catch (e) { + failure = e; + break; + } + } + const original = this.models[schemaName].originalSchema; + for (const name of dropped) { + removeIndexFromSchemaFields(original, name); + } + if (!failure || dropped.length > 0) { + await this.persistIndexesAndPublish({ + schemaName, + originalSchema: original, + droppedNames: dropped, }); } + if (failure) { + throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); + } return 'Indexes deleted'; } @@ -983,10 +1100,6 @@ export class MongooseAdapter extends DatabaseAdapter { const newSchema = schemaConverter(compiledSchema); const indexes = newSchema.modelOptions.indexes; delete newSchema.modelOptions.indexes; - this.registeredSchemas.set( - schema.name, - Object.freeze(JSON.parse(JSON.stringify(schema))), - ); this.models[schema.name] = new MongooseSchema( this.grpcSdk, this.mongoose, @@ -994,16 +1107,28 @@ export class MongooseAdapter extends DatabaseAdapter { schema, this, ); - if (saveToDb) { - await this.compareAndStoreMigratedSchema(schema); - await this.saveSchemaToDatabase(schema); - } + try { + if (!isInstanceSync && schema.modelOptions.indexes?.length) { + const live = await this.listLiveIndexes(schema.name); + schema.modelOptions.indexes = bindDeclaredIndexesToLive( + schema.modelOptions.indexes, + live, + indexNameCollection(schema), + ); + } + if (saveToDb) { + await this.compareAndStoreMigratedSchema(schema); + await this.saveSchemaToDatabase(schema); + } - if (indexes && !isInstanceSync) { - await this.createIndexes(schema.name, indexes, schema.ownerModule); - } - if (!isInstanceSync) { - await this.createMongooseFieldIndexes(schema.name); + if (indexes && !isInstanceSync) { + await this.createIndexes(schema.name, indexes, schema.ownerModule); + } + if (!isInstanceSync) { + await this.createMongooseFieldIndexes(schema.name); + } + } finally { + this.snapshotRegisteredSchema(schema); } await this.applyDeclaredVectorIndexes(schema.name, isInstanceSync); return this.models[schema.name]; @@ -1013,35 +1138,47 @@ export class MongooseAdapter extends DatabaseAdapter { schemaName: string, indexes: readonly ModelOptionsIndexes[], callerModule: string, - ) { - for (const index of indexes) { + privileged?: boolean, + ): ModelOptionsIndexes[] { + const schema = this.models[schemaName].originalSchema; + const collectionName = indexNameCollection(schema); + const prepared: ModelOptionsIndexes[] = []; + for (const raw of toMutableIndexes(indexes)) { + const index = ensureIndexName(raw, collectionName); + validateIndexFields(schema, index); const options = index.options; const types = index.types; - if (!options && !types) continue; if (options) { if (!checkIfMongoOptions(options)) { - throw new GrpcError(status.INTERNAL, 'Invalid index options for mongoDB'); - } - if ( - Object.keys(options).includes('unique') && - this.models[schemaName].originalSchema.ownerModule !== callerModule - ) { throw new GrpcError( - status.PERMISSION_DENIED, - 'Not authorized to create unique index', + status.INVALID_ARGUMENT, + 'Invalid index options for mongoDB', ); } + assertUniqueIndexPrivilege({ + unique: options.unique === true, + schemaOwner: schema.ownerModule, + callerModule, + privileged, + }); } if (types) { - if (!Array.isArray(types) || types.length !== index.fields.length) { - throw new GrpcError(status.INTERNAL, 'Invalid index types format'); + const typeList = normalizeIndexTypes(types, index.fields.length) ?? []; + if (Array.isArray(types) && typeList.length !== index.fields.length) { + throw new GrpcError(status.INVALID_ARGUMENT, 'Invalid index types format'); } - for (const type of types) { - if (!Object.values(MongoIndexType).includes(type)) { - throw new GrpcError(status.INTERNAL, 'Invalid index type for mongoDB'); + for (const type of typeList) { + if (!mongoAllowsIndexType(type)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Invalid index type for mongoDB', + ); } } + index.types = typeList.map(mapCompatibleToMongo); } + prepared.push(index); } + return prepared; } } diff --git a/modules/database/src/adapters/sequelize-adapter/SequelizeSchema.ts b/modules/database/src/adapters/sequelize-adapter/SequelizeSchema.ts index c7dec1ecd..09b062ba6 100644 --- a/modules/database/src/adapters/sequelize-adapter/SequelizeSchema.ts +++ b/modules/database/src/adapters/sequelize-adapter/SequelizeSchema.ts @@ -93,32 +93,36 @@ export class SequelizeSchema extends SchemaAdapter> { } : undefined, }); - // if a relation is to self, then it will be undefined inside the extractedRelations - // so we set it manually to self - for (const relation in extractedRelations) { - if ( - Array.isArray(extractedRelations[relation]) && - (extractedRelations[relation] as SequelizeSchema[])[0] === undefined - ) { - extractedRelations[relation] = [this]; - } else if ( - !Array.isArray(extractedRelations[relation]) && - extractedRelations[relation] === undefined - ) { - extractedRelations[relation] = this; - } - } this.objectDotPaths = []; for (const concatenatedKey of Object.keys(objectPaths)) { const { parentKey, childKey } = objectPaths[concatenatedKey]; this.objectDotPaths.push(`${parentKey}.${childKey}`); this.objectDotPathMapping[concatenatedKey] = `${parentKey}.${childKey}`; } + if (Object.keys(extractedRelations).length > 0) { + this.bindExtractedRelations(); + } + } + + bindExtractedRelations() { + for (const relation in this.extractedRelations) { + if ( + Array.isArray(this.extractedRelations[relation]) && + (this.extractedRelations[relation] as SequelizeSchema[])[0] === undefined + ) { + this.extractedRelations[relation] = [this]; + } else if ( + !Array.isArray(this.extractedRelations[relation]) && + this.extractedRelations[relation] === undefined + ) { + this.extractedRelations[relation] = this; + } + } extractRelations( this.originalSchema.name, - originalSchema, + this.originalSchema, this.model, - extractedRelations, + this.extractedRelations, ); } @@ -339,6 +343,7 @@ export class SequelizeSchema extends SchemaAdapter> { scope?: string; select?: string; populate?: string[]; + readPreference?: string; }, ) { const filter = await this.getAuthorizedQuery( diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 5c3a95048..a268b9fe0 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -6,7 +6,6 @@ import { GrpcError, Indexable, ModelOptionsIndexes, - PostgresIndexOptions, PostgresIndexType, RawSQLQuery, UntypedArray, @@ -51,6 +50,31 @@ import { assertPostgresVectorIndexDropTarget, type PostgresCatalogIndex, } from '../utils/index.js'; +import { + assertUniqueIndexPrivilege, + bindDeclaredIndexesToLive, + canonicalizeDeclaredIndexFields, + collectSchemaIndexFields, + ensureIndexName, + findLiveIndex, + indexNameCollection, + inferSqlIndexType, + isIndexAlreadyExistsError, + isIndexKeySpecsConflictError, + isPostgresIndexType, + liveIndexFromSql, + liveNameConflictAllowsReuse, + overlayDeclaredOnLive, + normalizeIndexTypes, + removeIndexFromSchemaFields, + sqlDeclaredIndexFieldName, + sqlDialectAllowsIndexType, + sqlEngineIndexFields, + sqlIndexFieldNormalizer, + sqlIndexUnsupportedReason, + toMutableIndexes, + validateIndexFields, +} from '../utils/indexes.js'; const sqlSchemaName = process.env.SQL_SCHEMA ?? 'public'; @@ -267,19 +291,36 @@ export abstract class SequelizeAdapter extends DatabaseAdapter this.sequelize.models, ); const dialect = this.sequelize.getDialect(); + const collectionName = this.getCollectionName(schema); + const schemaFields = collectSchemaIndexFields(schema); + const normalizeFields = sqlIndexFieldNormalizer(schemaFields); + const live = isInstanceSync + ? [] + : await this.listLiveIndexesForCollection(collectionName); + if (!isInstanceSync && schema.modelOptions.indexes?.length) { + schema.modelOptions.indexes = bindDeclaredIndexesToLive( + schema.modelOptions.indexes, + live, + collectionName, + normalizeFields, + ); + compiledSchema.modelOptions.indexes = schema.modelOptions.indexes; + } const [newSchema, objectPaths, extractedRelations] = dialect === 'postgres' ? pgSchemaConverter(compiledSchema) - : sqlSchemaConverter(compiledSchema); - this.registeredSchemas.set( - schema.name, - Object.freeze(JSON.parse(JSON.stringify(schema))), - ); - const relatedSchemas = await resolveRelatedSchemas( - schema, - extractedRelations, - this.models, - ); + : sqlSchemaConverter(compiledSchema, dialect as 'mysql' | 'mariadb' | 'sqlite'); + if (!isInstanceSync && newSchema.modelOptions.indexes?.length) { + newSchema.modelOptions.indexes = bindDeclaredIndexesToLive( + newSchema.modelOptions.indexes, + live, + collectionName, + normalizeFields, + ); + } + const relatedSchemas: { + [key: string]: SequelizeSchema | SequelizeSchema[]; + } = {}; this.models[schema.name] = new SequelizeSchema( this.grpcSdk, this.sequelize, @@ -289,20 +330,29 @@ export abstract class SequelizeAdapter extends DatabaseAdapter relatedSchemas, objectPaths, ); + Object.assign( + relatedSchemas, + await resolveRelatedSchemas(schema, extractedRelations, this.models), + ); + this.models[schema.name].bindExtractedRelations(); - const noSync = - this.models[schema.name].originalSchema.modelOptions.conduit!.noSync || - isInstanceSync; - // do not sync extracted schemas - if (isNil(noSync) || !noSync) { - await this.models[schema.name].sync(); - } else { - this.models[schema.name].synced = true; - } - // do not store extracted schemas to db - if (saveToDb && !isInstanceSync) { - await this.compareAndStoreMigratedSchema(schema); - await this.saveSchemaToDatabase(schema); + try { + const noSync = + this.models[schema.name].originalSchema.modelOptions.conduit!.noSync || + isInstanceSync; + // do not sync extracted schemas + if (isNil(noSync) || !noSync) { + await this.models[schema.name].sync(); + } else { + this.models[schema.name].synced = true; + } + // do not store extracted schemas to db + if (saveToDb && !isInstanceSync) { + await this.compareAndStoreMigratedSchema(schema); + await this.saveSchemaToDatabase(schema); + } + } finally { + this.snapshotRegisteredSchema(schema); } await this.applyDeclaredVectorIndexes(schema.name, isInstanceSync); return this.models[schema.name]; @@ -379,69 +429,169 @@ export abstract class SequelizeAdapter extends DatabaseAdapter async createIndexes( schemaName: string, - indexes: ModelOptionsIndexes[], + indexes: readonly ModelOptionsIndexes[], callerModule: string, + options?: { privileged?: boolean }, ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - indexes = this.checkAndConvertIndexes(schemaName, indexes, callerModule); + const collectionName = this.models[schemaName].originalSchema.collectionName; + const schemaFields = collectSchemaIndexFields(this.models[schemaName].originalSchema); + const normalizeFields = sqlIndexFieldNormalizer(schemaFields); + const live = await this.listLiveIndexesForCollection(collectionName); + const prepared = bindDeclaredIndexesToLive( + this.checkAndConvertIndexes(schemaName, indexes, callerModule, options?.privileged), + live, + collectionName, + normalizeFields, + ); const queryInterface = this.sequelize.getQueryInterface(); - for (const index of indexes) { - await queryInterface - .addIndex('cnd_' + schemaName, [...index.fields], index.options) - .catch(() => { - throw new GrpcError(status.INTERNAL, 'Unsuccessful index creation'); + const applied: ModelOptionsIndexes[] = []; + let failure: unknown; + for (const index of prepared) { + const existing = findLiveIndex(live, index, normalizeFields); + if (existing) { + applied.push(index); + continue; + } + try { + await queryInterface.addIndex(collectionName, { + fields: sqlEngineIndexFields(index, schemaFields), + ...index.options, }); + applied.push(index); + live.push(index); + } catch (e) { + if (isIndexAlreadyExistsError(e)) { + const relisted = await this.listLiveIndexesForCollection(collectionName); + if (liveNameConflictAllowsReuse(index, relisted, normalizeFields)) { + applied.push(index); + live.splice(0, live.length, ...relisted); + continue; + } + const match = findLiveIndex(relisted, index, normalizeFields); + if (match) { + applied.push( + bindDeclaredIndexesToLive( + [index], + relisted, + collectionName, + normalizeFields, + )[0], + ); + live.splice(0, live.length, ...relisted); + continue; + } + failure = e; + break; + } + if (isIndexKeySpecsConflictError(e)) { + const relisted = await this.listLiveIndexesForCollection(collectionName); + const match = findLiveIndex(relisted, index, normalizeFields); + if (match) { + applied.push( + bindDeclaredIndexesToLive( + [index], + relisted, + collectionName, + normalizeFields, + )[0], + ); + live.splice(0, live.length, ...relisted); + continue; + } + } + failure = e; + break; + } + } + if (!failure || applied.length > 0) { + await this.persistIndexesAndPublish({ + schemaName, + originalSchema: this.models[schemaName].originalSchema, + applied, + }); + } + if (failure) { + throw new GrpcError(status.INTERNAL, 'Unsuccessful index creation'); } - await this.models[schemaName].sync(); return 'Indexes created!'; } + private async listLiveIndexesForCollection( + collectionName: string, + ): Promise { + try { + const result = (await this.sequelize + .getQueryInterface() + .showIndex(collectionName)) as UntypedArray; + return result.map(liveIndexFromSql); + } catch { + return []; + } + } + async getIndexes(schemaName: string): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const collectionName = this.models[schemaName].originalSchema.collectionName; const queryInterface = this.sequelize.getQueryInterface(); - const result = (await queryInterface.showIndex('cnd_' + schemaName)) as UntypedArray; - result.filter(index => { - const fieldNames = []; - for (const field of index.fields) { - fieldNames.push(field.attribute); - } - index.fields = fieldNames; - // extract index type from index definition - let tmp = index.definition.split('USING '); - tmp = tmp[1].split(' '); - index.types = tmp[0]; - delete index.definition; - index.options = {}; - for (const indexEntry of Object.entries(index)) { - if ( - indexEntry[0] === 'options' || - indexEntry[0] === 'types' || - indexEntry[0] === 'fields' - ) { - continue; - } - if (indexEntry[0] === 'indkey') { - delete index.indkey; - continue; - } - index.options[indexEntry[0]] = indexEntry[1]; - delete index[indexEntry[0]]; - } + const result = (await queryInterface.showIndex(collectionName)) as UntypedArray; + const dialect = this.sequelize.getDialect(); + const originalSchema = this.models[schemaName].originalSchema; + const schemaFields = collectSchemaIndexFields(originalSchema); + const declared = originalSchema.modelOptions.indexes; + return result.map(row => { + const engineFields = (row.fields ?? []).map((field: unknown) => + typeof field === 'string' ? field : (field as { attribute?: string }).attribute, + ); + const fields = engineFields.map((field: string) => + sqlDeclaredIndexFieldName(field, schemaFields), + ); + const name = row.name as string; + return overlayDeclaredOnLive( + { + name, + fields, + types: inferSqlIndexType(row, dialect), + options: { name, unique: !!row.unique }, + ...(row.primary ? { primary: true } : {}), + }, + declared, + ); }); - return result; } async deleteIndexes(schemaName: string, indexNames: string[]): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const collectionName = this.models[schemaName].originalSchema.collectionName; const queryInterface = this.sequelize.getQueryInterface(); + const dropped: string[] = []; + let failure: unknown; for (const name of indexNames) { - queryInterface.removeIndex('cnd_' + schemaName, name).catch(() => { - throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); + try { + await queryInterface.removeIndex(collectionName, name); + dropped.push(name); + } catch (e) { + failure = e; + break; + } + } + const original = this.models[schemaName].originalSchema; + for (const name of dropped) { + removeIndexFromSchemaFields(original, name); + } + if (!failure || dropped.length > 0) { + await this.persistIndexesAndPublish({ + schemaName, + originalSchema: original, + droppedNames: dropped, }); } + if (failure) { + throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); + } return 'Indexes deleted'; } @@ -690,42 +840,67 @@ export abstract class SequelizeAdapter extends DatabaseAdapter private checkAndConvertIndexes( schemaName: string, - indexes: ModelOptionsIndexes[], + indexes: readonly ModelOptionsIndexes[], callerModule: string, - ) { - for (const index of indexes) { - if (!index.types && !index.options) continue; + privileged?: boolean, + ): ModelOptionsIndexes[] { + const schema = this.models[schemaName].originalSchema; + const dialect = this.sequelize.getDialect(); + const collectionName = indexNameCollection(schema); + const schemaFields = collectSchemaIndexFields(schema); + const prepared: ModelOptionsIndexes[] = []; + for (const raw of toMutableIndexes(indexes)) { + const index = ensureIndexName( + canonicalizeDeclaredIndexFields(raw, schemaFields), + collectionName, + ); + validateIndexFields(schema, index); + const unsupported = sqlIndexUnsupportedReason( + dialect, + index, + { + ...(schema.fields as Record), + ...(schema.compiledFields as Record), + }, + { timestamps: schema.modelOptions?.timestamps }, + ); + if (unsupported) { + throw new GrpcError(status.INVALID_ARGUMENT, unsupported); + } if (index.types) { - if ( - Array.isArray(index.types) || - !Object.values(PostgresIndexType).includes(index.types) - ) { + const types = normalizeIndexTypes(index.types, index.fields.length) ?? []; + if (types.some(type => !sqlDialectAllowsIndexType(dialect, type))) { throw new GrpcError( status.INVALID_ARGUMENT, - 'Invalid index type for PostgreSQL', + `Invalid index type for ${dialect}`, ); } - (index.options as PostgresIndexOptions).using = index.types; - delete index.types; + const first = types[0]; + const using = + types.length === 1 && isPostgresIndexType(first) + ? first + : PostgresIndexType.BTREE; + index.options = { + ...(index.options ?? {}), + using, + }; } if (index.options) { if (!checkIfPostgresOptions(index.options)) { throw new GrpcError( status.INVALID_ARGUMENT, - 'Invalid index options for PostgreSQL', - ); - } - if ( - Object.keys(index.options).includes('unique') && - this.models[schemaName].originalSchema.ownerModule !== callerModule - ) { - throw new GrpcError( - status.PERMISSION_DENIED, - 'Not authorized to create unique index', + `Invalid index options for ${dialect}`, ); } + assertUniqueIndexPrivilege({ + unique: index.options.unique === true, + schemaOwner: schema.ownerModule, + callerModule, + privileged, + }); } + prepared.push(index); } - return indexes; + return prepared; } } diff --git a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts index 6504e65ba..069118953 100644 --- a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts @@ -11,6 +11,7 @@ import { convertModelOptionsIndexes, convertSchemaFieldIndexes, extractFieldProperties, + liftSqlScalarRelationFieldIndexes, } from '../../utils/index.js'; import { sqlDataTypeMap } from '../utils/sqlTypeMap.js'; import { @@ -38,14 +39,15 @@ export function pgSchemaConverter(jsonSchema: ConduitSchema): [ if (copy.fields.hasOwnProperty('_id')) { delete copy.fields['_id']; } + copy = liftSqlScalarRelationFieldIndexes(copy); if (copy.modelOptions.indexes) { - copy = convertModelOptionsIndexes(copy); + copy = convertModelOptionsIndexes(copy, 'postgres'); } const objectPaths: any = {}; convertObjectToDotNotation(jsonSchema.fields, copy.fields, objectPaths); const secondaryCopy = cloneDeep(copy.fields); const extractedRelations = extractRelations(secondaryCopy, copy.fields); - copy = convertSchemaFieldIndexes(copy); + copy = convertSchemaFieldIndexes(copy, 'postgres'); iterDeep(secondaryCopy, copy.fields); return [copy, objectPaths, extractedRelations]; } diff --git a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts index 7f418d43a..35d45e67f 100644 --- a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts @@ -11,6 +11,7 @@ import { convertModelOptionsIndexes, convertSchemaFieldIndexes, extractFieldProperties, + liftSqlScalarRelationFieldIndexes, } from '../../utils/index.js'; import { sqlDataTypeMap } from '../utils/sqlTypeMap.js'; import { @@ -23,7 +24,10 @@ import { * This function should take as an input a JSON schema and convert it to the sequelize equivalent * @param jsonSchema */ -export function sqlSchemaConverter(jsonSchema: ConduitSchema): [ +export function sqlSchemaConverter( + jsonSchema: ConduitSchema, + dialect: 'mysql' | 'mariadb' | 'sqlite' = 'mysql', +): [ ConduitSchema, { [key: string]: { parentKey: string; childKey: string }; @@ -34,14 +38,15 @@ export function sqlSchemaConverter(jsonSchema: ConduitSchema): [ if (copy.fields.hasOwnProperty('_id')) { delete copy.fields['_id']; } + copy = liftSqlScalarRelationFieldIndexes(copy); if (copy.modelOptions.indexes) { - copy = convertModelOptionsIndexes(copy); + copy = convertModelOptionsIndexes(copy, dialect); } const objectPaths: any = {}; convertObjectToDotNotation(jsonSchema.fields, copy.fields, objectPaths); const secondaryCopy = cloneDeep(copy.fields); const extractedRelations = extractRelations(secondaryCopy, copy.fields); - copy = convertSchemaFieldIndexes(copy); + copy = convertSchemaFieldIndexes(copy, dialect); iterDeep(secondaryCopy, copy.fields); return [copy, objectPaths, extractedRelations]; } diff --git a/modules/database/src/adapters/sequelize-adapter/utils/schema.ts b/modules/database/src/adapters/sequelize-adapter/utils/schema.ts index 4138af8ca..f96bcad25 100644 --- a/modules/database/src/adapters/sequelize-adapter/utils/schema.ts +++ b/modules/database/src/adapters/sequelize-adapter/utils/schema.ts @@ -232,10 +232,10 @@ export async function resolveRelatedSchemas( const rel = Array.isArray(extractedRelations[relation]) ? (extractedRelations[relation] as UntypedArray)[0] : extractedRelations[relation]; - if ( - (!models[rel.model] || !models[rel.model].synced) && - schema.name !== rel.model - ) { + // Wait until the related adapter exists, not until it has synced. + // Same-module cycles construct in parallel under Promise.all; waiting + // for .synced deadlocks because neither side can sync until this returns. + if (!models[rel.model] && schema.name !== rel.model) { if (!pendingModels.includes(rel.model)) { pendingModels.push(rel.model); } @@ -255,19 +255,18 @@ export async function resolveRelatedSchemas( while (pendingModels.length > 0) { await ConduitGrpcSdk.Sleep(500); pendingModels = pendingModels.filter(model => { - if (!models[model] || !models[model].synced) { + if (!models[model]) { return true; - } else { - for (const schema in relatedSchemas) { - const simple = Array.isArray(relatedSchemas[schema]) - ? (relatedSchemas[schema] as SequelizeSchema[])[0] - : relatedSchemas[schema]; - // @ts-ignore - if (simple === model) { - relatedSchemas[schema] = Array.isArray(relatedSchemas[schema]) - ? [models[model]] - : models[model]; - } + } + for (const schemaName in relatedSchemas) { + const simple = Array.isArray(relatedSchemas[schemaName]) + ? (relatedSchemas[schemaName] as SequelizeSchema[])[0] + : relatedSchemas[schemaName]; + // @ts-ignore + if (simple === model) { + relatedSchemas[schemaName] = Array.isArray(relatedSchemas[schemaName]) + ? [models[model]] + : models[model]; } } }); diff --git a/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts b/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts index e48c16439..98c06da6e 100644 --- a/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts +++ b/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts @@ -50,12 +50,13 @@ describe('mongoose vector index lifecycle', () => { toArray: async () => [], })); const createIndex = jest.fn(async () => 'title_1'); + const indexes = jest.fn(async () => [{ v: 2, key: { _id: 1 }, name: '_id_' }]); const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; Object.assign(adapter, { models: { Article: articleModel() }, mongoose: { model: () => ({ - collection: { createSearchIndex, listSearchIndexes, createIndex }, + collection: { createSearchIndex, listSearchIndexes, createIndex, indexes }, }), }, }); diff --git a/modules/database/src/adapters/utils/database-transform-utils.ts b/modules/database/src/adapters/utils/database-transform-utils.ts index 72956f938..ad0bb4467 100644 --- a/modules/database/src/adapters/utils/database-transform-utils.ts +++ b/modules/database/src/adapters/utils/database-transform-utils.ts @@ -1,13 +1,36 @@ -import { isArray, isBoolean, isNumber, isString } from 'lodash-es'; +import { isBoolean, isNumber, isString } from 'lodash-es'; import { ConduitGrpcSdk, ConduitModelField, ConduitSchema, Indexable, - PostgresIndexOptions, + ModelOptionsIndexes, PostgresIndexType, } from '@conduitplatform/grpc-sdk'; import { checkIfPostgresOptions } from '../sequelize-adapter/utils/index.js'; +import { + ensureIndexName, + indexNameCollection, + isPortableDirection, + isPostgresIndexType, + isScalarRelationField, + mapCompatibleToSqlOrder, + mapIndexFieldsToSqlEngine, + normalizeIndexTypes, + sqlDialectAllowsIndexType, + sqlIndexUnsupportedReason, + type SqlIndexField, +} from './indexes.js'; + +type SqlEngineIndex = Omit & { + fields: SqlIndexField[]; + using?: PostgresIndexType; +}; + +function setSqlEngineIndexes(copy: ConduitSchema, indexes: SqlEngineIndex[]) { + // Sequelize reads these through define(..., schema.modelOptions). + copy.modelOptions.indexes = indexes as ModelOptionsIndexes[]; +} export function checkDefaultValue(type: string, value: string) { switch (type) { @@ -28,78 +51,139 @@ export function checkDefaultValue(type: string, value: string) { } } -export function convertModelOptionsIndexes(copy: ConduitSchema) { - for (const index of copy.modelOptions.indexes!) { - if (index.types) { - if ( - isArray(index.types) || - !Object.values(PostgresIndexType).includes(index.types as PostgresIndexType) - ) { - // ignore index instead of error - ConduitGrpcSdk.Logger.warn('Invalid index type for PostgreSQL, ignoring index'); - continue; - // throw new Error('Incorrect index type for PostgreSQL'); - } - index.using = index.types as PostgresIndexType; - delete index.types; +function skipIndex(schemaName: string, dialect: string, reason: string) { + ConduitGrpcSdk.Logger.warn( + `Invalid index ${reason} for ${dialect} found in '${schemaName}', ignoring index`, + ); +} + +function toSqlEngineIndex( + raw: ModelOptionsIndexes, + dialect: string, + schemaName: string, + collectionName: string, + schemaFields: Record, +): SqlEngineIndex | null { + const index = ensureIndexName({ ...raw, fields: [...raw.fields] }, collectionName); + if (index.options && !checkIfPostgresOptions(index.options)) { + skipIndex(schemaName, dialect, 'options'); + return null; + } + + const engineFieldNames = mapIndexFieldsToSqlEngine(index, schemaFields); + let fields: SqlIndexField[] = [...engineFieldNames]; + let using = PostgresIndexType.BTREE; + if (index.types) { + const types = normalizeIndexTypes(index.types, index.fields.length) ?? []; + if (types.some(type => !sqlDialectAllowsIndexType(dialect, type))) { + skipIndex(schemaName, dialect, 'type'); + return null; } - if (index.options) { - if (!checkIfPostgresOptions(index.options)) { - // ignore index instead of error - ConduitGrpcSdk.Logger.warn( - 'Invalid index options for PostgreSQL, ignoring index', - ); - continue; - // throw new Error('Incorrect index options for PostgreSQL'); - } - for (const [option, value] of Object.entries(index.options)) { - index[option as keyof PostgresIndexOptions] = value; - } - delete index.options; + if (types.some(isPortableDirection)) { + fields = engineFieldNames.map((field, i) => ({ + name: field, + order: mapCompatibleToSqlOrder(types[i]), + })); + } else if (types.length === 1 && isPostgresIndexType(types[0])) { + using = types[0]; + } else { + skipIndex(schemaName, dialect, 'type'); + return null; } } + + return { + ...index.options, + name: index.name, + fields, + using, + unique: index.options?.unique, + }; +} + +export function liftSqlScalarRelationFieldIndexes(copy: ConduitSchema) { + const lifted: ModelOptionsIndexes[] = []; + for (const [fieldName, fieldValue] of Object.entries(copy.fields)) { + if (!isScalarRelationField(fieldValue)) continue; + const field = fieldValue as ConduitModelField; + const index = field.index; + if (!index) continue; + lifted.push({ + fields: [fieldName], + types: index.type === undefined ? undefined : [index.type], + options: index.options, + name: index.name, + }); + delete field.index; + } + if (lifted.length) { + copy.modelOptions.indexes = [...(copy.modelOptions.indexes ?? []), ...lifted]; + } return copy; } -export function convertSchemaFieldIndexes(copy: ConduitSchema) { - const indexes = []; - for (const field of Object.entries(copy.fields)) { - const fieldName = field[0]; - const index = (copy.fields[fieldName] as ConduitModelField).index; +export function convertModelOptionsIndexes(copy: ConduitSchema, dialect = 'postgres') { + const collectionName = indexNameCollection(copy); + const converted: SqlEngineIndex[] = []; + for (const raw of copy.modelOptions.indexes ?? []) { + const unsupported = sqlIndexUnsupportedReason(dialect, raw, copy.fields, { + timestamps: copy.modelOptions.timestamps, + }); + if (unsupported) { + ConduitGrpcSdk.Logger.warn( + `Skipping index on '${copy.name}' for ${dialect}: ${unsupported}`, + ); + continue; + } + const index = toSqlEngineIndex(raw, dialect, copy.name, collectionName, copy.fields); + if (index) converted.push(index); + } + setSqlEngineIndexes(copy, converted); + return copy; +} + +export function convertSchemaFieldIndexes(copy: ConduitSchema, dialect = 'postgres') { + const collectionName = indexNameCollection(copy); + const indexes: SqlEngineIndex[] = []; + for (const [fieldName, fieldValue] of Object.entries(copy.fields)) { + const field = fieldValue as ConduitModelField; + const index = field.index; if (!index) continue; - const newIndex: any = { + if (index.type && !sqlDialectAllowsIndexType(dialect, index.type)) { + skipIndex(copy.name, dialect, 'type'); + delete field.index; + continue; + } + const raw = { fields: [fieldName], + types: index.type === undefined ? undefined : [index.type], + options: index.options, + name: index.name, }; - if (index.type) { - if (!Object.values(PostgresIndexType).includes(index.type as PostgresIndexType)) { - // ignore index instead of error - ConduitGrpcSdk.Logger.warn('Invalid index type for PostgreSQL, ignoring index'); - continue; - // throw new Error('Invalid index type for PostgreSQL'); - } - newIndex.using = index.type; + const unsupported = sqlIndexUnsupportedReason(dialect, raw, copy.fields, { + timestamps: copy.modelOptions.timestamps, + }); + if (unsupported) { + ConduitGrpcSdk.Logger.warn( + `Skipping index on '${copy.name}' for ${dialect}: ${unsupported}`, + ); + delete field.index; + continue; } - if (index.options) { - if (!checkIfPostgresOptions(index.options)) { - // ignore index instead of error - ConduitGrpcSdk.Logger.warn( - 'Invalid index options for PostgreSQL, ignoring index', - ); - continue; - // throw new Error('Invalid index options for PostgreSQL'); - } - for (const [option, value] of Object.entries(index.options)) { - newIndex[option] = value; - } - } - indexes.push(newIndex); - delete copy.fields[fieldName]; - } - if (copy.modelOptions.indexes) { - copy.modelOptions.indexes = [...copy.modelOptions.indexes, ...indexes]; - } else { - copy.modelOptions.indexes = indexes; + const converted = toSqlEngineIndex( + raw, + dialect, + copy.name, + collectionName, + copy.fields, + ); + delete field.index; + if (converted) indexes.push(converted); } + setSqlEngineIndexes(copy, [ + ...((copy.modelOptions.indexes ?? []) as SqlEngineIndex[]), + ...indexes, + ]); return copy; } diff --git a/modules/database/src/adapters/utils/index.ts b/modules/database/src/adapters/utils/index.ts index a514cd7ad..5f28a8e86 100644 --- a/modules/database/src/adapters/utils/index.ts +++ b/modules/database/src/adapters/utils/index.ts @@ -2,6 +2,7 @@ export * from './validateFieldChanges.js'; export * from './validateFieldConstraints.js'; export * from './database-transform-utils.js'; export * from './extensions.js'; +export * from './indexes.js'; export * from './vectorField.js'; export * from './vectorCapabilities.js'; export * from './vectorMappings.js'; diff --git a/modules/database/src/adapters/utils/indexes.ts b/modules/database/src/adapters/utils/indexes.ts new file mode 100644 index 000000000..14590877f --- /dev/null +++ b/modules/database/src/adapters/utils/indexes.ts @@ -0,0 +1,766 @@ +import { createHash } from 'crypto'; +import { + CompatibleIndexType, + ConduitModelField, + GrpcError, + ModelOptionsIndexes, + MongoIndexType, + PostgresIndexType, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { ConduitDatabaseSchema } from '../../interfaces/index.js'; + +export const ADMIN_INDEX_CALLER = 'database'; + +const MONGO_INDEX_TYPE_VALUES: ReadonlySet = new Set([ + MongoIndexType.Ascending, + MongoIndexType.Descending, + MongoIndexType.GeoSpatial2d, + MongoIndexType.GeoSpatial2dSphere, + MongoIndexType.GeoHaystack, + MongoIndexType.Hashed, + MongoIndexType.Text, +]); + +const SQL_IDENTIFIER_MAX_LEN = 63; + +export type SqlIndexField = string | { name: string; order: 'ASC' | 'DESC' }; + +export function isCompatibleIndexType(value: unknown): value is CompatibleIndexType { + return ( + value === CompatibleIndexType.Ascending || value === CompatibleIndexType.Descending + ); +} + +export function isMongoIndexType(value: unknown): value is MongoIndexType { + return MONGO_INDEX_TYPE_VALUES.has(value); +} + +export function isPostgresIndexType(value: unknown): value is PostgresIndexType { + return Object.values(PostgresIndexType).includes(value as PostgresIndexType); +} + +export function resolveIndexName(index: ModelOptionsIndexes): string | undefined { + if (typeof index.name === 'string' && index.name.length > 0) return index.name; + const optionsName = index.options?.name; + if (typeof optionsName === 'string' && optionsName.length > 0) return optionsName; + return undefined; +} + +export function isUniqueIndex(index: ModelOptionsIndexes): boolean { + return index.options?.unique === true || index.unique === true; +} + +export type IndexIdentity = { fields: string[]; unique: boolean }; + +export function indexFieldNames( + index: Pick | { fields?: readonly unknown[] }, +): string[] { + return (index.fields ?? []) + .map(field => { + if (typeof field === 'string') return field; + if (field && typeof field === 'object') { + const obj = field as { name?: string; attribute?: string }; + if (typeof obj.name === 'string' && obj.name.length > 0) return obj.name; + if (typeof obj.attribute === 'string' && obj.attribute.length > 0) { + return obj.attribute; + } + } + return ''; + }) + .filter(name => name.length > 0); +} + +export type IndexFieldNormalizer = (fields: string[]) => string[]; + +export function indexIdentity( + index: ModelOptionsIndexes, + normalizeFields?: IndexFieldNormalizer, +): IndexIdentity { + const fields = indexFieldNames(index); + return { + fields: normalizeFields ? normalizeFields(fields) : fields, + unique: isUniqueIndex(index), + }; +} + +export function indexIdentitiesEqual(a: IndexIdentity, b: IndexIdentity): boolean { + return ( + a.unique === b.unique && + a.fields.length === b.fields.length && + a.fields.every((field, i) => field === b.fields[i]) + ); +} + +export function indexIdentityKey(identity: IndexIdentity): string { + return `${identity.unique ? 'u' : 'n'}:${identity.fields.join('\0')}`; +} + +export function isSkippedLiveIndex(index: ModelOptionsIndexes): boolean { + if (index.primary === true) return true; + const name = resolveIndexName(index); + return name === '_id_' || name === 'PRIMARY'; +} + +export function findLiveIndex( + live: readonly ModelOptionsIndexes[], + declared: ModelOptionsIndexes, + normalizeFields?: IndexFieldNormalizer, +): ModelOptionsIndexes | undefined { + const wanted = indexIdentity(declared, normalizeFields); + return live.find( + row => + !isSkippedLiveIndex(row) && + indexIdentitiesEqual(indexIdentity(row, normalizeFields), wanted), + ); +} + +export function findIndexByName( + indexes: readonly ModelOptionsIndexes[], + name: string, +): ModelOptionsIndexes | undefined { + return indexes.find(index => resolveIndexName(index) === name); +} + +export function liveNameConflictAllowsReuse( + declared: ModelOptionsIndexes, + live: readonly ModelOptionsIndexes[], + normalizeFields?: IndexFieldNormalizer, +): boolean { + const name = resolveIndexName(declared); + if (!name) return false; + const row = findIndexByName(live, name); + if (!row) return false; + return indexIdentitiesEqual( + indexIdentity(row, normalizeFields), + indexIdentity(declared, normalizeFields), + ); +} + +export function indexNameCollection(schema: { + collectionName?: string; + name?: string; +}): string { + if (schema.collectionName && schema.collectionName.length > 0) { + return schema.collectionName; + } + return schema.name ?? ''; +} + +export function bindDeclaredIndexesToLive( + declared: readonly T[], + live: readonly ModelOptionsIndexes[], + collectionName: string, + normalizeFields?: IndexFieldNormalizer, +): T[] { + return declared.map(index => { + const match = findLiveIndex(live, index, normalizeFields); + if (match) { + const name = resolveIndexName(match); + if (name) { + return { + ...index, + name, + options: { ...index.options, name }, + }; + } + } + const fields = indexFieldNames(index); + const stringFields = Array.isArray(index.fields) + ? index.fields.every(field => typeof field === 'string') + : false; + if (stringFields && fields.length === index.fields.length) { + return ensureIndexName(index, collectionName) as T; + } + return index; + }); +} + +export function keepDeclaredIndexExtras( + incomingBound: readonly ModelOptionsIndexes[], + existingDb: readonly ModelOptionsIndexes[] | undefined, + collectionName: string, +): ModelOptionsIndexes[] { + const incoming = incomingBound.map(index => { + const name = resolveIndexName(index); + return name + ? { ...index, name, options: { ...index.options, name } } + : ensureIndexName(index, collectionName); + }); + const incomingNames = new Set( + incoming.map(resolveIndexName).filter((name): name is string => Boolean(name)), + ); + const incomingIdentities = new Set( + incoming.map(index => indexIdentityKey(indexIdentity(index))), + ); + const extras: ModelOptionsIndexes[] = []; + for (const index of existingDb ?? []) { + const name = resolveIndexName(index); + if (name && incomingNames.has(name)) continue; + if (incomingIdentities.has(indexIdentityKey(indexIdentity(index)))) continue; + extras.push(ensureIndexName(index, collectionName)); + } + return [...incoming, ...extras]; +} + +export function overlayDeclaredOnLive( + live: ModelOptionsIndexes, + declared: readonly ModelOptionsIndexes[] | undefined, +): ModelOptionsIndexes { + if (!declared?.length) return live; + const name = resolveIndexName(live); + const byName = name ? declaredIndexMap(declared).get(name) : undefined; + const match = byName ?? findLiveIndex(declared, live); + if (!match) return live; + const liveName = name ?? resolveIndexName(match); + return { + ...live, + types: match.types ?? live.types, + options: { ...match.options, ...live.options, name: liveName }, + }; +} + +export function liveIndexFromMongo(index: { + key?: Record; + name?: string; + unique?: boolean; +}): ModelOptionsIndexes { + return { + name: index.name, + fields: Object.keys(index.key ?? {}), + options: { name: index.name, unique: !!index.unique }, + }; +} + +export function liveIndexFromSql(row: { + name?: string; + unique?: boolean; + primary?: boolean; + fields?: Array; +}): ModelOptionsIndexes { + return { + name: row.name, + fields: indexFieldNames({ fields: row.fields ?? [] }), + options: { name: row.name, unique: !!row.unique }, + ...(row.primary ? { primary: true } : {}), + }; +} + +export function normalizeIndexTypes( + types: ModelOptionsIndexes['types'], + fieldCount: number, +): unknown[] | undefined { + if (types === undefined) return undefined; + if (Array.isArray(types)) return [...types]; + return Array.from({ length: fieldCount }, () => types); +} + +function typeToken(type: unknown): string { + if (isPortableDirection(type) || type === undefined) { + return mapCompatibleToSqlOrder(type).toLowerCase(); + } + if (typeof type === 'string') return type.toLowerCase().replace(/[^a-z0-9]+/g, ''); + return String(type); +} + +function sanitizeIdentifierPart(value: string): string { + return value + .replace(/[^A-Za-z0-9_]+/g, '_') + .replace(/_+/g, '_') + .replace(/^_|_$/g, ''); +} + +export function generateIndexName( + fields: readonly string[], + types?: ModelOptionsIndexes['types'], + unique = false, + collectionName = '', +): string { + const tokens = ( + normalizeIndexTypes(types, fields.length) ?? fields.map(() => undefined) + ) + .map(typeToken) + .join('_'); + const prefix = unique ? 'cnd_uidx' : 'cnd_idx'; + const table = sanitizeIdentifierPart(collectionName); + const raw = `${prefix}_${table}_${fields.join('_')}_${tokens}`.replace( + /[^A-Za-z0-9_]+/g, + '_', + ); + const sanitized = raw.replace(/_+/g, '_').replace(/^_|_$/g, ''); + if (sanitized.length <= SQL_IDENTIFIER_MAX_LEN) return sanitized; + const identityKey = `${collectionName}\0${unique}\0${fields.join('\0')}\0${tokens}`; + const hash = createHash('sha1').update(identityKey).digest('hex').slice(0, 8); + const maxTableLen = SQL_IDENTIFIER_MAX_LEN - prefix.length - hash.length - 2; + const tableSlice = table.slice(0, Math.max(1, maxTableLen)); + return `${prefix}_${tableSlice}_${hash}`.slice(0, SQL_IDENTIFIER_MAX_LEN); +} + +export function ensureIndexName( + index: ModelOptionsIndexes, + collectionName: string, +): ModelOptionsIndexes { + const existing = resolveIndexName(index); + const name = + existing ?? + generateIndexName( + indexFieldNames(index), + index.types, + isUniqueIndex(index), + collectionName, + ); + return { + ...index, + name, + options: { ...index.options, name }, + }; +} + +export function mapCompatibleToMongo(type: unknown): MongoIndexType { + if (type === CompatibleIndexType.Descending || type === MongoIndexType.Descending) { + return MongoIndexType.Descending; + } + if ( + type === undefined || + type === CompatibleIndexType.Ascending || + type === MongoIndexType.Ascending + ) { + return MongoIndexType.Ascending; + } + if (isMongoIndexType(type)) return type; + throw new GrpcError(status.INVALID_ARGUMENT, `Invalid index type for MongoDB: ${type}`); +} + +export function mapCompatibleToSqlOrder(type: unknown): 'ASC' | 'DESC' { + if (type === CompatibleIndexType.Descending || type === MongoIndexType.Descending) { + return 'DESC'; + } + return 'ASC'; +} + +export function isPortableDirection(type: unknown): boolean { + return ( + isCompatibleIndexType(type) || + type === MongoIndexType.Ascending || + type === MongoIndexType.Descending + ); +} + +export function sqlDialectAllowsIndexType(dialect: string, type: unknown): boolean { + if (type === undefined || isPortableDirection(type)) return true; + if (type === PostgresIndexType.BTREE) return true; + if (type === PostgresIndexType.HASH) { + return dialect === 'postgres' || dialect === 'mysql' || dialect === 'mariadb'; + } + return isPostgresIndexType(type) && dialect === 'postgres'; +} + +export function mongoAllowsIndexType(type: unknown): boolean { + return type === undefined || isCompatibleIndexType(type) || isMongoIndexType(type); +} + +export function mergeDeclaredIndexes( + existing: readonly ModelOptionsIndexes[] | undefined, + incoming: readonly ModelOptionsIndexes[], + collectionName: string, +): ModelOptionsIndexes[] { + const merged = declaredIndexMap(existing, collectionName); + for (const index of incoming) { + const named = ensureIndexName(index, collectionName); + const name = resolveIndexName(named); + if (name && !merged.has(name)) merged.set(name, named); + } + return [...merged.values()]; +} + +export function removeDeclaredIndexes( + existing: readonly ModelOptionsIndexes[] | undefined, + names: readonly string[], +): ModelOptionsIndexes[] { + const drop = new Set(names); + return (existing ?? []).filter(index => { + const name = resolveIndexName(index); + return !name || !drop.has(name); + }); +} + +export function removeIndexFromSchemaFields( + schema: { fields?: Record; compiledFields?: Record }, + indexName: string, +): boolean { + let removed = false; + for (const bag of [schema.fields, schema.compiledFields]) { + if (!bag) continue; + for (const value of Object.values(bag)) { + if (!value || typeof value !== 'object') continue; + const field = value as ConduitModelField; + const name = field.index?.options?.name ?? (field.index as { name?: string })?.name; + if (name === indexName) { + delete field.index; + removed = true; + } + } + } + return removed; +} + +export function validateIndexFields( + schema: Pick, + index: ModelOptionsIndexes, +) { + if (!index.fields || index.fields.length === 0) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Index fields must be a non-empty array', + ); + } + const available = new Set([ + ...Object.keys(schema.compiledFields ?? {}), + ...Object.keys(schema.fields ?? {}), + ]); + const missing = index.fields.filter(field => !available.has(field)); + if (missing.length > 0) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Invalid fields for index creation: ${missing.join(', ')}`, + ); + } +} + +export function assertUniqueIndexPrivilege(args: { + unique: boolean; + schemaOwner: string; + callerModule: string; + privileged?: boolean; +}) { + if (!args.unique) return; + if (args.privileged || args.schemaOwner === args.callerModule) return; + throw new GrpcError(status.PERMISSION_DENIED, 'Not authorized to create unique index'); +} + +type ErrorPart = { code?: number | string; message?: string; name?: string }; + +function walkError(error: unknown): ErrorPart[] { + const parts: ErrorPart[] = []; + const seen = new Set(); + let current: unknown = error; + while (current && typeof current === 'object' && !seen.has(current)) { + seen.add(current); + const err = current as ErrorPart & { + original?: unknown; + parent?: unknown; + cause?: unknown; + }; + parts.push({ code: err.code, message: err.message, name: err.name }); + current = err.original ?? err.parent ?? err.cause; + } + return parts; +} + +const UNIQUE_OR_OPTIONS_CONFLICT_CODES = new Set([ + 85, + '85', + 86, + '86', + 11000, + '11000', + 23505, + '23505', + 1062, + '1062', +]); + +export function isIndexKeySpecsConflictError(error: unknown): boolean { + return walkError(error).some(part => part.code === 86 || part.code === '86'); +} + +export function isIndexAlreadyExistsError(error: unknown): boolean { + const parts = walkError(error); + if (parts.some(part => part.name === 'SequelizeUniqueConstraintError')) { + return false; + } + if ( + parts.some( + part => part.code !== undefined && UNIQUE_OR_OPTIONS_CONFLICT_CODES.has(part.code), + ) + ) { + return false; + } + for (const part of parts) { + if (part.code === '42P07' || part.code === 1061 || part.code === '1061') { + return true; + } + const message = part.message ?? ''; + if (/index .+ already exists/i.test(message)) return true; + if (/duplicate key name/i.test(message)) return true; + if (/relation .+ already exists/i.test(message)) return true; + } + return false; +} + +export async function persistDeclaredSchemaIndexes(args: { + declaredSchemaModel: { + findOne: ( + query: Record, + options?: { readPreference?: string }, + ) => Promise<{ + _id: string; + modelOptions?: { indexes?: ModelOptionsIndexes[] }; + } | null>; + findByIdAndUpdate: (id: string, update: Record) => Promise; + }; + schemaName: string; + originalSchema: { + modelOptions: { indexes?: ModelOptionsIndexes[] | readonly ModelOptionsIndexes[] }; + fields?: Record; + compiledFields?: Record; + collectionName?: string; + name?: string; + }; + applied?: ModelOptionsIndexes[]; + droppedNames?: string[]; +}): Promise { + const found = await args.declaredSchemaModel.findOne( + { name: args.schemaName }, + { readPreference: 'primary' }, + ); + const memoryIndexes = (args.originalSchema.modelOptions.indexes ?? + []) as ModelOptionsIndexes[]; + const dbIndexes = found?.modelOptions?.indexes ?? memoryIndexes; + const collectionName = indexNameCollection(args.originalSchema) || args.schemaName; + const next = args.droppedNames + ? removeDeclaredIndexes(dbIndexes, args.droppedNames) + : mergeDeclaredIndexes(dbIndexes, args.applied ?? [], collectionName); + args.originalSchema.modelOptions.indexes = next; + if (!found) return false; + await args.declaredSchemaModel.findByIdAndUpdate(found._id, { + modelOptions: args.originalSchema.modelOptions, + fields: args.originalSchema.fields, + compiledFields: args.originalSchema.compiledFields, + }); + return true; +} + +export function collectExistingIndexNames( + indexes: readonly ModelOptionsIndexes[], +): Set { + return new Set( + indexes.map(resolveIndexName).filter((name): name is string => Boolean(name)), + ); +} + +export function declaredIndexMap( + indexes: readonly ModelOptionsIndexes[] | undefined, + collectionName = '', +): Map { + const map = new Map(); + for (const index of indexes ?? []) { + const named = ensureIndexName(index, collectionName); + const name = resolveIndexName(named); + if (name) map.set(name, named); + } + return map; +} + +export function toMutableIndexes( + indexes: readonly ModelOptionsIndexes[], +): ModelOptionsIndexes[] { + return indexes.map(index => ({ + ...index, + fields: [...index.fields], + options: index.options ? { ...index.options } : index.options, + })); +} + +export function sqlIndexFields(index: ModelOptionsIndexes): SqlIndexField[] { + const types = normalizeIndexTypes(index.types, index.fields.length); + if (!types || !types.some(isPortableDirection)) { + return [...index.fields]; + } + return index.fields.map((field, i) => ({ + name: field, + order: mapCompatibleToSqlOrder(types[i]), + })); +} + +export function sqlEngineIndexFields( + index: ModelOptionsIndexes, + schemaFields: Record, +): SqlIndexField[] { + return sqlIndexFields({ + ...index, + fields: mapIndexFieldsToSqlEngine(index, schemaFields), + }); +} + +export function inferSqlIndexType( + row: { type?: string; definition?: string }, + dialect: string, +): PostgresIndexType | undefined { + if (typeof row.type === 'string' && isPostgresIndexType(row.type.toUpperCase())) { + return row.type.toUpperCase() as PostgresIndexType; + } + if (typeof row.definition === 'string') { + const match = /USING\s+(\w+)/i.exec(row.definition); + const using = match?.[1]?.toUpperCase(); + if (using && isPostgresIndexType(using)) return using; + } + if (['postgres', 'mysql', 'mariadb', 'sqlite'].includes(dialect)) { + return PostgresIndexType.BTREE; + } + return undefined; +} + +export function isMongoNamespaceMissingError(error: unknown): boolean { + const err = error as { code?: number | string; codeName?: string; message?: string }; + if (err?.code === 26 || err?.code === '26' || err?.codeName === 'NamespaceNotFound') { + return true; + } + return /ns does not exist|ns not found|namespace not found/i.test(err?.message ?? ''); +} + +export function isArrayLikeConduitField(field: unknown): boolean { + if (Array.isArray(field)) return true; + return Boolean( + field && + typeof field === 'object' && + Array.isArray((field as { type?: unknown }).type), + ); +} + +function isRelationElement(value: unknown): boolean { + return Boolean( + value && + typeof value === 'object' && + (value as { type?: unknown }).type === 'Relation', + ); +} + +export function isExtractedArrayRelationField(field: unknown): boolean { + if (Array.isArray(field)) return isRelationElement(field[0]); + if (field && typeof field === 'object') { + const type = (field as { type?: unknown }).type; + if (Array.isArray(type)) return isRelationElement(type[0]); + } + return false; +} + +export function isMysqlJsonLikeField(field: unknown): boolean { + if (!field || typeof field !== 'object') return false; + if (Array.isArray(field)) { + const first = field[0]; + if (isRelationElement(first)) return false; + if (typeof first === 'string') return first === 'JSON'; + return Boolean(first && typeof first === 'object'); + } + const type = (field as { type?: unknown }).type; + if (type === 'JSON') return true; + return Array.isArray(type); +} + +export function isScalarRelationField(field: unknown): boolean { + return Boolean( + field && + typeof field === 'object' && + !Array.isArray(field) && + (field as { type?: unknown }).type === 'Relation', + ); +} + +export function collectSchemaIndexFields(schema: { + fields?: Record; + compiledFields?: Record; +}): Record { + return { + ...(schema.fields ?? {}), + ...(schema.compiledFields ?? {}), + }; +} + +export function sqlEngineIndexFieldName( + declaredField: string, + fields: Record, +): string { + if (isScalarRelationField(fields[declaredField])) { + return `${declaredField}Id`; + } + return declaredField; +} + +export function sqlDeclaredIndexFieldName( + engineField: string, + fields: Record, +): string { + if (Object.prototype.hasOwnProperty.call(fields, engineField)) { + return engineField; + } + if (engineField.endsWith('Id')) { + const declared = engineField.slice(0, -2); + if (declared.length > 0 && isScalarRelationField(fields[declared])) { + return declared; + } + } + return engineField; +} + +export function mapIndexFieldsToSqlEngine( + index: Pick | { fields?: readonly unknown[] }, + fields: Record, +): string[] { + return indexFieldNames(index).map(name => sqlEngineIndexFieldName(name, fields)); +} + +export function mapIndexFieldsToDeclared( + index: Pick | { fields?: readonly unknown[] }, + fields: Record, +): string[] { + return indexFieldNames(index).map(name => sqlDeclaredIndexFieldName(name, fields)); +} + +export function canonicalizeDeclaredIndexFields( + index: T, + fields: Record, +): T { + const mapped = mapIndexFieldsToDeclared(index, fields); + const current = indexFieldNames(index); + if ( + mapped.length === current.length && + mapped.every((name, i) => name === current[i]) + ) { + return index; + } + return { ...index, fields: mapped }; +} + +export function sqlIndexFieldNormalizer( + fields: Record, +): IndexFieldNormalizer { + return names => names.map(name => sqlEngineIndexFieldName(name, fields)); +} + +export function sqlIndexUnsupportedReason( + dialect: string, + index: Pick, + fields: Record, + options?: { timestamps?: boolean }, +): string | undefined { + const present = new Set(Object.keys(fields)); + if (options?.timestamps) { + present.add('createdAt'); + present.add('updatedAt'); + } + const mysqlJson = dialect === 'mysql' || dialect === 'mariadb'; + for (const name of indexFieldNames(index)) { + const field = fields[name]; + if (isExtractedArrayRelationField(field)) { + return `Field '${name}' is stored as a relation join table and cannot be indexed on SQL`; + } + if (mysqlJson && isMysqlJsonLikeField(field)) { + return `Compatible btree indexes are not supported on MySQL JSON field '${name}'`; + } + if (field === undefined && !present.has(name)) { + return `Field '${name}' is stored as a relation join table and cannot be indexed on SQL`; + } + } + return undefined; +} diff --git a/modules/database/src/admin/index.ts b/modules/database/src/admin/index.ts index 5a4605e9f..6db05353a 100644 --- a/modules/database/src/admin/index.ts +++ b/modules/database/src/admin/index.ts @@ -633,6 +633,35 @@ export class AdminHandlers { }), this.customEndpointsAdmin.schemaDetailsForOperation.bind(this.customEndpointsAdmin), ); + this.routingManager.route( + { + path: '/indexes/export', + action: ConduitRouteActions.GET, + description: `Exports schema indexes. Admin-only and paginated.`, + queryParams: { + skip: ConduitNumber.OptionalWith({ min: 0, integer: true }), + limit: ConduitNumber.OptionalWith({ min: 1, max: 1000, integer: true }), + }, + }, + new ConduitRouteReturnDefinition('ExportSchemaIndexes', { + indexes: [ConduitJson.Required], + count: ConduitNumber.Required, + }), + this.schemaAdmin.exportIndexes.bind(this.schemaAdmin), + ); + this.routingManager.route( + { + path: '/indexes/import', + action: ConduitRouteActions.POST, + description: `Imports schema indexes. Skips indexes that already exist by name. Unique indexes respect schema owner privilege.`, + mcp: false, + bodyParams: { + indexes: { type: [TYPE.JSON], required: true }, + }, + }, + new ConduitRouteReturnDefinition('ImportSchemaIndexes', 'String'), + this.schemaAdmin.importIndexes.bind(this.schemaAdmin), + ); this.routingManager.route( { path: '/schemas/:id/indexes', diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index 9e8ae74cc..5ae3f165e 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -3,6 +3,7 @@ import { ConduitSchema, GrpcError, Indexable, + ModelOptionsIndexes, ParsedRouterRequest, UnparsedRouterResponse, } from '@conduitplatform/grpc-sdk'; @@ -23,6 +24,14 @@ import { import { SchemaConverter } from '../utils/SchemaConverter.js'; import { parseSortParam } from '../handlers/utils.js'; import escapeStringRegexp from 'escape-string-regexp'; +import { + ADMIN_INDEX_CALLER, + canonicalizeDeclaredIndexFields, + collectExistingIndexNames, + collectSchemaIndexFields, + ensureIndexName, + resolveIndexName, +} from '../adapters/utils/indexes.js'; type ExportedCmsSchema = Pick< ConduitDatabaseSchema, @@ -743,34 +752,27 @@ export class SchemaAdmin { async createIndexes(call: ParsedRouterRequest): Promise { const { id, indexes } = call.request.params; - const requestedSchema = await this.database - .getSchemaModel('_DeclaredSchema') - .model.findOne({ _id: id }); - if (isNil(requestedSchema)) { - throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); - } - return await this.database.createIndexes(requestedSchema.name, indexes, 'database'); + const requestedSchema = await this.findDeclaredSchemaById(id); + const schemaFields = collectSchemaIndexFields(requestedSchema); + return await this.database.createIndexes( + requestedSchema.name, + (indexes as ModelOptionsIndexes[]).map(index => + canonicalizeDeclaredIndexFields(index, schemaFields), + ), + ADMIN_INDEX_CALLER, + { privileged: true }, + ); } async getIndexes(call: ParsedRouterRequest): Promise { - const id = call.request.params.id; - const requestedSchema = await this.database - .getSchemaModel('_DeclaredSchema') - .model.findOne({ _id: id }); - if (isNil(requestedSchema)) { - throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); - } - return this.database.getIndexes(requestedSchema.name); + const requestedSchema = await this.findDeclaredSchemaById(call.request.params.id); + const indexes = await this.database.getIndexes(requestedSchema.name); + return { indexes }; } async deleteIndexes(call: ParsedRouterRequest): Promise { const { id, indexNames } = call.request.params; - const requestedSchema = await this.database - .getSchemaModel('_DeclaredSchema') - .model.findOne({ _id: id }); - if (isNil(requestedSchema)) { - throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); - } + const requestedSchema = await this.findDeclaredSchemaById(id); if (isNil(indexNames) || indexNames.length === 0) { throw new GrpcError( status.INVALID_ARGUMENT, @@ -780,6 +782,110 @@ export class SchemaAdmin { return this.database.deleteIndexes(requestedSchema.name, indexNames); } + async exportIndexes(call: ParsedRouterRequest): Promise { + const skip = call.request.params.skip ?? 0; + const limit = call.request.params.limit ?? 25; + const query: Indexable = { + name: { $nin: this.database.systemSchemas }, + $or: [ + { parentSchema: { $exists: false } }, + { parentSchema: { $eq: null } }, + { parentSchema: { $eq: '' } }, + ], + }; + const schemaAdapter = this.database.getSchemaModel('_DeclaredSchema'); + const [schemas, count] = await Promise.all([ + schemaAdapter.model.findMany(query, { + skip, + limit, + select: 'name', + sort: { name: 1 }, + }), + schemaAdapter.model.countDocuments(query), + ]); + const indexes: Array = []; + for (const schema of schemas) { + if (!this.database.models[schema.name]) continue; + let schemaIndexes: ModelOptionsIndexes[]; + try { + schemaIndexes = await this.database.getIndexes(schema.name); + } catch (e) { + ConduitGrpcSdk.Logger.warn( + `Skipping indexes export for schema '${schema.name}': ${(e as Error).message}`, + ); + continue; + } + if (isNil(schemaIndexes) || isEmpty(schemaIndexes)) continue; + indexes.push( + ...schemaIndexes.map(index => ({ ...index, schemaName: schema.name })), + ); + } + return { indexes, count }; + } + + async importIndexes(call: ParsedRouterRequest): Promise { + const { indexes } = call.request.params as { + indexes: Array; + }; + if (!Array.isArray(indexes) || indexes.length === 0) { + throw new GrpcError(status.INVALID_ARGUMENT, 'indexes must be a non-empty array'); + } + const bySchema = new Map(); + for (const entry of indexes) { + const { schemaName, ...rest } = entry; + if (!schemaName) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Each imported index needs schemaName', + ); + } + const bucket = bySchema.get(schemaName) ?? []; + bucket.push(rest); + bySchema.set(schemaName, bucket); + } + for (const [schemaName, schemaIndexes] of bySchema) { + if (!this.database.models[schemaName]) { + throw new GrpcError( + status.NOT_FOUND, + `Requested schema not found: ${schemaName}`, + ); + } + const collectionName = + this.database.models[schemaName].originalSchema.collectionName ?? schemaName; + const schemaFields = collectSchemaIndexFields( + this.database.models[schemaName].originalSchema, + ); + const existing = await this.database.getIndexes(schemaName); + const existingNames = collectExistingIndexNames(existing); + const toCreate = schemaIndexes + .map(index => + ensureIndexName( + canonicalizeDeclaredIndexFields(index, schemaFields), + collectionName, + ), + ) + .filter(index => { + const name = resolveIndexName(index); + return !name || !existingNames.has(name); + }); + if (toCreate.length === 0) continue; + await this.database.createIndexes(schemaName, toCreate, ADMIN_INDEX_CALLER, { + privileged: false, + }); + } + return 'Indexes imported successfully'; + } + + private async findDeclaredSchemaById(id: string) { + const requestedSchema = await this.database + .getSchemaModel('_DeclaredSchema') + .model.findOne({ _id: id }); + if (isNil(requestedSchema)) { + throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); + } + return requestedSchema; + } + async checkRequestedSchema(id: string) { const requestedSchema = await this.database .getSchemaModel('_DeclaredSchema') diff --git a/modules/database/src/utils/utilities.ts b/modules/database/src/utils/utilities.ts index 15f6e4d4f..aa31a1f4a 100644 --- a/modules/database/src/utils/utilities.ts +++ b/modules/database/src/utils/utilities.ts @@ -158,10 +158,12 @@ const ALLOWED_CONDUIT_READ_PREFERENCES = [ function validateModelOptions(modelOptions: ConduitSchemaOptions) { if (!isPlainObject(modelOptions)) throw new Error('Model options must be an object'); Object.keys(modelOptions).forEach(key => { - if (key !== 'conduit' && key !== 'timestamps') - throw new Error("Only 'conduit' and 'timestamps' options allowed"); + if (key !== 'conduit' && key !== 'timestamps' && key !== 'indexes') + throw new Error("Only 'conduit', 'timestamps', and 'indexes' options allowed"); else if (key === 'timestamps' && !isBoolean(modelOptions.timestamps)) throw new Error("Option 'timestamps' must be of type Boolean"); + else if (key === 'indexes' && !isArray(modelOptions.indexes)) + throw new Error("Option 'indexes' must be of type Array"); else if (key === 'conduit') { if (!isObject(modelOptions.conduit)) throw new Error("Option 'conduit' must be of type Object");