From 7373fba177469dc5d974b588a565934d4c92961a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 14 Sep 2026 11:41:05 +0000 Subject: [PATCH 1/9] feat(database): add Compatible indexes for Mongo and SQL dialects Make schema indexes first-class on Mongo, PostgreSQL, MySQL, MariaDB, and SQLite. Platform models use Compatible Ascending/Descending so the same declarations create live indexes on every dialect. Admin create/get/delete/import/export persist into _DeclaredSchema without a full schema rebuild. --- libraries/grpc-sdk/src/interfaces/Model.ts | 20 +- .../src/models/ActorIndex.schema.ts | 7 +- .../src/models/ObjectIndex.schema.ts | 23 +- .../src/models/Permission.schema.ts | 8 +- .../src/models/Relationship.schema.ts | 8 +- modules/chat/src/models/ChatRoom.schema.ts | 9 +- modules/chat/src/models/Message.schema.ts | 17 +- .../src/__tests__/no-old-pr-bugs.test.ts | 36 ++ .../__tests__/platform-models.indexes.test.ts | 33 ++ .../database/src/adapters/DatabaseAdapter.ts | 3 +- .../mongoose-adapter/SchemaConverter.ts | 65 +++- .../__tests__/SchemaConverter.indexes.test.ts | 66 ++++ .../__tests__/indexes.adapter.test.ts | 123 +++++++ .../src/adapters/mongoose-adapter/index.ts | 163 ++++++--- .../__tests__/indexes.adapter.test.ts | 128 +++++++ .../src/adapters/sequelize-adapter/index.ts | 183 +++++++--- .../postgres-adapter/PgSchemaConverter.ts | 4 +- .../sql-adapter/SqlSchemaConverter.ts | 9 +- .../adapters/utils/__tests__/indexes.test.ts | 198 +++++++++++ .../__tests__/sql-index-converters.test.ts | 125 +++++++ .../utils/database-transform-utils.ts | 158 ++++++--- modules/database/src/adapters/utils/index.ts | 1 + .../database/src/adapters/utils/indexes.ts | 329 ++++++++++++++++++ .../__tests__/schema.admin.indexes.test.ts | 129 +++++++ modules/database/src/admin/index.ts | 29 ++ modules/database/src/admin/schema.admin.ts | 93 ++++- .../validateModelOptions.indexes.test.ts | 45 +++ modules/database/src/utils/utilities.ts | 6 +- 28 files changed, 1810 insertions(+), 208 deletions(-) create mode 100644 modules/database/src/__tests__/no-old-pr-bugs.test.ts create mode 100644 modules/database/src/__tests__/platform-models.indexes.test.ts create mode 100644 modules/database/src/adapters/mongoose-adapter/__tests__/SchemaConverter.indexes.test.ts create mode 100644 modules/database/src/adapters/mongoose-adapter/__tests__/indexes.adapter.test.ts create mode 100644 modules/database/src/adapters/sequelize-adapter/__tests__/indexes.adapter.test.ts create mode 100644 modules/database/src/adapters/utils/__tests__/indexes.test.ts create mode 100644 modules/database/src/adapters/utils/__tests__/sql-index-converters.test.ts create mode 100644 modules/database/src/adapters/utils/indexes.ts create mode 100644 modules/database/src/admin/__tests__/schema.admin.indexes.test.ts create mode 100644 modules/database/src/utils/__tests__/validateModelOptions.indexes.test.ts diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 894ca419e..099eb424f 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -79,6 +79,20 @@ export enum PostgresIndexType { BRIN = 'BRIN', } +/** + * Portable ascending/descending indexes that exist on MongoDB and every SQL dialect. + * Map to Mongo 1/-1 and SQL BTREE ASC/DESC. Optional uniqueness lives on index options. + */ +export enum CompatibleIndexType { + Ascending = 'Ascending', + Descending = 'Descending', +} + +export type IndexType = MongoIndexType | PostgresIndexType | CompatibleIndexType; + +export type ModelOptionsIndexTypes = + MongoIndexType[] | PostgresIndexType | CompatibleIndexType | CompatibleIndexType[]; + export type Array = any[]; export interface ConduitStringValidation { @@ -244,7 +258,7 @@ export interface ConduitSchemaOptions { } export interface SchemaFieldIndex { - type?: MongoIndexType | PostgresIndexType; + type?: IndexType; options?: MongoIndexOptions | PostgresIndexOptions; [field: string]: any; @@ -252,8 +266,10 @@ export interface SchemaFieldIndex { export interface ModelOptionsIndexes { fields: string[] | readonly string[]; - types?: MongoIndexType[] | PostgresIndexType; + types?: ModelOptionsIndexTypes; options?: MongoIndexOptions | PostgresIndexOptions; + /** Optional. Generated deterministically when omitted. */ + name?: string; [field: string]: any; } diff --git a/modules/authorization/src/models/ActorIndex.schema.ts b/modules/authorization/src/models/ActorIndex.schema.ts index 2663dee2b..30cce8a83 100644 --- a/modules/authorization/src/models/ActorIndex.schema.ts +++ b/modules/authorization/src/models/ActorIndex.schema.ts @@ -1,8 +1,8 @@ import { + CompatibleIndexType, ConduitModel, ConduitSchemaOptions, DatabaseProvider, - MongoIndexType, TYPE, } from '@conduitplatform/grpc-sdk'; import { ConduitActiveSchema } from '@conduitplatform/module-tools'; @@ -23,7 +23,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, subjectId: { @@ -41,7 +41,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, entityId: { @@ -69,6 +69,7 @@ const schemaOptions: ConduitSchemaOptions = { indexes: [ { fields: ['subject', 'entity'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], }, ], conduit: { diff --git a/modules/authorization/src/models/ObjectIndex.schema.ts b/modules/authorization/src/models/ObjectIndex.schema.ts index 7ef08abd1..0366dd08a 100644 --- a/modules/authorization/src/models/ObjectIndex.schema.ts +++ b/modules/authorization/src/models/ObjectIndex.schema.ts @@ -1,8 +1,8 @@ import { + CompatibleIndexType, ConduitModel, ConduitSchemaOptions, DatabaseProvider, - MongoIndexType, TYPE, } from '@conduitplatform/grpc-sdk'; import { ConduitActiveSchema } from '@conduitplatform/module-tools'; @@ -23,7 +23,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, subjectId: { @@ -47,7 +47,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, entityId: { @@ -75,7 +75,7 @@ const schema: ConduitModel = { type: [TYPE.String], default: [], index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, createdAt: TYPE.Date, @@ -86,12 +86,27 @@ const schemaOptions: ConduitSchemaOptions = { indexes: [ { fields: ['subjectType', 'subjectPermission', 'entity'], + types: [ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ], }, { fields: ['entity', 'subjectType', 'subjectPermission'], + types: [ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ], }, { fields: ['entityType', 'entityId', 'entityPermission'], + types: [ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ], }, ], conduit: { diff --git a/modules/authorization/src/models/Permission.schema.ts b/modules/authorization/src/models/Permission.schema.ts index 76dd2e3a5..ad1502649 100644 --- a/modules/authorization/src/models/Permission.schema.ts +++ b/modules/authorization/src/models/Permission.schema.ts @@ -1,7 +1,7 @@ import { + CompatibleIndexType, ConduitModel, DatabaseProvider, - MongoIndexType, TYPE, } from '@conduitplatform/grpc-sdk'; import { ConduitActiveSchema } from '@conduitplatform/module-tools'; @@ -18,7 +18,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, resourceId: { @@ -37,7 +37,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, subjectId: { @@ -61,7 +61,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, options: { unique: true, }, diff --git a/modules/authorization/src/models/Relationship.schema.ts b/modules/authorization/src/models/Relationship.schema.ts index 4961f5813..88cff5fc7 100644 --- a/modules/authorization/src/models/Relationship.schema.ts +++ b/modules/authorization/src/models/Relationship.schema.ts @@ -1,7 +1,7 @@ import { + CompatibleIndexType, ConduitModel, DatabaseProvider, - MongoIndexType, TYPE, } from '@conduitplatform/grpc-sdk'; import { ConduitActiveSchema } from '@conduitplatform/module-tools'; @@ -17,7 +17,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, resourceId: { @@ -36,7 +36,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, }, }, // user:1adasdas @@ -61,7 +61,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { - type: MongoIndexType.Ascending, + type: CompatibleIndexType.Ascending, options: { unique: true, }, diff --git a/modules/chat/src/models/ChatRoom.schema.ts b/modules/chat/src/models/ChatRoom.schema.ts index 6a7c10f61..92eae55e7 100644 --- a/modules/chat/src/models/ChatRoom.schema.ts +++ b/modules/chat/src/models/ChatRoom.schema.ts @@ -1,4 +1,5 @@ import { + CompatibleIndexType, ConduitModel, ConduitSchemaOptions, DatabaseProvider, @@ -40,7 +41,13 @@ const schema: ConduitModel = { }; const modelOptions: ConduitSchemaOptions = { timestamps: true, - indexes: [{ fields: ['participants'] }, { fields: ['participants', 'deleted'] }], + indexes: [ + { fields: ['participants'], types: [CompatibleIndexType.Ascending] }, + { + fields: ['participants', 'deleted'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], conduit: { permissions: { extendable: true, diff --git a/modules/chat/src/models/Message.schema.ts b/modules/chat/src/models/Message.schema.ts index 7f5bf5546..752928a49 100644 --- a/modules/chat/src/models/Message.schema.ts +++ b/modules/chat/src/models/Message.schema.ts @@ -1,8 +1,9 @@ import { + CompatibleIndexType, ConduitModel, + ConduitSchemaOptions, DatabaseProvider, TYPE, - ConduitSchemaOptions, } from '@conduitplatform/grpc-sdk'; import { ConduitActiveSchema } from '@conduitplatform/module-tools'; import { ChatRoom } from './ChatRoom.schema.js'; @@ -53,8 +54,18 @@ const schema: ConduitModel = { const modelOptions: ConduitSchemaOptions = { timestamps: true, indexes: [ - { fields: ['room', 'createdAt'] }, - { fields: ['room', 'deleted', 'createdAt'] }, + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + { + fields: ['room', 'deleted', 'createdAt'], + types: [ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ], + }, ], conduit: { permissions: { diff --git a/modules/database/src/__tests__/no-old-pr-bugs.test.ts b/modules/database/src/__tests__/no-old-pr-bugs.test.ts new file mode 100644 index 000000000..68a155907 --- /dev/null +++ b/modules/database/src/__tests__/no-old-pr-bugs.test.ts @@ -0,0 +1,36 @@ +import { readFileSync } from 'fs'; +import { resolve } from 'path'; +import { describe, expect, it } from '@jest/globals'; + +describe('do not port old PR #643 bugs', () => { + it("does not use `case 'mysql' || 'mariadb'`", () => { + const files = [ + resolve(process.cwd(), 'src/adapters/utils/indexes.ts'), + resolve(process.cwd(), 'src/adapters/sequelize-adapter/index.ts'), + resolve(process.cwd(), 'src/adapters/utils/database-transform-utils.ts'), + ]; + for (const file of files) { + const source = readFileSync(file, 'utf8'); + expect(source).not.toMatch(/case ['"]mysql['"]\s*\|\|/); + } + }); + + it('does not rename getDatabaseType PostgreSQL to postgres', () => { + const source = readFileSync( + resolve(process.cwd(), 'src/adapters/sequelize-adapter/index.ts'), + 'utf8', + ); + expect(source).toContain("return 'PostgreSQL'"); + }); + + it('does not use metadata-only getIndexes or createSchemaFromAdapter rebuild for indexes', () => { + const sequelize = readFileSync( + resolve(process.cwd(), 'src/adapters/sequelize-adapter/index.ts'), + 'utf8', + ); + expect(sequelize).toContain('showIndex'); + expect(sequelize).toContain('addIndex'); + expect(sequelize).not.toMatch(/createIndexes[\s\S]*createSchemaFromAdapter/); + expect(sequelize).not.toMatch(/await this\.models\[schemaName\]\.sync\(\)/); + }); +}); diff --git a/modules/database/src/__tests__/platform-models.indexes.test.ts b/modules/database/src/__tests__/platform-models.indexes.test.ts new file mode 100644 index 000000000..e5baea9a5 --- /dev/null +++ b/modules/database/src/__tests__/platform-models.indexes.test.ts @@ -0,0 +1,33 @@ +import { existsSync, readFileSync } from 'fs'; +import { resolve } from 'path'; +import { describe, expect, it } from '@jest/globals'; + +function repoFile(...parts: string[]) { + const candidates = [ + resolve(process.cwd(), '..', ...parts), + resolve(process.cwd(), ...parts), + resolve(process.cwd(), '../..', ...parts), + ]; + const found = candidates.find(existsSync); + if (!found) throw new Error(`Missing ${parts.join('/')}`); + return found; +} + +const files = [ + repoFile('authorization', 'src', 'models', 'Permission.schema.ts'), + repoFile('authorization', 'src', 'models', 'Relationship.schema.ts'), + repoFile('authorization', 'src', 'models', 'ActorIndex.schema.ts'), + repoFile('authorization', 'src', 'models', 'ObjectIndex.schema.ts'), + repoFile('chat', 'src', 'models', 'ChatRoom.schema.ts'), + repoFile('chat', 'src', 'models', 'Message.schema.ts'), +]; + +describe('platform models T7 CompatibleIndexType', () => { + it('authz + chat schemas declare Compatible indexes, not Mongo-only types', () => { + for (const file of files) { + const source = readFileSync(file, 'utf8'); + expect(source).toContain('CompatibleIndexType'); + expect(source).not.toContain('MongoIndexType'); + } + }); +}); diff --git a/modules/database/src/adapters/DatabaseAdapter.ts b/modules/database/src/adapters/DatabaseAdapter.ts index 96d33b4dc..da415e949 100644 --- a/modules/database/src/adapters/DatabaseAdapter.ts +++ b/modules/database/src/adapters/DatabaseAdapter.ts @@ -204,8 +204,9 @@ export abstract class DatabaseAdapter { abstract createIndexes( schemaName: string, - indexes: ModelOptionsIndexes[], + indexes: readonly ModelOptionsIndexes[], callerModule: string, + options?: { privileged?: boolean }, ): Promise; abstract getIndexes(schemaName: string): Promise; diff --git a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts index fe9dab7ba..edf889775 100644 --- a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts +++ b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts @@ -1,5 +1,6 @@ import { Schema } from 'mongoose'; import { + ConduitGrpcSdk, ConduitModelField, ConduitSchema, ModelOptionsIndexes, @@ -10,6 +11,12 @@ import { cloneDeep, isArray, isNil, isObject } from 'lodash-es'; import { checkIfMongoOptions } from './utils.js'; import { applyMongoVectorField } from '../utils/vectorMappings.js'; import { isVectorTypeName } from '../utils/vectorField.js'; +import { + isCompatibleIndexType, + isMongoIndexType, + mapCompatibleToMongo, + mongoAllowsIndexType, +} from '../utils/indexes.js'; import * as deepdash from 'deepdash-es/standalone'; @@ -109,12 +116,23 @@ function convertSchemaFieldIndexes(copy: ConduitSchema) { if (!index) continue; const type = index.type; const options = index.options; - if (type && !Object.values(MongoIndexType).includes(type)) { - throw new Error('Incorrect index type for MongoDB'); + if (type && !mongoAllowsIndexType(type)) { + ConduitGrpcSdk.Logger.warn( + `Invalid index type for MongoDB found in '${copy.name}', ignoring index`, + ); + delete (field[1] as ConduitModelField).index; + continue; + } + if (type && isCompatibleIndexType(type)) { + index.type = mapCompatibleToMongo(type); } if (options) { if (!checkIfMongoOptions(options)) { - throw new Error('Incorrect index options for MongoDB'); + ConduitGrpcSdk.Logger.warn( + `Invalid index options for MongoDB found in '${copy.name}', ignoring index`, + ); + delete (field[1] as ConduitModelField).index; + continue; } for (const [option, optionValue] of Object.entries(options)) { index[option as keyof SchemaFieldIndex] = optionValue; @@ -128,31 +146,48 @@ function convertSchemaFieldIndexes(copy: ConduitSchema) { function convertModelOptionsIndexes(copy: ConduitSchema) { if (!copy.modelOptions.indexes?.length) return copy; const mutIndexes = copy.modelOptions.indexes as ModelOptionsIndexes[]; - for (const index of mutIndexes) { + for (const index of [...mutIndexes]) { + if (index.types) { + const types = isArray(index.types) + ? index.types + : index.fields.map(() => index.types); + if ( + types.some(type => !mongoAllowsIndexType(type)) || + (isArray(index.types) && index.fields.length !== index.types.length) + ) { + ConduitGrpcSdk.Logger.warn( + `Invalid index type for MongoDB found in '${copy.name}', ignoring index`, + ); + mutIndexes.splice(mutIndexes.indexOf(index), 1); + continue; + } + index.types = types.map(type => + isCompatibleIndexType(type) || isMongoIndexType(type) + ? mapCompatibleToMongo(type) + : (type as MongoIndexType), + ) as MongoIndexType[]; + } // compound indexes are maintained in modelOptions in order to be created after schema creation // single field index => add it to specified schema field if (index.fields.length !== 1) continue; const modelField = copy.fields[index.fields[0]] as ConduitModelField; if (!modelField) { - throw new Error(`Field ${modelField} in index definition doesn't exist`); + throw new Error(`Field ${index.fields[0]} in index definition doesn't exist`); } if (index.types) { - if ( - !isArray(index.types) || - !Object.values(MongoIndexType).includes(index.types[0]) || - index.fields.length !== index.types.length - ) { - throw new Error('Invalid index type for MongoDB'); - } - const type = index.types[0] as MongoIndexType; modelField.index = { - type: type, + type: (index.types as MongoIndexType[])[0], }; } if (index.options) { if (!checkIfMongoOptions(index.options)) { - throw new Error('Incorrect index options for MongoDB'); + ConduitGrpcSdk.Logger.warn( + `Invalid index options for MongoDB found in '${copy.name}', ignoring index`, + ); + mutIndexes.splice(mutIndexes.indexOf(index), 1); + continue; } + if (!modelField.index) modelField.index = {}; for (const [option, optionValue] of Object.entries(index.options)) { modelField.index![option as keyof SchemaFieldIndex] = optionValue; } diff --git a/modules/database/src/adapters/mongoose-adapter/__tests__/SchemaConverter.indexes.test.ts b/modules/database/src/adapters/mongoose-adapter/__tests__/SchemaConverter.indexes.test.ts new file mode 100644 index 000000000..daeb74055 --- /dev/null +++ b/modules/database/src/adapters/mongoose-adapter/__tests__/SchemaConverter.indexes.test.ts @@ -0,0 +1,66 @@ +import { beforeEach, describe, expect, it, jest } from '@jest/globals'; +import { + CompatibleIndexType, + ConduitGrpcSdk, + ConduitSchema, + MongoIndexType, + PostgresIndexType, + TYPE, +} from '@conduitplatform/grpc-sdk'; +import { schemaConverter } from '../SchemaConverter.js'; + +describe('mongoose SchemaConverter indexes T17 T23', () => { + beforeEach(() => { + jest.spyOn(ConduitGrpcSdk.Logger, 'warn').mockImplementation(() => {}); + }); + + it('T17 treats Compatible as Mongo 1/-1', () => { + const converted = schemaConverter( + new ConduitSchema( + 'User', + { + email: { + type: TYPE.String, + index: { type: CompatibleIndexType.Descending }, + }, + } as any, + {}, + ), + ); + expect((converted.fields.email as any).index.type).toBe(MongoIndexType.Descending); + }); + + it('T23 recover: postgres leftovers on Mongo are warned and skipped', () => { + const warn = ConduitGrpcSdk.Logger.warn as jest.Mock; + const converted = schemaConverter( + new ConduitSchema( + 'User', + { + email: { + type: TYPE.String, + index: { type: PostgresIndexType.GIN }, + }, + } as any, + {}, + ), + ); + expect((converted.fields.email as any).index).toBeUndefined(); + expect(warn).toHaveBeenCalled(); + }); + + it('does not treat the Mongo enum key "Ascending" as a valid Mongo type', () => { + const converted = schemaConverter( + new ConduitSchema( + 'User', + { + email: { + type: TYPE.String, + index: { type: 'Ascending' as any }, + }, + } as any, + {}, + ), + ); + expect((converted.fields.email as any).index.type).toBe(MongoIndexType.Ascending); + }); +}); diff --git a/modules/database/src/adapters/mongoose-adapter/__tests__/indexes.adapter.test.ts b/modules/database/src/adapters/mongoose-adapter/__tests__/indexes.adapter.test.ts new file mode 100644 index 000000000..681228b26 --- /dev/null +++ b/modules/database/src/adapters/mongoose-adapter/__tests__/indexes.adapter.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { status } from '@grpc/grpc-js'; +import { CompatibleIndexType, MongoIndexType } from '@conduitplatform/grpc-sdk'; +import { MongooseAdapter } from '../index.js'; + +function makeAdapter(overrides: Record = {}) { + const createIndex = jest.fn().mockResolvedValue('email_1'); + const dropIndex = jest.fn().mockResolvedValue(undefined); + const indexes = jest.fn().mockResolvedValue([ + { v: 2, key: { _id: 1 }, name: '_id_' }, + { v: 2, key: { email: 1 }, name: 'cnd_idx_email_asc', unique: false }, + ]); + const findOne = jest.fn().mockResolvedValue({ _id: 'declared-1' }); + const findByIdAndUpdate = jest.fn().mockResolvedValue({}); + const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; + adapter.mongoose = { + model: () => ({ collection: { createIndex, dropIndex, indexes } }), + } as any; + const originalSchema = { + name: 'User', + ownerModule: 'chat', + collectionName: 'cnd_User', + fields: { email: { type: 'String' } }, + compiledFields: { email: { type: 'String' } }, + modelOptions: { indexes: [] as unknown[] }, + ...((overrides.originalSchema as object) ?? {}), + }; + adapter.models = { + User: { originalSchema }, + _DeclaredSchema: { findOne, findByIdAndUpdate }, + } as any; + return { adapter, createIndex, dropIndex, indexes, findByIdAndUpdate, originalSchema }; +} + +describe('mongoose adapter indexes T26–T29 T34 T38', () => { + it('T26 createIndex uses a single key spec object, not an array of objects', async () => { + const { adapter, createIndex } = makeAdapter(); + await adapter.createIndexes( + 'User', + [ + { + fields: ['email'], + types: [CompatibleIndexType.Ascending], + }, + ], + 'chat', + ); + expect(createIndex).toHaveBeenCalledTimes(1); + expect(createIndex.mock.calls[0][0]).toEqual({ email: MongoIndexType.Ascending }); + expect(Array.isArray(createIndex.mock.calls[0][0])).toBe(false); + }); + + it('T27 create persists metadata into _DeclaredSchema', async () => { + const { adapter, findByIdAndUpdate } = makeAdapter(); + await adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'chat', + ); + expect(findByIdAndUpdate).toHaveBeenCalledTimes(1); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { name?: string }[] }; + }; + expect(update.modelOptions.indexes[0].name).toMatch(/email/); + }); + + it('T28 delete awaits dropIndex', async () => { + const { adapter, dropIndex } = makeAdapter(); + let resolveDrop: () => void = () => {}; + const dropped = new Promise(resolve => { + resolveDrop = resolve; + }); + dropIndex.mockImplementation(async () => { + resolveDrop(); + }); + await adapter.deleteIndexes('User', ['cnd_idx_email_asc']); + await dropped; + expect(dropIndex).toHaveBeenCalledWith('cnd_idx_email_asc'); + }); + + it('T29 delete persists removal', async () => { + const { adapter, findByIdAndUpdate, originalSchema } = makeAdapter({ + originalSchema: { + modelOptions: { indexes: [{ fields: ['email'], name: 'cnd_idx_email_asc' }] }, + }, + }); + await adapter.deleteIndexes('User', ['cnd_idx_email_asc']); + expect(originalSchema.modelOptions.indexes).toEqual([]); + expect(findByIdAndUpdate).toHaveBeenCalled(); + }); + + it('T34 getIndexes uses the live engine as source of truth', async () => { + const { adapter, indexes } = makeAdapter(); + const result = await adapter.getIndexes('User'); + expect(indexes).toHaveBeenCalled(); + expect(result.map(i => i.name)).toEqual(['_id_', 'cnd_idx_email_asc']); + expect(result[1].fields).toEqual(['email']); + }); + + it('T38 Admin-bound invalid types throw', async () => { + const { adapter } = makeAdapter(); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], types: ['GIST'] as any }], + 'database', + { privileged: true }, + ), + ).rejects.toMatchObject({ code: status.INVALID_ARGUMENT }); + }); + + it('T15 Admin privileged unique is allowed on a foreign-owned schema', async () => { + const { adapter } = makeAdapter(); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], options: { unique: true } }], + 'database', + { privileged: true }, + ), + ).resolves.toBe('Indexes created!'); + }); +}); diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 47c3b1a4b..a81052f35 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -30,6 +30,22 @@ import { planMongoVectorIndexCreate, assertMongoVectorSearchIndexDropTarget, } from '../utils/index.js'; +import { + assertUniqueIndexPrivilege, + ensureIndexName, + isCompatibleIndexType, + isIndexAlreadyExistsError, + isMongoIndexType, + mapCompatibleToMongo, + mergeDeclaredIndexes, + mongoAllowsIndexType, + persistDeclaredSchemaIndexes, + removeDeclaredIndexes, + removeIndexFromSchemaFields, + resolveIndexName, + toMutableIndexes, + validateIndexFields, +} from '../utils/indexes.js'; import pluralize from '../../utils/pluralize.js'; import { mongoSchemaConverter } from '../../introspection/mongoose/utils.js'; import { status } from '@grpc/grpc-js'; @@ -640,22 +656,40 @@ export class MongooseAdapter extends DatabaseAdapter { schemaName: string, indexes: readonly ModelOptionsIndexes[], callerModule: string, + options?: { privileged?: boolean }, ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - this.checkIndexes(schemaName, indexes, callerModule); + const prepared = this.checkIndexes( + schemaName, + indexes, + callerModule, + options?.privileged, + ); const collection = this.mongoose.model(schemaName).collection; - for (const index of indexes) { - const indexSpecs = []; + for (const index of prepared) { + const spec: Record = {}; + const types = Array.isArray(index.types) ? index.types : undefined; for (let i = 0; i < index.fields.length; i++) { - const spec: any = {}; - spec[index.fields[i]] = index.types ? index.types[i] : 1; - indexSpecs.push(spec); + spec[index.fields[i]] = types + ? mapCompatibleToMongo(types[i]) + : MongoIndexType.Ascending; + } + try { + await collection.createIndex(spec, index.options); + } catch (e) { + if (isIndexAlreadyExistsError(e)) continue; + throw new GrpcError(status.INTERNAL, (e as Error).message); } - await collection.createIndex(indexSpecs, index.options).catch((e: Error) => { - throw new GrpcError(status.INTERNAL, e.message); - }); } + const original = this.models[schemaName].originalSchema; + const merged = mergeDeclaredIndexes(original.modelOptions.indexes, prepared); + await persistDeclaredSchemaIndexes({ + declaredSchemaModel: this.models['_DeclaredSchema'], + schemaName, + originalSchema: original, + indexes: merged, + }); return 'Indexes created!'; } @@ -701,29 +735,36 @@ export class MongooseAdapter extends DatabaseAdapter { throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); const collection = this.mongoose.model(schemaName).collection; const result = await collection.indexes(); - result.filter(index => { - index.options = {}; - for (const indexEntry of Object.entries(index)) { - if (indexEntry[0] === 'key' || indexEntry[0] === 'options') { - continue; - } - if (indexEntry[0] === 'v') { - delete index.v; - continue; - } - index.options[indexEntry[0]] = indexEntry[1]; - delete index[indexEntry[0]]; + const declared = this.models[schemaName].originalSchema.modelOptions.indexes ?? []; + const declaredByName = new Map( + declared.map((index: ModelOptionsIndexes) => [resolveIndexName(index), index]), + ); + return result.map(index => { + const options: Record = {}; + for (const [key, value] of Object.entries(index)) { + if (key === 'key' || key === 'options') continue; + if (key === 'v') continue; + options[key] = value; } - index.fields = []; - index.types = []; - for (const keyEntry of Object.entries(index.key)) { - index.fields.push(keyEntry[0]); - index.types.push(keyEntry[1]); - //@ts-expect-error - delete index.key; + const fields: string[] = []; + const types: MongoIndexType[] = []; + for (const [field, type] of Object.entries(index.key ?? {})) { + fields.push(field); + types.push(type as MongoIndexType); } + const name = (options.name as string | undefined) ?? index.name; + const declaredIndex = name ? declaredByName.get(name) : undefined; + return { + name, + fields, + types: declaredIndex?.types ?? types, + options: { + ...declaredIndex?.options, + ...options, + name, + }, + } as ModelOptionsIndexes; }); - return result as unknown as ModelOptionsIndexes[]; } async deleteIndexes(schemaName: string, indexNames: string[]): Promise { @@ -731,10 +772,23 @@ export class MongooseAdapter extends DatabaseAdapter { throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); const collection = this.mongoose.model(schemaName).collection; for (const name of indexNames) { - collection.dropIndex(name).catch(() => { + try { + await collection.dropIndex(name); + } catch { throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); - }); + } + } + const original = this.models[schemaName].originalSchema; + for (const name of indexNames) { + removeIndexFromSchemaFields(original, name); } + const remaining = removeDeclaredIndexes(original.modelOptions.indexes, indexNames); + await persistDeclaredSchemaIndexes({ + declaredSchemaModel: this.models['_DeclaredSchema'], + schemaName, + originalSchema: original, + indexes: remaining, + }); return 'Indexes deleted'; } @@ -1013,35 +1067,50 @@ export class MongooseAdapter extends DatabaseAdapter { schemaName: string, indexes: readonly ModelOptionsIndexes[], callerModule: string, - ) { - for (const index of indexes) { + privileged?: boolean, + ): ModelOptionsIndexes[] { + const schema = this.models[schemaName].originalSchema; + const prepared: ModelOptionsIndexes[] = []; + for (const raw of toMutableIndexes(indexes)) { + const index = ensureIndexName(raw); + validateIndexFields(schema, index); const options = index.options; const types = index.types; - if (!options && !types) continue; if (options) { if (!checkIfMongoOptions(options)) { - throw new GrpcError(status.INTERNAL, 'Invalid index options for mongoDB'); - } - if ( - Object.keys(options).includes('unique') && - this.models[schemaName].originalSchema.ownerModule !== callerModule - ) { throw new GrpcError( - status.PERMISSION_DENIED, - 'Not authorized to create unique index', + status.INVALID_ARGUMENT, + 'Invalid index options for mongoDB', ); } + assertUniqueIndexPrivilege({ + unique: options.unique === true, + schemaOwner: schema.ownerModule, + callerModule, + privileged, + }); } if (types) { - if (!Array.isArray(types) || types.length !== index.fields.length) { - throw new GrpcError(status.INTERNAL, 'Invalid index types format'); + const typeList = Array.isArray(types) ? types : index.fields.map(() => types); + if (Array.isArray(types) && typeList.length !== index.fields.length) { + throw new GrpcError(status.INVALID_ARGUMENT, 'Invalid index types format'); } - for (const type of types) { - if (!Object.values(MongoIndexType).includes(type)) { - throw new GrpcError(status.INTERNAL, 'Invalid index type for mongoDB'); + for (const type of typeList) { + if (!mongoAllowsIndexType(type)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Invalid index type for mongoDB', + ); } } + index.types = typeList.map(type => + isCompatibleIndexType(type) || isMongoIndexType(type) + ? mapCompatibleToMongo(type) + : type, + ) as MongoIndexType[]; } + prepared.push(index); } + return prepared; } } diff --git a/modules/database/src/adapters/sequelize-adapter/__tests__/indexes.adapter.test.ts b/modules/database/src/adapters/sequelize-adapter/__tests__/indexes.adapter.test.ts new file mode 100644 index 000000000..fe7cf5cd1 --- /dev/null +++ b/modules/database/src/adapters/sequelize-adapter/__tests__/indexes.adapter.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { CompatibleIndexType, PostgresIndexType } from '@conduitplatform/grpc-sdk'; +import { SequelizeAdapter } from '../index.js'; + +class TestSequelizeAdapter extends SequelizeAdapter { + protected async hasLegacyCollections(): Promise { + return false; + } +} + +function makeAdapter(dialect: string = 'postgres') { + const addIndex = jest.fn().mockResolvedValue(undefined); + const removeIndex = jest.fn().mockResolvedValue(undefined); + const showIndex = jest.fn().mockResolvedValue([ + { + name: 'cnd_idx_email_asc', + unique: false, + fields: [{ attribute: 'email', order: 'ASC' }], + definition: 'CREATE INDEX cnd_idx_email_asc ON cnd_User USING btree (email)', + }, + ]); + const findOne = jest.fn().mockResolvedValue({ _id: 'declared-1' }); + const findByIdAndUpdate = jest.fn().mockResolvedValue({}); + const sync = jest.fn().mockResolvedValue(undefined); + const adapter = Object.create(TestSequelizeAdapter.prototype) as SequelizeAdapter; + adapter.sequelize = { + getDialect: () => dialect, + getQueryInterface: () => ({ addIndex, removeIndex, showIndex }), + } as any; + const originalSchema = { + name: 'User', + ownerModule: 'database', + collectionName: 'custom_users', + fields: { email: { type: 'String' } }, + compiledFields: { email: { type: 'String' } }, + modelOptions: { indexes: [] as unknown[] }, + }; + adapter.models = { + User: { originalSchema, sync }, + _DeclaredSchema: { findOne, findByIdAndUpdate }, + } as any; + return { + adapter, + addIndex, + removeIndex, + showIndex, + sync, + findByIdAndUpdate, + originalSchema, + }; +} + +describe('sequelize adapter indexes T24 T30–T33', () => { + it('T24 getDatabaseType still returns PostgreSQL, not postgres', () => { + const { adapter } = makeAdapter('postgres'); + expect(adapter.getDatabaseType()).toBe('PostgreSQL'); + }); + + it('T30 create/get/delete use originalSchema.collectionName, not a hardcoded cnd_ prefix', async () => { + const { adapter, addIndex, removeIndex, showIndex } = makeAdapter(); + await adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'database', + { privileged: true }, + ); + expect(addIndex.mock.calls[0][0]).toBe('custom_users'); + expect(addIndex.mock.calls[0][0]).not.toBe('cnd_User'); + await adapter.getIndexes('User'); + expect(showIndex).toHaveBeenCalledWith('custom_users'); + await adapter.deleteIndexes('User', ['cnd_idx_email_asc']); + expect(removeIndex).toHaveBeenCalledWith('custom_users', 'cnd_idx_email_asc'); + }); + + it('T31 create does not rebuild/sync the schema', async () => { + const { adapter, sync } = makeAdapter(); + await adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'database', + ); + expect(sync).not.toHaveBeenCalled(); + }); + + it('T32 getIndexes reads the live engine and overlays declared Compatible types', async () => { + const { adapter, showIndex, originalSchema } = makeAdapter(); + originalSchema.modelOptions.indexes = [ + { + fields: ['email'], + name: 'cnd_idx_email_asc', + types: [CompatibleIndexType.Ascending], + }, + ]; + const result = await adapter.getIndexes('User'); + expect(showIndex).toHaveBeenCalledWith('custom_users'); + expect(result[0].types).toEqual([CompatibleIndexType.Ascending]); + expect(result[0].fields).toEqual(['email']); + }); + + it('T33 delete awaits removeIndex and persists', async () => { + const { adapter, removeIndex, findByIdAndUpdate } = makeAdapter(); + await adapter.deleteIndexes('User', ['cnd_idx_email_asc']); + expect(removeIndex).toHaveBeenCalledTimes(1); + expect(findByIdAndUpdate).toHaveBeenCalled(); + }); + + it('mysql HASH is allowed; sqlite HASH throws on Admin create', async () => { + const mysql = makeAdapter('mysql'); + await expect( + mysql.adapter.createIndexes( + 'User', + [{ fields: ['email'], types: PostgresIndexType.HASH }], + 'database', + { privileged: true }, + ), + ).resolves.toBe('Indexes created!'); + + const sqlite = makeAdapter('sqlite'); + await expect( + sqlite.adapter.createIndexes( + 'User', + [{ fields: ['email'], types: PostgresIndexType.HASH }], + 'database', + { privileged: true }, + ), + ).rejects.toMatchObject({ message: expect.stringMatching(/sqlite/i) }); + }); +}); diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 5c3a95048..6a9a77d2f 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -51,6 +51,21 @@ import { assertPostgresVectorIndexDropTarget, type PostgresCatalogIndex, } from '../utils/index.js'; +import { + assertUniqueIndexPrivilege, + ensureIndexName, + inferSqlIndexType, + isIndexAlreadyExistsError, + mergeDeclaredIndexes, + persistDeclaredSchemaIndexes, + removeDeclaredIndexes, + removeIndexFromSchemaFields, + resolveIndexName, + sqlDialectAllowsIndexType, + sqlIndexFields, + toMutableIndexes, + validateIndexFields, +} from '../utils/indexes.js'; const sqlSchemaName = process.env.SQL_SCHEMA ?? 'public'; @@ -270,7 +285,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter const [newSchema, objectPaths, extractedRelations] = dialect === 'postgres' ? pgSchemaConverter(compiledSchema) - : sqlSchemaConverter(compiledSchema); + : sqlSchemaConverter(compiledSchema, dialect as 'mysql' | 'mariadb' | 'sqlite'); this.registeredSchemas.set( schema.name, Object.freeze(JSON.parse(JSON.stringify(schema))), @@ -379,69 +394,108 @@ export abstract class SequelizeAdapter extends DatabaseAdapter async createIndexes( schemaName: string, - indexes: ModelOptionsIndexes[], + indexes: readonly ModelOptionsIndexes[], callerModule: string, + options?: { privileged?: boolean }, ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - indexes = this.checkAndConvertIndexes(schemaName, indexes, callerModule); + const prepared = this.checkAndConvertIndexes( + schemaName, + indexes, + callerModule, + options?.privileged, + ); + const collectionName = this.models[schemaName].originalSchema.collectionName; const queryInterface = this.sequelize.getQueryInterface(); - for (const index of indexes) { - await queryInterface - .addIndex('cnd_' + schemaName, [...index.fields], index.options) - .catch(() => { - throw new GrpcError(status.INTERNAL, 'Unsuccessful index creation'); + for (const index of prepared) { + try { + await queryInterface.addIndex(collectionName, { + fields: sqlIndexFields(index), + ...index.options, }); + } catch (e) { + if (isIndexAlreadyExistsError(e)) continue; + throw new GrpcError(status.INTERNAL, 'Unsuccessful index creation'); + } } - await this.models[schemaName].sync(); + const original = this.models[schemaName].originalSchema; + const merged = mergeDeclaredIndexes(original.modelOptions.indexes, prepared); + await persistDeclaredSchemaIndexes({ + declaredSchemaModel: this.models['_DeclaredSchema'], + schemaName, + originalSchema: original, + indexes: merged, + }); return 'Indexes created!'; } async getIndexes(schemaName: string): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const collectionName = this.models[schemaName].originalSchema.collectionName; const queryInterface = this.sequelize.getQueryInterface(); - const result = (await queryInterface.showIndex('cnd_' + schemaName)) as UntypedArray; - result.filter(index => { - const fieldNames = []; - for (const field of index.fields) { - fieldNames.push(field.attribute); - } - index.fields = fieldNames; - // extract index type from index definition - let tmp = index.definition.split('USING '); - tmp = tmp[1].split(' '); - index.types = tmp[0]; - delete index.definition; - index.options = {}; - for (const indexEntry of Object.entries(index)) { + const result = (await queryInterface.showIndex(collectionName)) as UntypedArray; + const dialect = this.sequelize.getDialect(); + const declared = this.models[schemaName].originalSchema.modelOptions.indexes ?? []; + const declaredByName = new Map( + declared.map((index: ModelOptionsIndexes) => [resolveIndexName(index), index]), + ); + return result.map(row => { + const fields = (row.fields ?? []).map((field: unknown) => + typeof field === 'string' ? field : (field as { attribute?: string }).attribute, + ); + const name = row.name as string; + const declaredIndex = declaredByName.get(name); + const options: Record = { + name, + unique: !!row.unique, + ...(declaredIndex?.options ?? {}), + }; + for (const [key, value] of Object.entries(row)) { if ( - indexEntry[0] === 'options' || - indexEntry[0] === 'types' || - indexEntry[0] === 'fields' + key === 'options' || + key === 'types' || + key === 'fields' || + key === 'definition' || + key === 'indkey' ) { continue; } - if (indexEntry[0] === 'indkey') { - delete index.indkey; - continue; - } - index.options[indexEntry[0]] = indexEntry[1]; - delete index[indexEntry[0]]; + if (options[key] === undefined) options[key] = value; } + return { + name, + fields, + types: declaredIndex?.types ?? inferSqlIndexType(row, dialect), + options, + } as ModelOptionsIndexes; }); - return result; } async deleteIndexes(schemaName: string, indexNames: string[]): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const collectionName = this.models[schemaName].originalSchema.collectionName; const queryInterface = this.sequelize.getQueryInterface(); for (const name of indexNames) { - queryInterface.removeIndex('cnd_' + schemaName, name).catch(() => { + try { + await queryInterface.removeIndex(collectionName, name); + } catch { throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); - }); + } + } + const original = this.models[schemaName].originalSchema; + for (const name of indexNames) { + removeIndexFromSchemaFields(original, name); } + const remaining = removeDeclaredIndexes(original.modelOptions.indexes, indexNames); + await persistDeclaredSchemaIndexes({ + declaredSchemaModel: this.models['_DeclaredSchema'], + schemaName, + originalSchema: original, + indexes: remaining, + }); return 'Indexes deleted'; } @@ -690,42 +744,57 @@ export abstract class SequelizeAdapter extends DatabaseAdapter private checkAndConvertIndexes( schemaName: string, - indexes: ModelOptionsIndexes[], + indexes: readonly ModelOptionsIndexes[], callerModule: string, - ) { - for (const index of indexes) { - if (!index.types && !index.options) continue; + privileged?: boolean, + ): ModelOptionsIndexes[] { + const schema = this.models[schemaName].originalSchema; + const dialect = this.sequelize.getDialect(); + const prepared: ModelOptionsIndexes[] = []; + for (const raw of toMutableIndexes(indexes)) { + const index = ensureIndexName(raw); + validateIndexFields(schema, index); if (index.types) { - if ( - Array.isArray(index.types) || - !Object.values(PostgresIndexType).includes(index.types) - ) { + const types = Array.isArray(index.types) ? index.types : [index.types]; + if (types.some(type => !sqlDialectAllowsIndexType(dialect, type))) { throw new GrpcError( status.INVALID_ARGUMENT, - 'Invalid index type for PostgreSQL', + `Invalid index type for ${dialect}`, ); } - (index.options as PostgresIndexOptions).using = index.types; - delete index.types; + const first = types[0]; + if ( + typeof first === 'string' && + Object.values(PostgresIndexType).includes(first as PostgresIndexType) && + types.length === 1 + ) { + index.options = { + ...(index.options ?? {}), + using: first as PostgresIndexType, + } as PostgresIndexOptions; + } else { + index.options = { + ...(index.options ?? {}), + using: PostgresIndexType.BTREE, + } as PostgresIndexOptions; + } } if (index.options) { if (!checkIfPostgresOptions(index.options)) { throw new GrpcError( status.INVALID_ARGUMENT, - 'Invalid index options for PostgreSQL', - ); - } - if ( - Object.keys(index.options).includes('unique') && - this.models[schemaName].originalSchema.ownerModule !== callerModule - ) { - throw new GrpcError( - status.PERMISSION_DENIED, - 'Not authorized to create unique index', + `Invalid index options for ${dialect}`, ); } + assertUniqueIndexPrivilege({ + unique: index.options.unique === true, + schemaOwner: schema.ownerModule, + callerModule, + privileged, + }); } + prepared.push(index); } - return indexes; + return prepared; } } diff --git a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts index 6504e65ba..a1c18bb92 100644 --- a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts @@ -39,13 +39,13 @@ export function pgSchemaConverter(jsonSchema: ConduitSchema): [ delete copy.fields['_id']; } if (copy.modelOptions.indexes) { - copy = convertModelOptionsIndexes(copy); + copy = convertModelOptionsIndexes(copy, 'postgres'); } const objectPaths: any = {}; convertObjectToDotNotation(jsonSchema.fields, copy.fields, objectPaths); const secondaryCopy = cloneDeep(copy.fields); const extractedRelations = extractRelations(secondaryCopy, copy.fields); - copy = convertSchemaFieldIndexes(copy); + copy = convertSchemaFieldIndexes(copy, 'postgres'); iterDeep(secondaryCopy, copy.fields); return [copy, objectPaths, extractedRelations]; } diff --git a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts index 7f418d43a..3709a774e 100644 --- a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts @@ -23,7 +23,10 @@ import { * This function should take as an input a JSON schema and convert it to the sequelize equivalent * @param jsonSchema */ -export function sqlSchemaConverter(jsonSchema: ConduitSchema): [ +export function sqlSchemaConverter( + jsonSchema: ConduitSchema, + dialect: 'mysql' | 'mariadb' | 'sqlite' = 'mysql', +): [ ConduitSchema, { [key: string]: { parentKey: string; childKey: string }; @@ -35,13 +38,13 @@ export function sqlSchemaConverter(jsonSchema: ConduitSchema): [ delete copy.fields['_id']; } if (copy.modelOptions.indexes) { - copy = convertModelOptionsIndexes(copy); + copy = convertModelOptionsIndexes(copy, dialect); } const objectPaths: any = {}; convertObjectToDotNotation(jsonSchema.fields, copy.fields, objectPaths); const secondaryCopy = cloneDeep(copy.fields); const extractedRelations = extractRelations(secondaryCopy, copy.fields); - copy = convertSchemaFieldIndexes(copy); + copy = convertSchemaFieldIndexes(copy, dialect); iterDeep(secondaryCopy, copy.fields); return [copy, objectPaths, extractedRelations]; } diff --git a/modules/database/src/adapters/utils/__tests__/indexes.test.ts b/modules/database/src/adapters/utils/__tests__/indexes.test.ts new file mode 100644 index 000000000..c5dbc5335 --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/indexes.test.ts @@ -0,0 +1,198 @@ +import { describe, expect, it } from '@jest/globals'; +import { status } from '@grpc/grpc-js'; +import { + CompatibleIndexType, + MongoIndexType, + PostgresIndexType, +} from '@conduitplatform/grpc-sdk'; +import { + assertUniqueIndexPrivilege, + collectExistingIndexNames, + ensureIndexName, + generateIndexName, + isCompatibleIndexType, + isIndexAlreadyExistsError, + isMongoIndexType, + mapCompatibleToMongo, + mapCompatibleToSqlOrder, + mergeDeclaredIndexes, + mongoAllowsIndexType, + removeDeclaredIndexes, + removeIndexFromSchemaFields, + resolveIndexName, + sqlDialectAllowsIndexType, + sqlIndexFields, + validateIndexFields, +} from '../indexes.js'; + +describe('index helpers T1–T16', () => { + it('T1 CompatibleIndexType uses portable string values, not Mongo 1/-1', () => { + expect(CompatibleIndexType.Ascending).toBe('Ascending'); + expect(CompatibleIndexType.Descending).toBe('Descending'); + expect(CompatibleIndexType.Ascending).not.toBe(MongoIndexType.Ascending); + expect(isCompatibleIndexType(CompatibleIndexType.Ascending)).toBe(true); + expect(isCompatibleIndexType(1)).toBe(false); + expect(isMongoIndexType('Ascending')).toBe(false); + }); + + it('T4 generates a name when missing', () => { + const name = generateIndexName(['email'], [CompatibleIndexType.Ascending], false); + expect(name).toMatch(/^cnd_idx_email_asc$/); + }); + + it('T5 keeps a provided name', () => { + const named = ensureIndexName({ + fields: ['email'], + name: 'custom_email_idx', + }); + expect(resolveIndexName(named)).toBe('custom_email_idx'); + expect(named.options?.name).toBe('custom_email_idx'); + }); + + it('T6 is deterministic for the same input', () => { + const a = generateIndexName( + ['room', 'createdAt'], + [CompatibleIndexType.Ascending, CompatibleIndexType.Descending], + true, + ); + const b = generateIndexName( + ['room', 'createdAt'], + [CompatibleIndexType.Ascending, CompatibleIndexType.Descending], + true, + ); + expect(a).toBe(b); + expect(a).toMatch(/^cnd_uidx_/); + }); + + it('T7 maps Compatible to Mongo 1/-1', () => { + expect(mapCompatibleToMongo(CompatibleIndexType.Ascending)).toBe(1); + expect(mapCompatibleToMongo(CompatibleIndexType.Descending)).toBe(-1); + expect(mapCompatibleToMongo(undefined)).toBe(1); + }); + + it('T8 maps Compatible to SQL BTREE ASC/DESC field order', () => { + expect(mapCompatibleToSqlOrder(CompatibleIndexType.Ascending)).toBe('ASC'); + expect(mapCompatibleToSqlOrder(CompatibleIndexType.Descending)).toBe('DESC'); + const fields = sqlIndexFields({ + fields: ['createdAt', 'room'], + types: [CompatibleIndexType.Descending, CompatibleIndexType.Ascending], + }); + expect(fields).toEqual([ + { name: 'createdAt', order: 'DESC' }, + { name: 'room', order: 'ASC' }, + ]); + }); + + it('T9 preserves the unique option on generated names', () => { + const unique = ensureIndexName({ + fields: ['email'], + types: [CompatibleIndexType.Ascending], + options: { unique: true }, + }); + expect(unique.options?.unique).toBe(true); + expect(resolveIndexName(unique)).toMatch(/uidx/); + }); + + it('T10 persist helper merges incoming indexes by name and skips duplicates', () => { + const merged = mergeDeclaredIndexes( + [{ fields: ['a'], name: 'idx_a' }], + [ + { fields: ['a'], name: 'idx_a', options: { unique: true } }, + { fields: ['b'], name: 'idx_b' }, + ], + ); + expect(merged.map(i => i.name)).toEqual(['idx_a', 'idx_b']); + expect(merged[0].options?.unique).toBeUndefined(); + }); + + it('T11 persist helper removes indexes by name', () => { + const remaining = removeDeclaredIndexes( + [ + { fields: ['a'], name: 'idx_a' }, + { fields: ['b'], name: 'idx_b' }, + ], + ['idx_a'], + ); + expect(remaining).toEqual([{ fields: ['b'], name: 'idx_b' }]); + }); + + it('T12 validateIndexFields rejects unknown fields', () => { + expect(() => + validateIndexFields( + { compiledFields: { email: 'String' }, fields: {} }, + { + fields: ['missing'], + }, + ), + ).toThrow(/Invalid fields/); + }); + + it('T13 unique is denied for a non-owner, non-admin caller', () => { + expect(() => + assertUniqueIndexPrivilege({ + unique: true, + schemaOwner: 'chat', + callerModule: 'database', + privileged: false, + }), + ).toThrow(expect.objectContaining({ code: status.PERMISSION_DENIED })); + }); + + it('T14 unique is allowed for the schema owner', () => { + expect(() => + assertUniqueIndexPrivilege({ + unique: true, + schemaOwner: 'chat', + callerModule: 'chat', + privileged: false, + }), + ).not.toThrow(); + }); + + it('T15 unique is allowed for privileged Admin', () => { + expect(() => + assertUniqueIndexPrivilege({ + unique: true, + schemaOwner: 'chat', + callerModule: 'database', + privileged: true, + }), + ).not.toThrow(); + }); + + it('T16 import unique respects owner privilege (not Admin-privileged)', () => { + expect(() => + assertUniqueIndexPrivilege({ + unique: true, + schemaOwner: 'authorization', + callerModule: 'database', + privileged: false, + }), + ).toThrow(/Not authorized to create unique index/); + }); + + it('collects existing names and detects already-exists errors', () => { + expect( + collectExistingIndexNames([{ fields: ['a'], options: { name: 'idx_a' } }]).has( + 'idx_a', + ), + ).toBe(true); + expect(isIndexAlreadyExistsError(new Error('index already exists'))).toBe(true); + expect( + removeIndexFromSchemaFields({ fields: { a: { index: { name: 'x' } } } }, 'x'), + ).toBe(true); + }); + + it('T25 dialect checks use real switches, not `mysql || mariadb`', () => { + expect(sqlDialectAllowsIndexType('mysql', CompatibleIndexType.Ascending)).toBe(true); + expect(sqlDialectAllowsIndexType('mariadb', CompatibleIndexType.Descending)).toBe( + true, + ); + expect(sqlDialectAllowsIndexType('sqlite', PostgresIndexType.BTREE)).toBe(true); + expect(sqlDialectAllowsIndexType('sqlite', PostgresIndexType.HASH)).toBe(false); + expect(sqlDialectAllowsIndexType('mysql', PostgresIndexType.GIST)).toBe(false); + expect(sqlDialectAllowsIndexType('postgres', PostgresIndexType.GIN)).toBe(true); + expect(mongoAllowsIndexType(CompatibleIndexType.Ascending)).toBe(true); + expect(mongoAllowsIndexType(PostgresIndexType.BTREE)).toBe(false); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/sql-index-converters.test.ts b/modules/database/src/adapters/utils/__tests__/sql-index-converters.test.ts new file mode 100644 index 000000000..95e061b0e --- /dev/null +++ b/modules/database/src/adapters/utils/__tests__/sql-index-converters.test.ts @@ -0,0 +1,125 @@ +import { beforeEach, describe, expect, it, jest } from '@jest/globals'; +import { + CompatibleIndexType, + ConduitGrpcSdk, + ConduitSchema, + MongoIndexType, + PostgresIndexType, + TYPE, +} from '@conduitplatform/grpc-sdk'; +import { + convertModelOptionsIndexes, + convertSchemaFieldIndexes, +} from '../database-transform-utils.js'; +import { sqlSchemaConverter } from '../../sequelize-adapter/sql-adapter/SqlSchemaConverter.js'; +import { pgSchemaConverter } from '../../sequelize-adapter/postgres-adapter/PgSchemaConverter.js'; + +function schemaWithIndexes( + indexes: ConduitSchema['modelOptions']['indexes'], + fields: ConduitSchema['fields'] = { email: { type: TYPE.String } }, +) { + return new ConduitSchema('User', fields as any, { indexes }); +} + +describe('SQL dialect-aware converters T17–T24', () => { + beforeEach(() => { + jest.spyOn(ConduitGrpcSdk.Logger, 'warn').mockImplementation(() => {}); + }); + + it('T18 postgres maps Compatible to BTREE + ASC/DESC', () => { + const copy = convertModelOptionsIndexes( + schemaWithIndexes([ + { + fields: ['email'], + types: [CompatibleIndexType.Descending], + }, + ]), + 'postgres', + ); + const index = copy.modelOptions.indexes![0] as any; + expect(index.using).toBe(PostgresIndexType.BTREE); + expect(index.fields[0]).toEqual({ name: 'email', order: 'DESC' }); + }); + + it('T19 mysql maps Compatible to BTREE + ASC', () => { + const copy = convertModelOptionsIndexes( + schemaWithIndexes([{ fields: ['email'], types: [CompatibleIndexType.Ascending] }]), + 'mysql', + ); + const index = copy.modelOptions.indexes![0] as any; + expect(index.using).toBe(PostgresIndexType.BTREE); + expect(index.fields[0]).toEqual({ name: 'email', order: 'ASC' }); + }); + + it('T20 sqlite maps Compatible to BTREE + ASC', () => { + const copy = convertModelOptionsIndexes( + schemaWithIndexes([{ fields: ['email'], types: CompatibleIndexType.Ascending }]), + 'sqlite', + ); + const index = copy.modelOptions.indexes![0] as any; + expect(index.using).toBe(PostgresIndexType.BTREE); + }); + + it('T21 recover: Mongo-only leftovers on SQL are warned and skipped', () => { + const warn = ConduitGrpcSdk.Logger.warn as jest.Mock; + const copy = convertModelOptionsIndexes( + schemaWithIndexes([ + { fields: ['loc'], types: [MongoIndexType.GeoSpatial2dSphere] }, + { fields: ['email'], types: [CompatibleIndexType.Ascending] }, + ]), + 'postgres', + ); + expect(copy.modelOptions.indexes).toHaveLength(1); + expect(warn).toHaveBeenCalled(); + }); + + it('T22 recover: postgres-only types on mysql/mariadb/sqlite are skipped', () => { + for (const dialect of ['mysql', 'mariadb', 'sqlite'] as const) { + const copy = convertModelOptionsIndexes( + schemaWithIndexes([{ fields: ['email'], types: PostgresIndexType.GIST }]), + dialect, + ); + expect(copy.modelOptions.indexes).toHaveLength(0); + } + }); + + it('converts field-level Compatible indexes into sequelize model indexes', () => { + const copy = convertSchemaFieldIndexes( + new ConduitSchema( + 'Perm', + { + resource: { + type: TYPE.String, + index: { type: CompatibleIndexType.Ascending }, + }, + } as any, + {}, + ), + 'mysql', + ); + expect(copy.modelOptions.indexes).toHaveLength(1); + expect((copy.fields.resource as any).index).toBeUndefined(); + }); + + it('sqlSchemaConverter is dialect-aware for mysql vs sqlite', () => { + const schema = new ConduitSchema('User', { email: { type: TYPE.String } } as any, { + indexes: [ + { fields: ['email'], types: [CompatibleIndexType.Descending] }, + { fields: ['email'], types: PostgresIndexType.HASH }, + ], + }); + const [mysql] = sqlSchemaConverter(schema, 'mysql'); + const [sqlite] = sqlSchemaConverter(schema, 'sqlite'); + expect(mysql.modelOptions.indexes).toHaveLength(2); + expect(sqlite.modelOptions.indexes).toHaveLength(1); + }); + + it('pgSchemaConverter keeps postgres-only types', () => { + const schema = new ConduitSchema('User', { email: { type: TYPE.String } } as any, { + indexes: [{ fields: ['email'], types: PostgresIndexType.GIN }], + }); + const [pg] = pgSchemaConverter(schema); + expect(pg.modelOptions.indexes).toHaveLength(1); + expect((pg.modelOptions.indexes![0] as any).using).toBe(PostgresIndexType.GIN); + }); +}); diff --git a/modules/database/src/adapters/utils/database-transform-utils.ts b/modules/database/src/adapters/utils/database-transform-utils.ts index 72956f938..f0be2bed7 100644 --- a/modules/database/src/adapters/utils/database-transform-utils.ts +++ b/modules/database/src/adapters/utils/database-transform-utils.ts @@ -1,13 +1,23 @@ -import { isArray, isBoolean, isNumber, isString } from 'lodash-es'; +import { isBoolean, isNumber, isString } from 'lodash-es'; import { + CompatibleIndexType, ConduitGrpcSdk, ConduitModelField, ConduitSchema, Indexable, + ModelOptionsIndexes, PostgresIndexOptions, PostgresIndexType, } from '@conduitplatform/grpc-sdk'; import { checkIfPostgresOptions } from '../sequelize-adapter/utils/index.js'; +import { + ensureIndexName, + isPortableDirection, + isPostgresIndexType, + mapCompatibleToSqlOrder, + normalizeIndexTypes, + sqlDialectAllowsIndexType, +} from './indexes.js'; export function checkDefaultValue(type: string, value: string) { switch (type) { @@ -28,72 +38,104 @@ export function checkDefaultValue(type: string, value: string) { } } -export function convertModelOptionsIndexes(copy: ConduitSchema) { - for (const index of copy.modelOptions.indexes!) { - if (index.types) { - if ( - isArray(index.types) || - !Object.values(PostgresIndexType).includes(index.types as PostgresIndexType) - ) { - // ignore index instead of error - ConduitGrpcSdk.Logger.warn('Invalid index type for PostgreSQL, ignoring index'); - continue; - // throw new Error('Incorrect index type for PostgreSQL'); - } - index.using = index.types as PostgresIndexType; - delete index.types; - } - if (index.options) { - if (!checkIfPostgresOptions(index.options)) { - // ignore index instead of error - ConduitGrpcSdk.Logger.warn( - 'Invalid index options for PostgreSQL, ignoring index', - ); - continue; - // throw new Error('Incorrect index options for PostgreSQL'); - } - for (const [option, value] of Object.entries(index.options)) { - index[option as keyof PostgresIndexOptions] = value; - } - delete index.options; - } +function flattenSqlIndexOptions(index: ModelOptionsIndexes, dialect: string): boolean { + if (!index.options) return true; + if (!checkIfPostgresOptions(index.options)) { + ConduitGrpcSdk.Logger.warn( + `Invalid index options for ${dialect} found in '${copyName(index)}', ignoring index`, + ); + return false; + } + for (const [option, value] of Object.entries(index.options)) { + index[option as keyof PostgresIndexOptions] = value; + } + delete index.options; + return true; +} + +function copyName(index: ModelOptionsIndexes): string { + return index.name ?? index.fields?.join(',') ?? 'unnamed'; +} + +function applySqlIndexTypes( + index: ModelOptionsIndexes, + dialect: string, + schemaName: string, +): boolean { + if (!index.types) { + index.using = PostgresIndexType.BTREE; + return true; + } + const types = normalizeIndexTypes(index.types, index.fields.length) ?? []; + if (types.some(type => !sqlDialectAllowsIndexType(dialect, type))) { + ConduitGrpcSdk.Logger.warn( + `Invalid index type for ${dialect} found in '${schemaName}', ignoring index`, + ); + return false; + } + if (types.some(isPortableDirection)) { + index.fields = index.fields.map((field, i) => ({ + name: field, + order: mapCompatibleToSqlOrder(types[i]), + })) as unknown as string[]; + index.using = PostgresIndexType.BTREE; + } else if (types.length === 1 && isPostgresIndexType(types[0])) { + index.using = types[0]; + } else { + ConduitGrpcSdk.Logger.warn( + `Invalid index type for ${dialect} found in '${schemaName}', ignoring index`, + ); + return false; } + delete index.types; + return true; +} + +export function convertModelOptionsIndexes(copy: ConduitSchema, dialect = 'postgres') { + const converted: ModelOptionsIndexes[] = []; + for (const raw of copy.modelOptions.indexes ?? []) { + const index = ensureIndexName({ ...raw, fields: [...raw.fields] }); + if (!applySqlIndexTypes(index, dialect, copy.name)) continue; + if (!flattenSqlIndexOptions(index, dialect)) continue; + if (!index.using) index.using = PostgresIndexType.BTREE; + converted.push(index); + } + copy.modelOptions.indexes = converted; return copy; } -export function convertSchemaFieldIndexes(copy: ConduitSchema) { - const indexes = []; - for (const field of Object.entries(copy.fields)) { - const fieldName = field[0]; - const index = (copy.fields[fieldName] as ConduitModelField).index; +export function convertSchemaFieldIndexes(copy: ConduitSchema, dialect = 'postgres') { + const indexes: ModelOptionsIndexes[] = []; + for (const [fieldName, fieldValue] of Object.entries(copy.fields)) { + const index = (fieldValue as ConduitModelField).index; if (!index) continue; - const newIndex: any = { + const newIndex = ensureIndexName({ fields: [fieldName], - }; - if (index.type) { - if (!Object.values(PostgresIndexType).includes(index.type as PostgresIndexType)) { - // ignore index instead of error - ConduitGrpcSdk.Logger.warn('Invalid index type for PostgreSQL, ignoring index'); - continue; - // throw new Error('Invalid index type for PostgreSQL'); - } - newIndex.using = index.type; + types: index.type + ? isPortableDirection(index.type) + ? [index.type as CompatibleIndexType] + : (index.type as PostgresIndexType) + : undefined, + options: index.options, + name: (index as { name?: string }).name, + }); + if (index.type && !sqlDialectAllowsIndexType(dialect, index.type)) { + ConduitGrpcSdk.Logger.warn( + `Invalid index type for ${dialect} found in '${copy.name}', ignoring index`, + ); + delete (copy.fields[fieldName] as ConduitModelField).index; + continue; + } + if (!applySqlIndexTypes(newIndex, dialect, copy.name)) { + delete (copy.fields[fieldName] as ConduitModelField).index; + continue; } - if (index.options) { - if (!checkIfPostgresOptions(index.options)) { - // ignore index instead of error - ConduitGrpcSdk.Logger.warn( - 'Invalid index options for PostgreSQL, ignoring index', - ); - continue; - // throw new Error('Invalid index options for PostgreSQL'); - } - for (const [option, value] of Object.entries(index.options)) { - newIndex[option] = value; - } + if (!flattenSqlIndexOptions(newIndex, dialect)) { + delete (copy.fields[fieldName] as ConduitModelField).index; + continue; } indexes.push(newIndex); - delete copy.fields[fieldName]; + delete (copy.fields[fieldName] as ConduitModelField).index; } if (copy.modelOptions.indexes) { copy.modelOptions.indexes = [...copy.modelOptions.indexes, ...indexes]; diff --git a/modules/database/src/adapters/utils/index.ts b/modules/database/src/adapters/utils/index.ts index a514cd7ad..5f28a8e86 100644 --- a/modules/database/src/adapters/utils/index.ts +++ b/modules/database/src/adapters/utils/index.ts @@ -2,6 +2,7 @@ export * from './validateFieldChanges.js'; export * from './validateFieldConstraints.js'; export * from './database-transform-utils.js'; export * from './extensions.js'; +export * from './indexes.js'; export * from './vectorField.js'; export * from './vectorCapabilities.js'; export * from './vectorMappings.js'; diff --git a/modules/database/src/adapters/utils/indexes.ts b/modules/database/src/adapters/utils/indexes.ts new file mode 100644 index 000000000..2c1a2776a --- /dev/null +++ b/modules/database/src/adapters/utils/indexes.ts @@ -0,0 +1,329 @@ +import { createHash } from 'crypto'; +import { + CompatibleIndexType, + ConduitModelField, + GrpcError, + ModelOptionsIndexes, + MongoIndexType, + PostgresIndexType, +} from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { ConduitDatabaseSchema } from '../../interfaces/index.js'; + +export const ADMIN_INDEX_CALLER = 'database'; + +export const MONGO_INDEX_TYPE_VALUES: ReadonlyArray = [ + MongoIndexType.Ascending, + MongoIndexType.Descending, + MongoIndexType.GeoSpatial2d, + MongoIndexType.GeoSpatial2dSphere, + MongoIndexType.GeoHaystack, + MongoIndexType.Hashed, + MongoIndexType.Text, +]; + +const SQL_IDENTIFIER_MAX_LEN = 63; + +export function isCompatibleIndexType(value: unknown): value is CompatibleIndexType { + return ( + value === CompatibleIndexType.Ascending || value === CompatibleIndexType.Descending + ); +} + +export function isMongoIndexType(value: unknown): value is MongoIndexType { + return (MONGO_INDEX_TYPE_VALUES as readonly unknown[]).includes(value); +} + +export function isPostgresIndexType(value: unknown): value is PostgresIndexType { + return Object.values(PostgresIndexType).includes(value as PostgresIndexType); +} + +export function resolveIndexName(index: ModelOptionsIndexes): string | undefined { + if (typeof index.name === 'string' && index.name.length > 0) return index.name; + const optionsName = index.options?.name; + if (typeof optionsName === 'string' && optionsName.length > 0) return optionsName; + return undefined; +} + +export function isUniqueIndex(index: ModelOptionsIndexes): boolean { + return index.options?.unique === true; +} + +export function normalizeIndexTypes( + types: ModelOptionsIndexes['types'], + fieldCount: number, +): unknown[] | undefined { + if (types === undefined) return undefined; + if (Array.isArray(types)) { + return [...types]; + } + return Array.from({ length: fieldCount }, () => types); +} + +function typeToken(type: unknown): string { + if (type === undefined || type === CompatibleIndexType.Ascending) return 'asc'; + if (type === CompatibleIndexType.Descending) return 'desc'; + if (type === MongoIndexType.Ascending || type === 1) return 'asc'; + if (type === MongoIndexType.Descending || type === -1) return 'desc'; + if (typeof type === 'string') return type.toLowerCase().replace(/[^a-z0-9]+/g, ''); + return String(type); +} + +export function generateIndexName( + fields: readonly string[], + types?: ModelOptionsIndexes['types'], + unique = false, +): string { + const tokens = ( + normalizeIndexTypes(types, fields.length) ?? fields.map(() => undefined) + ) + .map(typeToken) + .join('_'); + const prefix = unique ? 'cnd_uidx' : 'cnd_idx'; + const raw = `${prefix}_${fields.join('_')}_${tokens}`.replace(/[^A-Za-z0-9_]+/g, '_'); + const sanitized = raw.replace(/_+/g, '_').replace(/^_|_$/g, ''); + if (sanitized.length <= SQL_IDENTIFIER_MAX_LEN) return sanitized; + const hash = createHash('sha1').update(sanitized).digest('hex').slice(0, 8); + return `${sanitized.slice(0, SQL_IDENTIFIER_MAX_LEN - 9)}_${hash}`; +} + +export function ensureIndexName(index: ModelOptionsIndexes): ModelOptionsIndexes { + const existing = resolveIndexName(index); + if (existing) { + return { + ...index, + name: existing, + options: { ...index.options, name: existing }, + }; + } + const name = generateIndexName(index.fields, index.types, isUniqueIndex(index)); + return { + ...index, + name, + options: { ...index.options, name }, + }; +} + +export function mapCompatibleToMongo(type: unknown): MongoIndexType { + if (type === CompatibleIndexType.Descending) return MongoIndexType.Descending; + if (type === CompatibleIndexType.Ascending || type === undefined) { + return MongoIndexType.Ascending; + } + if (isMongoIndexType(type)) return type; + throw new GrpcError(status.INVALID_ARGUMENT, `Invalid index type for MongoDB: ${type}`); +} + +export function mapCompatibleToSqlOrder(type: unknown): 'ASC' | 'DESC' { + if (type === CompatibleIndexType.Descending || type === MongoIndexType.Descending) { + return 'DESC'; + } + return 'ASC'; +} + +export function isPortableDirection(type: unknown): boolean { + return ( + isCompatibleIndexType(type) || + type === MongoIndexType.Ascending || + type === MongoIndexType.Descending + ); +} + +export function sqlDialectAllowsIndexType(dialect: string, type: unknown): boolean { + if (type === undefined || isPortableDirection(type)) return true; + if (type === PostgresIndexType.BTREE) return true; + if (type === PostgresIndexType.HASH) { + return dialect === 'postgres' || dialect === 'mysql' || dialect === 'mariadb'; + } + if (isPostgresIndexType(type)) return dialect === 'postgres'; + return false; +} + +export function mongoAllowsIndexType(type: unknown): boolean { + return type === undefined || isCompatibleIndexType(type) || isMongoIndexType(type); +} + +export function mergeDeclaredIndexes( + existing: readonly ModelOptionsIndexes[] | undefined, + incoming: readonly ModelOptionsIndexes[], +): ModelOptionsIndexes[] { + const merged = new Map(); + for (const index of existing ?? []) { + const named = ensureIndexName(index); + merged.set(resolveIndexName(named)!, named); + } + for (const index of incoming) { + const named = ensureIndexName(index); + const name = resolveIndexName(named)!; + if (!merged.has(name)) { + merged.set(name, named); + } + } + return [...merged.values()]; +} + +export function removeDeclaredIndexes( + existing: readonly ModelOptionsIndexes[] | undefined, + names: readonly string[], +): ModelOptionsIndexes[] { + const drop = new Set(names); + return (existing ?? []).filter(index => { + const name = resolveIndexName(index); + return !name || !drop.has(name); + }); +} + +export function removeIndexFromSchemaFields( + schema: { fields?: Record; compiledFields?: Record }, + indexName: string, +): boolean { + let removed = false; + for (const bag of [schema.fields, schema.compiledFields]) { + if (!bag) continue; + for (const value of Object.values(bag)) { + if (!value || typeof value !== 'object') continue; + const field = value as ConduitModelField; + const name = field.index?.options?.name ?? (field.index as { name?: string })?.name; + if (name === indexName) { + delete field.index; + removed = true; + } + } + } + return removed; +} + +export function validateIndexFields( + schema: Pick, + index: ModelOptionsIndexes, +) { + if (!index.fields || index.fields.length === 0) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Index fields must be a non-empty array', + ); + } + const available = new Set([ + ...Object.keys(schema.compiledFields ?? {}), + ...Object.keys(schema.fields ?? {}), + ]); + const missing = index.fields.filter(field => !available.has(field)); + if (missing.length > 0) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Invalid fields for index creation: ${missing.join(', ')}`, + ); + } +} + +export function assertUniqueIndexPrivilege(args: { + unique: boolean; + schemaOwner: string; + callerModule: string; + privileged?: boolean; +}) { + if (!args.unique) return; + if (args.privileged) return; + if (args.schemaOwner === args.callerModule) return; + throw new GrpcError(status.PERMISSION_DENIED, 'Not authorized to create unique index'); +} + +export function isIndexAlreadyExistsError(error: unknown): boolean { + const err = error as { message?: string; code?: number | string; name?: string }; + const message = (err.message ?? '').toLowerCase(); + if ( + message.includes('already exists') || + message.includes('already exist') || + message.includes('duplicate key name') || + message.includes('index already exists') + ) { + return true; + } + return ( + err.code === 85 || + err.code === '42P07' || + err.name === 'SequelizeUniqueConstraintError' + ); +} + +export async function persistDeclaredSchemaIndexes(args: { + declaredSchemaModel: { + findOne: (query: Record) => Promise<{ _id: string } | null>; + findByIdAndUpdate: (id: string, update: Record) => Promise; + }; + schemaName: string; + originalSchema: { + modelOptions: { indexes?: ModelOptionsIndexes[] | readonly ModelOptionsIndexes[] }; + fields?: Record; + compiledFields?: Record; + }; + indexes: ModelOptionsIndexes[]; +}): Promise { + args.originalSchema.modelOptions.indexes = args.indexes; + const found = await args.declaredSchemaModel.findOne({ name: args.schemaName }); + if (!found) return; + await args.declaredSchemaModel.findByIdAndUpdate(found._id, { + modelOptions: args.originalSchema.modelOptions, + fields: args.originalSchema.fields, + compiledFields: args.originalSchema.compiledFields, + }); +} + +export function collectExistingIndexNames( + indexes: readonly ModelOptionsIndexes[], +): Set { + const names = new Set(); + for (const index of indexes) { + const name = resolveIndexName(index); + if (name) names.add(name); + } + return names; +} + +export function toMutableIndexes( + indexes: readonly ModelOptionsIndexes[], +): ModelOptionsIndexes[] { + return indexes.map(index => ({ + ...index, + fields: [...index.fields], + types: Array.isArray(index.types) ? [...index.types] : index.types, + options: index.options ? { ...index.options } : index.options, + })); +} + +export function sqlIndexFields( + index: ModelOptionsIndexes, +): Array { + const types = normalizeIndexTypes(index.types, index.fields.length); + if (!types || !types.some(isCompatibleIndexType)) { + return [...index.fields]; + } + return index.fields.map((field, i) => ({ + name: field, + order: mapCompatibleToSqlOrder(types[i]), + })); +} + +export function inferSqlIndexType( + row: { type?: string; definition?: string }, + dialect: string, +): PostgresIndexType | undefined { + if (typeof row.type === 'string' && row.type.length > 0) { + const upper = row.type.toUpperCase(); + if (isPostgresIndexType(upper)) return upper; + } + if (typeof row.definition === 'string') { + const match = /USING\s+(\w+)/i.exec(row.definition); + if (match && isPostgresIndexType(match[1].toUpperCase())) { + return match[1].toUpperCase() as PostgresIndexType; + } + } + if ( + dialect === 'postgres' || + dialect === 'mysql' || + dialect === 'mariadb' || + dialect === 'sqlite' + ) { + return PostgresIndexType.BTREE; + } + return undefined; +} diff --git a/modules/database/src/admin/__tests__/schema.admin.indexes.test.ts b/modules/database/src/admin/__tests__/schema.admin.indexes.test.ts new file mode 100644 index 000000000..86627f6d5 --- /dev/null +++ b/modules/database/src/admin/__tests__/schema.admin.indexes.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { status } from '@grpc/grpc-js'; +import { ConduitGrpcSdk, ParsedRouterRequest } from '@conduitplatform/grpc-sdk'; +import { SchemaAdmin } from '../schema.admin.js'; +import { DatabaseAdapter } from '../../adapters/DatabaseAdapter.js'; +import { MongooseSchema } from '../../adapters/mongoose-adapter/MongooseSchema.js'; +import { SequelizeSchema } from '../../adapters/sequelize-adapter/SequelizeSchema.js'; +import { SchemaController } from '../../controllers/cms/schema.controller.js'; +import { CustomEndpointController } from '../../controllers/customEndpoints/customEndpoint.controller.js'; +import { ADMIN_INDEX_CALLER } from '../../adapters/utils/indexes.js'; + +function makeCall(params: Record): ParsedRouterRequest { + return { request: { params } } as unknown as ParsedRouterRequest; +} + +function setup() { + const findOne = jest.fn().mockResolvedValue({ + _id: 'schema-1', + name: 'User', + ownerModule: 'database', + }); + const findMany = jest.fn().mockResolvedValue([{ name: 'User' }, { name: 'ChatRoom' }]); + const countDocuments = jest.fn().mockResolvedValue(2); + const createIndexes = jest.fn().mockResolvedValue('Indexes created!'); + const getIndexes = jest + .fn() + .mockResolvedValue([{ name: 'cnd_idx_email_asc', fields: ['email'] }]); + const deleteIndexes = jest.fn().mockResolvedValue('Indexes deleted'); + const getSchemaModel = jest.fn().mockReturnValue({ + model: { findOne, findMany, countDocuments }, + }); + const database = { + getSchemaModel, + createIndexes, + getIndexes, + deleteIndexes, + systemSchemas: ['_DeclaredSchema'], + models: { + User: { originalSchema: { ownerModule: 'database' } }, + ChatRoom: { originalSchema: { ownerModule: 'chat' } }, + }, + } as unknown as DatabaseAdapter; + const admin = new SchemaAdmin( + {} as ConduitGrpcSdk, + database, + {} as SchemaController, + {} as CustomEndpointController, + ); + return { admin, createIndexes, getIndexes, findMany, countDocuments, findOne }; +} + +describe('SchemaAdmin indexes T35–T42', () => { + it('T37 Admin createIndexes is privileged', async () => { + const { admin, createIndexes } = setup(); + await admin.createIndexes( + makeCall({ + id: 'schema-1', + indexes: [{ fields: ['email'], options: { unique: true } }], + }), + ); + expect(createIndexes).toHaveBeenCalledWith( + 'User', + [{ fields: ['email'], options: { unique: true } }], + ADMIN_INDEX_CALLER, + { privileged: true }, + ); + }); + + it('T39 exportIndexes is paginated (skip/limit, no unbounded findMany)', async () => { + const { admin, findMany, countDocuments, getIndexes } = setup(); + const result = (await admin.exportIndexes(makeCall({ skip: 10, limit: 5 }))) as { + indexes: unknown[]; + count: number; + }; + expect(findMany).toHaveBeenCalledWith( + expect.any(Object), + expect.objectContaining({ skip: 10, limit: 5 }), + ); + expect(countDocuments).toHaveBeenCalled(); + expect(getIndexes).toHaveBeenCalled(); + expect(result.count).toBe(2); + expect(result.indexes.every(index => 'schemaName' in (index as object))).toBe(true); + }); + + it('T40 importIndexes skips same-name indexes', async () => { + const { admin, createIndexes, getIndexes } = setup(); + getIndexes.mockResolvedValue([{ name: 'keep_me', fields: ['email'] }]); + await admin.importIndexes( + makeCall({ + indexes: [ + { schemaName: 'User', fields: ['email'], name: 'keep_me' }, + { schemaName: 'User', fields: ['name'], name: 'new_name' }, + ], + }), + ); + expect(createIndexes).toHaveBeenCalledTimes(1); + const created = createIndexes.mock.calls[0][1] as { name?: string }[]; + expect(created.map(i => i.name)).toEqual(['new_name']); + }); + + it('T41 import unique is not Admin-privileged (respects owner)', async () => { + const { admin, createIndexes } = setup(); + await admin.importIndexes( + makeCall({ + indexes: [ + { + schemaName: 'ChatRoom', + fields: ['name'], + name: 'chat_name', + options: { unique: true }, + }, + ], + }), + ); + expect(createIndexes).toHaveBeenCalledWith( + 'ChatRoom', + expect.any(Array), + ADMIN_INDEX_CALLER, + { privileged: false }, + ); + }); + + it('T42 import/export are Admin handlers and reject empty import', async () => { + const { admin } = setup(); + await expect(admin.importIndexes(makeCall({ indexes: [] }))).rejects.toMatchObject({ + code: status.INVALID_ARGUMENT, + }); + }); +}); diff --git a/modules/database/src/admin/index.ts b/modules/database/src/admin/index.ts index 5a4605e9f..6db05353a 100644 --- a/modules/database/src/admin/index.ts +++ b/modules/database/src/admin/index.ts @@ -633,6 +633,35 @@ export class AdminHandlers { }), this.customEndpointsAdmin.schemaDetailsForOperation.bind(this.customEndpointsAdmin), ); + this.routingManager.route( + { + path: '/indexes/export', + action: ConduitRouteActions.GET, + description: `Exports schema indexes. Admin-only and paginated.`, + queryParams: { + skip: ConduitNumber.OptionalWith({ min: 0, integer: true }), + limit: ConduitNumber.OptionalWith({ min: 1, max: 1000, integer: true }), + }, + }, + new ConduitRouteReturnDefinition('ExportSchemaIndexes', { + indexes: [ConduitJson.Required], + count: ConduitNumber.Required, + }), + this.schemaAdmin.exportIndexes.bind(this.schemaAdmin), + ); + this.routingManager.route( + { + path: '/indexes/import', + action: ConduitRouteActions.POST, + description: `Imports schema indexes. Skips indexes that already exist by name. Unique indexes respect schema owner privilege.`, + mcp: false, + bodyParams: { + indexes: { type: [TYPE.JSON], required: true }, + }, + }, + new ConduitRouteReturnDefinition('ImportSchemaIndexes', 'String'), + this.schemaAdmin.importIndexes.bind(this.schemaAdmin), + ); this.routingManager.route( { path: '/schemas/:id/indexes', diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index 9e8ae74cc..0d53e8c82 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -3,6 +3,7 @@ import { ConduitSchema, GrpcError, Indexable, + ModelOptionsIndexes, ParsedRouterRequest, UnparsedRouterResponse, } from '@conduitplatform/grpc-sdk'; @@ -23,6 +24,12 @@ import { import { SchemaConverter } from '../utils/SchemaConverter.js'; import { parseSortParam } from '../handlers/utils.js'; import escapeStringRegexp from 'escape-string-regexp'; +import { + ADMIN_INDEX_CALLER, + collectExistingIndexNames, + ensureIndexName, + resolveIndexName, +} from '../adapters/utils/indexes.js'; type ExportedCmsSchema = Pick< ConduitDatabaseSchema, @@ -749,7 +756,12 @@ export class SchemaAdmin { if (isNil(requestedSchema)) { throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); } - return await this.database.createIndexes(requestedSchema.name, indexes, 'database'); + return await this.database.createIndexes( + requestedSchema.name, + indexes, + ADMIN_INDEX_CALLER, + { privileged: true }, + ); } async getIndexes(call: ParsedRouterRequest): Promise { @@ -760,7 +772,8 @@ export class SchemaAdmin { if (isNil(requestedSchema)) { throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); } - return this.database.getIndexes(requestedSchema.name); + const indexes = await this.database.getIndexes(requestedSchema.name); + return { indexes }; } async deleteIndexes(call: ParsedRouterRequest): Promise { @@ -780,6 +793,82 @@ export class SchemaAdmin { return this.database.deleteIndexes(requestedSchema.name, indexNames); } + async exportIndexes(call: ParsedRouterRequest): Promise { + const skip = call.request.params.skip ?? 0; + const limit = call.request.params.limit ?? 25; + const query: Indexable = { + name: { $nin: this.database.systemSchemas }, + $or: [ + { parentSchema: { $exists: false } }, + { parentSchema: { $eq: null } }, + { parentSchema: { $eq: '' } }, + ], + }; + const schemaAdapter = this.database.getSchemaModel('_DeclaredSchema'); + const [schemas, count] = await Promise.all([ + schemaAdapter.model.findMany(query, { + skip, + limit, + select: 'name', + sort: { name: 1 }, + }), + schemaAdapter.model.countDocuments(query), + ]); + const indexes: Array = []; + for (const schema of schemas) { + if (!this.database.models[schema.name]) continue; + const schemaIndexes = await this.database.getIndexes(schema.name); + if (isNil(schemaIndexes) || isEmpty(schemaIndexes)) continue; + indexes.push( + ...schemaIndexes.map(index => ({ ...index, schemaName: schema.name })), + ); + } + return { indexes, count }; + } + + async importIndexes(call: ParsedRouterRequest): Promise { + const { indexes } = call.request.params as { + indexes: Array; + }; + if (!Array.isArray(indexes) || indexes.length === 0) { + throw new GrpcError(status.INVALID_ARGUMENT, 'indexes must be a non-empty array'); + } + const bySchema = new Map(); + for (const entry of indexes) { + const { schemaName, ...rest } = entry; + if (!schemaName) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Each imported index needs schemaName', + ); + } + const bucket = bySchema.get(schemaName) ?? []; + bucket.push(rest); + bySchema.set(schemaName, bucket); + } + for (const [schemaName, schemaIndexes] of bySchema) { + if (!this.database.models[schemaName]) { + throw new GrpcError( + status.NOT_FOUND, + `Requested schema not found: ${schemaName}`, + ); + } + const existing = await this.database.getIndexes(schemaName); + const existingNames = collectExistingIndexNames(existing); + const toCreate = schemaIndexes + .map(index => ensureIndexName(index)) + .filter(index => { + const name = resolveIndexName(index); + return !name || !existingNames.has(name); + }); + if (toCreate.length === 0) continue; + await this.database.createIndexes(schemaName, toCreate, ADMIN_INDEX_CALLER, { + privileged: false, + }); + } + return 'Indexes imported successfully'; + } + async checkRequestedSchema(id: string) { const requestedSchema = await this.database .getSchemaModel('_DeclaredSchema') diff --git a/modules/database/src/utils/__tests__/validateModelOptions.indexes.test.ts b/modules/database/src/utils/__tests__/validateModelOptions.indexes.test.ts new file mode 100644 index 000000000..a2d6e52fa --- /dev/null +++ b/modules/database/src/utils/__tests__/validateModelOptions.indexes.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from '@jest/globals'; +import { CompatibleIndexType } from '@conduitplatform/grpc-sdk'; +import { validateSchemaInput } from '../utilities.js'; + +describe('validateModelOptions T35 T36', () => { + it('T35 accepts modelOptions.indexes', () => { + expect(() => + validateSchemaInput( + 'User', + { email: 'String' }, + { + timestamps: true, + indexes: [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + }, + ), + ).not.toThrow(); + }); + + it('T36 keeps conduit.readPreference while accepting indexes', () => { + expect(() => + validateSchemaInput( + 'User', + { email: 'String' }, + { + timestamps: true, + indexes: [{ fields: ['email'] }], + conduit: { readPreference: 'secondaryPreferred' }, + }, + ), + ).not.toThrow(); + }); + + it('still rejects unknown conduit keys and unknown model option keys', () => { + expect(() => + validateSchemaInput('User', { email: 'String' }, { + unknown: true, + } as any), + ).toThrow(/indexes/); + expect(() => + validateSchemaInput('User', { email: 'String' }, { + conduit: { notARealKey: true }, + } as any), + ).toThrow(/readPreference/); + }); +}); diff --git a/modules/database/src/utils/utilities.ts b/modules/database/src/utils/utilities.ts index 15f6e4d4f..aa31a1f4a 100644 --- a/modules/database/src/utils/utilities.ts +++ b/modules/database/src/utils/utilities.ts @@ -158,10 +158,12 @@ const ALLOWED_CONDUIT_READ_PREFERENCES = [ function validateModelOptions(modelOptions: ConduitSchemaOptions) { if (!isPlainObject(modelOptions)) throw new Error('Model options must be an object'); Object.keys(modelOptions).forEach(key => { - if (key !== 'conduit' && key !== 'timestamps') - throw new Error("Only 'conduit' and 'timestamps' options allowed"); + if (key !== 'conduit' && key !== 'timestamps' && key !== 'indexes') + throw new Error("Only 'conduit', 'timestamps', and 'indexes' options allowed"); else if (key === 'timestamps' && !isBoolean(modelOptions.timestamps)) throw new Error("Option 'timestamps' must be of type Boolean"); + else if (key === 'indexes' && !isArray(modelOptions.indexes)) + throw new Error("Option 'indexes' must be of type Array"); else if (key === 'conduit') { if (!isObject(modelOptions.conduit)) throw new Error("Option 'conduit' must be of type Object"); From 1b52e42254ed43e8bb6637c2a5a27a3054db8f7d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 14 Sep 2026 11:57:25 +0000 Subject: [PATCH 2/9] fix(database): typecheck dialect indexes and group tests Repair the ModelOptionsIndexTypes union that failed tsc, stop stuffing Sequelize field objects into Conduit index types, and move the index suites into src/__tests__/indexes. --- libraries/grpc-sdk/src/interfaces/Model.ts | 3 +- .../src/__tests__/indexes/adapters.test.ts | 242 ++++++++++++++++++ .../indexes/admin.test.ts} | 52 +++- .../src/__tests__/indexes/converters.test.ts | 195 ++++++++++++++ .../indexes/helpers.test.ts} | 116 ++++----- .../regressions.test.ts} | 31 ++- .../__tests__/platform-models.indexes.test.ts | 33 --- .../mongoose-adapter/SchemaConverter.ts | 55 ++-- .../__tests__/SchemaConverter.indexes.test.ts | 66 ----- .../__tests__/indexes.adapter.test.ts | 123 --------- .../src/adapters/mongoose-adapter/index.ts | 43 ++-- .../__tests__/indexes.adapter.test.ts | 128 --------- .../src/adapters/sequelize-adapter/index.ts | 64 ++--- .../__tests__/sql-index-converters.test.ts | 125 --------- .../utils/database-transform-utils.ts | 160 ++++++------ .../database/src/adapters/utils/indexes.ts | 108 ++++---- modules/database/src/admin/schema.admin.ts | 32 +-- .../validateModelOptions.indexes.test.ts | 45 ---- 18 files changed, 766 insertions(+), 855 deletions(-) create mode 100644 modules/database/src/__tests__/indexes/adapters.test.ts rename modules/database/src/{admin/__tests__/schema.admin.indexes.test.ts => __tests__/indexes/admin.test.ts} (71%) create mode 100644 modules/database/src/__tests__/indexes/converters.test.ts rename modules/database/src/{adapters/utils/__tests__/indexes.test.ts => __tests__/indexes/helpers.test.ts} (70%) rename modules/database/src/__tests__/{no-old-pr-bugs.test.ts => indexes/regressions.test.ts} (53%) delete mode 100644 modules/database/src/__tests__/platform-models.indexes.test.ts delete mode 100644 modules/database/src/adapters/mongoose-adapter/__tests__/SchemaConverter.indexes.test.ts delete mode 100644 modules/database/src/adapters/mongoose-adapter/__tests__/indexes.adapter.test.ts delete mode 100644 modules/database/src/adapters/sequelize-adapter/__tests__/indexes.adapter.test.ts delete mode 100644 modules/database/src/adapters/utils/__tests__/sql-index-converters.test.ts delete mode 100644 modules/database/src/utils/__tests__/validateModelOptions.indexes.test.ts diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 099eb424f..efcbf47b0 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -90,8 +90,7 @@ export enum CompatibleIndexType { export type IndexType = MongoIndexType | PostgresIndexType | CompatibleIndexType; -export type ModelOptionsIndexTypes = - MongoIndexType[] | PostgresIndexType | CompatibleIndexType | CompatibleIndexType[]; +export type ModelOptionsIndexTypes = IndexType | readonly IndexType[]; export type Array = any[]; diff --git a/modules/database/src/__tests__/indexes/adapters.test.ts b/modules/database/src/__tests__/indexes/adapters.test.ts new file mode 100644 index 000000000..cb0810497 --- /dev/null +++ b/modules/database/src/__tests__/indexes/adapters.test.ts @@ -0,0 +1,242 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { status } from '@grpc/grpc-js'; +import { + CompatibleIndexType, + MongoIndexType, + PostgresIndexType, +} from '@conduitplatform/grpc-sdk'; +import { MongooseAdapter } from '../../adapters/mongoose-adapter/index.js'; +import { SequelizeAdapter } from '../../adapters/sequelize-adapter/index.js'; + +function makeMongooseAdapter(overrides: Record = {}) { + const createIndex = jest.fn().mockResolvedValue('email_1'); + const dropIndex = jest.fn().mockResolvedValue(undefined); + const indexes = jest.fn().mockResolvedValue([ + { v: 2, key: { _id: 1 }, name: '_id_' }, + { v: 2, key: { email: 1 }, name: 'cnd_idx_email_asc', unique: false }, + ]); + const findOne = jest.fn().mockResolvedValue({ _id: 'declared-1' }); + const findByIdAndUpdate = jest.fn().mockResolvedValue({}); + const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; + adapter.mongoose = { + model: () => ({ collection: { createIndex, dropIndex, indexes } }), + } as MongooseAdapter['mongoose']; + const originalSchema = { + name: 'User', + ownerModule: 'chat', + collectionName: 'cnd_User', + fields: { email: { type: 'String' } }, + compiledFields: { email: { type: 'String' } }, + modelOptions: { indexes: [] as unknown[] }, + ...((overrides.originalSchema as object) ?? {}), + }; + adapter.models = { + User: { originalSchema }, + _DeclaredSchema: { findOne, findByIdAndUpdate }, + } as MongooseAdapter['models']; + return { adapter, createIndex, dropIndex, indexes, findByIdAndUpdate, originalSchema }; +} + +class TestSequelizeAdapter extends SequelizeAdapter { + protected async hasLegacyCollections(): Promise { + return false; + } +} + +function makeSequelizeAdapter(dialect = 'postgres') { + const addIndex = jest.fn().mockResolvedValue(undefined); + const removeIndex = jest.fn().mockResolvedValue(undefined); + const showIndex = jest.fn().mockResolvedValue([ + { + name: 'cnd_idx_email_asc', + unique: false, + fields: [{ attribute: 'email', order: 'ASC' }], + definition: 'CREATE INDEX cnd_idx_email_asc ON cnd_User USING btree (email)', + }, + ]); + const findOne = jest.fn().mockResolvedValue({ _id: 'declared-1' }); + const findByIdAndUpdate = jest.fn().mockResolvedValue({}); + const sync = jest.fn().mockResolvedValue(undefined); + const adapter = Object.create(TestSequelizeAdapter.prototype) as SequelizeAdapter; + adapter.sequelize = { + getDialect: () => dialect, + getQueryInterface: () => ({ addIndex, removeIndex, showIndex }), + } as SequelizeAdapter['sequelize']; + const originalSchema = { + name: 'User', + ownerModule: 'database', + collectionName: 'custom_users', + fields: { email: { type: 'String' } }, + compiledFields: { email: { type: 'String' } }, + modelOptions: { indexes: [] as unknown[] }, + }; + adapter.models = { + User: { originalSchema, sync }, + _DeclaredSchema: { findOne, findByIdAndUpdate }, + } as SequelizeAdapter['models']; + return { + adapter, + addIndex, + removeIndex, + showIndex, + sync, + findByIdAndUpdate, + originalSchema, + }; +} + +describe('mongoose adapter indexes', () => { + it('creates a single key spec object, not an array of objects', async () => { + const { adapter, createIndex } = makeMongooseAdapter(); + await adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'chat', + ); + expect(createIndex).toHaveBeenCalledTimes(1); + expect(createIndex.mock.calls[0][0]).toEqual({ email: MongoIndexType.Ascending }); + expect(Array.isArray(createIndex.mock.calls[0][0])).toBe(false); + }); + + it('persists created and deleted indexes on _DeclaredSchema', async () => { + const created = makeMongooseAdapter(); + await created.adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'chat', + ); + expect(created.findByIdAndUpdate).toHaveBeenCalledTimes(1); + const update = created.findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { name?: string }[] }; + }; + expect(update.modelOptions.indexes[0].name).toMatch(/email/); + + const deleted = makeMongooseAdapter({ + originalSchema: { + modelOptions: { indexes: [{ fields: ['email'], name: 'cnd_idx_email_asc' }] }, + }, + }); + await deleted.adapter.deleteIndexes('User', ['cnd_idx_email_asc']); + expect(deleted.originalSchema.modelOptions.indexes).toEqual([]); + expect(deleted.findByIdAndUpdate).toHaveBeenCalled(); + }); + + it('awaits dropIndex', async () => { + const { adapter, dropIndex } = makeMongooseAdapter(); + let resolveDrop: () => void = () => undefined; + const dropped = new Promise(resolve => { + resolveDrop = resolve; + }); + dropIndex.mockImplementation(async () => { + resolveDrop(); + }); + await adapter.deleteIndexes('User', ['cnd_idx_email_asc']); + await dropped; + expect(dropIndex).toHaveBeenCalledWith('cnd_idx_email_asc'); + }); + + it('reads live engine indexes', async () => { + const { adapter, indexes } = makeMongooseAdapter(); + const result = await adapter.getIndexes('User'); + expect(indexes).toHaveBeenCalled(); + expect(result.map(index => index.name)).toEqual(['_id_', 'cnd_idx_email_asc']); + expect(result[1].fields).toEqual(['email']); + }); + + it('throws on Admin-bound invalid types and allows privileged unique', async () => { + const { adapter } = makeMongooseAdapter(); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [PostgresIndexType.GIST] }], + 'database', + { privileged: true }, + ), + ).rejects.toMatchObject({ code: status.INVALID_ARGUMENT }); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], options: { unique: true } }], + 'database', + { privileged: true }, + ), + ).resolves.toBe('Indexes created!'); + }); +}); + +describe('sequelize adapter indexes', () => { + it('keeps getDatabaseType as PostgreSQL', () => { + const { adapter } = makeSequelizeAdapter('postgres'); + expect(adapter.getDatabaseType()).toBe('PostgreSQL'); + }); + + it('uses originalSchema.collectionName instead of a hardcoded cnd_ prefix', async () => { + const { adapter, addIndex, removeIndex, showIndex } = makeSequelizeAdapter(); + await adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'database', + { privileged: true }, + ); + expect(addIndex.mock.calls[0][0]).toBe('custom_users'); + expect(addIndex.mock.calls[0][0]).not.toBe('cnd_User'); + await adapter.getIndexes('User'); + expect(showIndex).toHaveBeenCalledWith('custom_users'); + await adapter.deleteIndexes('User', ['cnd_idx_email_asc']); + expect(removeIndex).toHaveBeenCalledWith('custom_users', 'cnd_idx_email_asc'); + }); + + it('does not rebuild or sync the schema when creating indexes', async () => { + const { adapter, sync } = makeSequelizeAdapter(); + await adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'database', + ); + expect(sync).not.toHaveBeenCalled(); + }); + + it('reads the live engine and overlays declared Compatible types', async () => { + const { adapter, showIndex, originalSchema } = makeSequelizeAdapter(); + originalSchema.modelOptions.indexes = [ + { + fields: ['email'], + name: 'cnd_idx_email_asc', + types: [CompatibleIndexType.Ascending], + }, + ]; + const result = await adapter.getIndexes('User'); + expect(showIndex).toHaveBeenCalledWith('custom_users'); + expect(result[0].types).toEqual([CompatibleIndexType.Ascending]); + expect(result[0].fields).toEqual(['email']); + }); + + it('awaits removeIndex and persists the deletion', async () => { + const { adapter, removeIndex, findByIdAndUpdate } = makeSequelizeAdapter(); + await adapter.deleteIndexes('User', ['cnd_idx_email_asc']); + expect(removeIndex).toHaveBeenCalledTimes(1); + expect(findByIdAndUpdate).toHaveBeenCalled(); + }); + + it('allows HASH on mysql and rejects it on sqlite', async () => { + const mysql = makeSequelizeAdapter('mysql'); + await expect( + mysql.adapter.createIndexes( + 'User', + [{ fields: ['email'], types: PostgresIndexType.HASH }], + 'database', + { privileged: true }, + ), + ).resolves.toBe('Indexes created!'); + + const sqlite = makeSequelizeAdapter('sqlite'); + await expect( + sqlite.adapter.createIndexes( + 'User', + [{ fields: ['email'], types: PostgresIndexType.HASH }], + 'database', + { privileged: true }, + ), + ).rejects.toMatchObject({ message: expect.stringMatching(/sqlite/i) }); + }); +}); diff --git a/modules/database/src/admin/__tests__/schema.admin.indexes.test.ts b/modules/database/src/__tests__/indexes/admin.test.ts similarity index 71% rename from modules/database/src/admin/__tests__/schema.admin.indexes.test.ts rename to modules/database/src/__tests__/indexes/admin.test.ts index 86627f6d5..757954193 100644 --- a/modules/database/src/admin/__tests__/schema.admin.indexes.test.ts +++ b/modules/database/src/__tests__/indexes/admin.test.ts @@ -1,13 +1,18 @@ import { describe, expect, it, jest } from '@jest/globals'; import { status } from '@grpc/grpc-js'; -import { ConduitGrpcSdk, ParsedRouterRequest } from '@conduitplatform/grpc-sdk'; -import { SchemaAdmin } from '../schema.admin.js'; +import { + CompatibleIndexType, + ConduitGrpcSdk, + ParsedRouterRequest, +} from '@conduitplatform/grpc-sdk'; +import { SchemaAdmin } from '../../admin/schema.admin.js'; import { DatabaseAdapter } from '../../adapters/DatabaseAdapter.js'; import { MongooseSchema } from '../../adapters/mongoose-adapter/MongooseSchema.js'; import { SequelizeSchema } from '../../adapters/sequelize-adapter/SequelizeSchema.js'; import { SchemaController } from '../../controllers/cms/schema.controller.js'; import { CustomEndpointController } from '../../controllers/customEndpoints/customEndpoint.controller.js'; import { ADMIN_INDEX_CALLER } from '../../adapters/utils/indexes.js'; +import { validateSchemaInput } from '../../utils/utilities.js'; function makeCall(params: Record): ParsedRouterRequest { return { request: { params } } as unknown as ParsedRouterRequest; @@ -49,8 +54,8 @@ function setup() { return { admin, createIndexes, getIndexes, findMany, countDocuments, findOne }; } -describe('SchemaAdmin indexes T35–T42', () => { - it('T37 Admin createIndexes is privileged', async () => { +describe('SchemaAdmin indexes', () => { + it('creates indexes as a privileged Admin caller', async () => { const { admin, createIndexes } = setup(); await admin.createIndexes( makeCall({ @@ -66,7 +71,7 @@ describe('SchemaAdmin indexes T35–T42', () => { ); }); - it('T39 exportIndexes is paginated (skip/limit, no unbounded findMany)', async () => { + it('exports indexes with skip/limit pagination', async () => { const { admin, findMany, countDocuments, getIndexes } = setup(); const result = (await admin.exportIndexes(makeCall({ skip: 10, limit: 5 }))) as { indexes: unknown[]; @@ -82,7 +87,7 @@ describe('SchemaAdmin indexes T35–T42', () => { expect(result.indexes.every(index => 'schemaName' in (index as object))).toBe(true); }); - it('T40 importIndexes skips same-name indexes', async () => { + it('skips same-name indexes on import', async () => { const { admin, createIndexes, getIndexes } = setup(); getIndexes.mockResolvedValue([{ name: 'keep_me', fields: ['email'] }]); await admin.importIndexes( @@ -95,10 +100,10 @@ describe('SchemaAdmin indexes T35–T42', () => { ); expect(createIndexes).toHaveBeenCalledTimes(1); const created = createIndexes.mock.calls[0][1] as { name?: string }[]; - expect(created.map(i => i.name)).toEqual(['new_name']); + expect(created.map(index => index.name)).toEqual(['new_name']); }); - it('T41 import unique is not Admin-privileged (respects owner)', async () => { + it('imports unique indexes without Admin privilege so owner rules apply', async () => { const { admin, createIndexes } = setup(); await admin.importIndexes( makeCall({ @@ -120,10 +125,39 @@ describe('SchemaAdmin indexes T35–T42', () => { ); }); - it('T42 import/export are Admin handlers and reject empty import', async () => { + it('rejects an empty import payload', async () => { const { admin } = setup(); await expect(admin.importIndexes(makeCall({ indexes: [] }))).rejects.toMatchObject({ code: status.INVALID_ARGUMENT, }); }); }); + +describe('validateModelOptions indexes', () => { + it('accepts modelOptions.indexes and conduit.readPreference together', () => { + expect(() => + validateSchemaInput( + 'User', + { email: 'String' }, + { + timestamps: true, + indexes: [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + conduit: { readPreference: 'secondaryPreferred' }, + }, + ), + ).not.toThrow(); + }); + + it('still rejects unknown conduit keys and unknown model option keys', () => { + expect(() => + validateSchemaInput('User', { email: 'String' }, { unknown: true } as Parameters< + typeof validateSchemaInput + >[2]), + ).toThrow(/indexes/); + expect(() => + validateSchemaInput('User', { email: 'String' }, { + conduit: { notARealKey: true }, + } as Parameters[2]), + ).toThrow(/readPreference/); + }); +}); diff --git a/modules/database/src/__tests__/indexes/converters.test.ts b/modules/database/src/__tests__/indexes/converters.test.ts new file mode 100644 index 000000000..88843de97 --- /dev/null +++ b/modules/database/src/__tests__/indexes/converters.test.ts @@ -0,0 +1,195 @@ +import { beforeEach, describe, expect, it, jest } from '@jest/globals'; +import { + CompatibleIndexType, + ConduitGrpcSdk, + ConduitSchema, + MongoIndexType, + PostgresIndexType, + TYPE, +} from '@conduitplatform/grpc-sdk'; +import { + convertModelOptionsIndexes, + convertSchemaFieldIndexes, +} from '../../adapters/utils/database-transform-utils.js'; +import { sqlSchemaConverter } from '../../adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.js'; +import { pgSchemaConverter } from '../../adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.js'; +import { schemaConverter } from '../../adapters/mongoose-adapter/SchemaConverter.js'; + +type ConvertedSqlIndex = { + fields: Array; + using?: PostgresIndexType; +}; + +function schemaWithIndexes( + indexes: ConduitSchema['modelOptions']['indexes'], + fields: ConduitSchema['fields'] = { email: { type: TYPE.String } }, +) { + return new ConduitSchema('User', fields, { indexes }); +} + +describe('SQL index converters', () => { + beforeEach(() => { + jest.spyOn(ConduitGrpcSdk.Logger, 'warn').mockImplementation(() => undefined); + }); + + it('maps Compatible types to BTREE plus ASC/DESC on postgres and mysql', () => { + const postgres = convertModelOptionsIndexes( + schemaWithIndexes([{ fields: ['email'], types: [CompatibleIndexType.Descending] }]), + 'postgres', + ); + const pgIndex = postgres.modelOptions.indexes![0] as ConvertedSqlIndex; + expect(pgIndex.using).toBe(PostgresIndexType.BTREE); + expect(pgIndex.fields[0]).toEqual({ name: 'email', order: 'DESC' }); + + const mysql = convertModelOptionsIndexes( + schemaWithIndexes([{ fields: ['email'], types: [CompatibleIndexType.Ascending] }]), + 'mysql', + ); + const mysqlIndex = mysql.modelOptions.indexes![0] as ConvertedSqlIndex; + expect(mysqlIndex.using).toBe(PostgresIndexType.BTREE); + expect(mysqlIndex.fields[0]).toEqual({ name: 'email', order: 'ASC' }); + }); + + it('maps Compatible types to BTREE on sqlite', () => { + const copy = convertModelOptionsIndexes( + schemaWithIndexes([{ fields: ['email'], types: CompatibleIndexType.Ascending }]), + 'sqlite', + ); + expect((copy.modelOptions.indexes![0] as ConvertedSqlIndex).using).toBe( + PostgresIndexType.BTREE, + ); + }); + + it('warns and skips Mongo-only leftovers on SQL', () => { + const warn = ConduitGrpcSdk.Logger.warn as jest.Mock; + const copy = convertModelOptionsIndexes( + schemaWithIndexes([ + { fields: ['loc'], types: [MongoIndexType.GeoSpatial2dSphere] }, + { fields: ['email'], types: [CompatibleIndexType.Ascending] }, + ]), + 'postgres', + ); + expect(copy.modelOptions.indexes).toHaveLength(1); + expect(warn).toHaveBeenCalled(); + }); + + it('skips postgres-only types on mysql, mariadb, and sqlite', () => { + for (const dialect of ['mysql', 'mariadb', 'sqlite'] as const) { + const copy = convertModelOptionsIndexes( + schemaWithIndexes([{ fields: ['email'], types: PostgresIndexType.GIST }]), + dialect, + ); + expect(copy.modelOptions.indexes).toHaveLength(0); + } + }); + + it('converts field-level Compatible indexes into sequelize model indexes', () => { + const copy = convertSchemaFieldIndexes( + new ConduitSchema( + 'Perm', + { + resource: { + type: TYPE.String, + index: { type: CompatibleIndexType.Ascending }, + }, + }, + {}, + ), + 'mysql', + ); + expect(copy.modelOptions.indexes).toHaveLength(1); + expect((copy.fields.resource as { index?: unknown }).index).toBeUndefined(); + }); + + it('keeps HASH on mysql and drops it on sqlite', () => { + const schema = new ConduitSchema( + 'User', + { email: { type: TYPE.String } }, + { + indexes: [ + { fields: ['email'], types: [CompatibleIndexType.Descending] }, + { fields: ['email'], types: PostgresIndexType.HASH }, + ], + }, + ); + const [mysql] = sqlSchemaConverter(schema, 'mysql'); + const [sqlite] = sqlSchemaConverter(schema, 'sqlite'); + expect(mysql.modelOptions.indexes).toHaveLength(2); + expect(sqlite.modelOptions.indexes).toHaveLength(1); + }); + + it('keeps postgres-only types in the pg converter', () => { + const schema = new ConduitSchema( + 'User', + { email: { type: TYPE.String } }, + { + indexes: [{ fields: ['email'], types: PostgresIndexType.GIN }], + }, + ); + const [pg] = pgSchemaConverter(schema); + expect(pg.modelOptions.indexes).toHaveLength(1); + expect((pg.modelOptions.indexes![0] as ConvertedSqlIndex).using).toBe( + PostgresIndexType.GIN, + ); + }); +}); + +describe('mongoose SchemaConverter indexes', () => { + beforeEach(() => { + jest.spyOn(ConduitGrpcSdk.Logger, 'warn').mockImplementation(() => undefined); + }); + + it('treats Compatible types as Mongo 1/-1', () => { + const converted = schemaConverter( + new ConduitSchema( + 'User', + { + email: { + type: TYPE.String, + index: { type: CompatibleIndexType.Descending }, + }, + }, + {}, + ), + ); + expect( + (converted.fields.email as { index: { type: MongoIndexType } }).index.type, + ).toBe(MongoIndexType.Descending); + }); + + it('warns and skips postgres leftovers on Mongo', () => { + const warn = ConduitGrpcSdk.Logger.warn as jest.Mock; + const converted = schemaConverter( + new ConduitSchema( + 'User', + { + email: { + type: TYPE.String, + index: { type: PostgresIndexType.GIN }, + }, + }, + {}, + ), + ); + expect((converted.fields.email as { index?: unknown }).index).toBeUndefined(); + expect(warn).toHaveBeenCalled(); + }); + + it('maps the Compatible string Ascending, not a Mongo enum key', () => { + const converted = schemaConverter( + new ConduitSchema( + 'User', + { + email: { + type: TYPE.String, + index: { type: CompatibleIndexType.Ascending }, + }, + }, + {}, + ), + ); + expect( + (converted.fields.email as { index: { type: MongoIndexType } }).index.type, + ).toBe(MongoIndexType.Ascending); + }); +}); diff --git a/modules/database/src/adapters/utils/__tests__/indexes.test.ts b/modules/database/src/__tests__/indexes/helpers.test.ts similarity index 70% rename from modules/database/src/adapters/utils/__tests__/indexes.test.ts rename to modules/database/src/__tests__/indexes/helpers.test.ts index c5dbc5335..74be105b6 100644 --- a/modules/database/src/adapters/utils/__tests__/indexes.test.ts +++ b/modules/database/src/__tests__/indexes/helpers.test.ts @@ -23,10 +23,10 @@ import { sqlDialectAllowsIndexType, sqlIndexFields, validateIndexFields, -} from '../indexes.js'; +} from '../../adapters/utils/indexes.js'; -describe('index helpers T1–T16', () => { - it('T1 CompatibleIndexType uses portable string values, not Mongo 1/-1', () => { +describe('index helpers', () => { + it('keeps CompatibleIndexType as portable strings, not Mongo 1/-1', () => { expect(CompatibleIndexType.Ascending).toBe('Ascending'); expect(CompatibleIndexType.Descending).toBe('Descending'); expect(CompatibleIndexType.Ascending).not.toBe(MongoIndexType.Ascending); @@ -35,12 +35,25 @@ describe('index helpers T1–T16', () => { expect(isMongoIndexType('Ascending')).toBe(false); }); - it('T4 generates a name when missing', () => { + it('generates a deterministic name when one is missing', () => { const name = generateIndexName(['email'], [CompatibleIndexType.Ascending], false); - expect(name).toMatch(/^cnd_idx_email_asc$/); + expect(name).toBe('cnd_idx_email_asc'); + const unique = generateIndexName( + ['room', 'createdAt'], + [CompatibleIndexType.Ascending, CompatibleIndexType.Descending], + true, + ); + expect(unique).toBe( + generateIndexName( + ['room', 'createdAt'], + [CompatibleIndexType.Ascending, CompatibleIndexType.Descending], + true, + ), + ); + expect(unique).toMatch(/^cnd_uidx_/); }); - it('T5 keeps a provided name', () => { + it('keeps a provided name on the index and options', () => { const named = ensureIndexName({ fields: ['email'], name: 'custom_email_idx', @@ -49,41 +62,34 @@ describe('index helpers T1–T16', () => { expect(named.options?.name).toBe('custom_email_idx'); }); - it('T6 is deterministic for the same input', () => { - const a = generateIndexName( - ['room', 'createdAt'], - [CompatibleIndexType.Ascending, CompatibleIndexType.Descending], - true, - ); - const b = generateIndexName( - ['room', 'createdAt'], - [CompatibleIndexType.Ascending, CompatibleIndexType.Descending], - true, - ); - expect(a).toBe(b); - expect(a).toMatch(/^cnd_uidx_/); - }); - - it('T7 maps Compatible to Mongo 1/-1', () => { + it('maps Compatible and Mongo directions to engine types', () => { expect(mapCompatibleToMongo(CompatibleIndexType.Ascending)).toBe(1); expect(mapCompatibleToMongo(CompatibleIndexType.Descending)).toBe(-1); expect(mapCompatibleToMongo(undefined)).toBe(1); - }); - - it('T8 maps Compatible to SQL BTREE ASC/DESC field order', () => { expect(mapCompatibleToSqlOrder(CompatibleIndexType.Ascending)).toBe('ASC'); expect(mapCompatibleToSqlOrder(CompatibleIndexType.Descending)).toBe('DESC'); - const fields = sqlIndexFields({ - fields: ['createdAt', 'room'], - types: [CompatibleIndexType.Descending, CompatibleIndexType.Ascending], - }); - expect(fields).toEqual([ + expect(sqlIndexFields({ fields: ['createdAt', 'room'] })).toEqual([ + 'createdAt', + 'room', + ]); + expect( + sqlIndexFields({ + fields: ['createdAt', 'room'], + types: [CompatibleIndexType.Descending, CompatibleIndexType.Ascending], + }), + ).toEqual([ { name: 'createdAt', order: 'DESC' }, { name: 'room', order: 'ASC' }, ]); + expect( + sqlIndexFields({ + fields: ['createdAt'], + types: [MongoIndexType.Descending], + }), + ).toEqual([{ name: 'createdAt', order: 'DESC' }]); }); - it('T9 preserves the unique option on generated names', () => { + it('preserves unique when generating a name', () => { const unique = ensureIndexName({ fields: ['email'], types: [CompatibleIndexType.Ascending], @@ -93,7 +99,7 @@ describe('index helpers T1–T16', () => { expect(resolveIndexName(unique)).toMatch(/uidx/); }); - it('T10 persist helper merges incoming indexes by name and skips duplicates', () => { + it('merges declared indexes by name without overwriting the first', () => { const merged = mergeDeclaredIndexes( [{ fields: ['a'], name: 'idx_a' }], [ @@ -101,33 +107,35 @@ describe('index helpers T1–T16', () => { { fields: ['b'], name: 'idx_b' }, ], ); - expect(merged.map(i => i.name)).toEqual(['idx_a', 'idx_b']); + expect(merged.map(index => index.name)).toEqual(['idx_a', 'idx_b']); expect(merged[0].options?.unique).toBeUndefined(); }); - it('T11 persist helper removes indexes by name', () => { - const remaining = removeDeclaredIndexes( - [ - { fields: ['a'], name: 'idx_a' }, - { fields: ['b'], name: 'idx_b' }, - ], - ['idx_a'], - ); - expect(remaining).toEqual([{ fields: ['b'], name: 'idx_b' }]); + it('removes declared indexes and field-level index metadata by name', () => { + expect( + removeDeclaredIndexes( + [ + { fields: ['a'], name: 'idx_a' }, + { fields: ['b'], name: 'idx_b' }, + ], + ['idx_a'], + ), + ).toEqual([{ fields: ['b'], name: 'idx_b' }]); + expect( + removeIndexFromSchemaFields({ fields: { a: { index: { name: 'x' } } } }, 'x'), + ).toBe(true); }); - it('T12 validateIndexFields rejects unknown fields', () => { + it('rejects unknown index fields', () => { expect(() => validateIndexFields( { compiledFields: { email: 'String' }, fields: {} }, - { - fields: ['missing'], - }, + { fields: ['missing'] }, ), ).toThrow(/Invalid fields/); }); - it('T13 unique is denied for a non-owner, non-admin caller', () => { + it('enforces unique-index privilege for owner, Admin, and import', () => { expect(() => assertUniqueIndexPrivilege({ unique: true, @@ -136,9 +144,6 @@ describe('index helpers T1–T16', () => { privileged: false, }), ).toThrow(expect.objectContaining({ code: status.PERMISSION_DENIED })); - }); - - it('T14 unique is allowed for the schema owner', () => { expect(() => assertUniqueIndexPrivilege({ unique: true, @@ -147,9 +152,6 @@ describe('index helpers T1–T16', () => { privileged: false, }), ).not.toThrow(); - }); - - it('T15 unique is allowed for privileged Admin', () => { expect(() => assertUniqueIndexPrivilege({ unique: true, @@ -158,9 +160,6 @@ describe('index helpers T1–T16', () => { privileged: true, }), ).not.toThrow(); - }); - - it('T16 import unique respects owner privilege (not Admin-privileged)', () => { expect(() => assertUniqueIndexPrivilege({ unique: true, @@ -178,12 +177,9 @@ describe('index helpers T1–T16', () => { ), ).toBe(true); expect(isIndexAlreadyExistsError(new Error('index already exists'))).toBe(true); - expect( - removeIndexFromSchemaFields({ fields: { a: { index: { name: 'x' } } } }, 'x'), - ).toBe(true); }); - it('T25 dialect checks use real switches, not `mysql || mariadb`', () => { + it('allows dialect-native types and rejects foreign leftovers', () => { expect(sqlDialectAllowsIndexType('mysql', CompatibleIndexType.Ascending)).toBe(true); expect(sqlDialectAllowsIndexType('mariadb', CompatibleIndexType.Descending)).toBe( true, diff --git a/modules/database/src/__tests__/no-old-pr-bugs.test.ts b/modules/database/src/__tests__/indexes/regressions.test.ts similarity index 53% rename from modules/database/src/__tests__/no-old-pr-bugs.test.ts rename to modules/database/src/__tests__/indexes/regressions.test.ts index 68a155907..5a13612c8 100644 --- a/modules/database/src/__tests__/no-old-pr-bugs.test.ts +++ b/modules/database/src/__tests__/indexes/regressions.test.ts @@ -1,7 +1,36 @@ -import { readFileSync } from 'fs'; +import { existsSync, readFileSync } from 'fs'; import { resolve } from 'path'; import { describe, expect, it } from '@jest/globals'; +function repoFile(...parts: string[]) { + const candidates = [ + resolve(process.cwd(), '..', ...parts), + resolve(process.cwd(), ...parts), + resolve(process.cwd(), '../..', ...parts), + ]; + const found = candidates.find(existsSync); + if (!found) throw new Error(`Missing ${parts.join('/')}`); + return found; +} + +describe('platform models use CompatibleIndexType', () => { + it('authz and chat schemas declare Compatible indexes, not Mongo-only types', () => { + const files = [ + repoFile('authorization', 'src', 'models', 'Permission.schema.ts'), + repoFile('authorization', 'src', 'models', 'Relationship.schema.ts'), + repoFile('authorization', 'src', 'models', 'ActorIndex.schema.ts'), + repoFile('authorization', 'src', 'models', 'ObjectIndex.schema.ts'), + repoFile('chat', 'src', 'models', 'ChatRoom.schema.ts'), + repoFile('chat', 'src', 'models', 'Message.schema.ts'), + ]; + for (const file of files) { + const source = readFileSync(file, 'utf8'); + expect(source).toContain('CompatibleIndexType'); + expect(source).not.toContain('MongoIndexType'); + } + }); +}); + describe('do not port old PR #643 bugs', () => { it("does not use `case 'mysql' || 'mariadb'`", () => { const files = [ diff --git a/modules/database/src/__tests__/platform-models.indexes.test.ts b/modules/database/src/__tests__/platform-models.indexes.test.ts deleted file mode 100644 index e5baea9a5..000000000 --- a/modules/database/src/__tests__/platform-models.indexes.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { existsSync, readFileSync } from 'fs'; -import { resolve } from 'path'; -import { describe, expect, it } from '@jest/globals'; - -function repoFile(...parts: string[]) { - const candidates = [ - resolve(process.cwd(), '..', ...parts), - resolve(process.cwd(), ...parts), - resolve(process.cwd(), '../..', ...parts), - ]; - const found = candidates.find(existsSync); - if (!found) throw new Error(`Missing ${parts.join('/')}`); - return found; -} - -const files = [ - repoFile('authorization', 'src', 'models', 'Permission.schema.ts'), - repoFile('authorization', 'src', 'models', 'Relationship.schema.ts'), - repoFile('authorization', 'src', 'models', 'ActorIndex.schema.ts'), - repoFile('authorization', 'src', 'models', 'ObjectIndex.schema.ts'), - repoFile('chat', 'src', 'models', 'ChatRoom.schema.ts'), - repoFile('chat', 'src', 'models', 'Message.schema.ts'), -]; - -describe('platform models T7 CompatibleIndexType', () => { - it('authz + chat schemas declare Compatible indexes, not Mongo-only types', () => { - for (const file of files) { - const source = readFileSync(file, 'utf8'); - expect(source).toContain('CompatibleIndexType'); - expect(source).not.toContain('MongoIndexType'); - } - }); -}); diff --git a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts index edf889775..aa423c2f7 100644 --- a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts +++ b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts @@ -13,9 +13,9 @@ import { applyMongoVectorField } from '../utils/vectorMappings.js'; import { isVectorTypeName } from '../utils/vectorField.js'; import { isCompatibleIndexType, - isMongoIndexType, mapCompatibleToMongo, mongoAllowsIndexType, + normalizeIndexTypes, } from '../utils/indexes.js'; import * as deepdash from 'deepdash-es/standalone'; @@ -145,12 +145,11 @@ function convertSchemaFieldIndexes(copy: ConduitSchema) { function convertModelOptionsIndexes(copy: ConduitSchema) { if (!copy.modelOptions.indexes?.length) return copy; - const mutIndexes = copy.modelOptions.indexes as ModelOptionsIndexes[]; - for (const index of [...mutIndexes]) { + const remaining: ModelOptionsIndexes[] = []; + for (const index of copy.modelOptions.indexes) { + let mappedTypes: MongoIndexType[] | undefined; if (index.types) { - const types = isArray(index.types) - ? index.types - : index.fields.map(() => index.types); + const types = normalizeIndexTypes(index.types, index.fields.length) ?? []; if ( types.some(type => !mongoAllowsIndexType(type)) || (isArray(index.types) && index.fields.length !== index.types.length) @@ -158,41 +157,31 @@ function convertModelOptionsIndexes(copy: ConduitSchema) { ConduitGrpcSdk.Logger.warn( `Invalid index type for MongoDB found in '${copy.name}', ignoring index`, ); - mutIndexes.splice(mutIndexes.indexOf(index), 1); continue; } - index.types = types.map(type => - isCompatibleIndexType(type) || isMongoIndexType(type) - ? mapCompatibleToMongo(type) - : (type as MongoIndexType), - ) as MongoIndexType[]; + mappedTypes = types.map(mapCompatibleToMongo); + index.types = mappedTypes; + } + // compound indexes stay on modelOptions and are created after schema creation + if (index.fields.length !== 1) { + remaining.push(index); + continue; } - // compound indexes are maintained in modelOptions in order to be created after schema creation - // single field index => add it to specified schema field - if (index.fields.length !== 1) continue; const modelField = copy.fields[index.fields[0]] as ConduitModelField; if (!modelField) { throw new Error(`Field ${index.fields[0]} in index definition doesn't exist`); } - if (index.types) { - modelField.index = { - type: (index.types as MongoIndexType[])[0], - }; - } - if (index.options) { - if (!checkIfMongoOptions(index.options)) { - ConduitGrpcSdk.Logger.warn( - `Invalid index options for MongoDB found in '${copy.name}', ignoring index`, - ); - mutIndexes.splice(mutIndexes.indexOf(index), 1); - continue; - } - if (!modelField.index) modelField.index = {}; - for (const [option, optionValue] of Object.entries(index.options)) { - modelField.index![option as keyof SchemaFieldIndex] = optionValue; - } + if (index.options && !checkIfMongoOptions(index.options)) { + ConduitGrpcSdk.Logger.warn( + `Invalid index options for MongoDB found in '${copy.name}', ignoring index`, + ); + continue; } - mutIndexes.splice(mutIndexes.indexOf(index), 1); + modelField.index = { + ...(mappedTypes ? { type: mappedTypes[0] } : {}), + ...index.options, + }; } + copy.modelOptions.indexes = remaining; return copy; } diff --git a/modules/database/src/adapters/mongoose-adapter/__tests__/SchemaConverter.indexes.test.ts b/modules/database/src/adapters/mongoose-adapter/__tests__/SchemaConverter.indexes.test.ts deleted file mode 100644 index daeb74055..000000000 --- a/modules/database/src/adapters/mongoose-adapter/__tests__/SchemaConverter.indexes.test.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { beforeEach, describe, expect, it, jest } from '@jest/globals'; -import { - CompatibleIndexType, - ConduitGrpcSdk, - ConduitSchema, - MongoIndexType, - PostgresIndexType, - TYPE, -} from '@conduitplatform/grpc-sdk'; -import { schemaConverter } from '../SchemaConverter.js'; - -describe('mongoose SchemaConverter indexes T17 T23', () => { - beforeEach(() => { - jest.spyOn(ConduitGrpcSdk.Logger, 'warn').mockImplementation(() => {}); - }); - - it('T17 treats Compatible as Mongo 1/-1', () => { - const converted = schemaConverter( - new ConduitSchema( - 'User', - { - email: { - type: TYPE.String, - index: { type: CompatibleIndexType.Descending }, - }, - } as any, - {}, - ), - ); - expect((converted.fields.email as any).index.type).toBe(MongoIndexType.Descending); - }); - - it('T23 recover: postgres leftovers on Mongo are warned and skipped', () => { - const warn = ConduitGrpcSdk.Logger.warn as jest.Mock; - const converted = schemaConverter( - new ConduitSchema( - 'User', - { - email: { - type: TYPE.String, - index: { type: PostgresIndexType.GIN }, - }, - } as any, - {}, - ), - ); - expect((converted.fields.email as any).index).toBeUndefined(); - expect(warn).toHaveBeenCalled(); - }); - - it('does not treat the Mongo enum key "Ascending" as a valid Mongo type', () => { - const converted = schemaConverter( - new ConduitSchema( - 'User', - { - email: { - type: TYPE.String, - index: { type: 'Ascending' as any }, - }, - } as any, - {}, - ), - ); - expect((converted.fields.email as any).index.type).toBe(MongoIndexType.Ascending); - }); -}); diff --git a/modules/database/src/adapters/mongoose-adapter/__tests__/indexes.adapter.test.ts b/modules/database/src/adapters/mongoose-adapter/__tests__/indexes.adapter.test.ts deleted file mode 100644 index 681228b26..000000000 --- a/modules/database/src/adapters/mongoose-adapter/__tests__/indexes.adapter.test.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { describe, expect, it, jest } from '@jest/globals'; -import { status } from '@grpc/grpc-js'; -import { CompatibleIndexType, MongoIndexType } from '@conduitplatform/grpc-sdk'; -import { MongooseAdapter } from '../index.js'; - -function makeAdapter(overrides: Record = {}) { - const createIndex = jest.fn().mockResolvedValue('email_1'); - const dropIndex = jest.fn().mockResolvedValue(undefined); - const indexes = jest.fn().mockResolvedValue([ - { v: 2, key: { _id: 1 }, name: '_id_' }, - { v: 2, key: { email: 1 }, name: 'cnd_idx_email_asc', unique: false }, - ]); - const findOne = jest.fn().mockResolvedValue({ _id: 'declared-1' }); - const findByIdAndUpdate = jest.fn().mockResolvedValue({}); - const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; - adapter.mongoose = { - model: () => ({ collection: { createIndex, dropIndex, indexes } }), - } as any; - const originalSchema = { - name: 'User', - ownerModule: 'chat', - collectionName: 'cnd_User', - fields: { email: { type: 'String' } }, - compiledFields: { email: { type: 'String' } }, - modelOptions: { indexes: [] as unknown[] }, - ...((overrides.originalSchema as object) ?? {}), - }; - adapter.models = { - User: { originalSchema }, - _DeclaredSchema: { findOne, findByIdAndUpdate }, - } as any; - return { adapter, createIndex, dropIndex, indexes, findByIdAndUpdate, originalSchema }; -} - -describe('mongoose adapter indexes T26–T29 T34 T38', () => { - it('T26 createIndex uses a single key spec object, not an array of objects', async () => { - const { adapter, createIndex } = makeAdapter(); - await adapter.createIndexes( - 'User', - [ - { - fields: ['email'], - types: [CompatibleIndexType.Ascending], - }, - ], - 'chat', - ); - expect(createIndex).toHaveBeenCalledTimes(1); - expect(createIndex.mock.calls[0][0]).toEqual({ email: MongoIndexType.Ascending }); - expect(Array.isArray(createIndex.mock.calls[0][0])).toBe(false); - }); - - it('T27 create persists metadata into _DeclaredSchema', async () => { - const { adapter, findByIdAndUpdate } = makeAdapter(); - await adapter.createIndexes( - 'User', - [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], - 'chat', - ); - expect(findByIdAndUpdate).toHaveBeenCalledTimes(1); - const update = findByIdAndUpdate.mock.calls[0][1] as { - modelOptions: { indexes: { name?: string }[] }; - }; - expect(update.modelOptions.indexes[0].name).toMatch(/email/); - }); - - it('T28 delete awaits dropIndex', async () => { - const { adapter, dropIndex } = makeAdapter(); - let resolveDrop: () => void = () => {}; - const dropped = new Promise(resolve => { - resolveDrop = resolve; - }); - dropIndex.mockImplementation(async () => { - resolveDrop(); - }); - await adapter.deleteIndexes('User', ['cnd_idx_email_asc']); - await dropped; - expect(dropIndex).toHaveBeenCalledWith('cnd_idx_email_asc'); - }); - - it('T29 delete persists removal', async () => { - const { adapter, findByIdAndUpdate, originalSchema } = makeAdapter({ - originalSchema: { - modelOptions: { indexes: [{ fields: ['email'], name: 'cnd_idx_email_asc' }] }, - }, - }); - await adapter.deleteIndexes('User', ['cnd_idx_email_asc']); - expect(originalSchema.modelOptions.indexes).toEqual([]); - expect(findByIdAndUpdate).toHaveBeenCalled(); - }); - - it('T34 getIndexes uses the live engine as source of truth', async () => { - const { adapter, indexes } = makeAdapter(); - const result = await adapter.getIndexes('User'); - expect(indexes).toHaveBeenCalled(); - expect(result.map(i => i.name)).toEqual(['_id_', 'cnd_idx_email_asc']); - expect(result[1].fields).toEqual(['email']); - }); - - it('T38 Admin-bound invalid types throw', async () => { - const { adapter } = makeAdapter(); - await expect( - adapter.createIndexes( - 'User', - [{ fields: ['email'], types: ['GIST'] as any }], - 'database', - { privileged: true }, - ), - ).rejects.toMatchObject({ code: status.INVALID_ARGUMENT }); - }); - - it('T15 Admin privileged unique is allowed on a foreign-owned schema', async () => { - const { adapter } = makeAdapter(); - await expect( - adapter.createIndexes( - 'User', - [{ fields: ['email'], options: { unique: true } }], - 'database', - { privileged: true }, - ), - ).resolves.toBe('Indexes created!'); - }); -}); diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index a81052f35..98c354eb7 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -32,17 +32,16 @@ import { } from '../utils/index.js'; import { assertUniqueIndexPrivilege, + declaredIndexMap, ensureIndexName, - isCompatibleIndexType, isIndexAlreadyExistsError, - isMongoIndexType, mapCompatibleToMongo, mergeDeclaredIndexes, mongoAllowsIndexType, + normalizeIndexTypes, persistDeclaredSchemaIndexes, removeDeclaredIndexes, removeIndexFromSchemaFields, - resolveIndexName, toMutableIndexes, validateIndexFields, } from '../utils/indexes.js'; @@ -669,7 +668,7 @@ export class MongooseAdapter extends DatabaseAdapter { const collection = this.mongoose.model(schemaName).collection; for (const index of prepared) { const spec: Record = {}; - const types = Array.isArray(index.types) ? index.types : undefined; + const types = normalizeIndexTypes(index.types, index.fields.length); for (let i = 0; i < index.fields.length; i++) { spec[index.fields[i]] = types ? mapCompatibleToMongo(types[i]) @@ -735,15 +734,13 @@ export class MongooseAdapter extends DatabaseAdapter { throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); const collection = this.mongoose.model(schemaName).collection; const result = await collection.indexes(); - const declared = this.models[schemaName].originalSchema.modelOptions.indexes ?? []; - const declaredByName = new Map( - declared.map((index: ModelOptionsIndexes) => [resolveIndexName(index), index]), + const declaredByName = declaredIndexMap( + this.models[schemaName].originalSchema.modelOptions.indexes, ); return result.map(index => { const options: Record = {}; for (const [key, value] of Object.entries(index)) { - if (key === 'key' || key === 'options') continue; - if (key === 'v') continue; + if (key === 'key' || key === 'options' || key === 'v') continue; options[key] = value; } const fields: string[] = []; @@ -752,18 +749,20 @@ export class MongooseAdapter extends DatabaseAdapter { fields.push(field); types.push(type as MongoIndexType); } - const name = (options.name as string | undefined) ?? index.name; + const name = typeof options.name === 'string' ? options.name : index.name; const declaredIndex = name ? declaredByName.get(name) : undefined; - return { + const live: ModelOptionsIndexes = { name, fields, - types: declaredIndex?.types ?? types, - options: { - ...declaredIndex?.options, - ...options, - name, - }, - } as ModelOptionsIndexes; + types, + options: { ...options, name }, + }; + if (!declaredIndex) return live; + return { + ...live, + types: declaredIndex.types ?? live.types, + options: { ...declaredIndex.options, ...live.options, name }, + }; }); } @@ -1091,7 +1090,7 @@ export class MongooseAdapter extends DatabaseAdapter { }); } if (types) { - const typeList = Array.isArray(types) ? types : index.fields.map(() => types); + const typeList = normalizeIndexTypes(types, index.fields.length) ?? []; if (Array.isArray(types) && typeList.length !== index.fields.length) { throw new GrpcError(status.INVALID_ARGUMENT, 'Invalid index types format'); } @@ -1103,11 +1102,7 @@ export class MongooseAdapter extends DatabaseAdapter { ); } } - index.types = typeList.map(type => - isCompatibleIndexType(type) || isMongoIndexType(type) - ? mapCompatibleToMongo(type) - : type, - ) as MongoIndexType[]; + index.types = typeList.map(mapCompatibleToMongo); } prepared.push(index); } diff --git a/modules/database/src/adapters/sequelize-adapter/__tests__/indexes.adapter.test.ts b/modules/database/src/adapters/sequelize-adapter/__tests__/indexes.adapter.test.ts deleted file mode 100644 index fe7cf5cd1..000000000 --- a/modules/database/src/adapters/sequelize-adapter/__tests__/indexes.adapter.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { describe, expect, it, jest } from '@jest/globals'; -import { CompatibleIndexType, PostgresIndexType } from '@conduitplatform/grpc-sdk'; -import { SequelizeAdapter } from '../index.js'; - -class TestSequelizeAdapter extends SequelizeAdapter { - protected async hasLegacyCollections(): Promise { - return false; - } -} - -function makeAdapter(dialect: string = 'postgres') { - const addIndex = jest.fn().mockResolvedValue(undefined); - const removeIndex = jest.fn().mockResolvedValue(undefined); - const showIndex = jest.fn().mockResolvedValue([ - { - name: 'cnd_idx_email_asc', - unique: false, - fields: [{ attribute: 'email', order: 'ASC' }], - definition: 'CREATE INDEX cnd_idx_email_asc ON cnd_User USING btree (email)', - }, - ]); - const findOne = jest.fn().mockResolvedValue({ _id: 'declared-1' }); - const findByIdAndUpdate = jest.fn().mockResolvedValue({}); - const sync = jest.fn().mockResolvedValue(undefined); - const adapter = Object.create(TestSequelizeAdapter.prototype) as SequelizeAdapter; - adapter.sequelize = { - getDialect: () => dialect, - getQueryInterface: () => ({ addIndex, removeIndex, showIndex }), - } as any; - const originalSchema = { - name: 'User', - ownerModule: 'database', - collectionName: 'custom_users', - fields: { email: { type: 'String' } }, - compiledFields: { email: { type: 'String' } }, - modelOptions: { indexes: [] as unknown[] }, - }; - adapter.models = { - User: { originalSchema, sync }, - _DeclaredSchema: { findOne, findByIdAndUpdate }, - } as any; - return { - adapter, - addIndex, - removeIndex, - showIndex, - sync, - findByIdAndUpdate, - originalSchema, - }; -} - -describe('sequelize adapter indexes T24 T30–T33', () => { - it('T24 getDatabaseType still returns PostgreSQL, not postgres', () => { - const { adapter } = makeAdapter('postgres'); - expect(adapter.getDatabaseType()).toBe('PostgreSQL'); - }); - - it('T30 create/get/delete use originalSchema.collectionName, not a hardcoded cnd_ prefix', async () => { - const { adapter, addIndex, removeIndex, showIndex } = makeAdapter(); - await adapter.createIndexes( - 'User', - [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], - 'database', - { privileged: true }, - ); - expect(addIndex.mock.calls[0][0]).toBe('custom_users'); - expect(addIndex.mock.calls[0][0]).not.toBe('cnd_User'); - await adapter.getIndexes('User'); - expect(showIndex).toHaveBeenCalledWith('custom_users'); - await adapter.deleteIndexes('User', ['cnd_idx_email_asc']); - expect(removeIndex).toHaveBeenCalledWith('custom_users', 'cnd_idx_email_asc'); - }); - - it('T31 create does not rebuild/sync the schema', async () => { - const { adapter, sync } = makeAdapter(); - await adapter.createIndexes( - 'User', - [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], - 'database', - ); - expect(sync).not.toHaveBeenCalled(); - }); - - it('T32 getIndexes reads the live engine and overlays declared Compatible types', async () => { - const { adapter, showIndex, originalSchema } = makeAdapter(); - originalSchema.modelOptions.indexes = [ - { - fields: ['email'], - name: 'cnd_idx_email_asc', - types: [CompatibleIndexType.Ascending], - }, - ]; - const result = await adapter.getIndexes('User'); - expect(showIndex).toHaveBeenCalledWith('custom_users'); - expect(result[0].types).toEqual([CompatibleIndexType.Ascending]); - expect(result[0].fields).toEqual(['email']); - }); - - it('T33 delete awaits removeIndex and persists', async () => { - const { adapter, removeIndex, findByIdAndUpdate } = makeAdapter(); - await adapter.deleteIndexes('User', ['cnd_idx_email_asc']); - expect(removeIndex).toHaveBeenCalledTimes(1); - expect(findByIdAndUpdate).toHaveBeenCalled(); - }); - - it('mysql HASH is allowed; sqlite HASH throws on Admin create', async () => { - const mysql = makeAdapter('mysql'); - await expect( - mysql.adapter.createIndexes( - 'User', - [{ fields: ['email'], types: PostgresIndexType.HASH }], - 'database', - { privileged: true }, - ), - ).resolves.toBe('Indexes created!'); - - const sqlite = makeAdapter('sqlite'); - await expect( - sqlite.adapter.createIndexes( - 'User', - [{ fields: ['email'], types: PostgresIndexType.HASH }], - 'database', - { privileged: true }, - ), - ).rejects.toMatchObject({ message: expect.stringMatching(/sqlite/i) }); - }); -}); diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 6a9a77d2f..bfc41925b 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -6,7 +6,6 @@ import { GrpcError, Indexable, ModelOptionsIndexes, - PostgresIndexOptions, PostgresIndexType, RawSQLQuery, UntypedArray, @@ -53,14 +52,16 @@ import { } from '../utils/index.js'; import { assertUniqueIndexPrivilege, + declaredIndexMap, ensureIndexName, inferSqlIndexType, isIndexAlreadyExistsError, + isPostgresIndexType, mergeDeclaredIndexes, + normalizeIndexTypes, persistDeclaredSchemaIndexes, removeDeclaredIndexes, removeIndexFromSchemaFields, - resolveIndexName, sqlDialectAllowsIndexType, sqlIndexFields, toMutableIndexes, @@ -437,9 +438,8 @@ export abstract class SequelizeAdapter extends DatabaseAdapter const queryInterface = this.sequelize.getQueryInterface(); const result = (await queryInterface.showIndex(collectionName)) as UntypedArray; const dialect = this.sequelize.getDialect(); - const declared = this.models[schemaName].originalSchema.modelOptions.indexes ?? []; - const declaredByName = new Map( - declared.map((index: ModelOptionsIndexes) => [resolveIndexName(index), index]), + const declaredByName = declaredIndexMap( + this.models[schemaName].originalSchema.modelOptions.indexes, ); return result.map(row => { const fields = (row.fields ?? []).map((field: unknown) => @@ -447,29 +447,18 @@ export abstract class SequelizeAdapter extends DatabaseAdapter ); const name = row.name as string; const declaredIndex = declaredByName.get(name); - const options: Record = { + const live: ModelOptionsIndexes = { name, - unique: !!row.unique, - ...(declaredIndex?.options ?? {}), + fields, + types: inferSqlIndexType(row, dialect), + options: { name, unique: !!row.unique }, }; - for (const [key, value] of Object.entries(row)) { - if ( - key === 'options' || - key === 'types' || - key === 'fields' || - key === 'definition' || - key === 'indkey' - ) { - continue; - } - if (options[key] === undefined) options[key] = value; - } + if (!declaredIndex) return live; return { - name, - fields, - types: declaredIndex?.types ?? inferSqlIndexType(row, dialect), - options, - } as ModelOptionsIndexes; + ...live, + types: declaredIndex.types ?? live.types, + options: { ...declaredIndex.options, ...live.options, name }, + }; }); } @@ -755,7 +744,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter const index = ensureIndexName(raw); validateIndexFields(schema, index); if (index.types) { - const types = Array.isArray(index.types) ? index.types : [index.types]; + const types = normalizeIndexTypes(index.types, index.fields.length) ?? []; if (types.some(type => !sqlDialectAllowsIndexType(dialect, type))) { throw new GrpcError( status.INVALID_ARGUMENT, @@ -763,21 +752,14 @@ export abstract class SequelizeAdapter extends DatabaseAdapter ); } const first = types[0]; - if ( - typeof first === 'string' && - Object.values(PostgresIndexType).includes(first as PostgresIndexType) && - types.length === 1 - ) { - index.options = { - ...(index.options ?? {}), - using: first as PostgresIndexType, - } as PostgresIndexOptions; - } else { - index.options = { - ...(index.options ?? {}), - using: PostgresIndexType.BTREE, - } as PostgresIndexOptions; - } + const using = + types.length === 1 && isPostgresIndexType(first) + ? first + : PostgresIndexType.BTREE; + index.options = { + ...(index.options ?? {}), + using, + }; } if (index.options) { if (!checkIfPostgresOptions(index.options)) { diff --git a/modules/database/src/adapters/utils/__tests__/sql-index-converters.test.ts b/modules/database/src/adapters/utils/__tests__/sql-index-converters.test.ts deleted file mode 100644 index 95e061b0e..000000000 --- a/modules/database/src/adapters/utils/__tests__/sql-index-converters.test.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { beforeEach, describe, expect, it, jest } from '@jest/globals'; -import { - CompatibleIndexType, - ConduitGrpcSdk, - ConduitSchema, - MongoIndexType, - PostgresIndexType, - TYPE, -} from '@conduitplatform/grpc-sdk'; -import { - convertModelOptionsIndexes, - convertSchemaFieldIndexes, -} from '../database-transform-utils.js'; -import { sqlSchemaConverter } from '../../sequelize-adapter/sql-adapter/SqlSchemaConverter.js'; -import { pgSchemaConverter } from '../../sequelize-adapter/postgres-adapter/PgSchemaConverter.js'; - -function schemaWithIndexes( - indexes: ConduitSchema['modelOptions']['indexes'], - fields: ConduitSchema['fields'] = { email: { type: TYPE.String } }, -) { - return new ConduitSchema('User', fields as any, { indexes }); -} - -describe('SQL dialect-aware converters T17–T24', () => { - beforeEach(() => { - jest.spyOn(ConduitGrpcSdk.Logger, 'warn').mockImplementation(() => {}); - }); - - it('T18 postgres maps Compatible to BTREE + ASC/DESC', () => { - const copy = convertModelOptionsIndexes( - schemaWithIndexes([ - { - fields: ['email'], - types: [CompatibleIndexType.Descending], - }, - ]), - 'postgres', - ); - const index = copy.modelOptions.indexes![0] as any; - expect(index.using).toBe(PostgresIndexType.BTREE); - expect(index.fields[0]).toEqual({ name: 'email', order: 'DESC' }); - }); - - it('T19 mysql maps Compatible to BTREE + ASC', () => { - const copy = convertModelOptionsIndexes( - schemaWithIndexes([{ fields: ['email'], types: [CompatibleIndexType.Ascending] }]), - 'mysql', - ); - const index = copy.modelOptions.indexes![0] as any; - expect(index.using).toBe(PostgresIndexType.BTREE); - expect(index.fields[0]).toEqual({ name: 'email', order: 'ASC' }); - }); - - it('T20 sqlite maps Compatible to BTREE + ASC', () => { - const copy = convertModelOptionsIndexes( - schemaWithIndexes([{ fields: ['email'], types: CompatibleIndexType.Ascending }]), - 'sqlite', - ); - const index = copy.modelOptions.indexes![0] as any; - expect(index.using).toBe(PostgresIndexType.BTREE); - }); - - it('T21 recover: Mongo-only leftovers on SQL are warned and skipped', () => { - const warn = ConduitGrpcSdk.Logger.warn as jest.Mock; - const copy = convertModelOptionsIndexes( - schemaWithIndexes([ - { fields: ['loc'], types: [MongoIndexType.GeoSpatial2dSphere] }, - { fields: ['email'], types: [CompatibleIndexType.Ascending] }, - ]), - 'postgres', - ); - expect(copy.modelOptions.indexes).toHaveLength(1); - expect(warn).toHaveBeenCalled(); - }); - - it('T22 recover: postgres-only types on mysql/mariadb/sqlite are skipped', () => { - for (const dialect of ['mysql', 'mariadb', 'sqlite'] as const) { - const copy = convertModelOptionsIndexes( - schemaWithIndexes([{ fields: ['email'], types: PostgresIndexType.GIST }]), - dialect, - ); - expect(copy.modelOptions.indexes).toHaveLength(0); - } - }); - - it('converts field-level Compatible indexes into sequelize model indexes', () => { - const copy = convertSchemaFieldIndexes( - new ConduitSchema( - 'Perm', - { - resource: { - type: TYPE.String, - index: { type: CompatibleIndexType.Ascending }, - }, - } as any, - {}, - ), - 'mysql', - ); - expect(copy.modelOptions.indexes).toHaveLength(1); - expect((copy.fields.resource as any).index).toBeUndefined(); - }); - - it('sqlSchemaConverter is dialect-aware for mysql vs sqlite', () => { - const schema = new ConduitSchema('User', { email: { type: TYPE.String } } as any, { - indexes: [ - { fields: ['email'], types: [CompatibleIndexType.Descending] }, - { fields: ['email'], types: PostgresIndexType.HASH }, - ], - }); - const [mysql] = sqlSchemaConverter(schema, 'mysql'); - const [sqlite] = sqlSchemaConverter(schema, 'sqlite'); - expect(mysql.modelOptions.indexes).toHaveLength(2); - expect(sqlite.modelOptions.indexes).toHaveLength(1); - }); - - it('pgSchemaConverter keeps postgres-only types', () => { - const schema = new ConduitSchema('User', { email: { type: TYPE.String } } as any, { - indexes: [{ fields: ['email'], types: PostgresIndexType.GIN }], - }); - const [pg] = pgSchemaConverter(schema); - expect(pg.modelOptions.indexes).toHaveLength(1); - expect((pg.modelOptions.indexes![0] as any).using).toBe(PostgresIndexType.GIN); - }); -}); diff --git a/modules/database/src/adapters/utils/database-transform-utils.ts b/modules/database/src/adapters/utils/database-transform-utils.ts index f0be2bed7..ef128195a 100644 --- a/modules/database/src/adapters/utils/database-transform-utils.ts +++ b/modules/database/src/adapters/utils/database-transform-utils.ts @@ -1,12 +1,10 @@ import { isBoolean, isNumber, isString } from 'lodash-es'; import { - CompatibleIndexType, ConduitGrpcSdk, ConduitModelField, ConduitSchema, Indexable, ModelOptionsIndexes, - PostgresIndexOptions, PostgresIndexType, } from '@conduitplatform/grpc-sdk'; import { checkIfPostgresOptions } from '../sequelize-adapter/utils/index.js'; @@ -17,8 +15,19 @@ import { mapCompatibleToSqlOrder, normalizeIndexTypes, sqlDialectAllowsIndexType, + type SqlIndexField, } from './indexes.js'; +type SqlEngineIndex = Omit & { + fields: SqlIndexField[]; + using?: PostgresIndexType; +}; + +function setSqlEngineIndexes(copy: ConduitSchema, indexes: SqlEngineIndex[]) { + // Sequelize reads these through define(..., schema.modelOptions). + copy.modelOptions.indexes = indexes as ModelOptionsIndexes[]; +} + export function checkDefaultValue(type: string, value: string) { switch (type) { case 'String': @@ -38,110 +47,91 @@ export function checkDefaultValue(type: string, value: string) { } } -function flattenSqlIndexOptions(index: ModelOptionsIndexes, dialect: string): boolean { - if (!index.options) return true; - if (!checkIfPostgresOptions(index.options)) { - ConduitGrpcSdk.Logger.warn( - `Invalid index options for ${dialect} found in '${copyName(index)}', ignoring index`, - ); - return false; - } - for (const [option, value] of Object.entries(index.options)) { - index[option as keyof PostgresIndexOptions] = value; - } - delete index.options; - return true; +function skipIndex(schemaName: string, dialect: string, reason: string) { + ConduitGrpcSdk.Logger.warn( + `Invalid index ${reason} for ${dialect} found in '${schemaName}', ignoring index`, + ); } -function copyName(index: ModelOptionsIndexes): string { - return index.name ?? index.fields?.join(',') ?? 'unnamed'; -} - -function applySqlIndexTypes( - index: ModelOptionsIndexes, +function toSqlEngineIndex( + raw: ModelOptionsIndexes, dialect: string, schemaName: string, -): boolean { - if (!index.types) { - index.using = PostgresIndexType.BTREE; - return true; +): SqlEngineIndex | null { + const index = ensureIndexName({ ...raw, fields: [...raw.fields] }); + if (index.options && !checkIfPostgresOptions(index.options)) { + skipIndex(schemaName, dialect, 'options'); + return null; } - const types = normalizeIndexTypes(index.types, index.fields.length) ?? []; - if (types.some(type => !sqlDialectAllowsIndexType(dialect, type))) { - ConduitGrpcSdk.Logger.warn( - `Invalid index type for ${dialect} found in '${schemaName}', ignoring index`, - ); - return false; - } - if (types.some(isPortableDirection)) { - index.fields = index.fields.map((field, i) => ({ - name: field, - order: mapCompatibleToSqlOrder(types[i]), - })) as unknown as string[]; - index.using = PostgresIndexType.BTREE; - } else if (types.length === 1 && isPostgresIndexType(types[0])) { - index.using = types[0]; - } else { - ConduitGrpcSdk.Logger.warn( - `Invalid index type for ${dialect} found in '${schemaName}', ignoring index`, - ); - return false; + + let fields: SqlIndexField[] = [...index.fields]; + let using = PostgresIndexType.BTREE; + if (index.types) { + const types = normalizeIndexTypes(index.types, index.fields.length) ?? []; + if (types.some(type => !sqlDialectAllowsIndexType(dialect, type))) { + skipIndex(schemaName, dialect, 'type'); + return null; + } + if (types.some(isPortableDirection)) { + fields = index.fields.map((field, i) => ({ + name: field, + order: mapCompatibleToSqlOrder(types[i]), + })); + } else if (types.length === 1 && isPostgresIndexType(types[0])) { + using = types[0]; + } else { + skipIndex(schemaName, dialect, 'type'); + return null; + } } - delete index.types; - return true; + + return { + ...index.options, + name: index.name, + fields, + using, + unique: index.options?.unique, + }; } export function convertModelOptionsIndexes(copy: ConduitSchema, dialect = 'postgres') { - const converted: ModelOptionsIndexes[] = []; + const converted: SqlEngineIndex[] = []; for (const raw of copy.modelOptions.indexes ?? []) { - const index = ensureIndexName({ ...raw, fields: [...raw.fields] }); - if (!applySqlIndexTypes(index, dialect, copy.name)) continue; - if (!flattenSqlIndexOptions(index, dialect)) continue; - if (!index.using) index.using = PostgresIndexType.BTREE; - converted.push(index); + const index = toSqlEngineIndex(raw, dialect, copy.name); + if (index) converted.push(index); } - copy.modelOptions.indexes = converted; + setSqlEngineIndexes(copy, converted); return copy; } export function convertSchemaFieldIndexes(copy: ConduitSchema, dialect = 'postgres') { - const indexes: ModelOptionsIndexes[] = []; + const indexes: SqlEngineIndex[] = []; for (const [fieldName, fieldValue] of Object.entries(copy.fields)) { - const index = (fieldValue as ConduitModelField).index; + const field = fieldValue as ConduitModelField; + const index = field.index; if (!index) continue; - const newIndex = ensureIndexName({ - fields: [fieldName], - types: index.type - ? isPortableDirection(index.type) - ? [index.type as CompatibleIndexType] - : (index.type as PostgresIndexType) - : undefined, - options: index.options, - name: (index as { name?: string }).name, - }); if (index.type && !sqlDialectAllowsIndexType(dialect, index.type)) { - ConduitGrpcSdk.Logger.warn( - `Invalid index type for ${dialect} found in '${copy.name}', ignoring index`, - ); - delete (copy.fields[fieldName] as ConduitModelField).index; - continue; - } - if (!applySqlIndexTypes(newIndex, dialect, copy.name)) { - delete (copy.fields[fieldName] as ConduitModelField).index; - continue; - } - if (!flattenSqlIndexOptions(newIndex, dialect)) { - delete (copy.fields[fieldName] as ConduitModelField).index; + skipIndex(copy.name, dialect, 'type'); + delete field.index; continue; } - indexes.push(newIndex); - delete (copy.fields[fieldName] as ConduitModelField).index; - } - if (copy.modelOptions.indexes) { - copy.modelOptions.indexes = [...copy.modelOptions.indexes, ...indexes]; - } else { - copy.modelOptions.indexes = indexes; + const converted = toSqlEngineIndex( + { + fields: [fieldName], + types: index.type === undefined ? undefined : [index.type], + options: index.options, + name: index.name, + }, + dialect, + copy.name, + ); + delete field.index; + if (converted) indexes.push(converted); } + setSqlEngineIndexes(copy, [ + ...((copy.modelOptions.indexes ?? []) as SqlEngineIndex[]), + ...indexes, + ]); return copy; } diff --git a/modules/database/src/adapters/utils/indexes.ts b/modules/database/src/adapters/utils/indexes.ts index 2c1a2776a..9338b6ecb 100644 --- a/modules/database/src/adapters/utils/indexes.ts +++ b/modules/database/src/adapters/utils/indexes.ts @@ -12,7 +12,7 @@ import { ConduitDatabaseSchema } from '../../interfaces/index.js'; export const ADMIN_INDEX_CALLER = 'database'; -export const MONGO_INDEX_TYPE_VALUES: ReadonlyArray = [ +const MONGO_INDEX_TYPE_VALUES: ReadonlySet = new Set([ MongoIndexType.Ascending, MongoIndexType.Descending, MongoIndexType.GeoSpatial2d, @@ -20,10 +20,12 @@ export const MONGO_INDEX_TYPE_VALUES: ReadonlyArray = [ MongoIndexType.GeoHaystack, MongoIndexType.Hashed, MongoIndexType.Text, -]; +]); const SQL_IDENTIFIER_MAX_LEN = 63; +export type SqlIndexField = string | { name: string; order: 'ASC' | 'DESC' }; + export function isCompatibleIndexType(value: unknown): value is CompatibleIndexType { return ( value === CompatibleIndexType.Ascending || value === CompatibleIndexType.Descending @@ -31,7 +33,7 @@ export function isCompatibleIndexType(value: unknown): value is CompatibleIndexT } export function isMongoIndexType(value: unknown): value is MongoIndexType { - return (MONGO_INDEX_TYPE_VALUES as readonly unknown[]).includes(value); + return MONGO_INDEX_TYPE_VALUES.has(value); } export function isPostgresIndexType(value: unknown): value is PostgresIndexType { @@ -54,17 +56,14 @@ export function normalizeIndexTypes( fieldCount: number, ): unknown[] | undefined { if (types === undefined) return undefined; - if (Array.isArray(types)) { - return [...types]; - } + if (Array.isArray(types)) return [...types]; return Array.from({ length: fieldCount }, () => types); } function typeToken(type: unknown): string { - if (type === undefined || type === CompatibleIndexType.Ascending) return 'asc'; - if (type === CompatibleIndexType.Descending) return 'desc'; - if (type === MongoIndexType.Ascending || type === 1) return 'asc'; - if (type === MongoIndexType.Descending || type === -1) return 'desc'; + if (isPortableDirection(type) || type === undefined) { + return mapCompatibleToSqlOrder(type).toLowerCase(); + } if (typeof type === 'string') return type.toLowerCase().replace(/[^a-z0-9]+/g, ''); return String(type); } @@ -89,14 +88,8 @@ export function generateIndexName( export function ensureIndexName(index: ModelOptionsIndexes): ModelOptionsIndexes { const existing = resolveIndexName(index); - if (existing) { - return { - ...index, - name: existing, - options: { ...index.options, name: existing }, - }; - } - const name = generateIndexName(index.fields, index.types, isUniqueIndex(index)); + const name = + existing ?? generateIndexName(index.fields, index.types, isUniqueIndex(index)); return { ...index, name, @@ -105,8 +98,14 @@ export function ensureIndexName(index: ModelOptionsIndexes): ModelOptionsIndexes } export function mapCompatibleToMongo(type: unknown): MongoIndexType { - if (type === CompatibleIndexType.Descending) return MongoIndexType.Descending; - if (type === CompatibleIndexType.Ascending || type === undefined) { + if (type === CompatibleIndexType.Descending || type === MongoIndexType.Descending) { + return MongoIndexType.Descending; + } + if ( + type === undefined || + type === CompatibleIndexType.Ascending || + type === MongoIndexType.Ascending + ) { return MongoIndexType.Ascending; } if (isMongoIndexType(type)) return type; @@ -134,8 +133,7 @@ export function sqlDialectAllowsIndexType(dialect: string, type: unknown): boole if (type === PostgresIndexType.HASH) { return dialect === 'postgres' || dialect === 'mysql' || dialect === 'mariadb'; } - if (isPostgresIndexType(type)) return dialect === 'postgres'; - return false; + return isPostgresIndexType(type) && dialect === 'postgres'; } export function mongoAllowsIndexType(type: unknown): boolean { @@ -146,17 +144,11 @@ export function mergeDeclaredIndexes( existing: readonly ModelOptionsIndexes[] | undefined, incoming: readonly ModelOptionsIndexes[], ): ModelOptionsIndexes[] { - const merged = new Map(); - for (const index of existing ?? []) { - const named = ensureIndexName(index); - merged.set(resolveIndexName(named)!, named); - } + const merged = declaredIndexMap(existing); for (const index of incoming) { const named = ensureIndexName(index); - const name = resolveIndexName(named)!; - if (!merged.has(name)) { - merged.set(name, named); - } + const name = resolveIndexName(named); + if (name && !merged.has(name)) merged.set(name, named); } return [...merged.values()]; } @@ -222,23 +214,18 @@ export function assertUniqueIndexPrivilege(args: { privileged?: boolean; }) { if (!args.unique) return; - if (args.privileged) return; - if (args.schemaOwner === args.callerModule) return; + if (args.privileged || args.schemaOwner === args.callerModule) return; throw new GrpcError(status.PERMISSION_DENIED, 'Not authorized to create unique index'); } export function isIndexAlreadyExistsError(error: unknown): boolean { const err = error as { message?: string; code?: number | string; name?: string }; const message = (err.message ?? '').toLowerCase(); - if ( + return ( message.includes('already exists') || message.includes('already exist') || message.includes('duplicate key name') || - message.includes('index already exists') - ) { - return true; - } - return ( + message.includes('index already exists') || err.code === 85 || err.code === '42P07' || err.name === 'SequelizeUniqueConstraintError' @@ -271,12 +258,21 @@ export async function persistDeclaredSchemaIndexes(args: { export function collectExistingIndexNames( indexes: readonly ModelOptionsIndexes[], ): Set { - const names = new Set(); - for (const index of indexes) { - const name = resolveIndexName(index); - if (name) names.add(name); + return new Set( + indexes.map(resolveIndexName).filter((name): name is string => Boolean(name)), + ); +} + +export function declaredIndexMap( + indexes: readonly ModelOptionsIndexes[] | undefined, +): Map { + const map = new Map(); + for (const index of indexes ?? []) { + const named = ensureIndexName(index); + const name = resolveIndexName(named); + if (name) map.set(name, named); } - return names; + return map; } export function toMutableIndexes( @@ -285,16 +281,13 @@ export function toMutableIndexes( return indexes.map(index => ({ ...index, fields: [...index.fields], - types: Array.isArray(index.types) ? [...index.types] : index.types, options: index.options ? { ...index.options } : index.options, })); } -export function sqlIndexFields( - index: ModelOptionsIndexes, -): Array { +export function sqlIndexFields(index: ModelOptionsIndexes): SqlIndexField[] { const types = normalizeIndexTypes(index.types, index.fields.length); - if (!types || !types.some(isCompatibleIndexType)) { + if (!types || !types.some(isPortableDirection)) { return [...index.fields]; } return index.fields.map((field, i) => ({ @@ -307,22 +300,15 @@ export function inferSqlIndexType( row: { type?: string; definition?: string }, dialect: string, ): PostgresIndexType | undefined { - if (typeof row.type === 'string' && row.type.length > 0) { - const upper = row.type.toUpperCase(); - if (isPostgresIndexType(upper)) return upper; + if (typeof row.type === 'string' && isPostgresIndexType(row.type.toUpperCase())) { + return row.type.toUpperCase() as PostgresIndexType; } if (typeof row.definition === 'string') { const match = /USING\s+(\w+)/i.exec(row.definition); - if (match && isPostgresIndexType(match[1].toUpperCase())) { - return match[1].toUpperCase() as PostgresIndexType; - } + const using = match?.[1]?.toUpperCase(); + if (using && isPostgresIndexType(using)) return using; } - if ( - dialect === 'postgres' || - dialect === 'mysql' || - dialect === 'mariadb' || - dialect === 'sqlite' - ) { + if (['postgres', 'mysql', 'mariadb', 'sqlite'].includes(dialect)) { return PostgresIndexType.BTREE; } return undefined; diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index 0d53e8c82..a21c92933 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -750,12 +750,7 @@ export class SchemaAdmin { async createIndexes(call: ParsedRouterRequest): Promise { const { id, indexes } = call.request.params; - const requestedSchema = await this.database - .getSchemaModel('_DeclaredSchema') - .model.findOne({ _id: id }); - if (isNil(requestedSchema)) { - throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); - } + const requestedSchema = await this.findDeclaredSchemaById(id); return await this.database.createIndexes( requestedSchema.name, indexes, @@ -765,25 +760,14 @@ export class SchemaAdmin { } async getIndexes(call: ParsedRouterRequest): Promise { - const id = call.request.params.id; - const requestedSchema = await this.database - .getSchemaModel('_DeclaredSchema') - .model.findOne({ _id: id }); - if (isNil(requestedSchema)) { - throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); - } + const requestedSchema = await this.findDeclaredSchemaById(call.request.params.id); const indexes = await this.database.getIndexes(requestedSchema.name); return { indexes }; } async deleteIndexes(call: ParsedRouterRequest): Promise { const { id, indexNames } = call.request.params; - const requestedSchema = await this.database - .getSchemaModel('_DeclaredSchema') - .model.findOne({ _id: id }); - if (isNil(requestedSchema)) { - throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); - } + const requestedSchema = await this.findDeclaredSchemaById(id); if (isNil(indexNames) || indexNames.length === 0) { throw new GrpcError( status.INVALID_ARGUMENT, @@ -869,6 +853,16 @@ export class SchemaAdmin { return 'Indexes imported successfully'; } + private async findDeclaredSchemaById(id: string) { + const requestedSchema = await this.database + .getSchemaModel('_DeclaredSchema') + .model.findOne({ _id: id }); + if (isNil(requestedSchema)) { + throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); + } + return requestedSchema; + } + async checkRequestedSchema(id: string) { const requestedSchema = await this.database .getSchemaModel('_DeclaredSchema') diff --git a/modules/database/src/utils/__tests__/validateModelOptions.indexes.test.ts b/modules/database/src/utils/__tests__/validateModelOptions.indexes.test.ts deleted file mode 100644 index a2d6e52fa..000000000 --- a/modules/database/src/utils/__tests__/validateModelOptions.indexes.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { describe, expect, it } from '@jest/globals'; -import { CompatibleIndexType } from '@conduitplatform/grpc-sdk'; -import { validateSchemaInput } from '../utilities.js'; - -describe('validateModelOptions T35 T36', () => { - it('T35 accepts modelOptions.indexes', () => { - expect(() => - validateSchemaInput( - 'User', - { email: 'String' }, - { - timestamps: true, - indexes: [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], - }, - ), - ).not.toThrow(); - }); - - it('T36 keeps conduit.readPreference while accepting indexes', () => { - expect(() => - validateSchemaInput( - 'User', - { email: 'String' }, - { - timestamps: true, - indexes: [{ fields: ['email'] }], - conduit: { readPreference: 'secondaryPreferred' }, - }, - ), - ).not.toThrow(); - }); - - it('still rejects unknown conduit keys and unknown model option keys', () => { - expect(() => - validateSchemaInput('User', { email: 'String' }, { - unknown: true, - } as any), - ).toThrow(/indexes/); - expect(() => - validateSchemaInput('User', { email: 'String' }, { - conduit: { notARealKey: true }, - } as any), - ).toThrow(/readPreference/); - }); -}); From 2f5f77513af77cd5a4454c1129a1b3908f208539 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 15 Sep 2026 09:57:49 +0000 Subject: [PATCH 3/9] fix(database): bind live index names and persist partial index changes Adopt engine names by fields+unique so upgrades and custom modules do not create duplicate indexes, keep Admin extras across re-register, persist only applied create/delete subsets, and publish bound names to replicas. --- .../src/__tests__/indexes/adapters.test.ts | 309 ++++++++++++++++-- .../src/__tests__/indexes/helpers.test.ts | 156 ++++++++- .../database/src/adapters/DatabaseAdapter.ts | 26 ++ .../src/adapters/mongoose-adapter/index.ts | 149 ++++++--- .../src/adapters/sequelize-adapter/index.ts | 147 ++++++--- .../database/src/adapters/utils/indexes.ts | 242 +++++++++++++- 6 files changed, 897 insertions(+), 132 deletions(-) diff --git a/modules/database/src/__tests__/indexes/adapters.test.ts b/modules/database/src/__tests__/indexes/adapters.test.ts index cb0810497..565624e53 100644 --- a/modules/database/src/__tests__/indexes/adapters.test.ts +++ b/modules/database/src/__tests__/indexes/adapters.test.ts @@ -11,22 +11,33 @@ import { SequelizeAdapter } from '../../adapters/sequelize-adapter/index.js'; function makeMongooseAdapter(overrides: Record = {}) { const createIndex = jest.fn().mockResolvedValue('email_1'); const dropIndex = jest.fn().mockResolvedValue(undefined); - const indexes = jest.fn().mockResolvedValue([ - { v: 2, key: { _id: 1 }, name: '_id_' }, - { v: 2, key: { email: 1 }, name: 'cnd_idx_email_asc', unique: false }, - ]); - const findOne = jest.fn().mockResolvedValue({ _id: 'declared-1' }); + const indexes = jest.fn().mockResolvedValue([{ v: 2, key: { _id: 1 }, name: '_id_' }]); + const findOne = jest + .fn() + .mockResolvedValue({ _id: 'declared-1', modelOptions: { indexes: [] } }); const findByIdAndUpdate = jest.fn().mockResolvedValue({}); + const publish = jest.fn(); const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; adapter.mongoose = { model: () => ({ collection: { createIndex, dropIndex, indexes } }), } as MongooseAdapter['mongoose']; + (adapter as unknown as { grpcSdk: { bus: { publish: typeof publish } } }).grpcSdk = { + bus: { publish }, + }; const originalSchema = { name: 'User', ownerModule: 'chat', collectionName: 'cnd_User', - fields: { email: { type: 'String' } }, - compiledFields: { email: { type: 'String' } }, + fields: { + email: { type: 'String' }, + room: { type: 'String' }, + createdAt: { type: 'Date' }, + }, + compiledFields: { + email: { type: 'String' }, + room: { type: 'String' }, + createdAt: { type: 'Date' }, + }, modelOptions: { indexes: [] as unknown[] }, ...((overrides.originalSchema as object) ?? {}), }; @@ -34,7 +45,20 @@ function makeMongooseAdapter(overrides: Record = {}) { User: { originalSchema }, _DeclaredSchema: { findOne, findByIdAndUpdate }, } as MongooseAdapter['models']; - return { adapter, createIndex, dropIndex, indexes, findByIdAndUpdate, originalSchema }; + findOne.mockImplementation(async () => ({ + _id: 'declared-1', + modelOptions: originalSchema.modelOptions, + })); + return { + adapter, + createIndex, + dropIndex, + indexes, + findOne, + findByIdAndUpdate, + publish, + originalSchema, + }; } class TestSequelizeAdapter extends SequelizeAdapter { @@ -46,41 +70,54 @@ class TestSequelizeAdapter extends SequelizeAdapter { function makeSequelizeAdapter(dialect = 'postgres') { const addIndex = jest.fn().mockResolvedValue(undefined); const removeIndex = jest.fn().mockResolvedValue(undefined); - const showIndex = jest.fn().mockResolvedValue([ - { - name: 'cnd_idx_email_asc', - unique: false, - fields: [{ attribute: 'email', order: 'ASC' }], - definition: 'CREATE INDEX cnd_idx_email_asc ON cnd_User USING btree (email)', - }, - ]); - const findOne = jest.fn().mockResolvedValue({ _id: 'declared-1' }); + const showIndex = jest.fn().mockResolvedValue([]); + const findOne = jest + .fn() + .mockResolvedValue({ _id: 'declared-1', modelOptions: { indexes: [] } }); const findByIdAndUpdate = jest.fn().mockResolvedValue({}); + const publish = jest.fn(); const sync = jest.fn().mockResolvedValue(undefined); const adapter = Object.create(TestSequelizeAdapter.prototype) as SequelizeAdapter; adapter.sequelize = { getDialect: () => dialect, getQueryInterface: () => ({ addIndex, removeIndex, showIndex }), } as SequelizeAdapter['sequelize']; + (adapter as unknown as { grpcSdk: { bus: { publish: typeof publish } } }).grpcSdk = { + bus: { publish }, + }; const originalSchema = { name: 'User', ownerModule: 'database', collectionName: 'custom_users', - fields: { email: { type: 'String' } }, - compiledFields: { email: { type: 'String' } }, + fields: { + email: { type: 'String' }, + room: { type: 'String' }, + createdAt: { type: 'Date' }, + }, + compiledFields: { + email: { type: 'String' }, + room: { type: 'String' }, + createdAt: { type: 'Date' }, + }, modelOptions: { indexes: [] as unknown[] }, }; adapter.models = { User: { originalSchema, sync }, _DeclaredSchema: { findOne, findByIdAndUpdate }, } as SequelizeAdapter['models']; + findOne.mockImplementation(async () => ({ + _id: 'declared-1', + modelOptions: originalSchema.modelOptions, + })); return { adapter, addIndex, removeIndex, showIndex, sync, + findOne, findByIdAndUpdate, + publish, originalSchema, }; } @@ -137,6 +174,10 @@ describe('mongoose adapter indexes', () => { it('reads live engine indexes', async () => { const { adapter, indexes } = makeMongooseAdapter(); + indexes.mockResolvedValue([ + { v: 2, key: { _id: 1 }, name: '_id_' }, + { v: 2, key: { email: 1 }, name: 'cnd_idx_email_asc', unique: false }, + ]); const result = await adapter.getIndexes('User'); expect(indexes).toHaveBeenCalled(); expect(result.map(index => index.name)).toEqual(['_id_', 'cnd_idx_email_asc']); @@ -162,6 +203,143 @@ describe('mongoose adapter indexes', () => { ), ).resolves.toBe('Indexes created!'); }); + + it('adopts a live compound name and skips createIndex', async () => { + const { adapter, createIndex, indexes, findByIdAndUpdate } = makeMongooseAdapter(); + indexes.mockResolvedValue([ + { v: 2, key: { _id: 1 }, name: '_id_' }, + { v: 2, key: { room: 1, createdAt: 1 }, name: 'room_1_createdAt_1', unique: false }, + ]); + await adapter.createIndexes( + 'User', + [ + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + 'chat', + ); + expect(createIndex).not.toHaveBeenCalled(); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { name?: string }[] }; + }; + expect(update.modelOptions.indexes[0].name).toBe('room_1_createdAt_1'); + }); + + it('throws on unique-data collisions and does not persist that index', async () => { + const { adapter, createIndex, findByIdAndUpdate } = makeMongooseAdapter(); + createIndex.mockRejectedValue({ code: 11000, message: 'E11000 duplicate key' }); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], options: { unique: true } }], + 'chat', + ), + ).rejects.toMatchObject({ code: status.INTERNAL }); + expect(findByIdAndUpdate).not.toHaveBeenCalled(); + }); + + it('persists the prefix and publishes after a later unique-data failure', async () => { + const { adapter, createIndex, findByIdAndUpdate, publish } = makeMongooseAdapter(); + createIndex + .mockResolvedValueOnce('cnd_idx_email_asc') + .mockRejectedValueOnce({ code: 11000, message: 'E11000 duplicate key' }); + await expect( + adapter.createIndexes( + 'User', + [ + { fields: ['email'], types: [CompatibleIndexType.Ascending] }, + { fields: ['room'], options: { unique: true } }, + ], + 'chat', + ), + ).rejects.toMatchObject({ code: status.INTERNAL }); + expect(findByIdAndUpdate).toHaveBeenCalledTimes(1); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { name?: string }[] }; + }; + expect(update.modelOptions.indexes.map(index => index.name)).toEqual([ + 'cnd_idx_email_asc', + ]); + expect(publish).toHaveBeenCalledWith('database:create:schema', expect.any(String)); + }); + + it('overlays declared Compatible types onto a live index by fields', async () => { + const { adapter, indexes, originalSchema } = makeMongooseAdapter(); + originalSchema.modelOptions.indexes = [ + { + fields: ['room', 'createdAt'], + name: 'cnd_idx_room_createdAt_asc_asc', + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ]; + indexes.mockResolvedValue([ + { v: 2, key: { _id: 1 }, name: '_id_' }, + { v: 2, key: { room: 1, createdAt: 1 }, name: 'room_1_createdAt_1', unique: false }, + ]); + const result = await adapter.getIndexes('User'); + expect(result[1].name).toBe('room_1_createdAt_1'); + expect(result[1].types).toEqual([ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ]); + }); + + it('publishes the schema after a successful persist', async () => { + const { adapter, publish } = makeMongooseAdapter(); + await adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'chat', + ); + expect(publish).toHaveBeenCalledWith('database:create:schema', expect.any(String)); + }); + + it('rebinds to the live name on Mongo 86 instead of persisting a generated name', async () => { + const { adapter, createIndex, indexes, findByIdAndUpdate } = makeMongooseAdapter(); + createIndex.mockRejectedValue({ code: 86, message: 'IndexKeySpecsConflict' }); + indexes + .mockResolvedValueOnce([{ v: 2, key: { _id: 1 }, name: '_id_' }]) + .mockResolvedValueOnce([ + { v: 2, key: { _id: 1 }, name: '_id_' }, + { v: 2, key: { email: 1 }, name: 'email_1', unique: false }, + ]); + await adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'chat', + ); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { name?: string }[] }; + }; + expect(update.modelOptions.indexes[0].name).toBe('email_1'); + }); + + it('persists only names that were actually dropped', async () => { + const { adapter, dropIndex, findByIdAndUpdate } = makeMongooseAdapter({ + originalSchema: { + modelOptions: { + indexes: [ + { fields: ['email'], name: 'idx_a' }, + { fields: ['room'], name: 'idx_b' }, + ], + }, + }, + }); + dropIndex + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error('missing')); + await expect(adapter.deleteIndexes('User', ['idx_a', 'idx_b'])).rejects.toMatchObject( + { + code: status.INTERNAL, + }, + ); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { name?: string }[] }; + }; + expect(update.modelOptions.indexes.map(index => index.name)).toEqual(['idx_b']); + }); }); describe('sequelize adapter indexes', () => { @@ -205,6 +383,14 @@ describe('sequelize adapter indexes', () => { types: [CompatibleIndexType.Ascending], }, ]; + showIndex.mockResolvedValue([ + { + name: 'cnd_idx_email_asc', + unique: false, + fields: [{ attribute: 'email', order: 'ASC' }], + definition: 'CREATE INDEX cnd_idx_email_asc ON custom_users USING btree (email)', + }, + ]); const result = await adapter.getIndexes('User'); expect(showIndex).toHaveBeenCalledWith('custom_users'); expect(result[0].types).toEqual([CompatibleIndexType.Ascending]); @@ -239,4 +425,89 @@ describe('sequelize adapter indexes', () => { ), ).rejects.toMatchObject({ message: expect.stringMatching(/sqlite/i) }); }); + + it('adopts a live compound name and skips addIndex', async () => { + const { adapter, addIndex, showIndex, findByIdAndUpdate } = makeSequelizeAdapter(); + showIndex.mockResolvedValue([ + { + name: 'room_createdAt', + unique: false, + fields: [{ attribute: 'room' }, { attribute: 'createdAt' }], + }, + ]); + await adapter.createIndexes( + 'User', + [ + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + 'database', + ); + expect(addIndex).not.toHaveBeenCalled(); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { name?: string }[] }; + }; + expect(update.modelOptions.indexes[0].name).toBe('room_createdAt'); + }); + + it('skips and persists on 42P07 name conflicts', async () => { + const { adapter, addIndex, findByIdAndUpdate, publish } = makeSequelizeAdapter(); + addIndex.mockRejectedValue({ + original: { code: '42P07', message: 'relation "cnd_idx_email_asc" already exists' }, + }); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'database', + ), + ).resolves.toBe('Indexes created!'); + expect(findByIdAndUpdate).toHaveBeenCalled(); + expect(publish).toHaveBeenCalled(); + }); + + it('throws on 23505 unique collisions and does not persist that index', async () => { + const { adapter, addIndex, findByIdAndUpdate } = makeSequelizeAdapter(); + addIndex.mockRejectedValue({ + name: 'SequelizeUniqueConstraintError', + original: { code: '23505' }, + }); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], options: { unique: true } }], + 'database', + { privileged: true }, + ), + ).rejects.toMatchObject({ code: status.INTERNAL }); + expect(findByIdAndUpdate).not.toHaveBeenCalled(); + }); + + it('overlays declared Compatible types onto a live SQL index by fields', async () => { + const { adapter, showIndex, originalSchema } = makeSequelizeAdapter(); + originalSchema.modelOptions.indexes = [ + { + fields: ['room', 'createdAt'], + name: 'cnd_idx_room_createdAt_asc_asc', + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ]; + showIndex.mockResolvedValue([ + { + name: 'room_createdAt', + unique: false, + fields: [{ attribute: 'room' }, { attribute: 'createdAt' }], + definition: + 'CREATE INDEX room_createdAt ON custom_users USING btree (room, createdAt)', + }, + ]); + const result = await adapter.getIndexes('User'); + expect(result[0].name).toBe('room_createdAt'); + expect(result[0].types).toEqual([ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ]); + }); }); diff --git a/modules/database/src/__tests__/indexes/helpers.test.ts b/modules/database/src/__tests__/indexes/helpers.test.ts index 74be105b6..48ec49ba8 100644 --- a/modules/database/src/__tests__/indexes/helpers.test.ts +++ b/modules/database/src/__tests__/indexes/helpers.test.ts @@ -7,16 +7,21 @@ import { } from '@conduitplatform/grpc-sdk'; import { assertUniqueIndexPrivilege, + bindDeclaredIndexesToLive, collectExistingIndexNames, ensureIndexName, generateIndexName, + indexIdentity, isCompatibleIndexType, isIndexAlreadyExistsError, isMongoIndexType, + keepDeclaredIndexExtras, mapCompatibleToMongo, mapCompatibleToSqlOrder, mergeDeclaredIndexes, mongoAllowsIndexType, + overlayDeclaredOnLive, + persistDeclaredSchemaIndexes, removeDeclaredIndexes, removeIndexFromSchemaFields, resolveIndexName, @@ -170,13 +175,160 @@ describe('index helpers', () => { ).toThrow(/Not authorized to create unique index/); }); - it('collects existing names and detects already-exists errors', () => { + it('collects existing names and detects name/relation already-exists errors only', () => { expect( collectExistingIndexNames([{ fields: ['a'], options: { name: 'idx_a' } }]).has( 'idx_a', ), ).toBe(true); - expect(isIndexAlreadyExistsError(new Error('index already exists'))).toBe(true); + expect( + isIndexAlreadyExistsError({ + code: '42P07', + message: 'relation "x" already exists', + }), + ).toBe(true); + expect( + isIndexAlreadyExistsError({ code: '1061', message: "Duplicate key name 'x'" }), + ).toBe(true); + expect( + isIndexAlreadyExistsError({ + original: { code: '42P07', message: 'relation "idx" already exists' }, + }), + ).toBe(true); + expect(isIndexAlreadyExistsError(new Error('index foo already exists'))).toBe(true); + expect(isIndexAlreadyExistsError(new Error('already exists'))).toBe(false); + expect( + isIndexAlreadyExistsError({ code: 11000, message: 'E11000 duplicate key' }), + ).toBe(false); + expect( + isIndexAlreadyExistsError({ + name: 'SequelizeUniqueConstraintError', + original: { code: '23505' }, + }), + ).toBe(false); + expect( + isIndexAlreadyExistsError({ + code: 85, + message: 'Index with name: x already exists', + }), + ).toBe(false); + expect( + isIndexAlreadyExistsError({ + code: 86, + message: 'Index already exists with different options', + }), + ).toBe(false); + }); + + it('binds unnamed declared indexes to live names by fields+unique', () => { + const bound = bindDeclaredIndexesToLive( + [ + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + [ + { + name: '_id_', + fields: ['_id'], + options: { name: '_id_' }, + }, + { + name: 'room_1_createdAt_1', + fields: ['room', 'createdAt'], + options: { name: 'room_1_createdAt_1', unique: false }, + }, + ], + ); + expect(resolveIndexName(bound[0])).toBe('room_1_createdAt_1'); + expect(indexIdentity(bound[0])).toEqual({ + fields: ['room', 'createdAt'], + unique: false, + }); + }); + + it('treats unique vs non-unique as different identities', () => { + const bound = bindDeclaredIndexesToLive( + [{ fields: ['email'], options: { unique: true } }], + [ + { + name: 'email_1', + fields: ['email'], + options: { name: 'email_1', unique: false }, + }, + ], + ); + expect(resolveIndexName(bound[0])).not.toBe('email_1'); + expect(resolveIndexName(bound[0])).toMatch(/uidx/); + }); + + it('keeps Admin extras and drops stale generated names for the same identity', () => { + const unioned = keepDeclaredIndexExtras( + [ + { + fields: ['room', 'createdAt'], + name: 'room_1_createdAt_1', + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + [ + { + fields: ['room', 'createdAt'], + name: 'cnd_idx_room_createdAt_asc_asc', + }, + { fields: ['email'], name: 'admin_email_idx' }, + ], + ); + expect(unioned.map(index => index.name)).toEqual([ + 'room_1_createdAt_1', + 'admin_email_idx', + ]); + }); + + it('overlays declared Compatible types onto live indexes by identity when names differ', () => { + const overlaid = overlayDeclaredOnLive( + { + name: 'room_1_createdAt_1', + fields: ['room', 'createdAt'], + types: [MongoIndexType.Ascending, MongoIndexType.Ascending], + options: { name: 'room_1_createdAt_1' }, + }, + [ + { + fields: ['room', 'createdAt'], + name: 'cnd_idx_room_createdAt_asc_asc', + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + ); + expect(overlaid.types).toEqual([ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ]); + expect(overlaid.name).toBe('room_1_createdAt_1'); + }); + + it('persists applied indexes against a re-read declared schema list', async () => { + const findOne = async () => ({ + _id: 'declared-1', + modelOptions: { indexes: [{ fields: ['keep'], name: 'keep_me' }] }, + }); + const findByIdAndUpdate = async () => ({}); + const originalSchema = { + modelOptions: { indexes: [] as { fields: string[]; name?: string }[] }, + }; + const persisted = await persistDeclaredSchemaIndexes({ + declaredSchemaModel: { findOne, findByIdAndUpdate }, + schemaName: 'User', + originalSchema, + applied: [{ fields: ['email'], name: 'cnd_idx_email_asc' }], + }); + expect(persisted).toBe(true); + expect(originalSchema.modelOptions.indexes.map(index => index.name)).toEqual([ + 'keep_me', + 'cnd_idx_email_asc', + ]); }); it('allows dialect-native types and rejects foreign leftovers', () => { diff --git a/modules/database/src/adapters/DatabaseAdapter.ts b/modules/database/src/adapters/DatabaseAdapter.ts index da415e949..ee3e13677 100644 --- a/modules/database/src/adapters/DatabaseAdapter.ts +++ b/modules/database/src/adapters/DatabaseAdapter.ts @@ -24,6 +24,10 @@ import { Schema, } from '../interfaces/index.js'; import { stitchSchema, validateExtensionFields } from './utils/extensions.js'; +import { + keepDeclaredIndexExtras, + persistDeclaredSchemaIndexes, +} from './utils/indexes.js'; import { status } from '@grpc/grpc-js'; import { isEqual, isNil } from 'lodash-es'; import ObjectHash from 'object-hash'; @@ -601,6 +605,24 @@ export abstract class DatabaseAdapter { instanceSync: boolean, ): Promise; + protected async persistIndexesAndPublish(args: { + schemaName: string; + originalSchema: ConduitDatabaseSchema; + applied?: ModelOptionsIndexes[]; + droppedNames?: string[]; + }): Promise { + if (!this.models['_DeclaredSchema']) return false; + const persisted = await persistDeclaredSchemaIndexes({ + declaredSchemaModel: this.models['_DeclaredSchema'], + schemaName: args.schemaName, + originalSchema: args.originalSchema, + applied: args.applied, + droppedNames: args.droppedNames, + }); + if (persisted) this.publishSchema(args.originalSchema); + return persisted; + } + protected async saveSchemaToDatabase(schema: ConduitSchema) { if (schema.name === '_DeclaredSchema') return; const model = await this.models['_DeclaredSchema'].findOne( @@ -608,6 +630,10 @@ export abstract class DatabaseAdapter { { readPreference: 'primary' }, ); if (model) { + schema.modelOptions.indexes = keepDeclaredIndexExtras( + schema.modelOptions.indexes ?? [], + model.modelOptions?.indexes, + ); await this.models['_DeclaredSchema'].findByIdAndUpdate(model._id, { name: schema.name, fields: schema.fields, diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 98c354eb7..39079342a 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -32,15 +32,16 @@ import { } from '../utils/index.js'; import { assertUniqueIndexPrivilege, - declaredIndexMap, + bindDeclaredIndexesToLive, ensureIndexName, + findLiveIndex, isIndexAlreadyExistsError, + isIndexKeySpecsConflictError, + liveIndexFromMongo, mapCompatibleToMongo, - mergeDeclaredIndexes, mongoAllowsIndexType, normalizeIndexTypes, - persistDeclaredSchemaIndexes, - removeDeclaredIndexes, + overlayDeclaredOnLive, removeIndexFromSchemaFields, toMutableIndexes, validateIndexFields, @@ -659,14 +660,20 @@ export class MongooseAdapter extends DatabaseAdapter { ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - const prepared = this.checkIndexes( - schemaName, - indexes, - callerModule, - options?.privileged, + const live = await this.listLiveIndexes(schemaName); + const prepared = bindDeclaredIndexesToLive( + this.checkIndexes(schemaName, indexes, callerModule, options?.privileged), + live, ); const collection = this.mongoose.model(schemaName).collection; + const applied: ModelOptionsIndexes[] = []; + let failure: unknown; for (const index of prepared) { + const existing = findLiveIndex(live, index); + if (existing) { + applied.push(index); + continue; + } const spec: Record = {}; const types = normalizeIndexTypes(index.types, index.fields.length); for (let i = 0; i < index.fields.length; i++) { @@ -676,22 +683,48 @@ export class MongooseAdapter extends DatabaseAdapter { } try { await collection.createIndex(spec, index.options); + applied.push(index); + live.push(index); } catch (e) { - if (isIndexAlreadyExistsError(e)) continue; - throw new GrpcError(status.INTERNAL, (e as Error).message); + if (isIndexAlreadyExistsError(e)) { + applied.push(index); + continue; + } + if (isIndexKeySpecsConflictError(e)) { + const relisted = await this.listLiveIndexes(schemaName); + const match = findLiveIndex(relisted, index); + if (match) { + applied.push(bindDeclaredIndexesToLive([index], relisted)[0]); + live.splice(0, live.length, ...relisted); + continue; + } + } + failure = e; + break; } } - const original = this.models[schemaName].originalSchema; - const merged = mergeDeclaredIndexes(original.modelOptions.indexes, prepared); - await persistDeclaredSchemaIndexes({ - declaredSchemaModel: this.models['_DeclaredSchema'], - schemaName, - originalSchema: original, - indexes: merged, - }); + if (!failure || applied.length > 0) { + await this.persistIndexesAndPublish({ + schemaName, + originalSchema: this.models[schemaName].originalSchema, + applied, + }); + } + if (failure) { + throw new GrpcError(status.INTERNAL, (failure as Error).message); + } return 'Indexes created!'; } + private async listLiveIndexes(schemaName: string): Promise { + try { + const result = await this.mongoose.model(schemaName).collection.indexes(); + return result.map(liveIndexFromMongo); + } catch { + return []; + } + } + private async createMongooseFieldIndexes(schemaName: string): Promise { const model = this.models[schemaName]; if (!model) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); @@ -700,6 +733,7 @@ export class MongooseAdapter extends DatabaseAdapter { const declaredIndexes = model.model.schema.indexes(); if (!declaredIndexes.length) return; + const live = await this.listLiveIndexes(schemaName); const collection = this.mongoose.model(schemaName).collection; for (const [keys, rawOptions] of declaredIndexes) { const indexKeys = keys as IndexSpecification; @@ -708,9 +742,20 @@ export class MongooseAdapter extends DatabaseAdapter { const options = this.sanitizeMongooseIndexOptions( rawOptions as Record, ); - await collection.createIndex(indexKeys, options).catch((e: Error) => { - throw new GrpcError(status.INTERNAL, e.message); - }); + const declared = { + fields: Object.keys((indexKeys as Record) ?? {}), + options: { + unique: options?.unique === true, + name: typeof options?.name === 'string' ? options.name : undefined, + }, + }; + if (findLiveIndex(live, declared)) continue; + try { + await collection.createIndex(indexKeys, options); + } catch (e) { + if (isIndexAlreadyExistsError(e) || isIndexKeySpecsConflictError(e)) continue; + throw new GrpcError(status.INTERNAL, (e as Error).message); + } } } @@ -734,9 +779,7 @@ export class MongooseAdapter extends DatabaseAdapter { throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); const collection = this.mongoose.model(schemaName).collection; const result = await collection.indexes(); - const declaredByName = declaredIndexMap( - this.models[schemaName].originalSchema.modelOptions.indexes, - ); + const declared = this.models[schemaName].originalSchema.modelOptions.indexes; return result.map(index => { const options: Record = {}; for (const [key, value] of Object.entries(index)) { @@ -750,19 +793,15 @@ export class MongooseAdapter extends DatabaseAdapter { types.push(type as MongoIndexType); } const name = typeof options.name === 'string' ? options.name : index.name; - const declaredIndex = name ? declaredByName.get(name) : undefined; - const live: ModelOptionsIndexes = { - name, - fields, - types, - options: { ...options, name }, - }; - if (!declaredIndex) return live; - return { - ...live, - types: declaredIndex.types ?? live.types, - options: { ...declaredIndex.options, ...live.options, name }, - }; + return overlayDeclaredOnLive( + { + name, + fields, + types, + options: { ...options, name }, + }, + declared, + ); }); } @@ -770,24 +809,31 @@ export class MongooseAdapter extends DatabaseAdapter { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); const collection = this.mongoose.model(schemaName).collection; + const dropped: string[] = []; + let failure: unknown; for (const name of indexNames) { try { await collection.dropIndex(name); - } catch { - throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); + dropped.push(name); + } catch (e) { + failure = e; + break; } } const original = this.models[schemaName].originalSchema; - for (const name of indexNames) { + for (const name of dropped) { removeIndexFromSchemaFields(original, name); } - const remaining = removeDeclaredIndexes(original.modelOptions.indexes, indexNames); - await persistDeclaredSchemaIndexes({ - declaredSchemaModel: this.models['_DeclaredSchema'], - schemaName, - originalSchema: original, - indexes: remaining, - }); + if (!failure || dropped.length > 0) { + await this.persistIndexesAndPublish({ + schemaName, + originalSchema: original, + droppedNames: dropped, + }); + } + if (failure) { + throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); + } return 'Indexes deleted'; } @@ -1047,6 +1093,13 @@ export class MongooseAdapter extends DatabaseAdapter { schema, this, ); + if (!isInstanceSync && schema.modelOptions.indexes?.length) { + const live = await this.listLiveIndexes(schema.name); + schema.modelOptions.indexes = bindDeclaredIndexesToLive( + schema.modelOptions.indexes, + live, + ); + } if (saveToDb) { await this.compareAndStoreMigratedSchema(schema); await this.saveSchemaToDatabase(schema); diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index bfc41925b..238038a93 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -52,15 +52,16 @@ import { } from '../utils/index.js'; import { assertUniqueIndexPrivilege, - declaredIndexMap, + bindDeclaredIndexesToLive, ensureIndexName, + findLiveIndex, inferSqlIndexType, isIndexAlreadyExistsError, + isIndexKeySpecsConflictError, isPostgresIndexType, - mergeDeclaredIndexes, + liveIndexFromSql, + overlayDeclaredOnLive, normalizeIndexTypes, - persistDeclaredSchemaIndexes, - removeDeclaredIndexes, removeIndexFromSchemaFields, sqlDialectAllowsIndexType, sqlIndexFields, @@ -283,10 +284,26 @@ export abstract class SequelizeAdapter extends DatabaseAdapter this.sequelize.models, ); const dialect = this.sequelize.getDialect(); + const live = isInstanceSync + ? [] + : await this.listLiveIndexesForCollection(this.getCollectionName(schema)); + if (!isInstanceSync && schema.modelOptions.indexes?.length) { + schema.modelOptions.indexes = bindDeclaredIndexesToLive( + schema.modelOptions.indexes, + live, + ); + compiledSchema.modelOptions.indexes = schema.modelOptions.indexes; + } const [newSchema, objectPaths, extractedRelations] = dialect === 'postgres' ? pgSchemaConverter(compiledSchema) : sqlSchemaConverter(compiledSchema, dialect as 'mysql' | 'mariadb' | 'sqlite'); + if (!isInstanceSync && newSchema.modelOptions.indexes?.length) { + newSchema.modelOptions.indexes = bindDeclaredIndexesToLive( + newSchema.modelOptions.indexes, + live, + ); + } this.registeredSchemas.set( schema.name, Object.freeze(JSON.parse(JSON.stringify(schema))), @@ -401,36 +418,72 @@ export abstract class SequelizeAdapter extends DatabaseAdapter ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - const prepared = this.checkAndConvertIndexes( - schemaName, - indexes, - callerModule, - options?.privileged, - ); const collectionName = this.models[schemaName].originalSchema.collectionName; + const live = await this.listLiveIndexesForCollection(collectionName); + const prepared = bindDeclaredIndexesToLive( + this.checkAndConvertIndexes(schemaName, indexes, callerModule, options?.privileged), + live, + ); const queryInterface = this.sequelize.getQueryInterface(); + const applied: ModelOptionsIndexes[] = []; + let failure: unknown; for (const index of prepared) { + const existing = findLiveIndex(live, index); + if (existing) { + applied.push(index); + continue; + } try { await queryInterface.addIndex(collectionName, { fields: sqlIndexFields(index), ...index.options, }); + applied.push(index); + live.push(index); } catch (e) { - if (isIndexAlreadyExistsError(e)) continue; - throw new GrpcError(status.INTERNAL, 'Unsuccessful index creation'); + if (isIndexAlreadyExistsError(e)) { + applied.push(index); + continue; + } + if (isIndexKeySpecsConflictError(e)) { + const relisted = await this.listLiveIndexesForCollection(collectionName); + const match = findLiveIndex(relisted, index); + if (match) { + applied.push(bindDeclaredIndexesToLive([index], relisted)[0]); + live.splice(0, live.length, ...relisted); + continue; + } + } + failure = e; + break; } } - const original = this.models[schemaName].originalSchema; - const merged = mergeDeclaredIndexes(original.modelOptions.indexes, prepared); - await persistDeclaredSchemaIndexes({ - declaredSchemaModel: this.models['_DeclaredSchema'], - schemaName, - originalSchema: original, - indexes: merged, - }); + if (!failure || applied.length > 0) { + await this.persistIndexesAndPublish({ + schemaName, + originalSchema: this.models[schemaName].originalSchema, + applied, + }); + } + if (failure) { + throw new GrpcError(status.INTERNAL, 'Unsuccessful index creation'); + } return 'Indexes created!'; } + private async listLiveIndexesForCollection( + collectionName: string, + ): Promise { + try { + const result = (await this.sequelize + .getQueryInterface() + .showIndex(collectionName)) as UntypedArray; + return result.map(liveIndexFromSql); + } catch { + return []; + } + } + async getIndexes(schemaName: string): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); @@ -438,27 +491,22 @@ export abstract class SequelizeAdapter extends DatabaseAdapter const queryInterface = this.sequelize.getQueryInterface(); const result = (await queryInterface.showIndex(collectionName)) as UntypedArray; const dialect = this.sequelize.getDialect(); - const declaredByName = declaredIndexMap( - this.models[schemaName].originalSchema.modelOptions.indexes, - ); + const declared = this.models[schemaName].originalSchema.modelOptions.indexes; return result.map(row => { const fields = (row.fields ?? []).map((field: unknown) => typeof field === 'string' ? field : (field as { attribute?: string }).attribute, ); const name = row.name as string; - const declaredIndex = declaredByName.get(name); - const live: ModelOptionsIndexes = { - name, - fields, - types: inferSqlIndexType(row, dialect), - options: { name, unique: !!row.unique }, - }; - if (!declaredIndex) return live; - return { - ...live, - types: declaredIndex.types ?? live.types, - options: { ...declaredIndex.options, ...live.options, name }, - }; + return overlayDeclaredOnLive( + { + name, + fields, + types: inferSqlIndexType(row, dialect), + options: { name, unique: !!row.unique }, + ...(row.primary ? { primary: true } : {}), + }, + declared, + ); }); } @@ -467,24 +515,31 @@ export abstract class SequelizeAdapter extends DatabaseAdapter throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); const collectionName = this.models[schemaName].originalSchema.collectionName; const queryInterface = this.sequelize.getQueryInterface(); + const dropped: string[] = []; + let failure: unknown; for (const name of indexNames) { try { await queryInterface.removeIndex(collectionName, name); - } catch { - throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); + dropped.push(name); + } catch (e) { + failure = e; + break; } } const original = this.models[schemaName].originalSchema; - for (const name of indexNames) { + for (const name of dropped) { removeIndexFromSchemaFields(original, name); } - const remaining = removeDeclaredIndexes(original.modelOptions.indexes, indexNames); - await persistDeclaredSchemaIndexes({ - declaredSchemaModel: this.models['_DeclaredSchema'], - schemaName, - originalSchema: original, - indexes: remaining, - }); + if (!failure || dropped.length > 0) { + await this.persistIndexesAndPublish({ + schemaName, + originalSchema: original, + droppedNames: dropped, + }); + } + if (failure) { + throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); + } return 'Indexes deleted'; } diff --git a/modules/database/src/adapters/utils/indexes.ts b/modules/database/src/adapters/utils/indexes.ts index 9338b6ecb..3c1964e1f 100644 --- a/modules/database/src/adapters/utils/indexes.ts +++ b/modules/database/src/adapters/utils/indexes.ts @@ -48,7 +48,158 @@ export function resolveIndexName(index: ModelOptionsIndexes): string | undefined } export function isUniqueIndex(index: ModelOptionsIndexes): boolean { - return index.options?.unique === true; + return index.options?.unique === true || index.unique === true; +} + +export type IndexIdentity = { fields: string[]; unique: boolean }; + +export function indexFieldNames( + index: Pick | { fields?: readonly unknown[] }, +): string[] { + return (index.fields ?? []) + .map(field => { + if (typeof field === 'string') return field; + if (field && typeof field === 'object') { + const obj = field as { name?: string; attribute?: string }; + if (typeof obj.name === 'string' && obj.name.length > 0) return obj.name; + if (typeof obj.attribute === 'string' && obj.attribute.length > 0) { + return obj.attribute; + } + } + return ''; + }) + .filter(name => name.length > 0); +} + +export function indexIdentity(index: ModelOptionsIndexes): IndexIdentity { + return { + fields: indexFieldNames(index), + unique: isUniqueIndex(index), + }; +} + +export function indexIdentitiesEqual(a: IndexIdentity, b: IndexIdentity): boolean { + return ( + a.unique === b.unique && + a.fields.length === b.fields.length && + a.fields.every((field, i) => field === b.fields[i]) + ); +} + +export function indexIdentityKey(identity: IndexIdentity): string { + return `${identity.unique ? 'u' : 'n'}:${identity.fields.join('\0')}`; +} + +export function isSkippedLiveIndex(index: ModelOptionsIndexes): boolean { + if (index.primary === true) return true; + const name = resolveIndexName(index); + return name === '_id_' || name === 'PRIMARY'; +} + +export function findLiveIndex( + live: readonly ModelOptionsIndexes[], + declared: ModelOptionsIndexes, +): ModelOptionsIndexes | undefined { + const wanted = indexIdentity(declared); + return live.find( + row => !isSkippedLiveIndex(row) && indexIdentitiesEqual(indexIdentity(row), wanted), + ); +} + +export function bindDeclaredIndexesToLive( + declared: readonly T[], + live: readonly ModelOptionsIndexes[], +): T[] { + return declared.map(index => { + const match = findLiveIndex(live, index); + if (match) { + const name = resolveIndexName(match); + if (name) { + return { + ...index, + name, + options: { ...index.options, name }, + }; + } + } + const fields = indexFieldNames(index); + const stringFields = Array.isArray(index.fields) + ? index.fields.every(field => typeof field === 'string') + : false; + if (stringFields && fields.length === index.fields.length) { + return ensureIndexName(index) as T; + } + return index; + }); +} + +export function keepDeclaredIndexExtras( + incomingBound: readonly ModelOptionsIndexes[], + existingDb: readonly ModelOptionsIndexes[] | undefined, +): ModelOptionsIndexes[] { + const incoming = incomingBound.map(index => { + const name = resolveIndexName(index); + return name + ? { ...index, name, options: { ...index.options, name } } + : ensureIndexName(index); + }); + const incomingNames = new Set( + incoming.map(resolveIndexName).filter((name): name is string => Boolean(name)), + ); + const incomingIdentities = new Set( + incoming.map(index => indexIdentityKey(indexIdentity(index))), + ); + const extras: ModelOptionsIndexes[] = []; + for (const index of existingDb ?? []) { + const name = resolveIndexName(index); + if (name && incomingNames.has(name)) continue; + if (incomingIdentities.has(indexIdentityKey(indexIdentity(index)))) continue; + extras.push(ensureIndexName(index)); + } + return [...incoming, ...extras]; +} + +export function overlayDeclaredOnLive( + live: ModelOptionsIndexes, + declared: readonly ModelOptionsIndexes[] | undefined, +): ModelOptionsIndexes { + if (!declared?.length) return live; + const name = resolveIndexName(live); + const byName = name ? declaredIndexMap(declared).get(name) : undefined; + const match = byName ?? findLiveIndex(declared, live); + if (!match) return live; + const liveName = name ?? resolveIndexName(match); + return { + ...live, + types: match.types ?? live.types, + options: { ...match.options, ...live.options, name: liveName }, + }; +} + +export function liveIndexFromMongo(index: { + key?: Record; + name?: string; + unique?: boolean; +}): ModelOptionsIndexes { + return { + name: index.name, + fields: Object.keys(index.key ?? {}), + options: { name: index.name, unique: !!index.unique }, + }; +} + +export function liveIndexFromSql(row: { + name?: string; + unique?: boolean; + primary?: boolean; + fields?: Array; +}): ModelOptionsIndexes { + return { + name: row.name, + fields: indexFieldNames({ fields: row.fields ?? [] }), + options: { name: row.name, unique: !!row.unique }, + ...(row.primary ? { primary: true } : {}), + }; } export function normalizeIndexTypes( @@ -218,23 +369,72 @@ export function assertUniqueIndexPrivilege(args: { throw new GrpcError(status.PERMISSION_DENIED, 'Not authorized to create unique index'); } +type ErrorPart = { code?: number | string; message?: string; name?: string }; + +function walkError(error: unknown): ErrorPart[] { + const parts: ErrorPart[] = []; + const seen = new Set(); + let current: unknown = error; + while (current && typeof current === 'object' && !seen.has(current)) { + seen.add(current); + const err = current as ErrorPart & { + original?: unknown; + parent?: unknown; + cause?: unknown; + }; + parts.push({ code: err.code, message: err.message, name: err.name }); + current = err.original ?? err.parent ?? err.cause; + } + return parts; +} + +const UNIQUE_OR_OPTIONS_CONFLICT_CODES = new Set([ + 85, + '85', + 86, + '86', + 11000, + '11000', + 23505, + '23505', + 1062, + '1062', +]); + +export function isIndexKeySpecsConflictError(error: unknown): boolean { + return walkError(error).some(part => part.code === 86 || part.code === '86'); +} + export function isIndexAlreadyExistsError(error: unknown): boolean { - const err = error as { message?: string; code?: number | string; name?: string }; - const message = (err.message ?? '').toLowerCase(); - return ( - message.includes('already exists') || - message.includes('already exist') || - message.includes('duplicate key name') || - message.includes('index already exists') || - err.code === 85 || - err.code === '42P07' || - err.name === 'SequelizeUniqueConstraintError' - ); + const parts = walkError(error); + if (parts.some(part => part.name === 'SequelizeUniqueConstraintError')) { + return false; + } + if ( + parts.some( + part => part.code !== undefined && UNIQUE_OR_OPTIONS_CONFLICT_CODES.has(part.code), + ) + ) { + return false; + } + for (const part of parts) { + if (part.code === '42P07' || part.code === 1061 || part.code === '1061') { + return true; + } + const message = part.message ?? ''; + if (/index .+ already exists/i.test(message)) return true; + if (/duplicate key name/i.test(message)) return true; + if (/relation .+ already exists/i.test(message)) return true; + } + return false; } export async function persistDeclaredSchemaIndexes(args: { declaredSchemaModel: { - findOne: (query: Record) => Promise<{ _id: string } | null>; + findOne: (query: Record) => Promise<{ + _id: string; + modelOptions?: { indexes?: ModelOptionsIndexes[] }; + } | null>; findByIdAndUpdate: (id: string, update: Record) => Promise; }; schemaName: string; @@ -243,16 +443,24 @@ export async function persistDeclaredSchemaIndexes(args: { fields?: Record; compiledFields?: Record; }; - indexes: ModelOptionsIndexes[]; -}): Promise { - args.originalSchema.modelOptions.indexes = args.indexes; + applied?: ModelOptionsIndexes[]; + droppedNames?: string[]; +}): Promise { const found = await args.declaredSchemaModel.findOne({ name: args.schemaName }); - if (!found) return; + const memoryIndexes = (args.originalSchema.modelOptions.indexes ?? + []) as ModelOptionsIndexes[]; + const dbIndexes = found?.modelOptions?.indexes ?? memoryIndexes; + const next = args.droppedNames + ? removeDeclaredIndexes(dbIndexes, args.droppedNames) + : mergeDeclaredIndexes(dbIndexes, args.applied ?? []); + args.originalSchema.modelOptions.indexes = next; + if (!found) return false; await args.declaredSchemaModel.findByIdAndUpdate(found._id, { modelOptions: args.originalSchema.modelOptions, fields: args.originalSchema.fields, compiledFields: args.originalSchema.compiledFields, }); + return true; } export function collectExistingIndexNames( From 364bfed81dfb5c7d5a4e86bf43fcf00571bb1cfc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 15 Sep 2026 10:34:54 +0000 Subject: [PATCH 4/9] fix(database): persist indexes from primary and snapshot after bind Read _DeclaredSchema from primary before index persist, reuse SQL name-exists errors only when fields+unique match, and freeze registeredSchemas after bind/save so replica catch-up gets live names. --- .../src/__tests__/indexes/adapters.test.ts | 83 ++++++++++++++++++- .../src/__tests__/indexes/helpers.test.ts | 52 +++++++++++- .../database/src/adapters/DatabaseAdapter.ts | 7 ++ .../src/adapters/mongoose-adapter/index.ts | 51 +++++++----- .../sequelize-adapter/SequelizeSchema.ts | 1 + .../src/adapters/sequelize-adapter/index.ts | 45 +++++----- .../database/src/adapters/utils/indexes.ts | 28 ++++++- 7 files changed, 218 insertions(+), 49 deletions(-) diff --git a/modules/database/src/__tests__/indexes/adapters.test.ts b/modules/database/src/__tests__/indexes/adapters.test.ts index 565624e53..fbf3b8fa5 100644 --- a/modules/database/src/__tests__/indexes/adapters.test.ts +++ b/modules/database/src/__tests__/indexes/adapters.test.ts @@ -143,6 +143,7 @@ describe('mongoose adapter indexes', () => { 'chat', ); expect(created.findByIdAndUpdate).toHaveBeenCalledTimes(1); + expect(created.findOne.mock.calls[0][1]).toEqual({ readPreference: 'primary' }); const update = created.findByIdAndUpdate.mock.calls[0][1] as { modelOptions: { indexes: { name?: string }[] }; }; @@ -296,6 +297,21 @@ describe('mongoose adapter indexes', () => { expect(publish).toHaveBeenCalledWith('database:create:schema', expect.any(String)); }); + it('throws on a name-already-exists error when the live name indexes different fields', async () => { + const { adapter, createIndex, indexes, findByIdAndUpdate } = makeMongooseAdapter(); + createIndex.mockRejectedValue({ message: 'index user_idx already exists' }); + indexes + .mockResolvedValueOnce([{ v: 2, key: { _id: 1 }, name: '_id_' }]) + .mockResolvedValueOnce([ + { v: 2, key: { _id: 1 }, name: '_id_' }, + { v: 2, key: { room: 1 }, name: 'user_idx', unique: false }, + ]); + await expect( + adapter.createIndexes('User', [{ fields: ['email'], name: 'user_idx' }], 'chat'), + ).rejects.toMatchObject({ code: status.INTERNAL }); + expect(findByIdAndUpdate).not.toHaveBeenCalled(); + }); + it('rebinds to the live name on Mongo 86 instead of persisting a generated name', async () => { const { adapter, createIndex, indexes, findByIdAndUpdate } = makeMongooseAdapter(); createIndex.mockRejectedValue({ code: 86, message: 'IndexKeySpecsConflict' }); @@ -452,8 +468,16 @@ describe('sequelize adapter indexes', () => { expect(update.modelOptions.indexes[0].name).toBe('room_createdAt'); }); - it('skips and persists on 42P07 name conflicts', async () => { - const { adapter, addIndex, findByIdAndUpdate, publish } = makeSequelizeAdapter(); + it('skips and persists on 42P07 when the live name has the same identity', async () => { + const { adapter, addIndex, showIndex, findByIdAndUpdate, publish } = + makeSequelizeAdapter(); + showIndex.mockResolvedValueOnce([]).mockResolvedValueOnce([ + { + name: 'cnd_idx_email_asc', + unique: false, + fields: [{ attribute: 'email', order: 'ASC' }], + }, + ]); addIndex.mockRejectedValue({ original: { code: '42P07', message: 'relation "cnd_idx_email_asc" already exists' }, }); @@ -468,6 +492,61 @@ describe('sequelize adapter indexes', () => { expect(publish).toHaveBeenCalled(); }); + it('throws on 42P07 when the live name indexes different fields', async () => { + const { adapter, addIndex, showIndex, findByIdAndUpdate } = makeSequelizeAdapter(); + showIndex.mockResolvedValueOnce([]).mockResolvedValueOnce([ + { + name: 'user_idx', + unique: false, + fields: [{ attribute: 'room' }], + }, + ]); + addIndex.mockRejectedValue({ + original: { code: '42P07', message: 'relation "user_idx" already exists' }, + }); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], name: 'user_idx' }], + 'database', + ), + ).rejects.toMatchObject({ code: status.INTERNAL }); + expect(findByIdAndUpdate).not.toHaveBeenCalled(); + }); + + it('persists the applied prefix when a later 42P07 name has different fields', async () => { + const { adapter, addIndex, showIndex, findByIdAndUpdate, publish } = + makeSequelizeAdapter(); + showIndex.mockResolvedValueOnce([]).mockResolvedValueOnce([ + { + name: 'user_idx', + unique: false, + fields: [{ attribute: 'username' }], + }, + ]); + addIndex.mockResolvedValueOnce(undefined).mockRejectedValueOnce({ + original: { code: '42P07', message: 'relation "user_idx" already exists' }, + }); + await expect( + adapter.createIndexes( + 'User', + [ + { fields: ['email'], types: [CompatibleIndexType.Ascending] }, + { fields: ['room'], name: 'user_idx' }, + ], + 'database', + ), + ).rejects.toMatchObject({ code: status.INTERNAL }); + expect(findByIdAndUpdate).toHaveBeenCalledTimes(1); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { name?: string }[] }; + }; + expect(update.modelOptions.indexes.map(index => index.name)).toEqual([ + 'cnd_idx_email_asc', + ]); + expect(publish).toHaveBeenCalled(); + }); + it('throws on 23505 unique collisions and does not persist that index', async () => { const { adapter, addIndex, findByIdAndUpdate } = makeSequelizeAdapter(); addIndex.mockRejectedValue({ diff --git a/modules/database/src/__tests__/indexes/helpers.test.ts b/modules/database/src/__tests__/indexes/helpers.test.ts index 48ec49ba8..c6e5aab40 100644 --- a/modules/database/src/__tests__/indexes/helpers.test.ts +++ b/modules/database/src/__tests__/indexes/helpers.test.ts @@ -16,6 +16,7 @@ import { isIndexAlreadyExistsError, isMongoIndexType, keepDeclaredIndexExtras, + liveNameConflictAllowsReuse, mapCompatibleToMongo, mapCompatibleToSqlOrder, mergeDeclaredIndexes, @@ -310,10 +311,14 @@ describe('index helpers', () => { }); it('persists applied indexes against a re-read declared schema list', async () => { - const findOne = async () => ({ - _id: 'declared-1', - modelOptions: { indexes: [{ fields: ['keep'], name: 'keep_me' }] }, - }); + let findOneOptions: unknown; + const findOne = async (_query: Record, options?: unknown) => { + findOneOptions = options; + return { + _id: 'declared-1', + modelOptions: { indexes: [{ fields: ['keep'], name: 'keep_me' }] }, + }; + }; const findByIdAndUpdate = async () => ({}); const originalSchema = { modelOptions: { indexes: [] as { fields: string[]; name?: string }[] }, @@ -325,12 +330,51 @@ describe('index helpers', () => { applied: [{ fields: ['email'], name: 'cnd_idx_email_asc' }], }); expect(persisted).toBe(true); + expect(findOneOptions).toEqual({ readPreference: 'primary' }); expect(originalSchema.modelOptions.indexes.map(index => index.name)).toEqual([ 'keep_me', 'cnd_idx_email_asc', ]); }); + it('reuses a live name-conflict only when identity matches', () => { + const live = [ + { + name: 'user_idx', + fields: ['email'], + options: { name: 'user_idx', unique: false }, + }, + ]; + expect( + liveNameConflictAllowsReuse( + { name: 'user_idx', fields: ['email'], options: { name: 'user_idx' } }, + live, + ), + ).toBe(true); + expect( + liveNameConflictAllowsReuse( + { name: 'user_idx', fields: ['username'], options: { name: 'user_idx' } }, + live, + ), + ).toBe(false); + expect( + liveNameConflictAllowsReuse( + { + name: 'user_idx', + fields: ['email'], + options: { name: 'user_idx', unique: true }, + }, + live, + ), + ).toBe(false); + expect( + liveNameConflictAllowsReuse( + { name: 'user_idx', fields: ['email'], options: { name: 'user_idx' } }, + [], + ), + ).toBe(false); + }); + it('allows dialect-native types and rejects foreign leftovers', () => { expect(sqlDialectAllowsIndexType('mysql', CompatibleIndexType.Ascending)).toBe(true); expect(sqlDialectAllowsIndexType('mariadb', CompatibleIndexType.Descending)).toBe( diff --git a/modules/database/src/adapters/DatabaseAdapter.ts b/modules/database/src/adapters/DatabaseAdapter.ts index ee3e13677..fa37cf1a7 100644 --- a/modules/database/src/adapters/DatabaseAdapter.ts +++ b/modules/database/src/adapters/DatabaseAdapter.ts @@ -605,6 +605,13 @@ export abstract class DatabaseAdapter { instanceSync: boolean, ): Promise; + protected snapshotRegisteredSchema(schema: ConduitSchema) { + this.registeredSchemas.set( + schema.name, + Object.freeze(JSON.parse(JSON.stringify(schema))), + ); + } + protected async persistIndexesAndPublish(args: { schemaName: string; originalSchema: ConduitDatabaseSchema; diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 39079342a..f4fcb80c3 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -38,6 +38,7 @@ import { isIndexAlreadyExistsError, isIndexKeySpecsConflictError, liveIndexFromMongo, + liveNameConflictAllowsReuse, mapCompatibleToMongo, mongoAllowsIndexType, normalizeIndexTypes, @@ -687,8 +688,14 @@ export class MongooseAdapter extends DatabaseAdapter { live.push(index); } catch (e) { if (isIndexAlreadyExistsError(e)) { - applied.push(index); - continue; + const relisted = await this.listLiveIndexes(schemaName); + if (liveNameConflictAllowsReuse(index, relisted)) { + applied.push(index); + live.splice(0, live.length, ...relisted); + continue; + } + failure = e; + break; } if (isIndexKeySpecsConflictError(e)) { const relisted = await this.listLiveIndexes(schemaName); @@ -1082,10 +1089,6 @@ export class MongooseAdapter extends DatabaseAdapter { const newSchema = schemaConverter(compiledSchema); const indexes = newSchema.modelOptions.indexes; delete newSchema.modelOptions.indexes; - this.registeredSchemas.set( - schema.name, - Object.freeze(JSON.parse(JSON.stringify(schema))), - ); this.models[schema.name] = new MongooseSchema( this.grpcSdk, this.mongoose, @@ -1093,23 +1096,27 @@ export class MongooseAdapter extends DatabaseAdapter { schema, this, ); - if (!isInstanceSync && schema.modelOptions.indexes?.length) { - const live = await this.listLiveIndexes(schema.name); - schema.modelOptions.indexes = bindDeclaredIndexesToLive( - schema.modelOptions.indexes, - live, - ); - } - if (saveToDb) { - await this.compareAndStoreMigratedSchema(schema); - await this.saveSchemaToDatabase(schema); - } + try { + if (!isInstanceSync && schema.modelOptions.indexes?.length) { + const live = await this.listLiveIndexes(schema.name); + schema.modelOptions.indexes = bindDeclaredIndexesToLive( + schema.modelOptions.indexes, + live, + ); + } + if (saveToDb) { + await this.compareAndStoreMigratedSchema(schema); + await this.saveSchemaToDatabase(schema); + } - if (indexes && !isInstanceSync) { - await this.createIndexes(schema.name, indexes, schema.ownerModule); - } - if (!isInstanceSync) { - await this.createMongooseFieldIndexes(schema.name); + if (indexes && !isInstanceSync) { + await this.createIndexes(schema.name, indexes, schema.ownerModule); + } + if (!isInstanceSync) { + await this.createMongooseFieldIndexes(schema.name); + } + } finally { + this.snapshotRegisteredSchema(schema); } await this.applyDeclaredVectorIndexes(schema.name, isInstanceSync); return this.models[schema.name]; diff --git a/modules/database/src/adapters/sequelize-adapter/SequelizeSchema.ts b/modules/database/src/adapters/sequelize-adapter/SequelizeSchema.ts index c7dec1ecd..47b93c6a2 100644 --- a/modules/database/src/adapters/sequelize-adapter/SequelizeSchema.ts +++ b/modules/database/src/adapters/sequelize-adapter/SequelizeSchema.ts @@ -339,6 +339,7 @@ export class SequelizeSchema extends SchemaAdapter> { scope?: string; select?: string; populate?: string[]; + readPreference?: string; }, ) { const filter = await this.getAuthorizedQuery( diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 238038a93..7f90688a2 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -60,6 +60,7 @@ import { isIndexKeySpecsConflictError, isPostgresIndexType, liveIndexFromSql, + liveNameConflictAllowsReuse, overlayDeclaredOnLive, normalizeIndexTypes, removeIndexFromSchemaFields, @@ -304,10 +305,6 @@ export abstract class SequelizeAdapter extends DatabaseAdapter live, ); } - this.registeredSchemas.set( - schema.name, - Object.freeze(JSON.parse(JSON.stringify(schema))), - ); const relatedSchemas = await resolveRelatedSchemas( schema, extractedRelations, @@ -323,19 +320,23 @@ export abstract class SequelizeAdapter extends DatabaseAdapter objectPaths, ); - const noSync = - this.models[schema.name].originalSchema.modelOptions.conduit!.noSync || - isInstanceSync; - // do not sync extracted schemas - if (isNil(noSync) || !noSync) { - await this.models[schema.name].sync(); - } else { - this.models[schema.name].synced = true; - } - // do not store extracted schemas to db - if (saveToDb && !isInstanceSync) { - await this.compareAndStoreMigratedSchema(schema); - await this.saveSchemaToDatabase(schema); + try { + const noSync = + this.models[schema.name].originalSchema.modelOptions.conduit!.noSync || + isInstanceSync; + // do not sync extracted schemas + if (isNil(noSync) || !noSync) { + await this.models[schema.name].sync(); + } else { + this.models[schema.name].synced = true; + } + // do not store extracted schemas to db + if (saveToDb && !isInstanceSync) { + await this.compareAndStoreMigratedSchema(schema); + await this.saveSchemaToDatabase(schema); + } + } finally { + this.snapshotRegisteredSchema(schema); } await this.applyDeclaredVectorIndexes(schema.name, isInstanceSync); return this.models[schema.name]; @@ -442,8 +443,14 @@ export abstract class SequelizeAdapter extends DatabaseAdapter live.push(index); } catch (e) { if (isIndexAlreadyExistsError(e)) { - applied.push(index); - continue; + const relisted = await this.listLiveIndexesForCollection(collectionName); + if (liveNameConflictAllowsReuse(index, relisted)) { + applied.push(index); + live.splice(0, live.length, ...relisted); + continue; + } + failure = e; + break; } if (isIndexKeySpecsConflictError(e)) { const relisted = await this.listLiveIndexesForCollection(collectionName); diff --git a/modules/database/src/adapters/utils/indexes.ts b/modules/database/src/adapters/utils/indexes.ts index 3c1964e1f..5717bae86 100644 --- a/modules/database/src/adapters/utils/indexes.ts +++ b/modules/database/src/adapters/utils/indexes.ts @@ -106,6 +106,24 @@ export function findLiveIndex( ); } +export function findIndexByName( + indexes: readonly ModelOptionsIndexes[], + name: string, +): ModelOptionsIndexes | undefined { + return indexes.find(index => resolveIndexName(index) === name); +} + +export function liveNameConflictAllowsReuse( + declared: ModelOptionsIndexes, + live: readonly ModelOptionsIndexes[], +): boolean { + const name = resolveIndexName(declared); + if (!name) return false; + const row = findIndexByName(live, name); + if (!row) return false; + return indexIdentitiesEqual(indexIdentity(row), indexIdentity(declared)); +} + export function bindDeclaredIndexesToLive( declared: readonly T[], live: readonly ModelOptionsIndexes[], @@ -431,7 +449,10 @@ export function isIndexAlreadyExistsError(error: unknown): boolean { export async function persistDeclaredSchemaIndexes(args: { declaredSchemaModel: { - findOne: (query: Record) => Promise<{ + findOne: ( + query: Record, + options?: { readPreference?: string }, + ) => Promise<{ _id: string; modelOptions?: { indexes?: ModelOptionsIndexes[] }; } | null>; @@ -446,7 +467,10 @@ export async function persistDeclaredSchemaIndexes(args: { applied?: ModelOptionsIndexes[]; droppedNames?: string[]; }): Promise { - const found = await args.declaredSchemaModel.findOne({ name: args.schemaName }); + const found = await args.declaredSchemaModel.findOne( + { name: args.schemaName }, + { readPreference: 'primary' }, + ); const memoryIndexes = (args.originalSchema.modelOptions.indexes ?? []) as ModelOptionsIndexes[]; const dbIndexes = found?.modelOptions?.indexes ?? memoryIndexes; From c98b78d07de2658fa0dd9972a03f4ba86a1a2d97 Mon Sep 17 00:00:00 2001 From: Christina Papadogianni <59121443+ChrisPdgn@users.noreply.github.com> Date: Wed, 16 Sep 2026 09:10:25 +0000 Subject: [PATCH 5/9] fix(database): make compatible indexes safe on Mongo, Postgres, and MySQL Fail-soft Mongo GET/export on missing collections. Qualify generated index names by table and bind live names without rename. Skip MySQL JSON btree and SQL extracted-relation indexes on register; reject them from Admin. Lift Mongo array Compatible indexes to createIndexes. Register Database, Authz, and Chat schemas sequentially. --- modules/authorization/src/Authorization.ts | 12 +- modules/chat/src/Chat.ts | 17 +- modules/database/src/Database.ts | 29 +-- .../src/__tests__/indexes/adapters.test.ts | 119 +++++++++++- .../src/__tests__/indexes/admin.test.ts | 15 ++ .../src/__tests__/indexes/converters.test.ts | 182 +++++++++++++++++- .../src/__tests__/indexes/helpers.test.ts | 145 ++++++++++++-- .../src/__tests__/indexes/regressions.test.ts | 17 ++ .../database/src/adapters/DatabaseAdapter.ts | 2 + .../mongoose-adapter/SchemaConverter.ts | 32 ++- .../src/adapters/mongoose-adapter/index.ts | 23 ++- .../src/adapters/sequelize-adapter/index.ts | 31 ++- .../utils/database-transform-utils.ts | 45 +++-- .../database/src/adapters/utils/indexes.ts | 151 +++++++++++++-- modules/database/src/admin/schema.admin.ts | 14 +- 15 files changed, 744 insertions(+), 90 deletions(-) diff --git a/modules/authorization/src/Authorization.ts b/modules/authorization/src/Authorization.ts index e9755e731..51a78aae6 100644 --- a/modules/authorization/src/Authorization.ts +++ b/modules/authorization/src/Authorization.ts @@ -380,13 +380,11 @@ export default class Authorization extends ManagedModule { return resource; } - protected registerSchemas(): Promise { - const promises = Object.values(models).map(model => { + protected async registerSchemas(): Promise { + for (const model of Object.values(models)) { const modelInstance = model.getInstance(this.database); - return this.database - .createSchemaFromAdapter(modelInstance) - .then(() => this.database.migrate(modelInstance.name)); - }); - return Promise.all(promises); + await this.database.createSchemaFromAdapter(modelInstance); + await this.database.migrate(modelInstance.name); + } } } diff --git a/modules/chat/src/Chat.ts b/modules/chat/src/Chat.ts index f817aa463..1c3ca0201 100644 --- a/modules/chat/src/Chat.ts +++ b/modules/chat/src/Chat.ts @@ -396,21 +396,18 @@ export default class Chat extends ManagedModule { }); } - protected registerSchemas(): Promise { - const promises = Object.values(models).map(model => { + protected async registerSchemas(): Promise { + for (const model of Object.values(models)) { const modelInstance = model.getInstance(this.database); - //TODO: add support for multiple schemas types if ( Object.keys((modelInstance as ConduitActiveSchema).fields) - .length !== 0 + .length === 0 ) { - // borrowed foreign model - return this.database - .createSchemaFromAdapter(modelInstance) - .then(() => this.database.migrate(modelInstance.name)); + continue; } - }); - return Promise.all(promises); + await this.database.createSchemaFromAdapter(modelInstance); + await this.database.migrate(modelInstance.name); + } } private scheduleAppRouteRefresh() { diff --git a/modules/database/src/Database.ts b/modules/database/src/Database.ts index ed46fa5bd..b6a8e67f9 100644 --- a/modules/database/src/Database.ts +++ b/modules/database/src/Database.ts @@ -167,26 +167,27 @@ export default class DatabaseModule extends ManagedModule { const isReplica = this.grpcSdk.isAvailable('database'); await this._activeAdapter.registerSystemSchema(models.DeclaredSchema, isReplica); await this._activeAdapter.registerSystemSchema(models.MigratedSchemas, isReplica); - let modelPromises = DATABASE_SYSTEM_SCHEMAS.filter( - model => - model.name !== models.DeclaredSchema.name && - model.name !== models.MigratedSchemas.name, - ).map(model => this._activeAdapter.registerSystemSchema(model, isReplica)); - await Promise.all(modelPromises); + for (const model of DATABASE_SYSTEM_SCHEMAS) { + if ( + model.name === models.DeclaredSchema.name || + model.name === models.MigratedSchemas.name + ) { + continue; + } + await this._activeAdapter.registerSystemSchema(model, isReplica); + } await this._activeAdapter.retrieveForeignSchemas(); await this._activeAdapter.recoverSchemasFromDatabase(); await this._activeAdapter.recoverViewsFromDatabase(); if (!isReplica) { await runMigrations(this._activeAdapter); } - modelPromises = DATABASE_SYSTEM_SCHEMAS.map(model => { - return this._activeAdapter.registerSystemSchema(model, isReplica).then(() => { - if (this._activeAdapter.getDatabaseType() !== 'MongoDB' && !isReplica) { - return this._activeAdapter.syncSchema(model.name); - } - }); - }); - await Promise.all(modelPromises); + for (const model of DATABASE_SYSTEM_SCHEMAS) { + await this._activeAdapter.registerSystemSchema(model, isReplica); + if (this._activeAdapter.getDatabaseType() !== 'MongoDB' && !isReplica) { + await this._activeAdapter.syncSchema(model.name); + } + } this.updateHealth(HealthCheckStatus.SERVING); } diff --git a/modules/database/src/__tests__/indexes/adapters.test.ts b/modules/database/src/__tests__/indexes/adapters.test.ts index fbf3b8fa5..924b9cea8 100644 --- a/modules/database/src/__tests__/indexes/adapters.test.ts +++ b/modules/database/src/__tests__/indexes/adapters.test.ts @@ -4,6 +4,7 @@ import { CompatibleIndexType, MongoIndexType, PostgresIndexType, + TYPE, } from '@conduitplatform/grpc-sdk'; import { MongooseAdapter } from '../../adapters/mongoose-adapter/index.js'; import { SequelizeAdapter } from '../../adapters/sequelize-adapter/index.js'; @@ -261,7 +262,7 @@ describe('mongoose adapter indexes', () => { modelOptions: { indexes: { name?: string }[] }; }; expect(update.modelOptions.indexes.map(index => index.name)).toEqual([ - 'cnd_idx_email_asc', + 'cnd_idx_cnd_User_email_asc', ]); expect(publish).toHaveBeenCalledWith('database:create:schema', expect.any(String)); }); @@ -356,6 +357,68 @@ describe('mongoose adapter indexes', () => { }; expect(update.modelOptions.indexes.map(index => index.name)).toEqual(['idx_b']); }); + + it('returns empty indexes when the Mongo namespace is missing', async () => { + const { adapter, indexes } = makeMongooseAdapter(); + indexes.mockRejectedValue({ + code: 26, + codeName: 'NamespaceNotFound', + message: 'ns does not exist: test.cnd_adminapitokens', + }); + await expect(adapter.getIndexes('User')).resolves.toEqual([]); + }); + + it('creates indexes when listLiveIndexes hits a missing namespace', async () => { + const { adapter, createIndex, indexes } = makeMongooseAdapter(); + indexes.mockRejectedValue({ + code: 26, + codeName: 'NamespaceNotFound', + message: 'ns does not exist: test.cnd_adminapitokens', + }); + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['email'], types: [CompatibleIndexType.Ascending] }], + 'chat', + ), + ).resolves.toBe('Indexes created!'); + expect(createIndex).toHaveBeenCalledTimes(1); + }); + + it('rethrows non-namespace Mongo index listing errors', async () => { + const { adapter, indexes } = makeMongooseAdapter(); + indexes.mockRejectedValue({ code: 13, message: 'unauthorized' }); + await expect(adapter.getIndexes('User')).rejects.toMatchObject({ + message: 'unauthorized', + }); + }); + + it('creates both a single-field array index and a compound sharing the leading field', async () => { + const { adapter, createIndex, originalSchema } = makeMongooseAdapter(); + originalSchema.fields.participants = [{ type: TYPE.Relation, model: 'User' }]; + originalSchema.fields.deleted = { type: TYPE.Boolean }; + originalSchema.compiledFields.participants = [{ type: TYPE.Relation, model: 'User' }]; + originalSchema.compiledFields.deleted = { type: TYPE.Boolean }; + await adapter.createIndexes( + 'User', + [ + { fields: ['participants'], types: [CompatibleIndexType.Ascending] }, + { + fields: ['participants', 'deleted'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + 'chat', + ); + expect(createIndex).toHaveBeenCalledTimes(2); + expect(createIndex.mock.calls[0][0]).toEqual({ + participants: MongoIndexType.Ascending, + }); + expect(createIndex.mock.calls[1][0]).toEqual({ + participants: MongoIndexType.Ascending, + deleted: MongoIndexType.Ascending, + }); + }); }); describe('sequelize adapter indexes', () => { @@ -542,7 +605,7 @@ describe('sequelize adapter indexes', () => { modelOptions: { indexes: { name?: string }[] }; }; expect(update.modelOptions.indexes.map(index => index.name)).toEqual([ - 'cnd_idx_email_asc', + 'cnd_idx_custom_users_email_asc', ]); expect(publish).toHaveBeenCalled(); }); @@ -589,4 +652,56 @@ describe('sequelize adapter indexes', () => { CompatibleIndexType.Ascending, ]); }); + + it('rejects MySQL JSON btree indexes and allows them on postgres', async () => { + const mysql = makeSequelizeAdapter('mysql'); + mysql.originalSchema.fields.inheritanceTree = { type: [TYPE.String] }; + mysql.originalSchema.compiledFields.inheritanceTree = { type: [TYPE.String] }; + await expect( + mysql.adapter.createIndexes( + 'User', + [{ fields: ['inheritanceTree'], types: [CompatibleIndexType.Ascending] }], + 'database', + ), + ).rejects.toMatchObject({ + code: status.INVALID_ARGUMENT, + message: expect.stringMatching(/MySQL JSON field 'inheritanceTree'/), + }); + expect(mysql.addIndex).not.toHaveBeenCalled(); + expect(mysql.findByIdAndUpdate).not.toHaveBeenCalled(); + + const postgres = makeSequelizeAdapter('postgres'); + postgres.originalSchema.fields.payload = { type: TYPE.JSON }; + postgres.originalSchema.compiledFields.payload = { type: TYPE.JSON }; + await expect( + postgres.adapter.createIndexes( + 'User', + [{ fields: ['payload'], types: [CompatibleIndexType.Ascending] }], + 'database', + ), + ).resolves.toBe('Indexes created!'); + expect(postgres.addIndex).toHaveBeenCalled(); + }); + + it('rejects SQL indexes on extracted relation fields', async () => { + const { adapter, addIndex, findByIdAndUpdate } = makeSequelizeAdapter('postgres'); + adapter.models.User.originalSchema.fields.participants = [ + { type: TYPE.Relation, model: 'User' }, + ]; + adapter.models.User.originalSchema.compiledFields.participants = [ + { type: TYPE.Relation, model: 'User' }, + ]; + await expect( + adapter.createIndexes( + 'User', + [{ fields: ['participants'], types: [CompatibleIndexType.Ascending] }], + 'database', + ), + ).rejects.toMatchObject({ + code: status.INVALID_ARGUMENT, + message: expect.stringMatching(/relation join table/), + }); + expect(addIndex).not.toHaveBeenCalled(); + expect(findByIdAndUpdate).not.toHaveBeenCalled(); + }); }); diff --git a/modules/database/src/__tests__/indexes/admin.test.ts b/modules/database/src/__tests__/indexes/admin.test.ts index 757954193..5ee3c24e6 100644 --- a/modules/database/src/__tests__/indexes/admin.test.ts +++ b/modules/database/src/__tests__/indexes/admin.test.ts @@ -87,6 +87,21 @@ describe('SchemaAdmin indexes', () => { expect(result.indexes.every(index => 'schemaName' in (index as object))).toBe(true); }); + it('continues exporting when getIndexes throws for one schema', async () => { + const { admin, getIndexes } = setup(); + getIndexes + .mockRejectedValueOnce(new Error('ns does not exist: test.cnd_adminapitokens')) + .mockResolvedValueOnce([{ name: 'ok', fields: ['email'] }]); + jest.spyOn(ConduitGrpcSdk.Logger, 'warn').mockImplementation(() => undefined); + const result = (await admin.exportIndexes(makeCall({}))) as { + indexes: Array<{ schemaName: string }>; + count: number; + }; + expect(result.count).toBe(2); + expect(result.indexes).toHaveLength(1); + expect(result.indexes[0].schemaName).toBe('ChatRoom'); + }); + it('skips same-name indexes on import', async () => { const { admin, createIndexes, getIndexes } = setup(); getIndexes.mockResolvedValue([{ name: 'keep_me', fields: ['email'] }]); diff --git a/modules/database/src/__tests__/indexes/converters.test.ts b/modules/database/src/__tests__/indexes/converters.test.ts index 88843de97..a53643b6d 100644 --- a/modules/database/src/__tests__/indexes/converters.test.ts +++ b/modules/database/src/__tests__/indexes/converters.test.ts @@ -63,10 +63,13 @@ describe('SQL index converters', () => { it('warns and skips Mongo-only leftovers on SQL', () => { const warn = ConduitGrpcSdk.Logger.warn as jest.Mock; const copy = convertModelOptionsIndexes( - schemaWithIndexes([ - { fields: ['loc'], types: [MongoIndexType.GeoSpatial2dSphere] }, - { fields: ['email'], types: [CompatibleIndexType.Ascending] }, - ]), + schemaWithIndexes( + [ + { fields: ['loc'], types: [MongoIndexType.GeoSpatial2dSphere] }, + { fields: ['email'], types: [CompatibleIndexType.Ascending] }, + ], + { email: { type: TYPE.String }, loc: { type: TYPE.JSON } }, + ), 'postgres', ); expect(copy.modelOptions.indexes).toHaveLength(1); @@ -132,6 +135,129 @@ describe('SQL index converters', () => { PostgresIndexType.GIN, ); }); + + it('gives distinct generated names to two schemas with the same fields', () => { + const permission = convertModelOptionsIndexes( + new ConduitSchema( + 'Permission', + { resource: { type: TYPE.String } }, + { indexes: [{ fields: ['resource'], types: [CompatibleIndexType.Ascending] }] }, + ), + 'postgres', + ); + const relationship = convertModelOptionsIndexes( + new ConduitSchema( + 'Relationship', + { resource: { type: TYPE.String } }, + { indexes: [{ fields: ['resource'], types: [CompatibleIndexType.Ascending] }] }, + ), + 'postgres', + ); + expect((permission.modelOptions.indexes![0] as { name?: string }).name).toBe( + 'cnd_idx_Permission_resource_asc', + ); + expect((relationship.modelOptions.indexes![0] as { name?: string }).name).toBe( + 'cnd_idx_Relationship_resource_asc', + ); + }); + + it('skips MySQL JSON field-level indexes and keeps them on postgres', () => { + const mysql = convertSchemaFieldIndexes( + new ConduitSchema( + 'ObjectIndex', + { + inheritanceTree: { + type: [TYPE.String], + index: { type: CompatibleIndexType.Ascending }, + }, + }, + {}, + ), + 'mysql', + ); + const postgres = convertSchemaFieldIndexes( + new ConduitSchema( + 'ObjectIndex', + { + inheritanceTree: { + type: [TYPE.String], + index: { type: CompatibleIndexType.Ascending }, + }, + }, + {}, + ), + 'postgres', + ); + expect(mysql.modelOptions.indexes).toHaveLength(0); + expect(postgres.modelOptions.indexes).toHaveLength(1); + }); + + it('skips extracted array-relation indexes on SQL and relation-field compounds', () => { + const chatRoom = new ConduitSchema( + 'ChatRoom', + { + participants: [{ type: TYPE.Relation, model: 'User', required: true }], + deleted: { type: TYPE.Boolean }, + }, + { + timestamps: true, + indexes: [ + { fields: ['participants'], types: [CompatibleIndexType.Ascending] }, + { + fields: ['participants', 'deleted'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + }, + ); + const [mysql] = sqlSchemaConverter(chatRoom, 'mysql'); + const [pg] = pgSchemaConverter(chatRoom); + expect(mysql.modelOptions.indexes ?? []).toHaveLength(0); + expect(pg.modelOptions.indexes ?? []).toHaveLength(0); + + const message = new ConduitSchema( + 'Message', + { + room: { type: TYPE.Relation, model: 'ChatRoom', required: true }, + deleted: { type: TYPE.Boolean }, + createdAt: { type: TYPE.Date }, + }, + { + timestamps: true, + indexes: [ + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + }, + ); + const [mysqlMessage] = sqlSchemaConverter(message, 'mysql'); + const [pgMessage] = pgSchemaConverter(message); + expect(mysqlMessage.modelOptions.indexes ?? []).toHaveLength(0); + expect(pgMessage.modelOptions.indexes ?? []).toHaveLength(0); + + const scalar = new ConduitSchema( + 'Message', + { + deleted: { type: TYPE.Boolean }, + createdAt: { type: TYPE.Date }, + }, + { + timestamps: true, + indexes: [ + { + fields: ['deleted', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + }, + ); + const [mysqlScalar] = sqlSchemaConverter(scalar, 'mysql'); + const [pgScalar] = pgSchemaConverter(scalar); + expect(mysqlScalar.modelOptions.indexes).toHaveLength(1); + expect(pgScalar.modelOptions.indexes).toHaveLength(1); + }); }); describe('mongoose SchemaConverter indexes', () => { @@ -192,4 +318,52 @@ describe('mongoose SchemaConverter indexes', () => { (converted.fields.email as { index: { type: MongoIndexType } }).index.type, ).toBe(MongoIndexType.Ascending); }); + + it('keeps single-field array indexes on modelOptions with compounds', () => { + const converted = schemaConverter( + new ConduitSchema( + 'ChatRoom', + { + participants: [{ type: TYPE.Relation, model: 'User', required: true }], + deleted: { type: TYPE.Boolean }, + }, + { + indexes: [ + { fields: ['participants'], types: [CompatibleIndexType.Ascending] }, + { + fields: ['participants', 'deleted'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + }, + ), + ); + const remaining = converted.modelOptions.indexes ?? []; + expect(remaining.map(index => index.fields)).toEqual([ + ['participants'], + ['participants', 'deleted'], + ]); + }); + + it('lifts field-level array Compatible indexes onto modelOptions', () => { + const converted = schemaConverter( + new ConduitSchema( + 'ObjectIndex', + { + inheritanceTree: { + type: [TYPE.String], + default: [], + index: { type: CompatibleIndexType.Ascending }, + }, + }, + {}, + ), + ); + expect( + (converted.fields.inheritanceTree as { index?: unknown }).index, + ).toBeUndefined(); + expect(converted.modelOptions.indexes?.map(index => index.fields)).toEqual([ + ['inheritanceTree'], + ]); + }); }); diff --git a/modules/database/src/__tests__/indexes/helpers.test.ts b/modules/database/src/__tests__/indexes/helpers.test.ts index c6e5aab40..702d44c45 100644 --- a/modules/database/src/__tests__/indexes/helpers.test.ts +++ b/modules/database/src/__tests__/indexes/helpers.test.ts @@ -4,6 +4,7 @@ import { CompatibleIndexType, MongoIndexType, PostgresIndexType, + TYPE, } from '@conduitplatform/grpc-sdk'; import { assertUniqueIndexPrivilege, @@ -15,6 +16,7 @@ import { isCompatibleIndexType, isIndexAlreadyExistsError, isMongoIndexType, + isMongoNamespaceMissingError, keepDeclaredIndexExtras, liveNameConflictAllowsReuse, mapCompatibleToMongo, @@ -28,6 +30,7 @@ import { resolveIndexName, sqlDialectAllowsIndexType, sqlIndexFields, + sqlIndexUnsupportedReason, validateIndexFields, } from '../../adapters/utils/indexes.js'; @@ -41,29 +44,64 @@ describe('index helpers', () => { expect(isMongoIndexType('Ascending')).toBe(false); }); - it('generates a deterministic name when one is missing', () => { - const name = generateIndexName(['email'], [CompatibleIndexType.Ascending], false); - expect(name).toBe('cnd_idx_email_asc'); + it('generates a deterministic name unique per collection', () => { + const name = generateIndexName( + ['email'], + [CompatibleIndexType.Ascending], + false, + 'cnd_User', + ); + expect(name).toBe('cnd_idx_cnd_User_email_asc'); + const permission = generateIndexName( + ['resource'], + [CompatibleIndexType.Ascending], + false, + 'cnd_Permission', + ); + const relationship = generateIndexName( + ['resource'], + [CompatibleIndexType.Ascending], + false, + 'cnd_Relationship', + ); + expect(permission).toBe('cnd_idx_cnd_Permission_resource_asc'); + expect(relationship).toBe('cnd_idx_cnd_Relationship_resource_asc'); + expect(permission).not.toBe(relationship); const unique = generateIndexName( ['room', 'createdAt'], [CompatibleIndexType.Ascending, CompatibleIndexType.Descending], true, + 'cnd_User', ); expect(unique).toBe( generateIndexName( ['room', 'createdAt'], [CompatibleIndexType.Ascending, CompatibleIndexType.Descending], true, + 'cnd_User', ), ); expect(unique).toMatch(/^cnd_uidx_/); + expect(unique).toContain('cnd_User'); + }); + + it('hashes long names with collectionName in the identity', () => { + const fields = ['one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight']; + const left = generateIndexName(fields, undefined, false, 'cnd_VeryLongTableNameOne'); + const right = generateIndexName(fields, undefined, false, 'cnd_VeryLongTableNameTwo'); + expect(left.length).toBeLessThanOrEqual(63); + expect(right.length).toBeLessThanOrEqual(63); + expect(left).not.toBe(right); }); it('keeps a provided name on the index and options', () => { - const named = ensureIndexName({ - fields: ['email'], - name: 'custom_email_idx', - }); + const named = ensureIndexName( + { + fields: ['email'], + name: 'custom_email_idx', + }, + 'cnd_User', + ); expect(resolveIndexName(named)).toBe('custom_email_idx'); expect(named.options?.name).toBe('custom_email_idx'); }); @@ -96,11 +134,14 @@ describe('index helpers', () => { }); it('preserves unique when generating a name', () => { - const unique = ensureIndexName({ - fields: ['email'], - types: [CompatibleIndexType.Ascending], - options: { unique: true }, - }); + const unique = ensureIndexName( + { + fields: ['email'], + types: [CompatibleIndexType.Ascending], + options: { unique: true }, + }, + 'cnd_User', + ); expect(unique.options?.unique).toBe(true); expect(resolveIndexName(unique)).toMatch(/uidx/); }); @@ -112,6 +153,7 @@ describe('index helpers', () => { { fields: ['a'], name: 'idx_a', options: { unique: true } }, { fields: ['b'], name: 'idx_b' }, ], + 'cnd_User', ); expect(merged.map(index => index.name)).toEqual(['idx_a', 'idx_b']); expect(merged[0].options?.unique).toBeUndefined(); @@ -241,6 +283,7 @@ describe('index helpers', () => { options: { name: 'room_1_createdAt_1', unique: false }, }, ], + 'cnd_User', ); expect(resolveIndexName(bound[0])).toBe('room_1_createdAt_1'); expect(indexIdentity(bound[0])).toEqual({ @@ -259,6 +302,7 @@ describe('index helpers', () => { options: { name: 'email_1', unique: false }, }, ], + 'cnd_User', ); expect(resolveIndexName(bound[0])).not.toBe('email_1'); expect(resolveIndexName(bound[0])).toMatch(/uidx/); @@ -280,6 +324,7 @@ describe('index helpers', () => { }, { fields: ['email'], name: 'admin_email_idx' }, ], + 'cnd_User', ); expect(unioned.map(index => index.name)).toEqual([ 'room_1_createdAt_1', @@ -387,4 +432,80 @@ describe('index helpers', () => { expect(mongoAllowsIndexType(CompatibleIndexType.Ascending)).toBe(true); expect(mongoAllowsIndexType(PostgresIndexType.BTREE)).toBe(false); }); + + it('adopts a live old global name and generates a table-qualified name when unmatched', () => { + const adopted = bindDeclaredIndexesToLive( + [{ fields: ['resource'], types: [CompatibleIndexType.Ascending] }], + [ + { + name: 'cnd_idx_resource_asc', + fields: ['resource'], + options: { name: 'cnd_idx_resource_asc', unique: false }, + }, + ], + 'cnd_Permission', + ); + expect(resolveIndexName(adopted[0])).toBe('cnd_idx_resource_asc'); + const generated = bindDeclaredIndexesToLive( + [{ fields: ['resource'], types: [CompatibleIndexType.Ascending] }], + [], + 'cnd_Relationship', + ); + expect(resolveIndexName(generated[0])).toBe('cnd_idx_cnd_Relationship_resource_asc'); + }); + + it('does not treat a missing-ns Mongo error as a live name conflict', () => { + expect( + isMongoNamespaceMissingError({ + code: 26, + codeName: 'NamespaceNotFound', + message: 'ns does not exist: test.cnd_adminapitokens', + }), + ).toBe(true); + expect(isMongoNamespaceMissingError({ message: 'unauthorized' })).toBe(false); + }); + + it('reports unsupported SQL JSON and extracted-relation indexes', () => { + expect( + sqlIndexUnsupportedReason( + 'mysql', + { fields: ['inheritanceTree'] }, + { inheritanceTree: { type: [TYPE.String] } }, + ), + ).toMatch(/MySQL JSON field 'inheritanceTree'/); + expect( + sqlIndexUnsupportedReason( + 'postgres', + { fields: ['inheritanceTree'] }, + { inheritanceTree: { type: [TYPE.String] } }, + ), + ).toBeUndefined(); + expect( + sqlIndexUnsupportedReason( + 'postgres', + { fields: ['participants'] }, + { + participants: [{ type: TYPE.Relation, model: 'User' }], + }, + ), + ).toMatch(/relation join table/); + expect( + sqlIndexUnsupportedReason( + 'postgres', + { fields: ['room', 'createdAt'] }, + { + room: { type: TYPE.Relation, model: 'ChatRoom' }, + createdAt: { type: TYPE.Date }, + }, + { timestamps: true }, + ), + ).toMatch(/relation and cannot be indexed/); + expect( + sqlIndexUnsupportedReason( + 'mysql', + { fields: ['email'] }, + { email: { type: TYPE.String } }, + ), + ).toBeUndefined(); + }); }); diff --git a/modules/database/src/__tests__/indexes/regressions.test.ts b/modules/database/src/__tests__/indexes/regressions.test.ts index 5a13612c8..385cbd536 100644 --- a/modules/database/src/__tests__/indexes/regressions.test.ts +++ b/modules/database/src/__tests__/indexes/regressions.test.ts @@ -62,4 +62,21 @@ describe('do not port old PR #643 bugs', () => { expect(sequelize).not.toMatch(/createIndexes[\s\S]*createSchemaFromAdapter/); expect(sequelize).not.toMatch(/await this\.models\[schemaName\]\.sync\(\)/); }); + + it('skips index creation on replica instanceSync paths', () => { + const mongoose = readFileSync( + resolve(process.cwd(), 'src/adapters/mongoose-adapter/index.ts'), + 'utf8', + ); + const sequelize = readFileSync( + resolve(process.cwd(), 'src/adapters/sequelize-adapter/index.ts'), + 'utf8', + ); + expect(mongoose).toMatch(/if \(indexes && !isInstanceSync\)/); + expect(mongoose).toMatch( + /if \(!isInstanceSync\) \{\s*await this\.createMongooseFieldIndexes/, + ); + expect(sequelize).toMatch(/isInstanceSync/); + expect(sequelize).toMatch(/noSync/); + }); }); diff --git a/modules/database/src/adapters/DatabaseAdapter.ts b/modules/database/src/adapters/DatabaseAdapter.ts index fa37cf1a7..85c7a3314 100644 --- a/modules/database/src/adapters/DatabaseAdapter.ts +++ b/modules/database/src/adapters/DatabaseAdapter.ts @@ -25,6 +25,7 @@ import { } from '../interfaces/index.js'; import { stitchSchema, validateExtensionFields } from './utils/extensions.js'; import { + indexNameCollection, keepDeclaredIndexExtras, persistDeclaredSchemaIndexes, } from './utils/indexes.js'; @@ -640,6 +641,7 @@ export abstract class DatabaseAdapter { schema.modelOptions.indexes = keepDeclaredIndexExtras( schema.modelOptions.indexes ?? [], model.modelOptions?.indexes, + indexNameCollection(schema), ); await this.models['_DeclaredSchema'].findByIdAndUpdate(model._id, { name: schema.name, diff --git a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts index aa423c2f7..e087905c2 100644 --- a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts +++ b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts @@ -12,6 +12,7 @@ import { checkIfMongoOptions } from './utils.js'; import { applyMongoVectorField } from '../utils/vectorMappings.js'; import { isVectorTypeName } from '../utils/vectorField.js'; import { + isArrayLikeConduitField, isCompatibleIndexType, mapCompatibleToMongo, mongoAllowsIndexType, @@ -30,10 +31,10 @@ export function schemaConverter(jsonSchema: ConduitSchema) { delete copy.fields['_id']; } copy = convertSchemaFieldIndexes(copy); - deepdash.eachDeep(copy.fields, convert); if (copy.modelOptions.indexes) { copy = convertModelOptionsIndexes(copy); } + deepdash.eachDeep(copy.fields, convert); iterDeep(copy.fields); return copy; } @@ -111,8 +112,10 @@ function convert(value: any, key: any, parentValue: any) { } function convertSchemaFieldIndexes(copy: ConduitSchema) { + const lifted: ModelOptionsIndexes[] = []; for (const field of Object.entries(copy.fields)) { - const index = (field[1] as ConduitModelField).index; + const modelField = field[1] as ConduitModelField; + const index = modelField.index; if (!index) continue; const type = index.type; const options = index.options; @@ -120,7 +123,7 @@ function convertSchemaFieldIndexes(copy: ConduitSchema) { ConduitGrpcSdk.Logger.warn( `Invalid index type for MongoDB found in '${copy.name}', ignoring index`, ); - delete (field[1] as ConduitModelField).index; + delete modelField.index; continue; } if (type && isCompatibleIndexType(type)) { @@ -131,7 +134,7 @@ function convertSchemaFieldIndexes(copy: ConduitSchema) { ConduitGrpcSdk.Logger.warn( `Invalid index options for MongoDB found in '${copy.name}', ignoring index`, ); - delete (field[1] as ConduitModelField).index; + delete modelField.index; continue; } for (const [option, optionValue] of Object.entries(options)) { @@ -139,6 +142,21 @@ function convertSchemaFieldIndexes(copy: ConduitSchema) { } delete index.options; } + if (isArrayLikeConduitField(field[1])) { + lifted.push({ + fields: [field[0]], + types: index.type !== undefined ? [index.type] : undefined, + options: { + ...(typeof index.unique === 'boolean' ? { unique: index.unique } : {}), + ...(typeof index.name === 'string' ? { name: index.name } : {}), + }, + name: index.name, + }); + delete modelField.index; + } + } + if (lifted.length) { + copy.modelOptions.indexes = [...(copy.modelOptions.indexes ?? []), ...lifted]; } return copy; } @@ -162,14 +180,14 @@ function convertModelOptionsIndexes(copy: ConduitSchema) { mappedTypes = types.map(mapCompatibleToMongo); index.types = mappedTypes; } - // compound indexes stay on modelOptions and are created after schema creation if (index.fields.length !== 1) { remaining.push(index); continue; } const modelField = copy.fields[index.fields[0]] as ConduitModelField; - if (!modelField) { - throw new Error(`Field ${index.fields[0]} in index definition doesn't exist`); + if (!modelField || isArrayLikeConduitField(modelField)) { + remaining.push(index); + continue; } if (index.options && !checkIfMongoOptions(index.options)) { ConduitGrpcSdk.Logger.warn( diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index f4fcb80c3..65832aad3 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -35,8 +35,10 @@ import { bindDeclaredIndexesToLive, ensureIndexName, findLiveIndex, + indexNameCollection, isIndexAlreadyExistsError, isIndexKeySpecsConflictError, + isMongoNamespaceMissingError, liveIndexFromMongo, liveNameConflictAllowsReuse, mapCompatibleToMongo, @@ -661,10 +663,12 @@ export class MongooseAdapter extends DatabaseAdapter { ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const collectionName = indexNameCollection(this.models[schemaName].originalSchema); const live = await this.listLiveIndexes(schemaName); const prepared = bindDeclaredIndexesToLive( this.checkIndexes(schemaName, indexes, callerModule, options?.privileged), live, + collectionName, ); const collection = this.mongoose.model(schemaName).collection; const applied: ModelOptionsIndexes[] = []; @@ -701,7 +705,7 @@ export class MongooseAdapter extends DatabaseAdapter { const relisted = await this.listLiveIndexes(schemaName); const match = findLiveIndex(relisted, index); if (match) { - applied.push(bindDeclaredIndexesToLive([index], relisted)[0]); + applied.push(bindDeclaredIndexesToLive([index], relisted, collectionName)[0]); live.splice(0, live.length, ...relisted); continue; } @@ -727,8 +731,9 @@ export class MongooseAdapter extends DatabaseAdapter { try { const result = await this.mongoose.model(schemaName).collection.indexes(); return result.map(liveIndexFromMongo); - } catch { - return []; + } catch (e) { + if (isMongoNamespaceMissingError(e)) return []; + throw e; } } @@ -785,7 +790,13 @@ export class MongooseAdapter extends DatabaseAdapter { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); const collection = this.mongoose.model(schemaName).collection; - const result = await collection.indexes(); + let result: Awaited>; + try { + result = await collection.indexes(); + } catch (e) { + if (isMongoNamespaceMissingError(e)) return []; + throw e; + } const declared = this.models[schemaName].originalSchema.modelOptions.indexes; return result.map(index => { const options: Record = {}; @@ -1102,6 +1113,7 @@ export class MongooseAdapter extends DatabaseAdapter { schema.modelOptions.indexes = bindDeclaredIndexesToLive( schema.modelOptions.indexes, live, + indexNameCollection(schema), ); } if (saveToDb) { @@ -1129,9 +1141,10 @@ export class MongooseAdapter extends DatabaseAdapter { privileged?: boolean, ): ModelOptionsIndexes[] { const schema = this.models[schemaName].originalSchema; + const collectionName = indexNameCollection(schema); const prepared: ModelOptionsIndexes[] = []; for (const raw of toMutableIndexes(indexes)) { - const index = ensureIndexName(raw); + const index = ensureIndexName(raw, collectionName); validateIndexFields(schema, index); const options = index.options; const types = index.types; diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 7f90688a2..0eec318ab 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -55,6 +55,7 @@ import { bindDeclaredIndexesToLive, ensureIndexName, findLiveIndex, + indexNameCollection, inferSqlIndexType, isIndexAlreadyExistsError, isIndexKeySpecsConflictError, @@ -66,6 +67,7 @@ import { removeIndexFromSchemaFields, sqlDialectAllowsIndexType, sqlIndexFields, + sqlIndexUnsupportedReason, toMutableIndexes, validateIndexFields, } from '../utils/indexes.js'; @@ -285,13 +287,15 @@ export abstract class SequelizeAdapter extends DatabaseAdapter this.sequelize.models, ); const dialect = this.sequelize.getDialect(); + const collectionName = this.getCollectionName(schema); const live = isInstanceSync ? [] - : await this.listLiveIndexesForCollection(this.getCollectionName(schema)); + : await this.listLiveIndexesForCollection(collectionName); if (!isInstanceSync && schema.modelOptions.indexes?.length) { schema.modelOptions.indexes = bindDeclaredIndexesToLive( schema.modelOptions.indexes, live, + collectionName, ); compiledSchema.modelOptions.indexes = schema.modelOptions.indexes; } @@ -303,6 +307,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter newSchema.modelOptions.indexes = bindDeclaredIndexesToLive( newSchema.modelOptions.indexes, live, + collectionName, ); } const relatedSchemas = await resolveRelatedSchemas( @@ -424,6 +429,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter const prepared = bindDeclaredIndexesToLive( this.checkAndConvertIndexes(schemaName, indexes, callerModule, options?.privileged), live, + collectionName, ); const queryInterface = this.sequelize.getQueryInterface(); const applied: ModelOptionsIndexes[] = []; @@ -449,6 +455,12 @@ export abstract class SequelizeAdapter extends DatabaseAdapter live.splice(0, live.length, ...relisted); continue; } + const match = findLiveIndex(relisted, index); + if (match) { + applied.push(bindDeclaredIndexesToLive([index], relisted, collectionName)[0]); + live.splice(0, live.length, ...relisted); + continue; + } failure = e; break; } @@ -456,7 +468,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter const relisted = await this.listLiveIndexesForCollection(collectionName); const match = findLiveIndex(relisted, index); if (match) { - applied.push(bindDeclaredIndexesToLive([index], relisted)[0]); + applied.push(bindDeclaredIndexesToLive([index], relisted, collectionName)[0]); live.splice(0, live.length, ...relisted); continue; } @@ -801,10 +813,23 @@ export abstract class SequelizeAdapter extends DatabaseAdapter ): ModelOptionsIndexes[] { const schema = this.models[schemaName].originalSchema; const dialect = this.sequelize.getDialect(); + const collectionName = indexNameCollection(schema); const prepared: ModelOptionsIndexes[] = []; for (const raw of toMutableIndexes(indexes)) { - const index = ensureIndexName(raw); + const index = ensureIndexName(raw, collectionName); validateIndexFields(schema, index); + const unsupported = sqlIndexUnsupportedReason( + dialect, + index, + { + ...(schema.fields as Record), + ...(schema.compiledFields as Record), + }, + { timestamps: schema.modelOptions?.timestamps }, + ); + if (unsupported) { + throw new GrpcError(status.INVALID_ARGUMENT, unsupported); + } if (index.types) { const types = normalizeIndexTypes(index.types, index.fields.length) ?? []; if (types.some(type => !sqlDialectAllowsIndexType(dialect, type))) { diff --git a/modules/database/src/adapters/utils/database-transform-utils.ts b/modules/database/src/adapters/utils/database-transform-utils.ts index ef128195a..3ca69144c 100644 --- a/modules/database/src/adapters/utils/database-transform-utils.ts +++ b/modules/database/src/adapters/utils/database-transform-utils.ts @@ -10,11 +10,13 @@ import { import { checkIfPostgresOptions } from '../sequelize-adapter/utils/index.js'; import { ensureIndexName, + indexNameCollection, isPortableDirection, isPostgresIndexType, mapCompatibleToSqlOrder, normalizeIndexTypes, sqlDialectAllowsIndexType, + sqlIndexUnsupportedReason, type SqlIndexField, } from './indexes.js'; @@ -57,8 +59,9 @@ function toSqlEngineIndex( raw: ModelOptionsIndexes, dialect: string, schemaName: string, + collectionName: string, ): SqlEngineIndex | null { - const index = ensureIndexName({ ...raw, fields: [...raw.fields] }); + const index = ensureIndexName({ ...raw, fields: [...raw.fields] }, collectionName); if (index.options && !checkIfPostgresOptions(index.options)) { skipIndex(schemaName, dialect, 'options'); return null; @@ -95,9 +98,19 @@ function toSqlEngineIndex( } export function convertModelOptionsIndexes(copy: ConduitSchema, dialect = 'postgres') { + const collectionName = indexNameCollection(copy); const converted: SqlEngineIndex[] = []; for (const raw of copy.modelOptions.indexes ?? []) { - const index = toSqlEngineIndex(raw, dialect, copy.name); + const unsupported = sqlIndexUnsupportedReason(dialect, raw, copy.fields, { + timestamps: copy.modelOptions.timestamps, + }); + if (unsupported) { + ConduitGrpcSdk.Logger.warn( + `Skipping index on '${copy.name}' for ${dialect}: ${unsupported}`, + ); + continue; + } + const index = toSqlEngineIndex(raw, dialect, copy.name, collectionName); if (index) converted.push(index); } setSqlEngineIndexes(copy, converted); @@ -105,6 +118,7 @@ export function convertModelOptionsIndexes(copy: ConduitSchema, dialect = 'postg } export function convertSchemaFieldIndexes(copy: ConduitSchema, dialect = 'postgres') { + const collectionName = indexNameCollection(copy); const indexes: SqlEngineIndex[] = []; for (const [fieldName, fieldValue] of Object.entries(copy.fields)) { const field = fieldValue as ConduitModelField; @@ -115,16 +129,23 @@ export function convertSchemaFieldIndexes(copy: ConduitSchema, dialect = 'postgr delete field.index; continue; } - const converted = toSqlEngineIndex( - { - fields: [fieldName], - types: index.type === undefined ? undefined : [index.type], - options: index.options, - name: index.name, - }, - dialect, - copy.name, - ); + const raw = { + fields: [fieldName], + types: index.type === undefined ? undefined : [index.type], + options: index.options, + name: index.name, + }; + const unsupported = sqlIndexUnsupportedReason(dialect, raw, copy.fields, { + timestamps: copy.modelOptions.timestamps, + }); + if (unsupported) { + ConduitGrpcSdk.Logger.warn( + `Skipping index on '${copy.name}' for ${dialect}: ${unsupported}`, + ); + delete field.index; + continue; + } + const converted = toSqlEngineIndex(raw, dialect, copy.name, collectionName); delete field.index; if (converted) indexes.push(converted); } diff --git a/modules/database/src/adapters/utils/indexes.ts b/modules/database/src/adapters/utils/indexes.ts index 5717bae86..e1db4cab7 100644 --- a/modules/database/src/adapters/utils/indexes.ts +++ b/modules/database/src/adapters/utils/indexes.ts @@ -124,9 +124,20 @@ export function liveNameConflictAllowsReuse( return indexIdentitiesEqual(indexIdentity(row), indexIdentity(declared)); } +export function indexNameCollection(schema: { + collectionName?: string; + name?: string; +}): string { + if (schema.collectionName && schema.collectionName.length > 0) { + return schema.collectionName; + } + return schema.name ?? ''; +} + export function bindDeclaredIndexesToLive( declared: readonly T[], live: readonly ModelOptionsIndexes[], + collectionName: string, ): T[] { return declared.map(index => { const match = findLiveIndex(live, index); @@ -145,7 +156,7 @@ export function bindDeclaredIndexesToLive( ? index.fields.every(field => typeof field === 'string') : false; if (stringFields && fields.length === index.fields.length) { - return ensureIndexName(index) as T; + return ensureIndexName(index, collectionName) as T; } return index; }); @@ -154,12 +165,13 @@ export function bindDeclaredIndexesToLive( export function keepDeclaredIndexExtras( incomingBound: readonly ModelOptionsIndexes[], existingDb: readonly ModelOptionsIndexes[] | undefined, + collectionName: string, ): ModelOptionsIndexes[] { const incoming = incomingBound.map(index => { const name = resolveIndexName(index); return name ? { ...index, name, options: { ...index.options, name } } - : ensureIndexName(index); + : ensureIndexName(index, collectionName); }); const incomingNames = new Set( incoming.map(resolveIndexName).filter((name): name is string => Boolean(name)), @@ -172,7 +184,7 @@ export function keepDeclaredIndexExtras( const name = resolveIndexName(index); if (name && incomingNames.has(name)) continue; if (incomingIdentities.has(indexIdentityKey(indexIdentity(index)))) continue; - extras.push(ensureIndexName(index)); + extras.push(ensureIndexName(index, collectionName)); } return [...incoming, ...extras]; } @@ -237,10 +249,18 @@ function typeToken(type: unknown): string { return String(type); } +function sanitizeIdentifierPart(value: string): string { + return value + .replace(/[^A-Za-z0-9_]+/g, '_') + .replace(/_+/g, '_') + .replace(/^_|_$/g, ''); +} + export function generateIndexName( fields: readonly string[], types?: ModelOptionsIndexes['types'], unique = false, + collectionName = '', ): string { const tokens = ( normalizeIndexTypes(types, fields.length) ?? fields.map(() => undefined) @@ -248,17 +268,33 @@ export function generateIndexName( .map(typeToken) .join('_'); const prefix = unique ? 'cnd_uidx' : 'cnd_idx'; - const raw = `${prefix}_${fields.join('_')}_${tokens}`.replace(/[^A-Za-z0-9_]+/g, '_'); + const table = sanitizeIdentifierPart(collectionName); + const raw = `${prefix}_${table}_${fields.join('_')}_${tokens}`.replace( + /[^A-Za-z0-9_]+/g, + '_', + ); const sanitized = raw.replace(/_+/g, '_').replace(/^_|_$/g, ''); if (sanitized.length <= SQL_IDENTIFIER_MAX_LEN) return sanitized; - const hash = createHash('sha1').update(sanitized).digest('hex').slice(0, 8); - return `${sanitized.slice(0, SQL_IDENTIFIER_MAX_LEN - 9)}_${hash}`; + const identityKey = `${collectionName}\0${unique}\0${fields.join('\0')}\0${tokens}`; + const hash = createHash('sha1').update(identityKey).digest('hex').slice(0, 8); + const maxTableLen = SQL_IDENTIFIER_MAX_LEN - prefix.length - hash.length - 2; + const tableSlice = table.slice(0, Math.max(1, maxTableLen)); + return `${prefix}_${tableSlice}_${hash}`.slice(0, SQL_IDENTIFIER_MAX_LEN); } -export function ensureIndexName(index: ModelOptionsIndexes): ModelOptionsIndexes { +export function ensureIndexName( + index: ModelOptionsIndexes, + collectionName: string, +): ModelOptionsIndexes { const existing = resolveIndexName(index); const name = - existing ?? generateIndexName(index.fields, index.types, isUniqueIndex(index)); + existing ?? + generateIndexName( + indexFieldNames(index), + index.types, + isUniqueIndex(index), + collectionName, + ); return { ...index, name, @@ -312,10 +348,11 @@ export function mongoAllowsIndexType(type: unknown): boolean { export function mergeDeclaredIndexes( existing: readonly ModelOptionsIndexes[] | undefined, incoming: readonly ModelOptionsIndexes[], + collectionName: string, ): ModelOptionsIndexes[] { - const merged = declaredIndexMap(existing); + const merged = declaredIndexMap(existing, collectionName); for (const index of incoming) { - const named = ensureIndexName(index); + const named = ensureIndexName(index, collectionName); const name = resolveIndexName(named); if (name && !merged.has(name)) merged.set(name, named); } @@ -463,6 +500,8 @@ export async function persistDeclaredSchemaIndexes(args: { modelOptions: { indexes?: ModelOptionsIndexes[] | readonly ModelOptionsIndexes[] }; fields?: Record; compiledFields?: Record; + collectionName?: string; + name?: string; }; applied?: ModelOptionsIndexes[]; droppedNames?: string[]; @@ -474,9 +513,10 @@ export async function persistDeclaredSchemaIndexes(args: { const memoryIndexes = (args.originalSchema.modelOptions.indexes ?? []) as ModelOptionsIndexes[]; const dbIndexes = found?.modelOptions?.indexes ?? memoryIndexes; + const collectionName = indexNameCollection(args.originalSchema) || args.schemaName; const next = args.droppedNames ? removeDeclaredIndexes(dbIndexes, args.droppedNames) - : mergeDeclaredIndexes(dbIndexes, args.applied ?? []); + : mergeDeclaredIndexes(dbIndexes, args.applied ?? [], collectionName); args.originalSchema.modelOptions.indexes = next; if (!found) return false; await args.declaredSchemaModel.findByIdAndUpdate(found._id, { @@ -497,10 +537,11 @@ export function collectExistingIndexNames( export function declaredIndexMap( indexes: readonly ModelOptionsIndexes[] | undefined, + collectionName = '', ): Map { const map = new Map(); for (const index of indexes ?? []) { - const named = ensureIndexName(index); + const named = ensureIndexName(index, collectionName); const name = resolveIndexName(named); if (name) map.set(name, named); } @@ -545,3 +586,89 @@ export function inferSqlIndexType( } return undefined; } + +export function isMongoNamespaceMissingError(error: unknown): boolean { + const err = error as { code?: number | string; codeName?: string; message?: string }; + if (err?.code === 26 || err?.code === '26' || err?.codeName === 'NamespaceNotFound') { + return true; + } + return /ns does not exist|ns not found|namespace not found/i.test(err?.message ?? ''); +} + +export function isArrayLikeConduitField(field: unknown): boolean { + if (Array.isArray(field)) return true; + return Boolean( + field && + typeof field === 'object' && + Array.isArray((field as { type?: unknown }).type), + ); +} + +function isRelationElement(value: unknown): boolean { + return Boolean( + value && + typeof value === 'object' && + (value as { type?: unknown }).type === 'Relation', + ); +} + +export function isExtractedArrayRelationField(field: unknown): boolean { + if (Array.isArray(field)) return isRelationElement(field[0]); + if (field && typeof field === 'object') { + const type = (field as { type?: unknown }).type; + if (Array.isArray(type)) return isRelationElement(type[0]); + } + return false; +} + +export function isMysqlJsonLikeField(field: unknown): boolean { + if (!field || typeof field !== 'object') return false; + if (Array.isArray(field)) { + const first = field[0]; + if (isRelationElement(first)) return false; + if (typeof first === 'string') return first === 'JSON'; + return Boolean(first && typeof first === 'object'); + } + const type = (field as { type?: unknown }).type; + if (type === 'JSON') return true; + return Array.isArray(type); +} + +export function isScalarRelationField(field: unknown): boolean { + return Boolean( + field && + typeof field === 'object' && + !Array.isArray(field) && + (field as { type?: unknown }).type === 'Relation', + ); +} + +export function sqlIndexUnsupportedReason( + dialect: string, + index: Pick, + fields: Record, + options?: { timestamps?: boolean }, +): string | undefined { + const present = new Set(Object.keys(fields)); + if (options?.timestamps) { + present.add('createdAt'); + present.add('updatedAt'); + } + const mysqlJson = dialect === 'mysql' || dialect === 'mariadb'; + for (const name of indexFieldNames(index)) { + const field = fields[name]; + if (isExtractedArrayRelationField(field)) { + return `Field '${name}' is stored as a relation join table and cannot be indexed on SQL`; + } + if (isScalarRelationField(field)) { + return `Field '${name}' is a relation and cannot be indexed on SQL`; + } + if (mysqlJson && isMysqlJsonLikeField(field)) { + return `Compatible btree indexes are not supported on MySQL JSON field '${name}'`; + } + if (field === undefined && !present.has(name)) { + return `Field '${name}' is stored as a relation join table and cannot be indexed on SQL`; + } + } + return undefined; +} diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index a21c92933..cde27cee7 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -801,7 +801,15 @@ export class SchemaAdmin { const indexes: Array = []; for (const schema of schemas) { if (!this.database.models[schema.name]) continue; - const schemaIndexes = await this.database.getIndexes(schema.name); + let schemaIndexes: ModelOptionsIndexes[]; + try { + schemaIndexes = await this.database.getIndexes(schema.name); + } catch (e) { + ConduitGrpcSdk.Logger.warn( + `Skipping indexes export for schema '${schema.name}': ${(e as Error).message}`, + ); + continue; + } if (isNil(schemaIndexes) || isEmpty(schemaIndexes)) continue; indexes.push( ...schemaIndexes.map(index => ({ ...index, schemaName: schema.name })), @@ -837,10 +845,12 @@ export class SchemaAdmin { `Requested schema not found: ${schemaName}`, ); } + const collectionName = + this.database.models[schemaName].originalSchema.collectionName ?? schemaName; const existing = await this.database.getIndexes(schemaName); const existingNames = collectExistingIndexNames(existing); const toCreate = schemaIndexes - .map(index => ensureIndexName(index)) + .map(index => ensureIndexName(index, collectionName)) .filter(index => { const name = resolveIndexName(index); return !name || !existingNames.has(name); From 7d4c194ceda8acd9e61396dceff923e0e362ca47 Mon Sep 17 00:00:00 2001 From: Christina Papadogianni <59121443+ChrisPdgn@users.noreply.github.com> Date: Wed, 16 Sep 2026 09:19:51 +0000 Subject: [PATCH 6/9] fix(chat): restore Promise.all schema register Keep Chat and Authorization registerSchemas on Promise.all, matching other Conduit modules. Sequential register stays in Database onServerStart. --- modules/authorization/src/Authorization.ts | 12 +++++++----- modules/chat/src/Chat.ts | 17 ++++++++++------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/modules/authorization/src/Authorization.ts b/modules/authorization/src/Authorization.ts index 51a78aae6..e9755e731 100644 --- a/modules/authorization/src/Authorization.ts +++ b/modules/authorization/src/Authorization.ts @@ -380,11 +380,13 @@ export default class Authorization extends ManagedModule { return resource; } - protected async registerSchemas(): Promise { - for (const model of Object.values(models)) { + protected registerSchemas(): Promise { + const promises = Object.values(models).map(model => { const modelInstance = model.getInstance(this.database); - await this.database.createSchemaFromAdapter(modelInstance); - await this.database.migrate(modelInstance.name); - } + return this.database + .createSchemaFromAdapter(modelInstance) + .then(() => this.database.migrate(modelInstance.name)); + }); + return Promise.all(promises); } } diff --git a/modules/chat/src/Chat.ts b/modules/chat/src/Chat.ts index 1c3ca0201..f817aa463 100644 --- a/modules/chat/src/Chat.ts +++ b/modules/chat/src/Chat.ts @@ -396,18 +396,21 @@ export default class Chat extends ManagedModule { }); } - protected async registerSchemas(): Promise { - for (const model of Object.values(models)) { + protected registerSchemas(): Promise { + const promises = Object.values(models).map(model => { const modelInstance = model.getInstance(this.database); + //TODO: add support for multiple schemas types if ( Object.keys((modelInstance as ConduitActiveSchema).fields) - .length === 0 + .length !== 0 ) { - continue; + // borrowed foreign model + return this.database + .createSchemaFromAdapter(modelInstance) + .then(() => this.database.migrate(modelInstance.name)); } - await this.database.createSchemaFromAdapter(modelInstance); - await this.database.migrate(modelInstance.name); - } + }); + return Promise.all(promises); } private scheduleAppRouteRefresh() { From dbbd7c3e4a297a0e2a6e47e7d91c3c7100de525f Mon Sep 17 00:00:00 2001 From: Christina Papadogianni <59121443+ChrisPdgn@users.noreply.github.com> Date: Wed, 16 Sep 2026 10:11:08 +0000 Subject: [PATCH 7/9] fix(database): unlock SQL ChatRoom ChatParticipantsLog register Publish the Sequelize adapter into models before resolving relations and wait for the related adapter to exist instead of .synced so Promise.all Chat register cannot deadlock on that cycle. --- .../src/__tests__/sequelize-relations.test.ts | 38 +++++++++++++++++++ .../sequelize-adapter/SequelizeSchema.ts | 38 ++++++++++--------- .../src/adapters/sequelize-adapter/index.ts | 13 ++++--- .../sequelize-adapter/utils/schema.ts | 31 ++++++++------- 4 files changed, 82 insertions(+), 38 deletions(-) create mode 100644 modules/database/src/__tests__/sequelize-relations.test.ts diff --git a/modules/database/src/__tests__/sequelize-relations.test.ts b/modules/database/src/__tests__/sequelize-relations.test.ts new file mode 100644 index 000000000..eacd60cf7 --- /dev/null +++ b/modules/database/src/__tests__/sequelize-relations.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from '@jest/globals'; +import { ConduitDatabaseSchema } from '../interfaces/index.js'; +import { resolveRelatedSchemas } from '../adapters/sequelize-adapter/utils/schema.js'; + +describe('resolveRelatedSchemas', () => { + it('accepts a related model that exists but is not yet synced', async () => { + const chatRoom = { synced: false }; + const models = { + ChatRoom: chatRoom, + ChatParticipantsLog: { synced: false }, + }; + const started = Date.now(); + const related = await resolveRelatedSchemas( + { name: 'ChatParticipantsLog' } as ConduitDatabaseSchema, + { chatRoom: { type: 'Relation', model: 'ChatRoom' } }, + models, + ); + expect(Date.now() - started).toBeLessThan(400); + expect(related.chatRoom).toBe(chatRoom); + }); + + it('resolves a cyclic peer that appears in models before sync', async () => { + const chatRoom = { synced: false }; + const models: Record = { ChatRoom: chatRoom }; + const pending = resolveRelatedSchemas( + { name: 'ChatRoom' } as ConduitDatabaseSchema, + { + participantsLog: [{ type: 'Relation', model: 'ChatParticipantsLog' }], + }, + models, + ); + const chatParticipantsLog = { synced: false }; + models.ChatParticipantsLog = chatParticipantsLog; + await expect(pending).resolves.toEqual({ + participantsLog: [chatParticipantsLog], + }); + }); +}); diff --git a/modules/database/src/adapters/sequelize-adapter/SequelizeSchema.ts b/modules/database/src/adapters/sequelize-adapter/SequelizeSchema.ts index 47b93c6a2..09b062ba6 100644 --- a/modules/database/src/adapters/sequelize-adapter/SequelizeSchema.ts +++ b/modules/database/src/adapters/sequelize-adapter/SequelizeSchema.ts @@ -93,32 +93,36 @@ export class SequelizeSchema extends SchemaAdapter> { } : undefined, }); - // if a relation is to self, then it will be undefined inside the extractedRelations - // so we set it manually to self - for (const relation in extractedRelations) { - if ( - Array.isArray(extractedRelations[relation]) && - (extractedRelations[relation] as SequelizeSchema[])[0] === undefined - ) { - extractedRelations[relation] = [this]; - } else if ( - !Array.isArray(extractedRelations[relation]) && - extractedRelations[relation] === undefined - ) { - extractedRelations[relation] = this; - } - } this.objectDotPaths = []; for (const concatenatedKey of Object.keys(objectPaths)) { const { parentKey, childKey } = objectPaths[concatenatedKey]; this.objectDotPaths.push(`${parentKey}.${childKey}`); this.objectDotPathMapping[concatenatedKey] = `${parentKey}.${childKey}`; } + if (Object.keys(extractedRelations).length > 0) { + this.bindExtractedRelations(); + } + } + + bindExtractedRelations() { + for (const relation in this.extractedRelations) { + if ( + Array.isArray(this.extractedRelations[relation]) && + (this.extractedRelations[relation] as SequelizeSchema[])[0] === undefined + ) { + this.extractedRelations[relation] = [this]; + } else if ( + !Array.isArray(this.extractedRelations[relation]) && + this.extractedRelations[relation] === undefined + ) { + this.extractedRelations[relation] = this; + } + } extractRelations( this.originalSchema.name, - originalSchema, + this.originalSchema, this.model, - extractedRelations, + this.extractedRelations, ); } diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 0eec318ab..d058847cb 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -310,11 +310,9 @@ export abstract class SequelizeAdapter extends DatabaseAdapter collectionName, ); } - const relatedSchemas = await resolveRelatedSchemas( - schema, - extractedRelations, - this.models, - ); + const relatedSchemas: { + [key: string]: SequelizeSchema | SequelizeSchema[]; + } = {}; this.models[schema.name] = new SequelizeSchema( this.grpcSdk, this.sequelize, @@ -324,6 +322,11 @@ export abstract class SequelizeAdapter extends DatabaseAdapter relatedSchemas, objectPaths, ); + Object.assign( + relatedSchemas, + await resolveRelatedSchemas(schema, extractedRelations, this.models), + ); + this.models[schema.name].bindExtractedRelations(); try { const noSync = diff --git a/modules/database/src/adapters/sequelize-adapter/utils/schema.ts b/modules/database/src/adapters/sequelize-adapter/utils/schema.ts index 4138af8ca..f96bcad25 100644 --- a/modules/database/src/adapters/sequelize-adapter/utils/schema.ts +++ b/modules/database/src/adapters/sequelize-adapter/utils/schema.ts @@ -232,10 +232,10 @@ export async function resolveRelatedSchemas( const rel = Array.isArray(extractedRelations[relation]) ? (extractedRelations[relation] as UntypedArray)[0] : extractedRelations[relation]; - if ( - (!models[rel.model] || !models[rel.model].synced) && - schema.name !== rel.model - ) { + // Wait until the related adapter exists, not until it has synced. + // Same-module cycles construct in parallel under Promise.all; waiting + // for .synced deadlocks because neither side can sync until this returns. + if (!models[rel.model] && schema.name !== rel.model) { if (!pendingModels.includes(rel.model)) { pendingModels.push(rel.model); } @@ -255,19 +255,18 @@ export async function resolveRelatedSchemas( while (pendingModels.length > 0) { await ConduitGrpcSdk.Sleep(500); pendingModels = pendingModels.filter(model => { - if (!models[model] || !models[model].synced) { + if (!models[model]) { return true; - } else { - for (const schema in relatedSchemas) { - const simple = Array.isArray(relatedSchemas[schema]) - ? (relatedSchemas[schema] as SequelizeSchema[])[0] - : relatedSchemas[schema]; - // @ts-ignore - if (simple === model) { - relatedSchemas[schema] = Array.isArray(relatedSchemas[schema]) - ? [models[model]] - : models[model]; - } + } + for (const schemaName in relatedSchemas) { + const simple = Array.isArray(relatedSchemas[schemaName]) + ? (relatedSchemas[schemaName] as SequelizeSchema[])[0] + : relatedSchemas[schemaName]; + // @ts-ignore + if (simple === model) { + relatedSchemas[schemaName] = Array.isArray(relatedSchemas[schemaName]) + ? [models[model]] + : models[model]; } } }); From d6ac2ec425d5717bfdb84aac781a9aa3b58b93f6 Mon Sep 17 00:00:00 2001 From: Christina Papadogianni <59121443+ChrisPdgn@users.noreply.github.com> Date: Thu, 17 Sep 2026 08:24:55 +0000 Subject: [PATCH 8/9] fix(database): index SQL scalar Relations via declared *Id mapping Map ChatMessage.room to engine roomId for SQL bind/create/get while keeping index names on declared fields. Array/extracted Relations (ChatRoom.participants) and MySQL JSON btree stay skipped. Authz String *Id fields are left unchanged. --- .../src/__tests__/indexes/adapters.test.ts | 103 ++++++++++++++++ .../src/__tests__/indexes/admin.test.ts | 81 ++++++++++++- .../src/__tests__/indexes/converters.test.ts | 84 ++++++++++++- .../src/__tests__/indexes/helpers.test.ts | 108 ++++++++++++++++- .../src/adapters/sequelize-adapter/index.ts | 56 +++++++-- .../postgres-adapter/PgSchemaConverter.ts | 2 + .../sql-adapter/SqlSchemaConverter.ts | 2 + .../utils/database-transform-utils.ts | 39 ++++++- .../database/src/adapters/utils/indexes.ts | 110 ++++++++++++++++-- modules/database/src/admin/schema.admin.ts | 17 ++- 10 files changed, 571 insertions(+), 31 deletions(-) diff --git a/modules/database/src/__tests__/indexes/adapters.test.ts b/modules/database/src/__tests__/indexes/adapters.test.ts index 924b9cea8..918bfd699 100644 --- a/modules/database/src/__tests__/indexes/adapters.test.ts +++ b/modules/database/src/__tests__/indexes/adapters.test.ts @@ -704,4 +704,107 @@ describe('sequelize adapter indexes', () => { expect(addIndex).not.toHaveBeenCalled(); expect(findByIdAndUpdate).not.toHaveBeenCalled(); }); + + it('creates scalar Relation compounds on roomId while persisting declared room', async () => { + const { adapter, addIndex, findByIdAndUpdate, originalSchema } = + makeSequelizeAdapter('postgres'); + originalSchema.fields.room = { type: TYPE.Relation, model: 'ChatRoom' }; + originalSchema.compiledFields.room = { type: TYPE.Relation, model: 'ChatRoom' }; + await adapter.createIndexes( + 'User', + [ + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + 'database', + ); + expect(addIndex.mock.calls[0][1].fields).toEqual([ + { name: 'roomId', order: 'ASC' }, + { name: 'createdAt', order: 'ASC' }, + ]); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { fields: string[]; name?: string }[] }; + }; + expect(update.modelOptions.indexes[0].fields).toEqual(['room', 'createdAt']); + expect(update.modelOptions.indexes[0].name).toBe( + 'cnd_idx_custom_users_room_createdAt_asc_asc', + ); + }); + + it('adopts a live roomId compound for declared room and skips addIndex', async () => { + const { adapter, addIndex, showIndex, findByIdAndUpdate, originalSchema } = + makeSequelizeAdapter(); + originalSchema.fields.room = { type: TYPE.Relation, model: 'ChatRoom' }; + originalSchema.compiledFields.room = { type: TYPE.Relation, model: 'ChatRoom' }; + showIndex.mockResolvedValue([ + { + name: 'roomId_createdAt', + unique: false, + fields: [{ attribute: 'roomId' }, { attribute: 'createdAt' }], + }, + ]); + await adapter.createIndexes( + 'User', + [ + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + 'database', + ); + expect(addIndex).not.toHaveBeenCalled(); + const update = findByIdAndUpdate.mock.calls[0][1] as { + modelOptions: { indexes: { fields: string[]; name?: string }[] }; + }; + expect(update.modelOptions.indexes[0].name).toBe('roomId_createdAt'); + expect(update.modelOptions.indexes[0].fields).toEqual(['room', 'createdAt']); + }); + + it('returns declared room names from live roomId indexes', async () => { + const { adapter, showIndex, originalSchema } = makeSequelizeAdapter(); + originalSchema.fields.room = { type: TYPE.Relation, model: 'ChatRoom' }; + originalSchema.compiledFields.room = { type: TYPE.Relation, model: 'ChatRoom' }; + originalSchema.modelOptions.indexes = [ + { + fields: ['room', 'createdAt'], + name: 'cnd_idx_custom_users_room_createdAt_asc_asc', + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ]; + showIndex.mockResolvedValue([ + { + name: 'roomId_createdAt', + unique: false, + fields: [{ attribute: 'roomId' }, { attribute: 'createdAt' }], + definition: + 'CREATE INDEX roomId_createdAt ON custom_users USING btree ("roomId", "createdAt")', + }, + ]); + const result = await adapter.getIndexes('User'); + expect(result[0].fields).toEqual(['room', 'createdAt']); + expect(result[0].name).toBe('roomId_createdAt'); + expect(result[0].types).toEqual([ + CompatibleIndexType.Ascending, + CompatibleIndexType.Ascending, + ]); + }); + + it('leaves Authz String *Id fields unchanged on SQL create', async () => { + const { adapter, addIndex, originalSchema } = makeSequelizeAdapter(); + originalSchema.fields.resource = { type: TYPE.String }; + originalSchema.fields.resourceId = { type: TYPE.String }; + originalSchema.compiledFields.resource = { type: TYPE.String }; + originalSchema.compiledFields.resourceId = { type: TYPE.String }; + await adapter.createIndexes( + 'User', + [{ fields: ['resource'], types: [CompatibleIndexType.Ascending] }], + 'database', + ); + expect(addIndex.mock.calls[0][1].fields).toEqual([ + { name: 'resource', order: 'ASC' }, + ]); + }); }); diff --git a/modules/database/src/__tests__/indexes/admin.test.ts b/modules/database/src/__tests__/indexes/admin.test.ts index 5ee3c24e6..7eaf4f5e0 100644 --- a/modules/database/src/__tests__/indexes/admin.test.ts +++ b/modules/database/src/__tests__/indexes/admin.test.ts @@ -23,6 +23,20 @@ function setup() { _id: 'schema-1', name: 'User', ownerModule: 'database', + fields: { + email: { type: 'String' }, + room: { type: 'Relation', model: 'ChatRoom' }, + createdAt: { type: 'Date' }, + resource: { type: 'String' }, + resourceId: { type: 'String' }, + }, + compiledFields: { + email: { type: 'String' }, + room: { type: 'Relation', model: 'ChatRoom' }, + createdAt: { type: 'Date' }, + resource: { type: 'String' }, + resourceId: { type: 'String' }, + }, }); const findMany = jest.fn().mockResolvedValue([{ name: 'User' }, { name: 'ChatRoom' }]); const countDocuments = jest.fn().mockResolvedValue(2); @@ -41,7 +55,25 @@ function setup() { deleteIndexes, systemSchemas: ['_DeclaredSchema'], models: { - User: { originalSchema: { ownerModule: 'database' } }, + User: { + originalSchema: { + ownerModule: 'database', + fields: { + email: { type: 'String' }, + room: { type: 'Relation', model: 'ChatRoom' }, + createdAt: { type: 'Date' }, + resource: { type: 'String' }, + resourceId: { type: 'String' }, + }, + compiledFields: { + email: { type: 'String' }, + room: { type: 'Relation', model: 'ChatRoom' }, + createdAt: { type: 'Date' }, + resource: { type: 'String' }, + resourceId: { type: 'String' }, + }, + }, + }, ChatRoom: { originalSchema: { ownerModule: 'chat' } }, }, } as unknown as DatabaseAdapter; @@ -71,6 +103,38 @@ describe('SchemaAdmin indexes', () => { ); }); + it('canonicalizes inbound roomId to room when room is a scalar Relation', async () => { + const { admin, createIndexes } = setup(); + await admin.createIndexes( + makeCall({ + id: 'schema-1', + indexes: [{ fields: ['roomId', 'createdAt'] }], + }), + ); + expect(createIndexes).toHaveBeenCalledWith( + 'User', + [{ fields: ['room', 'createdAt'] }], + ADMIN_INDEX_CALLER, + { privileged: true }, + ); + }); + + it('does not rewrite Authz String *Id fields on Admin create', async () => { + const { admin, createIndexes } = setup(); + await admin.createIndexes( + makeCall({ + id: 'schema-1', + indexes: [{ fields: ['resourceId'] }], + }), + ); + expect(createIndexes).toHaveBeenCalledWith( + 'User', + [{ fields: ['resourceId'] }], + ADMIN_INDEX_CALLER, + { privileged: true }, + ); + }); + it('exports indexes with skip/limit pagination', async () => { const { admin, findMany, countDocuments, getIndexes } = setup(); const result = (await admin.exportIndexes(makeCall({ skip: 10, limit: 5 }))) as { @@ -140,6 +204,21 @@ describe('SchemaAdmin indexes', () => { ); }); + it('canonicalizes inbound roomId on import when room is a scalar Relation', async () => { + const { admin, createIndexes } = setup(); + await admin.importIndexes( + makeCall({ + indexes: [{ schemaName: 'User', fields: ['roomId'], name: 'msg_room' }], + }), + ); + expect(createIndexes).toHaveBeenCalledWith( + 'User', + [expect.objectContaining({ fields: ['room'], name: 'msg_room' })], + ADMIN_INDEX_CALLER, + { privileged: false }, + ); + }); + it('rejects an empty import payload', async () => { const { admin } = setup(); await expect(admin.importIndexes(makeCall({ indexes: [] }))).rejects.toMatchObject({ diff --git a/modules/database/src/__tests__/indexes/converters.test.ts b/modules/database/src/__tests__/indexes/converters.test.ts index a53643b6d..0a803f30f 100644 --- a/modules/database/src/__tests__/indexes/converters.test.ts +++ b/modules/database/src/__tests__/indexes/converters.test.ts @@ -192,7 +192,7 @@ describe('SQL index converters', () => { expect(postgres.modelOptions.indexes).toHaveLength(1); }); - it('skips extracted array-relation indexes on SQL and relation-field compounds', () => { + it('skips extracted array-relation indexes on SQL and keeps scalar relation compounds', () => { const chatRoom = new ConduitSchema( 'ChatRoom', { @@ -234,8 +234,24 @@ describe('SQL index converters', () => { ); const [mysqlMessage] = sqlSchemaConverter(message, 'mysql'); const [pgMessage] = pgSchemaConverter(message); - expect(mysqlMessage.modelOptions.indexes ?? []).toHaveLength(0); - expect(pgMessage.modelOptions.indexes ?? []).toHaveLength(0); + const expectedFields = [ + { name: 'roomId', order: 'ASC' }, + { name: 'createdAt', order: 'ASC' }, + ]; + expect(mysqlMessage.modelOptions.indexes).toHaveLength(1); + expect(pgMessage.modelOptions.indexes).toHaveLength(1); + expect((mysqlMessage.modelOptions.indexes![0] as ConvertedSqlIndex).fields).toEqual( + expectedFields, + ); + expect((pgMessage.modelOptions.indexes![0] as ConvertedSqlIndex).fields).toEqual( + expectedFields, + ); + expect((mysqlMessage.modelOptions.indexes![0] as { name?: string }).name).toBe( + 'cnd_idx_Message_room_createdAt_asc_asc', + ); + expect((pgMessage.modelOptions.indexes![0] as { name?: string }).name).toBe( + 'cnd_idx_Message_room_createdAt_asc_asc', + ); const scalar = new ConduitSchema( 'Message', @@ -258,6 +274,68 @@ describe('SQL index converters', () => { expect(mysqlScalar.modelOptions.indexes).toHaveLength(1); expect(pgScalar.modelOptions.indexes).toHaveLength(1); }); + + it('lifts field-level scalar Relation indexes before extract and skips field-level arrays', () => { + const message = new ConduitSchema( + 'Message', + { + room: { + type: TYPE.Relation, + model: 'ChatRoom', + required: true, + index: { type: CompatibleIndexType.Ascending }, + }, + createdAt: { type: TYPE.Date }, + }, + { timestamps: true }, + ); + const [pg] = pgSchemaConverter(message); + const [mysql] = sqlSchemaConverter(message, 'mysql'); + expect(pg.modelOptions.indexes).toHaveLength(1); + expect(mysql.modelOptions.indexes).toHaveLength(1); + expect((pg.modelOptions.indexes![0] as { name?: string }).name).toBe( + 'cnd_idx_Message_room_asc', + ); + expect((pg.modelOptions.indexes![0] as ConvertedSqlIndex).fields).toEqual([ + { name: 'roomId', order: 'ASC' }, + ]); + expect(pg.fields.room).toBeUndefined(); + + const chatRoom = new ConduitSchema( + 'ChatRoom', + { + participants: { + type: [{ type: TYPE.Relation, model: 'User', required: true }], + index: { type: CompatibleIndexType.Ascending }, + }, + deleted: { type: TYPE.Boolean }, + }, + { timestamps: true }, + ); + const [pgRoom] = pgSchemaConverter(chatRoom); + const [mysqlRoom] = sqlSchemaConverter(chatRoom, 'mysql'); + expect(pgRoom.modelOptions.indexes ?? []).toHaveLength(0); + expect(mysqlRoom.modelOptions.indexes ?? []).toHaveLength(0); + }); + + it('does not rewrite Authz String *Id fields to a second Id suffix', () => { + const permission = convertModelOptionsIndexes( + new ConduitSchema( + 'Permission', + { + resource: { type: TYPE.String }, + resourceId: { type: TYPE.String }, + }, + { indexes: [{ fields: ['resource'], types: [CompatibleIndexType.Ascending] }] }, + ), + 'postgres', + ); + const index = permission.modelOptions.indexes![0] as ConvertedSqlIndex & { + name?: string; + }; + expect(index.name).toBe('cnd_idx_Permission_resource_asc'); + expect(index.fields[0]).toEqual({ name: 'resource', order: 'ASC' }); + }); }); describe('mongoose SchemaConverter indexes', () => { diff --git a/modules/database/src/__tests__/indexes/helpers.test.ts b/modules/database/src/__tests__/indexes/helpers.test.ts index 702d44c45..8544398c7 100644 --- a/modules/database/src/__tests__/indexes/helpers.test.ts +++ b/modules/database/src/__tests__/indexes/helpers.test.ts @@ -9,7 +9,9 @@ import { import { assertUniqueIndexPrivilege, bindDeclaredIndexesToLive, + canonicalizeDeclaredIndexFields, collectExistingIndexNames, + collectSchemaIndexFields, ensureIndexName, generateIndexName, indexIdentity, @@ -21,6 +23,8 @@ import { liveNameConflictAllowsReuse, mapCompatibleToMongo, mapCompatibleToSqlOrder, + mapIndexFieldsToDeclared, + mapIndexFieldsToSqlEngine, mergeDeclaredIndexes, mongoAllowsIndexType, overlayDeclaredOnLive, @@ -28,7 +32,10 @@ import { removeDeclaredIndexes, removeIndexFromSchemaFields, resolveIndexName, + sqlDeclaredIndexFieldName, sqlDialectAllowsIndexType, + sqlEngineIndexFieldName, + sqlIndexFieldNormalizer, sqlIndexFields, sqlIndexUnsupportedReason, validateIndexFields, @@ -499,7 +506,7 @@ describe('index helpers', () => { }, { timestamps: true }, ), - ).toMatch(/relation and cannot be indexed/); + ).toBeUndefined(); expect( sqlIndexUnsupportedReason( 'mysql', @@ -508,4 +515,103 @@ describe('index helpers', () => { ), ).toBeUndefined(); }); + + it('maps declared scalar relations to engine *Id and never Authz String *Id fields', () => { + const fields = collectSchemaIndexFields({ + fields: { + room: { type: TYPE.Relation, model: 'ChatRoom' }, + createdAt: { type: TYPE.Date }, + resource: { type: TYPE.String }, + resourceId: { type: TYPE.String }, + subject: { type: TYPE.String }, + subjectId: { type: TYPE.String }, + }, + }); + expect(sqlEngineIndexFieldName('room', fields)).toBe('roomId'); + expect(sqlDeclaredIndexFieldName('roomId', fields)).toBe('room'); + expect(sqlEngineIndexFieldName('resource', fields)).toBe('resource'); + expect(sqlEngineIndexFieldName('resourceId', fields)).toBe('resourceId'); + expect(sqlDeclaredIndexFieldName('resourceId', fields)).toBe('resourceId'); + expect(sqlDeclaredIndexFieldName('subjectId', fields)).toBe('subjectId'); + expect(mapIndexFieldsToSqlEngine({ fields: ['room', 'createdAt'] }, fields)).toEqual([ + 'roomId', + 'createdAt', + ]); + expect(mapIndexFieldsToDeclared({ fields: ['roomId', 'createdAt'] }, fields)).toEqual( + ['room', 'createdAt'], + ); + expect( + canonicalizeDeclaredIndexFields({ fields: ['roomId', 'createdAt'] }, fields).fields, + ).toEqual(['room', 'createdAt']); + expect( + canonicalizeDeclaredIndexFields({ fields: ['resourceId'] }, fields).fields, + ).toEqual(['resourceId']); + }); + + it('generates index names from declared fields, not engine *Id columns', () => { + expect( + generateIndexName( + ['room', 'createdAt'], + [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + false, + 'cnd_ChatMessage', + ), + ).toBe('cnd_idx_cnd_ChatMessage_room_createdAt_asc_asc'); + expect( + generateIndexName( + ['roomId', 'createdAt'], + [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + false, + 'cnd_ChatMessage', + ), + ).not.toBe('cnd_idx_cnd_ChatMessage_room_createdAt_asc_asc'); + }); + + it('binds live engine roomId to declared room without renaming Authz String *Id', () => { + const fields = { + room: { type: TYPE.Relation, model: 'ChatRoom' }, + createdAt: { type: TYPE.Date }, + resource: { type: TYPE.String }, + resourceId: { type: TYPE.String }, + }; + const normalize = sqlIndexFieldNormalizer(fields); + const bound = bindDeclaredIndexesToLive( + [ + { + fields: ['room', 'createdAt'], + types: [CompatibleIndexType.Ascending, CompatibleIndexType.Ascending], + }, + ], + [ + { + name: 'roomId_createdAt', + fields: ['roomId', 'createdAt'], + options: { name: 'roomId_createdAt', unique: false }, + }, + ], + 'cnd_ChatMessage', + normalize, + ); + expect(resolveIndexName(bound[0])).toBe('roomId_createdAt'); + expect(bound[0].fields).toEqual(['room', 'createdAt']); + expect(indexIdentity(bound[0])).toEqual({ + fields: ['room', 'createdAt'], + unique: false, + }); + + const authz = bindDeclaredIndexesToLive( + [{ fields: ['resource'], types: [CompatibleIndexType.Ascending] }], + [ + { + name: 'cnd_idx_cnd_Permission_resource_asc', + fields: ['resource'], + options: { name: 'cnd_idx_cnd_Permission_resource_asc', unique: false }, + }, + ], + 'cnd_Permission', + normalize, + ); + expect(resolveIndexName(authz[0])).toBe('cnd_idx_cnd_Permission_resource_asc'); + expect(authz[0].fields).toEqual(['resource']); + }); }); diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index d058847cb..a268b9fe0 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -53,6 +53,8 @@ import { import { assertUniqueIndexPrivilege, bindDeclaredIndexesToLive, + canonicalizeDeclaredIndexFields, + collectSchemaIndexFields, ensureIndexName, findLiveIndex, indexNameCollection, @@ -65,8 +67,10 @@ import { overlayDeclaredOnLive, normalizeIndexTypes, removeIndexFromSchemaFields, + sqlDeclaredIndexFieldName, sqlDialectAllowsIndexType, - sqlIndexFields, + sqlEngineIndexFields, + sqlIndexFieldNormalizer, sqlIndexUnsupportedReason, toMutableIndexes, validateIndexFields, @@ -288,6 +292,8 @@ export abstract class SequelizeAdapter extends DatabaseAdapter ); const dialect = this.sequelize.getDialect(); const collectionName = this.getCollectionName(schema); + const schemaFields = collectSchemaIndexFields(schema); + const normalizeFields = sqlIndexFieldNormalizer(schemaFields); const live = isInstanceSync ? [] : await this.listLiveIndexesForCollection(collectionName); @@ -296,6 +302,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter schema.modelOptions.indexes, live, collectionName, + normalizeFields, ); compiledSchema.modelOptions.indexes = schema.modelOptions.indexes; } @@ -308,6 +315,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter newSchema.modelOptions.indexes, live, collectionName, + normalizeFields, ); } const relatedSchemas: { @@ -428,24 +436,27 @@ export abstract class SequelizeAdapter extends DatabaseAdapter if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); const collectionName = this.models[schemaName].originalSchema.collectionName; + const schemaFields = collectSchemaIndexFields(this.models[schemaName].originalSchema); + const normalizeFields = sqlIndexFieldNormalizer(schemaFields); const live = await this.listLiveIndexesForCollection(collectionName); const prepared = bindDeclaredIndexesToLive( this.checkAndConvertIndexes(schemaName, indexes, callerModule, options?.privileged), live, collectionName, + normalizeFields, ); const queryInterface = this.sequelize.getQueryInterface(); const applied: ModelOptionsIndexes[] = []; let failure: unknown; for (const index of prepared) { - const existing = findLiveIndex(live, index); + const existing = findLiveIndex(live, index, normalizeFields); if (existing) { applied.push(index); continue; } try { await queryInterface.addIndex(collectionName, { - fields: sqlIndexFields(index), + fields: sqlEngineIndexFields(index, schemaFields), ...index.options, }); applied.push(index); @@ -453,14 +464,21 @@ export abstract class SequelizeAdapter extends DatabaseAdapter } catch (e) { if (isIndexAlreadyExistsError(e)) { const relisted = await this.listLiveIndexesForCollection(collectionName); - if (liveNameConflictAllowsReuse(index, relisted)) { + if (liveNameConflictAllowsReuse(index, relisted, normalizeFields)) { applied.push(index); live.splice(0, live.length, ...relisted); continue; } - const match = findLiveIndex(relisted, index); + const match = findLiveIndex(relisted, index, normalizeFields); if (match) { - applied.push(bindDeclaredIndexesToLive([index], relisted, collectionName)[0]); + applied.push( + bindDeclaredIndexesToLive( + [index], + relisted, + collectionName, + normalizeFields, + )[0], + ); live.splice(0, live.length, ...relisted); continue; } @@ -469,9 +487,16 @@ export abstract class SequelizeAdapter extends DatabaseAdapter } if (isIndexKeySpecsConflictError(e)) { const relisted = await this.listLiveIndexesForCollection(collectionName); - const match = findLiveIndex(relisted, index); + const match = findLiveIndex(relisted, index, normalizeFields); if (match) { - applied.push(bindDeclaredIndexesToLive([index], relisted, collectionName)[0]); + applied.push( + bindDeclaredIndexesToLive( + [index], + relisted, + collectionName, + normalizeFields, + )[0], + ); live.splice(0, live.length, ...relisted); continue; } @@ -513,11 +538,16 @@ export abstract class SequelizeAdapter extends DatabaseAdapter const queryInterface = this.sequelize.getQueryInterface(); const result = (await queryInterface.showIndex(collectionName)) as UntypedArray; const dialect = this.sequelize.getDialect(); - const declared = this.models[schemaName].originalSchema.modelOptions.indexes; + const originalSchema = this.models[schemaName].originalSchema; + const schemaFields = collectSchemaIndexFields(originalSchema); + const declared = originalSchema.modelOptions.indexes; return result.map(row => { - const fields = (row.fields ?? []).map((field: unknown) => + const engineFields = (row.fields ?? []).map((field: unknown) => typeof field === 'string' ? field : (field as { attribute?: string }).attribute, ); + const fields = engineFields.map((field: string) => + sqlDeclaredIndexFieldName(field, schemaFields), + ); const name = row.name as string; return overlayDeclaredOnLive( { @@ -817,9 +847,13 @@ export abstract class SequelizeAdapter extends DatabaseAdapter const schema = this.models[schemaName].originalSchema; const dialect = this.sequelize.getDialect(); const collectionName = indexNameCollection(schema); + const schemaFields = collectSchemaIndexFields(schema); const prepared: ModelOptionsIndexes[] = []; for (const raw of toMutableIndexes(indexes)) { - const index = ensureIndexName(raw, collectionName); + const index = ensureIndexName( + canonicalizeDeclaredIndexFields(raw, schemaFields), + collectionName, + ); validateIndexFields(schema, index); const unsupported = sqlIndexUnsupportedReason( dialect, diff --git a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts index a1c18bb92..069118953 100644 --- a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts @@ -11,6 +11,7 @@ import { convertModelOptionsIndexes, convertSchemaFieldIndexes, extractFieldProperties, + liftSqlScalarRelationFieldIndexes, } from '../../utils/index.js'; import { sqlDataTypeMap } from '../utils/sqlTypeMap.js'; import { @@ -38,6 +39,7 @@ export function pgSchemaConverter(jsonSchema: ConduitSchema): [ if (copy.fields.hasOwnProperty('_id')) { delete copy.fields['_id']; } + copy = liftSqlScalarRelationFieldIndexes(copy); if (copy.modelOptions.indexes) { copy = convertModelOptionsIndexes(copy, 'postgres'); } diff --git a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts index 3709a774e..35d45e67f 100644 --- a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts @@ -11,6 +11,7 @@ import { convertModelOptionsIndexes, convertSchemaFieldIndexes, extractFieldProperties, + liftSqlScalarRelationFieldIndexes, } from '../../utils/index.js'; import { sqlDataTypeMap } from '../utils/sqlTypeMap.js'; import { @@ -37,6 +38,7 @@ export function sqlSchemaConverter( if (copy.fields.hasOwnProperty('_id')) { delete copy.fields['_id']; } + copy = liftSqlScalarRelationFieldIndexes(copy); if (copy.modelOptions.indexes) { copy = convertModelOptionsIndexes(copy, dialect); } diff --git a/modules/database/src/adapters/utils/database-transform-utils.ts b/modules/database/src/adapters/utils/database-transform-utils.ts index 3ca69144c..ad0bb4467 100644 --- a/modules/database/src/adapters/utils/database-transform-utils.ts +++ b/modules/database/src/adapters/utils/database-transform-utils.ts @@ -13,7 +13,9 @@ import { indexNameCollection, isPortableDirection, isPostgresIndexType, + isScalarRelationField, mapCompatibleToSqlOrder, + mapIndexFieldsToSqlEngine, normalizeIndexTypes, sqlDialectAllowsIndexType, sqlIndexUnsupportedReason, @@ -60,6 +62,7 @@ function toSqlEngineIndex( dialect: string, schemaName: string, collectionName: string, + schemaFields: Record, ): SqlEngineIndex | null { const index = ensureIndexName({ ...raw, fields: [...raw.fields] }, collectionName); if (index.options && !checkIfPostgresOptions(index.options)) { @@ -67,7 +70,8 @@ function toSqlEngineIndex( return null; } - let fields: SqlIndexField[] = [...index.fields]; + const engineFieldNames = mapIndexFieldsToSqlEngine(index, schemaFields); + let fields: SqlIndexField[] = [...engineFieldNames]; let using = PostgresIndexType.BTREE; if (index.types) { const types = normalizeIndexTypes(index.types, index.fields.length) ?? []; @@ -76,7 +80,7 @@ function toSqlEngineIndex( return null; } if (types.some(isPortableDirection)) { - fields = index.fields.map((field, i) => ({ + fields = engineFieldNames.map((field, i) => ({ name: field, order: mapCompatibleToSqlOrder(types[i]), })); @@ -97,6 +101,27 @@ function toSqlEngineIndex( }; } +export function liftSqlScalarRelationFieldIndexes(copy: ConduitSchema) { + const lifted: ModelOptionsIndexes[] = []; + for (const [fieldName, fieldValue] of Object.entries(copy.fields)) { + if (!isScalarRelationField(fieldValue)) continue; + const field = fieldValue as ConduitModelField; + const index = field.index; + if (!index) continue; + lifted.push({ + fields: [fieldName], + types: index.type === undefined ? undefined : [index.type], + options: index.options, + name: index.name, + }); + delete field.index; + } + if (lifted.length) { + copy.modelOptions.indexes = [...(copy.modelOptions.indexes ?? []), ...lifted]; + } + return copy; +} + export function convertModelOptionsIndexes(copy: ConduitSchema, dialect = 'postgres') { const collectionName = indexNameCollection(copy); const converted: SqlEngineIndex[] = []; @@ -110,7 +135,7 @@ export function convertModelOptionsIndexes(copy: ConduitSchema, dialect = 'postg ); continue; } - const index = toSqlEngineIndex(raw, dialect, copy.name, collectionName); + const index = toSqlEngineIndex(raw, dialect, copy.name, collectionName, copy.fields); if (index) converted.push(index); } setSqlEngineIndexes(copy, converted); @@ -145,7 +170,13 @@ export function convertSchemaFieldIndexes(copy: ConduitSchema, dialect = 'postgr delete field.index; continue; } - const converted = toSqlEngineIndex(raw, dialect, copy.name, collectionName); + const converted = toSqlEngineIndex( + raw, + dialect, + copy.name, + collectionName, + copy.fields, + ); delete field.index; if (converted) indexes.push(converted); } diff --git a/modules/database/src/adapters/utils/indexes.ts b/modules/database/src/adapters/utils/indexes.ts index e1db4cab7..14590877f 100644 --- a/modules/database/src/adapters/utils/indexes.ts +++ b/modules/database/src/adapters/utils/indexes.ts @@ -71,9 +71,15 @@ export function indexFieldNames( .filter(name => name.length > 0); } -export function indexIdentity(index: ModelOptionsIndexes): IndexIdentity { +export type IndexFieldNormalizer = (fields: string[]) => string[]; + +export function indexIdentity( + index: ModelOptionsIndexes, + normalizeFields?: IndexFieldNormalizer, +): IndexIdentity { + const fields = indexFieldNames(index); return { - fields: indexFieldNames(index), + fields: normalizeFields ? normalizeFields(fields) : fields, unique: isUniqueIndex(index), }; } @@ -99,10 +105,13 @@ export function isSkippedLiveIndex(index: ModelOptionsIndexes): boolean { export function findLiveIndex( live: readonly ModelOptionsIndexes[], declared: ModelOptionsIndexes, + normalizeFields?: IndexFieldNormalizer, ): ModelOptionsIndexes | undefined { - const wanted = indexIdentity(declared); + const wanted = indexIdentity(declared, normalizeFields); return live.find( - row => !isSkippedLiveIndex(row) && indexIdentitiesEqual(indexIdentity(row), wanted), + row => + !isSkippedLiveIndex(row) && + indexIdentitiesEqual(indexIdentity(row, normalizeFields), wanted), ); } @@ -116,12 +125,16 @@ export function findIndexByName( export function liveNameConflictAllowsReuse( declared: ModelOptionsIndexes, live: readonly ModelOptionsIndexes[], + normalizeFields?: IndexFieldNormalizer, ): boolean { const name = resolveIndexName(declared); if (!name) return false; const row = findIndexByName(live, name); if (!row) return false; - return indexIdentitiesEqual(indexIdentity(row), indexIdentity(declared)); + return indexIdentitiesEqual( + indexIdentity(row, normalizeFields), + indexIdentity(declared, normalizeFields), + ); } export function indexNameCollection(schema: { @@ -138,9 +151,10 @@ export function bindDeclaredIndexesToLive( declared: readonly T[], live: readonly ModelOptionsIndexes[], collectionName: string, + normalizeFields?: IndexFieldNormalizer, ): T[] { return declared.map(index => { - const match = findLiveIndex(live, index); + const match = findLiveIndex(live, index, normalizeFields); if (match) { const name = resolveIndexName(match); if (name) { @@ -569,6 +583,16 @@ export function sqlIndexFields(index: ModelOptionsIndexes): SqlIndexField[] { })); } +export function sqlEngineIndexFields( + index: ModelOptionsIndexes, + schemaFields: Record, +): SqlIndexField[] { + return sqlIndexFields({ + ...index, + fields: mapIndexFieldsToSqlEngine(index, schemaFields), + }); +} + export function inferSqlIndexType( row: { type?: string; definition?: string }, dialect: string, @@ -643,6 +667,77 @@ export function isScalarRelationField(field: unknown): boolean { ); } +export function collectSchemaIndexFields(schema: { + fields?: Record; + compiledFields?: Record; +}): Record { + return { + ...(schema.fields ?? {}), + ...(schema.compiledFields ?? {}), + }; +} + +export function sqlEngineIndexFieldName( + declaredField: string, + fields: Record, +): string { + if (isScalarRelationField(fields[declaredField])) { + return `${declaredField}Id`; + } + return declaredField; +} + +export function sqlDeclaredIndexFieldName( + engineField: string, + fields: Record, +): string { + if (Object.prototype.hasOwnProperty.call(fields, engineField)) { + return engineField; + } + if (engineField.endsWith('Id')) { + const declared = engineField.slice(0, -2); + if (declared.length > 0 && isScalarRelationField(fields[declared])) { + return declared; + } + } + return engineField; +} + +export function mapIndexFieldsToSqlEngine( + index: Pick | { fields?: readonly unknown[] }, + fields: Record, +): string[] { + return indexFieldNames(index).map(name => sqlEngineIndexFieldName(name, fields)); +} + +export function mapIndexFieldsToDeclared( + index: Pick | { fields?: readonly unknown[] }, + fields: Record, +): string[] { + return indexFieldNames(index).map(name => sqlDeclaredIndexFieldName(name, fields)); +} + +export function canonicalizeDeclaredIndexFields( + index: T, + fields: Record, +): T { + const mapped = mapIndexFieldsToDeclared(index, fields); + const current = indexFieldNames(index); + if ( + mapped.length === current.length && + mapped.every((name, i) => name === current[i]) + ) { + return index; + } + return { ...index, fields: mapped }; +} + +export function sqlIndexFieldNormalizer( + fields: Record, +): IndexFieldNormalizer { + return names => names.map(name => sqlEngineIndexFieldName(name, fields)); +} + export function sqlIndexUnsupportedReason( dialect: string, index: Pick, @@ -660,9 +755,6 @@ export function sqlIndexUnsupportedReason( if (isExtractedArrayRelationField(field)) { return `Field '${name}' is stored as a relation join table and cannot be indexed on SQL`; } - if (isScalarRelationField(field)) { - return `Field '${name}' is a relation and cannot be indexed on SQL`; - } if (mysqlJson && isMysqlJsonLikeField(field)) { return `Compatible btree indexes are not supported on MySQL JSON field '${name}'`; } diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index cde27cee7..5ae3f165e 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -26,7 +26,9 @@ import { parseSortParam } from '../handlers/utils.js'; import escapeStringRegexp from 'escape-string-regexp'; import { ADMIN_INDEX_CALLER, + canonicalizeDeclaredIndexFields, collectExistingIndexNames, + collectSchemaIndexFields, ensureIndexName, resolveIndexName, } from '../adapters/utils/indexes.js'; @@ -751,9 +753,12 @@ export class SchemaAdmin { async createIndexes(call: ParsedRouterRequest): Promise { const { id, indexes } = call.request.params; const requestedSchema = await this.findDeclaredSchemaById(id); + const schemaFields = collectSchemaIndexFields(requestedSchema); return await this.database.createIndexes( requestedSchema.name, - indexes, + (indexes as ModelOptionsIndexes[]).map(index => + canonicalizeDeclaredIndexFields(index, schemaFields), + ), ADMIN_INDEX_CALLER, { privileged: true }, ); @@ -847,10 +852,18 @@ export class SchemaAdmin { } const collectionName = this.database.models[schemaName].originalSchema.collectionName ?? schemaName; + const schemaFields = collectSchemaIndexFields( + this.database.models[schemaName].originalSchema, + ); const existing = await this.database.getIndexes(schemaName); const existingNames = collectExistingIndexNames(existing); const toCreate = schemaIndexes - .map(index => ensureIndexName(index, collectionName)) + .map(index => + ensureIndexName( + canonicalizeDeclaredIndexFields(index, schemaFields), + collectionName, + ), + ) .filter(index => { const name = resolveIndexName(index); return !name || !existingNames.has(name); From a7fe0f4752896a3b523db7d4029973211c80db42 Mon Sep 17 00:00:00 2001 From: Christina Papadogianni <59121443+ChrisPdgn@users.noreply.github.com> Date: Thu, 17 Sep 2026 09:25:35 +0000 Subject: [PATCH 9/9] test(database): stub Mongo collection.indexes in vector adapter tests Compatible-index createIndexes binds against live Mongo indexes, so the vector lifecycle mock must expose collection.indexes() for both features to share the same adapter path after the rebase onto main. Co-authored-by: Christina Papadogianni --- .../src/adapters/utils/__tests__/vectorIndexAdapters.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts b/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts index e48c16439..98c06da6e 100644 --- a/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts +++ b/modules/database/src/adapters/utils/__tests__/vectorIndexAdapters.test.ts @@ -50,12 +50,13 @@ describe('mongoose vector index lifecycle', () => { toArray: async () => [], })); const createIndex = jest.fn(async () => 'title_1'); + const indexes = jest.fn(async () => [{ v: 2, key: { _id: 1 }, name: '_id_' }]); const adapter = Object.create(MongooseAdapter.prototype) as MongooseAdapter; Object.assign(adapter, { models: { Article: articleModel() }, mongoose: { model: () => ({ - collection: { createSearchIndex, listSearchIndexes, createIndex }, + collection: { createSearchIndex, listSearchIndexes, createIndex, indexes }, }), }, });