From c0668c6053d12a294265a2e6f4a8d83d4a8444cc Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Mon, 7 Oct 2024 11:06:15 +0200 Subject: [PATCH 1/9] wip --- .../field-metadata/field-metadata.service.ts | 2 - .../object-metadata/object-metadata.module.ts | 2 + .../object-metadata.service.ts | 104 +++-------- .../metadata-modules/search/search.module.ts | 23 +++ .../metadata-modules/search/search.service.ts | 167 ++++++++++++++++++ .../ts-vector-column-action.factory.ts | 34 ++-- .../get-ts-vector-column-expression.util.ts | 14 +- .../utils/is-searchable-field.util.ts | 17 ++ 8 files changed, 266 insertions(+), 97 deletions(-) create mode 100644 packages/twenty-server/src/engine/metadata-modules/search/search.module.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/search/search.service.ts create mode 100644 packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util.ts diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts index 019cfcea7bdf..855d63734f18 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts @@ -29,7 +29,6 @@ import { import { generateNullable } from 'src/engine/metadata-modules/field-metadata/utils/generate-nullable'; import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; import { assertMutationNotOnRemoteObject } from 'src/engine/metadata-modules/object-metadata/utils/assert-mutation-not-on-remote-object.util'; import { RelationMetadataEntity, @@ -75,7 +74,6 @@ export class FieldMetadataService extends TypeOrmQueryService, @InjectRepository(ObjectMetadataEntity, 'metadata') private readonly objectMetadataRepository: Repository, - private readonly objectMetadataService: ObjectMetadataService, private readonly workspaceMigrationFactory: WorkspaceMigrationFactory, private readonly workspaceMigrationService: WorkspaceMigrationService, private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService, diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts index 14d9d58c2200..b05d4aa4eb48 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts @@ -20,6 +20,7 @@ import { ObjectMetadataGraphqlApiExceptionInterceptor } from 'src/engine/metadat import { ObjectMetadataResolver } from 'src/engine/metadata-modules/object-metadata/object-metadata.resolver'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; import { RemoteTableRelationsModule } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-relations/remote-table-relations.module'; +import { SearchModule } from 'src/engine/metadata-modules/search/search.module'; import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; @@ -48,6 +49,7 @@ import { UpdateObjectPayload } from './dtos/update-object.input'; RemoteTableRelationsModule, IndexMetadataModule, FeatureFlagModule, + SearchModule, ], services: [ObjectMetadataService], resolvers: [ diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts index b4102db85746..a1342c7ca655 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts @@ -5,30 +5,21 @@ import console from 'console'; import { Query, QueryOptions } from '@ptc-org/nestjs-query-core'; import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm'; -import { isDefined } from 'class-validator'; import { FindManyOptions, FindOneOptions, In, Repository } from 'typeorm'; import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-settings.interface'; -import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; import { TypeORMService } from 'src/database/typeorm/typeorm.service'; import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; -import { SEARCH_VECTOR_FIELD } from 'src/engine/metadata-modules/constants/search-vector-field.constants'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { FieldMetadataEntity, FieldMetadataType, } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; -import { - computeColumnName, - FieldTypeAndNameMetadata, -} from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util'; -import { IndexType } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; -import { IndexMetadataService } from 'src/engine/metadata-modules/index-metadata/index-metadata.service'; +import { computeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util'; import { DeleteOneObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/delete-object.input'; import { UpdateOneObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/update-object.input'; -import { DEFAULT_LABEL_IDENTIFIER_FIELD_NAME } from 'src/engine/metadata-modules/object-metadata/object-metadata.constants'; import { ObjectMetadataException, ObjectMetadataExceptionCode, @@ -43,8 +34,8 @@ import { import { RelationToDelete } from 'src/engine/metadata-modules/relation-metadata/types/relation-to-delete'; import { RemoteTableRelationsService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-relations/remote-table-relations.service'; import { mapUdtNameToFieldType } from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util'; +import { SearchService } from 'src/engine/metadata-modules/search/search.service'; import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; -import { TsVectorColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory'; import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; import { WorkspaceMigrationColumnActionType, @@ -70,7 +61,6 @@ import { createForeignKeyDeterministicUuid, createRelationDeterministicUuid, } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/create-deterministic-uuid.util'; -import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity'; import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity'; @@ -92,17 +82,14 @@ export class ObjectMetadataService extends TypeOrmQueryService - field.id === createdObjectMetadata.labelIdentifierFieldMetadataId, - ) - : createdObjectMetadata.fields.find( - (field) => field.name === DEFAULT_LABEL_IDENTIFIER_FIELD_NAME, - ); - - if (!isDefined(searchableFieldForCustomObject)) { - throw new Error('No searchable field found for custom object'); - } - - this.workspaceMigrationService.createCustomMigration( - generateMigrationName(`create-${createdObjectMetadata.nameSingular}`), - createdObjectMetadata.workspaceId, - [ - { - name: computeTableName( - createdObjectMetadata.nameSingular, - createdObjectMetadata.isCustom, - ), - action: WorkspaceMigrationTableActionType.ALTER, - columns: this.tsVectorColumnActionFactory.handleCreateAction({ - ...searchVectorFieldMetadata, - defaultValue: undefined, - generatedType: 'STORED', - asExpression: getTsVectorColumnExpressionFromFields([ - searchableFieldForCustomObject as FieldTypeAndNameMetadata, - ]), - options: undefined, - } as FieldMetadataInterface), - }, - ], - ); - - await this.indexMetadataService.createIndex( - objectMetadataInput.workspaceId, - createdObjectMetadata, - [searchVectorFieldMetadata], - false, - IndexType.GIN, - ); - } - private async createActivityTargetRelation( workspaceId: string, createdObjectMetadata: ObjectMetadataEntity, diff --git a/packages/twenty-server/src/engine/metadata-modules/search/search.module.ts b/packages/twenty-server/src/engine/metadata-modules/search/search.module.ts new file mode 100644 index 000000000000..2260d72181f5 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/search/search.module.ts @@ -0,0 +1,23 @@ +import { Module } from '@nestjs/common'; + +import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm'; + +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { IndexMetadataModule } from 'src/engine/metadata-modules/index-metadata/index-metadata.module'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { SearchService } from 'src/engine/metadata-modules/search/search.service'; +import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; + +@Module({ + imports: [ + NestjsQueryTypeOrmModule.forFeature( + [ObjectMetadataEntity, FieldMetadataEntity], + 'metadata', + ), + IndexMetadataModule, + WorkspaceMigrationModule, + ], + providers: [SearchService], + exports: [SearchService], +}) +export class SearchModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts b/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts new file mode 100644 index 000000000000..37a4145d888a --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts @@ -0,0 +1,167 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; + +import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; + +import { SEARCH_VECTOR_FIELD } from 'src/engine/metadata-modules/constants/search-vector-field.constants'; +import { + FieldMetadataEntity, + FieldMetadataType, +} from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { IndexType } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { IndexMetadataService } from 'src/engine/metadata-modules/index-metadata/index-metadata.service'; +import { CreateObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/create-object.input'; +import { DEFAULT_LABEL_IDENTIFIER_FIELD_NAME } from 'src/engine/metadata-modules/object-metadata/object-metadata.constants'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { TsVectorColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory'; +import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; +import { + WorkspaceMigrationColumnActionType, + WorkspaceMigrationTableActionType, +} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; +import { WorkspaceMigrationFactory } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.factory'; +import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service'; +import { computeTableName } from 'src/engine/utils/compute-table-name.util'; +import { CUSTOM_OBJECT_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids'; +import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { + isSearchableFieldType, + SearchableFieldType, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util'; +import { isDefined } from 'src/utils/is-defined'; + +@Injectable() +export class SearchService { + constructor( + @InjectRepository(ObjectMetadataEntity, 'metadata') + private readonly objectMetadataRepository: Repository, + private readonly tsVectorColumnActionFactory: TsVectorColumnActionFactory, + private readonly indexMetadataService: IndexMetadataService, + @InjectRepository(FieldMetadataEntity, 'metadata') + private readonly fieldMetadataRepository: Repository, + private readonly workspaceMigrationService: WorkspaceMigrationService, + private readonly workspaceMigrationFactory: WorkspaceMigrationFactory, + ) {} + + public async createSearchVectorField( + objectMetadataInput: CreateObjectInput, + createdObjectMetadata: ObjectMetadataEntity, + ) { + const searchVectorFieldMetadata = await this.fieldMetadataRepository.save({ + standardId: CUSTOM_OBJECT_STANDARD_FIELD_IDS.searchVector, + objectMetadataId: createdObjectMetadata.id, + workspaceId: objectMetadataInput.workspaceId, + isCustom: false, + isActive: false, + isSystem: true, + type: FieldMetadataType.TS_VECTOR, + name: SEARCH_VECTOR_FIELD.name, + label: SEARCH_VECTOR_FIELD.label, + description: SEARCH_VECTOR_FIELD.description, + isNullable: true, + }); + + const searchableFieldForCustomObject = + createdObjectMetadata.labelIdentifierFieldMetadataId + ? createdObjectMetadata.fields.find( + (field) => + field.id === createdObjectMetadata.labelIdentifierFieldMetadataId, + ) + : createdObjectMetadata.fields.find( + (field) => field.name === DEFAULT_LABEL_IDENTIFIER_FIELD_NAME, + ); + + if (!isDefined(searchableFieldForCustomObject)) { + throw new Error('No searchable field found for custom object'); + } + + this.workspaceMigrationService.createCustomMigration( + generateMigrationName(`create-${createdObjectMetadata.nameSingular}`), + createdObjectMetadata.workspaceId, + [ + { + name: computeTableName( + createdObjectMetadata.nameSingular, + createdObjectMetadata.isCustom, + ), + action: WorkspaceMigrationTableActionType.ALTER, + columns: this.tsVectorColumnActionFactory.handleCreateAction({ + ...searchVectorFieldMetadata, + defaultValue: undefined, + generatedType: 'STORED', + asExpression: getTsVectorColumnExpressionFromFields([ + { + type: searchableFieldForCustomObject.type as SearchableFieldType, + name: searchableFieldForCustomObject.name, + }, + ]), + options: undefined, + } as FieldMetadataInterface), + }, + ], + ); + + await this.indexMetadataService.createIndex( + objectMetadataInput.workspaceId, + createdObjectMetadata, + [searchVectorFieldMetadata], + false, + IndexType.GIN, + ); + } + + public async updateSearchVector( + objectMetadataId: string, + labelIdentifierFieldMetadataId: string, + workspaceId: string, + ) { + const newLabelIdentifierField = + await this.fieldMetadataRepository.findOneByOrFail({ + id: labelIdentifierFieldMetadataId, + }); + + if (!isSearchableFieldType(newLabelIdentifierField.type)) { + return; + } + + const objectMetadata = await this.objectMetadataRepository.findOneByOrFail({ + id: objectMetadataId, + }); + + const existingSearchVectorFieldMetadata = + await this.fieldMetadataRepository.findOneByOrFail({ + name: SEARCH_VECTOR_FIELD.name, + objectMetadataId, + }); + + await this.workspaceMigrationService.createCustomMigration( + generateMigrationName(`update-${objectMetadata.nameSingular}`), + workspaceId, + [ + { + name: computeTableName( + objectMetadata.nameSingular, + objectMetadata.isCustom, + ), + action: WorkspaceMigrationTableActionType.ALTER, + columns: this.workspaceMigrationFactory.createColumnActions( + WorkspaceMigrationColumnActionType.ALTER, + existingSearchVectorFieldMetadata, + { + ...existingSearchVectorFieldMetadata, + asExpression: getTsVectorColumnExpressionFromFields([ + { + type: newLabelIdentifierField.type as SearchableFieldType, + name: newLabelIdentifierField.name, + }, + ]), + options: undefined, + }, + ), + }, + ], + ); + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts index dd72948f3264..afafdf3aa4b4 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts @@ -1,7 +1,6 @@ import { Injectable, Logger } from '@nestjs/common'; import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; -import { WorkspaceColumnActionOptions } from 'src/engine/metadata-modules/workspace-migration/interfaces/workspace-column-action-options.interface'; import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { computeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util'; @@ -12,10 +11,6 @@ import { WorkspaceMigrationColumnAlter, WorkspaceMigrationColumnCreate, } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; -import { - WorkspaceMigrationException, - WorkspaceMigrationExceptionCode, -} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.exception'; export type TsVectorFieldMetadataType = FieldMetadataType.TS_VECTOR; @@ -39,14 +34,27 @@ export class TsVectorColumnActionFactory extends ColumnActionAbstractFactory, - _alteredFieldMetadata: FieldMetadataInterface, - _options?: WorkspaceColumnActionOptions, + handleAlterAction( + currentFieldMetadata: FieldMetadataInterface, + alteredFieldMetadata: FieldMetadataInterface, ): WorkspaceMigrationColumnAlter[] { - throw new WorkspaceMigrationException( - `TsVectorColumnActionFactory.handleAlterAction has not been implemented yet.`, - WorkspaceMigrationExceptionCode.INVALID_FIELD_METADATA, - ); + return [ + { + action: WorkspaceMigrationColumnActionType.ALTER, + currentColumnDefinition: { + columnName: currentFieldMetadata.name, + columnType: fieldMetadataTypeToColumnType(currentFieldMetadata.type), + isNullable: currentFieldMetadata.isNullable ?? true, + defaultValue: undefined, + }, + alteredColumnDefinition: { + columnName: alteredFieldMetadata.name, + columnType: fieldMetadataTypeToColumnType(alteredFieldMetadata.type), + isNullable: alteredFieldMetadata.isNullable ?? true, + defaultValue: undefined, + asExpression: alteredFieldMetadata.asExpression, + }, + }, + ]; } } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts index 816c7b64d288..8a670de98af3 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts @@ -9,15 +9,27 @@ import { WorkspaceMigrationException, WorkspaceMigrationExceptionCode, } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.exception'; +import { + isSearchableFieldType, + SearchableFieldType, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util'; type FieldTypeAndNameMetadata = { name: string; - type: FieldMetadataType; + type: SearchableFieldType; }; export const getTsVectorColumnExpressionFromFields = ( fieldsUsedForSearch: FieldTypeAndNameMetadata[], ): string => { + const filteredFieldsUsedForSearch = fieldsUsedForSearch.filter((field) => + isSearchableFieldType(field.type), + ); + + if (filteredFieldsUsedForSearch.length < 1) { + throw new Error('No searchable fields found'); + } + const columnExpressions = fieldsUsedForSearch.flatMap( getColumnExpressionsFromField, ); diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util.ts new file mode 100644 index 000000000000..bb482ae4a83d --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util.ts @@ -0,0 +1,17 @@ +import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; + +const SEARCHABLE_FIELD_TYPES = [ + FieldMetadataType.TEXT, + FieldMetadataType.FULL_NAME, + FieldMetadataType.EMAILS, + FieldMetadataType.ADDRESS, + FieldMetadataType.LINKS, +] as const; + +export type SearchableFieldType = (typeof SEARCHABLE_FIELD_TYPES)[number]; + +export const isSearchableFieldType = ( + type: FieldMetadataType, +): type is SearchableFieldType => { + return SEARCHABLE_FIELD_TYPES.includes(type as SearchableFieldType); +}; From 7531e4fa297f0364ecace56997dada0c4eb583dd Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Fri, 11 Oct 2024 11:12:03 +0200 Subject: [PATCH 2/9] Add generatedType in migration --- .../object-metadata/object-metadata.module.ts | 2 - .../object-metadata.service.ts | 89 ++----------------- .../metadata-modules/search/search.service.ts | 1 + .../ts-vector-column-action.factory.ts | 1 + .../workspace-migration-runner.service.ts | 2 + 5 files changed, 12 insertions(+), 83 deletions(-) diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts index b05d4aa4eb48..46992a6e1556 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts @@ -14,7 +14,6 @@ import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature- import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; -import { IndexMetadataModule } from 'src/engine/metadata-modules/index-metadata/index-metadata.module'; import { BeforeUpdateOneObject } from 'src/engine/metadata-modules/object-metadata/hooks/before-update-one-object.hook'; import { ObjectMetadataGraphqlApiExceptionInterceptor } from 'src/engine/metadata-modules/object-metadata/interceptors/object-metadata-graphql-api-exception.interceptor'; import { ObjectMetadataResolver } from 'src/engine/metadata-modules/object-metadata/object-metadata.resolver'; @@ -47,7 +46,6 @@ import { UpdateObjectPayload } from './dtos/update-object.input'; WorkspaceMigrationRunnerModule, WorkspaceMetadataVersionModule, RemoteTableRelationsModule, - IndexMetadataModule, FeatureFlagModule, SearchModule, ], diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts index cfb696ada2e9..6d4e4a523049 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts @@ -5,30 +5,21 @@ import console from 'console'; import { Query, QueryOptions } from '@ptc-org/nestjs-query-core'; import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm'; -import { isDefined } from 'class-validator'; import { FindManyOptions, FindOneOptions, In, Repository } from 'typeorm'; import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-settings.interface'; -import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; import { TypeORMService } from 'src/database/typeorm/typeorm.service'; import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; -import { SEARCH_VECTOR_FIELD } from 'src/engine/metadata-modules/constants/search-vector-field.constants'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { FieldMetadataEntity, FieldMetadataType, } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; -import { - computeColumnName, - FieldTypeAndNameMetadata, -} from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util'; -import { IndexType } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; -import { IndexMetadataService } from 'src/engine/metadata-modules/index-metadata/index-metadata.service'; +import { computeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util'; import { DeleteOneObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/delete-object.input'; import { UpdateOneObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/update-object.input'; -import { DEFAULT_LABEL_IDENTIFIER_FIELD_NAME } from 'src/engine/metadata-modules/object-metadata/object-metadata.constants'; import { ObjectMetadataException, ObjectMetadataExceptionCode, @@ -45,7 +36,6 @@ import { RemoteTableRelationsService } from 'src/engine/metadata-modules/remote- import { mapUdtNameToFieldType } from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util'; import { SearchService } from 'src/engine/metadata-modules/search/search.service'; import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; -import { TsVectorColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory'; import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; import { WorkspaceMigrationColumnActionType, @@ -73,7 +63,6 @@ import { createForeignKeyDeterministicUuid, createRelationDeterministicUuid, } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/create-deterministic-uuid.util'; -import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity'; import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity'; @@ -104,8 +93,6 @@ export class ObjectMetadataService extends TypeOrmQueryService - field.id === createdObjectMetadata.labelIdentifierFieldMetadataId, - ) - : createdObjectMetadata.fields.find( - (field) => field.name === DEFAULT_LABEL_IDENTIFIER_FIELD_NAME, - ); - - if (!isDefined(searchableFieldForCustomObject)) { - throw new Error('No searchable field found for custom object'); - } - - this.workspaceMigrationService.createCustomMigration( - generateMigrationName( - `update-${createdObjectMetadata.nameSingular}-add-searchVector`, - ), - createdObjectMetadata.workspaceId, - [ - { - name: computeTableName( - createdObjectMetadata.nameSingular, - createdObjectMetadata.isCustom, - ), - action: WorkspaceMigrationTableActionType.ALTER, - columns: this.tsVectorColumnActionFactory.handleCreateAction({ - ...searchVectorFieldMetadata, - defaultValue: undefined, - generatedType: 'STORED', - asExpression: getTsVectorColumnExpressionFromFields([ - searchableFieldForCustomObject as FieldTypeAndNameMetadata, - ]), - options: undefined, - } as FieldMetadataInterface), - }, - ], - ); - - await this.indexMetadataService.createIndex( - objectMetadataInput.workspaceId, - createdObjectMetadata, - [searchVectorFieldMetadata], - false, - IndexType.GIN, - ); - } - private async createActivityTargetRelation( workspaceId: string, createdObjectMetadata: ObjectMetadataEntity, diff --git a/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts b/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts index 37a4145d888a..34ddf6714980 100644 --- a/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts @@ -157,6 +157,7 @@ export class SearchService { name: newLabelIdentifierField.name, }, ]), + generatedType: 'STORED', // Not stored on fieldMetadata options: undefined, }, ), diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts index afafdf3aa4b4..7ffa605d0fe5 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts @@ -53,6 +53,7 @@ export class TsVectorColumnActionFactory extends ColumnActionAbstractFactory Date: Fri, 11 Oct 2024 12:22:33 +0200 Subject: [PATCH 3/9] Recreate index at search vector update --- .../src/engine/metadata-modules/search/search.service.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts b/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts index 34ddf6714980..eb0eecfd40de 100644 --- a/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts @@ -164,5 +164,14 @@ export class SearchService { }, ], ); + + // index needs to be recreated as typeorm deletes then recreates searchVector column at alter + await this.indexMetadataService.createIndex( + workspaceId, + objectMetadata, + [existingSearchVectorFieldMetadata], + false, + IndexType.GIN, + ); } } From 6bb9aaa5d0aac7f386817b3ff38c57ec48e95ef1 Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Fri, 11 Oct 2024 18:28:55 +0200 Subject: [PATCH 4/9] Add command to simplify searchVector expression + refactor --- .../commands/database-command.module.ts | 2 + ...implify-search-vector-expression.module.ts | 21 ++++ .../0-32-simplify-search-vector-expression.ts | 115 ++++++++++++++++++ .../object-metadata.service.ts | 26 +++- .../metadata-modules/search/search.service.ts | 30 ++--- .../twenty-orm/custom.workspace-entity.ts | 20 +-- .../get-ts-vector-column-expression.util.ts | 2 +- .../company.workspace-entity.ts | 17 ++- .../opportunity.workspace-entity.ts | 15 ++- .../person.workspace-entity.ts | 19 ++- 10 files changed, 217 insertions(+), 50 deletions(-) create mode 100644 packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.module.ts create mode 100644 packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.ts diff --git a/packages/twenty-server/src/database/commands/database-command.module.ts b/packages/twenty-server/src/database/commands/database-command.module.ts index ec6414db9b94..4342a1b8ec19 100644 --- a/packages/twenty-server/src/database/commands/database-command.module.ts +++ b/packages/twenty-server/src/database/commands/database-command.module.ts @@ -8,6 +8,7 @@ import { DataSeedDemoWorkspaceModule } from 'src/database/commands/data-seed-dem import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-workspace.command'; import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question'; import { UpgradeTo0_31CommandModule } from 'src/database/commands/upgrade-version/0-31/0-31-upgrade-version.module'; +import { SimplifySearchVectorExpressionCommandModule } from 'src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.module'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { BillingSubscription } from 'src/engine/core-modules/billing/entities/billing-subscription.entity'; import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; @@ -47,6 +48,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp WorkspaceCacheStorageModule, WorkspaceMetadataVersionModule, UpgradeTo0_31CommandModule, + SimplifySearchVectorExpressionCommandModule, FeatureFlagModule, ], providers: [ diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.module.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.module.ts new file mode 100644 index 000000000000..9e6ea5e2be04 --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.module.ts @@ -0,0 +1,21 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { SimplifySearchVectorExpressionCommand } from 'src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { SearchModule } from 'src/engine/metadata-modules/search/search.module'; +import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; +import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([Workspace], 'core'), + TypeOrmModule.forFeature([FieldMetadataEntity], 'metadata'), + WorkspaceSyncMetadataCommandsModule, + SearchModule, + WorkspaceMigrationRunnerModule, + ], + providers: [SimplifySearchVectorExpressionCommand], +}) +export class SimplifySearchVectorExpressionCommandModule {} diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.ts new file mode 100644 index 000000000000..1f38466f5ef3 --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.ts @@ -0,0 +1,115 @@ +import { InjectRepository } from '@nestjs/typeorm'; + +import chalk from 'chalk'; +import { Command } from 'nest-commander'; +import { Repository } from 'typeorm'; + +import { + ActiveWorkspacesCommandOptions, + ActiveWorkspacesCommandRunner, +} from 'src/database/commands/active-workspaces.command'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { + FieldMetadataEntity, + FieldMetadataType, +} from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { SearchService } from 'src/engine/metadata-modules/search/search.service'; +import { SEARCH_FIELDS_FOR_CUSTOM_OBJECT } from 'src/engine/twenty-orm/custom.workspace-entity'; +import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service'; +import { + COMPANY_STANDARD_FIELD_IDS, + CUSTOM_OBJECT_STANDARD_FIELD_IDS, + OPPORTUNITY_STANDARD_FIELD_IDS, + PERSON_STANDARD_FIELD_IDS, +} from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids'; +import { FieldTypeAndNameMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { SEARCH_FIELDS_FOR_COMPANY } from 'src/modules/company/standard-objects/company.workspace-entity'; +import { SEARCH_FIELDS_FOR_OPPORTUNITY } from 'src/modules/opportunity/standard-objects/opportunity.workspace-entity'; +import { SEARCH_FIELDS_FOR_PERSON } from 'src/modules/person/standard-objects/person.workspace-entity'; + +@Command({ + name: 'upgrade-0.32:simplify-search-vector-expression', + description: 'Replace searchVector with simpler expression', +}) +export class SimplifySearchVectorExpressionCommand extends ActiveWorkspacesCommandRunner { + constructor( + @InjectRepository(Workspace, 'core') + protected readonly workspaceRepository: Repository, + @InjectRepository(FieldMetadataEntity, 'metadata') + private readonly fieldMetadataRepository: Repository, + private readonly searchService: SearchService, + private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService, + ) { + super(workspaceRepository); + } + + async executeActiveWorkspacesCommand( + _passedParam: string[], + _options: ActiveWorkspacesCommandOptions, + workspaceIds: string[], + ): Promise { + this.logger.log('Running command to fix migration'); + + for (const workspaceId of workspaceIds) { + this.logger.log(`Running command for workspace ${workspaceId}`); + + try { + const searchVectorFields = await this.fieldMetadataRepository.findBy({ + workspaceId: workspaceId, + type: FieldMetadataType.TS_VECTOR, + }); + + for (const searchVectorField of searchVectorFields) { + let fieldsUsedForSearch: FieldTypeAndNameMetadata[] = []; + + switch (searchVectorField.standardId) { + case CUSTOM_OBJECT_STANDARD_FIELD_IDS.searchVector: { + fieldsUsedForSearch = SEARCH_FIELDS_FOR_CUSTOM_OBJECT; + break; + } + case PERSON_STANDARD_FIELD_IDS.searchVector: { + fieldsUsedForSearch = SEARCH_FIELDS_FOR_PERSON; + break; + } + case COMPANY_STANDARD_FIELD_IDS.searchVector: { + fieldsUsedForSearch = SEARCH_FIELDS_FOR_COMPANY; + break; + } + case OPPORTUNITY_STANDARD_FIELD_IDS.searchVector: { + fieldsUsedForSearch = SEARCH_FIELDS_FOR_OPPORTUNITY; + break; + } + default: { + throw new Error( + `search vector has unexpected standardId: ${searchVectorField.standardId}`, + ); + } + } + + await this.searchService.updateSearchVector( + searchVectorField.objectMetadataId, + fieldsUsedForSearch, + workspaceId, + ); + + await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( + workspaceId, + ); + } + } catch (error) { + this.logger.log( + chalk.red( + `Running command on workspace ${workspaceId} failed with error: ${error}`, + ), + ); + continue; + } finally { + this.logger.log( + chalk.green(`Finished running command for workspace ${workspaceId}.`), + ); + } + + this.logger.log(chalk.green(`Command completed!`)); + } + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts index 6d4e4a523049..66744ec0dce6 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts @@ -63,6 +63,7 @@ import { createForeignKeyDeterministicUuid, createRelationDeterministicUuid, } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/create-deterministic-uuid.util'; +import { isSearchableFieldType } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util'; import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity'; import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity'; @@ -365,7 +366,7 @@ export class ObjectMetadataService extends TypeOrmQueryService Date: Mon, 14 Oct 2024 13:55:54 +0200 Subject: [PATCH 5/9] Fix tests --- ...natedObjectMetadataItemsToObjectMetadataItems.ts | 2 +- .../metadata-modules/search/search.service.ts | 2 ++ .../get-ts-vectors-column-expression.utils.spec.ts | 13 ++++++++++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/twenty-front/src/modules/object-metadata/utils/mapPaginatedObjectMetadataItemsToObjectMetadataItems.ts b/packages/twenty-front/src/modules/object-metadata/utils/mapPaginatedObjectMetadataItemsToObjectMetadataItems.ts index db3506c2d12c..0ab2a8229854 100644 --- a/packages/twenty-front/src/modules/object-metadata/utils/mapPaginatedObjectMetadataItemsToObjectMetadataItems.ts +++ b/packages/twenty-front/src/modules/object-metadata/utils/mapPaginatedObjectMetadataItemsToObjectMetadataItems.ts @@ -11,7 +11,7 @@ export const mapPaginatedObjectMetadataItemsToObjectMetadataItems = ({ pagedObjectMetadataItems?.objects.edges.map((object) => ({ ...object.node, fields: object.node.fields.edges.map((field) => field.node), - indexMetadatas: object.node.indexMetadatas.edges.map((index) => ({ + indexMetadatas: object.node.indexMetadatas?.edges.map((index) => ({ ...index.node, indexFieldMetadatas: index.node.indexFieldMetadatas?.edges.map( (indexField) => indexField.node, diff --git a/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts b/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts index ed4e28dc56cf..2d875955d3ac 100644 --- a/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts @@ -108,6 +108,7 @@ export class SearchService { createdObjectMetadata, [searchVectorFieldMetadata], false, + false, IndexType.GIN, ); } @@ -159,6 +160,7 @@ export class SearchService { objectMetadata, [existingSearchVectorFieldMetadata], false, + false, IndexType.GIN, ); } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts index 8703879e5e64..a7a4f73bd3b7 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts @@ -1,5 +1,8 @@ import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; -import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { + FieldTypeAndNameMetadata, + getTsVectorColumnExpressionFromFields, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; const nameTextField = { name: 'name', type: FieldMetadataType.TEXT }; const nameFullNameField = { @@ -63,14 +66,18 @@ jest.mock( describe('getTsVectorColumnExpressionFromFields', () => { it('should generate correct expression for simple text field', () => { - const fields = [nameTextField]; + const fields = [nameTextField] as FieldTypeAndNameMetadata[]; const result = getTsVectorColumnExpressionFromFields(fields); expect(result).toContain("to_tsvector('simple', COALESCE(\"name\", ''))"); }); it('should handle multiple fields', () => { - const fields = [nameFullNameField, jobTitleTextField, emailsEmailsField]; + const fields = [ + nameFullNameField, + jobTitleTextField, + emailsEmailsField, + ] as FieldTypeAndNameMetadata[]; const result = getTsVectorColumnExpressionFromFields(fields); const expected = ` to_tsvector('simple', COALESCE("nameFirstName", '') || ' ' || COALESCE("nameLastName", '') || ' ' || COALESCE("jobTitle", '') || ' ' || From c9a86035f80564bfe81134b5e5562a2c37878e92 Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Tue, 15 Oct 2024 15:07:26 +0200 Subject: [PATCH 6/9] Remove feature flags --- .../modules/workspace/types/FeatureFlagKey.ts | 2 - .../typeorm-seeds/core/feature-flags.ts | 10 ---- .../graphql-query-search-resolver.service.ts | 23 -------- .../enums/feature-flag-key.enum.ts | 2 - .../object-metadata.service.ts | 56 ++++++------------- .../constants/default-feature-flags.ts | 7 +-- .../workspace-sync-field-metadata.service.ts | 31 +--------- .../workspace-sync-index-metadata.service.ts | 18 +----- .../workspace-sync-metadata.service.ts | 8 --- 9 files changed, 24 insertions(+), 133 deletions(-) diff --git a/packages/twenty-front/src/modules/workspace/types/FeatureFlagKey.ts b/packages/twenty-front/src/modules/workspace/types/FeatureFlagKey.ts index cde44977047b..5471c5d4d59e 100644 --- a/packages/twenty-front/src/modules/workspace/types/FeatureFlagKey.ts +++ b/packages/twenty-front/src/modules/workspace/types/FeatureFlagKey.ts @@ -10,9 +10,7 @@ export type FeatureFlagKey = | 'IS_MESSAGE_THREAD_SUBSCRIBER_ENABLED' | 'IS_WORKFLOW_ENABLED' | 'IS_WORKSPACE_FAVORITE_ENABLED' - | 'IS_SEARCH_ENABLED' | 'IS_QUERY_RUNNER_TWENTY_ORM_ENABLED' | 'IS_GMAIL_SEND_EMAIL_SCOPE_ENABLED' - | 'IS_WORKSPACE_MIGRATED_FOR_SEARCH' | 'IS_ANALYTICS_V2_ENABLED' | 'IS_UNIQUE_INDEXES_ENABLED'; diff --git a/packages/twenty-server/src/database/typeorm-seeds/core/feature-flags.ts b/packages/twenty-server/src/database/typeorm-seeds/core/feature-flags.ts index c618677d3733..97b34a8844c7 100644 --- a/packages/twenty-server/src/database/typeorm-seeds/core/feature-flags.ts +++ b/packages/twenty-server/src/database/typeorm-seeds/core/feature-flags.ts @@ -55,16 +55,6 @@ export const seedFeatureFlags = async ( workspaceId: workspaceId, value: true, }, - { - key: FeatureFlagKey.IsSearchEnabled, - workspaceId: workspaceId, - value: true, - }, - { - key: FeatureFlagKey.IsWorkspaceMigratedForSearch, - workspaceId: workspaceId, - value: true, - }, { key: FeatureFlagKey.IsAnalyticsV2Enabled, workspaceId: workspaceId, diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts index a0fdbf0d377d..2914c20aae78 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts @@ -10,10 +10,6 @@ import { WorkspaceQueryRunnerOptions } from 'src/engine/api/graphql/workspace-qu import { SearchResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface'; import { QUERY_MAX_RECORDS } from 'src/engine/api/graphql/graphql-query-runner/constants/query-max-records.constant'; -import { - GraphqlQueryRunnerException, - GraphqlQueryRunnerExceptionCode, -} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception'; import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; import { SEARCH_VECTOR_FIELD } from 'src/engine/metadata-modules/constants/search-vector-field.constants'; @@ -97,23 +93,4 @@ export class GraphqlQuerySearchResolverService return formattedWords.join(' | '); } - - async validate( - _args: SearchResolverArgs, - options: WorkspaceQueryRunnerOptions, - ): Promise { - const featureFlagsForWorkspace = - await this.featureFlagService.getWorkspaceFeatureFlags( - options.authContext.workspace.id, - ); - - const isSearchEnabled = featureFlagsForWorkspace.IS_SEARCH_ENABLED; - - if (!isSearchEnabled) { - throw new GraphqlQueryRunnerException( - 'This endpoint is not available yet, please use findMany instead.', - GraphqlQueryRunnerExceptionCode.INVALID_QUERY_INPUT, - ); - } - } } diff --git a/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts b/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts index 8483f3089f6b..58282f9deca4 100644 --- a/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts +++ b/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts @@ -10,8 +10,6 @@ export enum FeatureFlagKey { IsMessageThreadSubscriberEnabled = 'IS_MESSAGE_THREAD_SUBSCRIBER_ENABLED', IsQueryRunnerTwentyORMEnabled = 'IS_QUERY_RUNNER_TWENTY_ORM_ENABLED', IsWorkspaceFavoriteEnabled = 'IS_WORKSPACE_FAVORITE_ENABLED', - IsSearchEnabled = 'IS_SEARCH_ENABLED', - IsWorkspaceMigratedForSearch = 'IS_WORKSPACE_MIGRATED_FOR_SEARCH', IsGmailSendEmailScopeEnabled = 'IS_GMAIL_SEND_EMAIL_SCOPE_ENABLED', IsAnalyticsV2Enabled = 'IS_ANALYTICS_V2_ENABLED', IsUniqueIndexesEnabled = 'IS_UNIQUE_INDEXES_ENABLED', diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts index 66744ec0dce6..8ec403f845a6 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts @@ -10,7 +10,6 @@ import { FindManyOptions, FindOneOptions, In, Repository } from 'typeorm'; import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-settings.interface'; import { TypeORMService } from 'src/database/typeorm/typeorm.service'; -import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { @@ -360,17 +359,10 @@ export class ObjectMetadataService extends TypeOrmQueryService field.type !== FieldMetadataType.TS_VECTOR, - ); - - originalObjectMetadataFields = originalObjectMetadataFields.filter( - (field) => field.type !== FieldMetadataType.TS_VECTOR, - ); - } - const fieldComparatorResults = this.workspaceFieldComparator.compare( originalObjectMetadata.id, originalObjectMetadata.fields, @@ -192,24 +178,11 @@ export class WorkspaceSyncFieldMetadataService { // Loop over all custom objects from the DB and compare their fields with standard fields for (const customObjectMetadata of customObjectMetadataCollection) { // Also, maybe it's better to refactor a bit and move generation part into a separate module ? - let standardFieldMetadataCollection = computeStandardFields( + const standardFieldMetadataCollection = computeStandardFields( customObjectStandardFieldMetadataCollection, customObjectMetadata, ); - let customObjectMetadataFields = customObjectMetadata.fields; - - if (!workspaceFeatureFlagsMap.IS_SEARCH_ENABLED) { - standardFieldMetadataCollection = - standardFieldMetadataCollection.filter( - (field) => field.type !== FieldMetadataType.TS_VECTOR, - ); - - customObjectMetadataFields = customObjectMetadataFields.filter( - (field) => field.type !== FieldMetadataType.TS_VECTOR, - ); - } - /** * COMPARE FIELD METADATA */ diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts index d9ee8908ce0e..0173d171e4a4 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts @@ -7,10 +7,7 @@ import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/wo import { ComparatorAction } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface'; import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; -import { - IndexMetadataEntity, - IndexType, -} from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceMigrationIndexFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory'; @@ -73,7 +70,7 @@ export class WorkspaceSyncIndexMetadataService { const indexMetadataRepository = manager.getRepository(IndexMetadataEntity); - let originalIndexMetadataCollection = await indexMetadataRepository.find({ + const originalIndexMetadataCollection = await indexMetadataRepository.find({ where: { workspaceId: context.workspaceId, objectMetadataId: Any( @@ -87,7 +84,7 @@ export class WorkspaceSyncIndexMetadataService { }); // Generate index metadata from models - let standardIndexMetadataCollection = this.standardIndexFactory.create( + const standardIndexMetadataCollection = this.standardIndexFactory.create( standardObjectMetadataDefinitions, context, originalStandardObjectMetadataMap, @@ -95,15 +92,6 @@ export class WorkspaceSyncIndexMetadataService { workspaceFeatureFlagsMap, ); - if (!workspaceFeatureFlagsMap.IS_SEARCH_ENABLED) { - originalIndexMetadataCollection = originalIndexMetadataCollection.filter( - (index) => index.indexType !== IndexType.GIN, - ); - - standardIndexMetadataCollection = standardIndexMetadataCollection.filter( - (index) => index.indexType !== IndexType.GIN, - ); - } const indexComparatorResults = this.workspaceIndexComparator.compare( originalIndexMetadataCollection, standardIndexMetadataCollection, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts index c090d413aa18..16f01b999ad4 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts @@ -5,7 +5,6 @@ import { DataSource, QueryFailedError, Repository } from 'typeorm'; import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; -import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; @@ -153,13 +152,6 @@ export class WorkspaceSyncMetadataService { await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( context.workspaceId, ); - - if (workspaceFeatureFlagsMap.IS_SEARCH_ENABLED) { - await this.featureFlagService.enableFeatureFlags( - [FeatureFlagKey.IsWorkspaceMigratedForSearch], - context.workspaceId, - ); - } } catch (error) { this.logger.error('Sync of standard objects failed with:', error); From 552eceecf6548304b32c70bc44923af1427495bc Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Tue, 15 Oct 2024 15:07:47 +0200 Subject: [PATCH 7/9] Add await and detail more in error message --- .../src/engine/metadata-modules/search/search.service.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts b/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts index 2d875955d3ac..b24ad3816a99 100644 --- a/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts @@ -74,10 +74,12 @@ export class SearchService { ); if (!isDefined(searchableFieldForCustomObject)) { - throw new Error('No searchable field found for custom object'); + throw new Error( + `No searchable field found for custom object (object name: ${createdObjectMetadata.nameSingular})`, + ); } - this.workspaceMigrationService.createCustomMigration( + await this.workspaceMigrationService.createCustomMigration( generateMigrationName(`create-${createdObjectMetadata.nameSingular}`), createdObjectMetadata.workspaceId, [ From ac4589a401f724fed7db18f4b04b1538befb4b4a Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Tue, 15 Oct 2024 15:18:39 +0200 Subject: [PATCH 8/9] change command name --- .../0-31/0-32/0-32-simplify-search-vector-expression.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.ts index 1f38466f5ef3..27d75e213c56 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.ts @@ -28,7 +28,7 @@ import { SEARCH_FIELDS_FOR_OPPORTUNITY } from 'src/modules/opportunity/standard- import { SEARCH_FIELDS_FOR_PERSON } from 'src/modules/person/standard-objects/person.workspace-entity'; @Command({ - name: 'upgrade-0.32:simplify-search-vector-expression', + name: 'fix-0.32:simplify-search-vector-expression', description: 'Replace searchVector with simpler expression', }) export class SimplifySearchVectorExpressionCommand extends ActiveWorkspacesCommandRunner { From a795da81b9f3c1bb8c264195949b72086eecff92 Mon Sep 17 00:00:00 2001 From: Marie Stoppa Date: Tue, 15 Oct 2024 15:49:27 +0200 Subject: [PATCH 9/9] Perform upsert at index creation and add constraint on indexMetadata --- ...8999374151-addConstraintOnIndexMetadata.ts | 17 ++++++++ .../graphql-query-search-resolver.service.ts | 5 +++ .../index-metadata/index-metadata.entity.ts | 6 +++ .../index-metadata/index-metadata.service.ts | 43 +++++++++++-------- 4 files changed, 52 insertions(+), 19 deletions(-) create mode 100644 packages/twenty-server/src/database/typeorm/metadata/migrations/1728999374151-addConstraintOnIndexMetadata.ts diff --git a/packages/twenty-server/src/database/typeorm/metadata/migrations/1728999374151-addConstraintOnIndexMetadata.ts b/packages/twenty-server/src/database/typeorm/metadata/migrations/1728999374151-addConstraintOnIndexMetadata.ts new file mode 100644 index 000000000000..9659750f889b --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/metadata/migrations/1728999374151-addConstraintOnIndexMetadata.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddConstraintOnIndex1728999374151 implements MigrationInterface { + name = 'AddConstraintOnIndexMetadata1728999374151'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "metadata"."indexMetadata" ADD CONSTRAINT "IndexOnNameAndWorkspaceIdAndObjectMetadataUnique" UNIQUE ("name", "workspaceId", "objectMetadataId")`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "metadata"."indexMetadata" DROP CONSTRAINT "IndexOnNameAndWorkspaceIdAndObjectMetadataUnique"`, + ); + } +} diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts index 2914c20aae78..cfd570281187 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts @@ -93,4 +93,9 @@ export class GraphqlQuerySearchResolverService return formattedWords.join(' | '); } + + async validate( + _args: SearchResolverArgs, + _options: WorkspaceQueryRunnerOptions, + ): Promise {} } diff --git a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts index fc2f991ce225..b748e2ea5042 100644 --- a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts @@ -7,6 +7,7 @@ import { OneToMany, PrimaryGeneratedColumn, Relation, + Unique, UpdateDateColumn, } from 'typeorm'; @@ -18,6 +19,11 @@ export enum IndexType { GIN = 'GIN', } +@Unique('IndexOnNameAndWorkspaceIdAndObjectMetadataUnique', [ + 'name', + 'workspaceId', + 'objectMetadataId', +]) @Entity('indexMetadata') export class IndexMetadataEntity { @PrimaryGeneratedColumn('uuid') diff --git a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.service.ts index 362519ef78a4..1d75cbd7ad3f 100644 --- a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { isDefined } from 'class-validator'; -import { Repository } from 'typeorm'; +import { InsertResult, Repository } from 'typeorm'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { @@ -45,32 +45,37 @@ export class IndexMetadataService { const indexName = `IDX_${generateDeterministicIndexName([tableName, ...columnNames])}`; - let savedIndexMetadata: IndexMetadataEntity; + let result: InsertResult; try { - savedIndexMetadata = await this.indexMetadataRepository.save({ - name: indexName, - tableName, - indexFieldMetadatas: fieldMetadataToIndex.map( - (fieldMetadata, index) => { - return { - fieldMetadataId: fieldMetadata.id, - order: index, - }; - }, - ), - workspaceId, - objectMetadataId: objectMetadata.id, - ...(isDefined(indexType) ? { indexType: indexType } : {}), - isCustom: isCustom, - }); + result = await this.indexMetadataRepository.upsert( + { + name: indexName, + indexFieldMetadatas: fieldMetadataToIndex.map( + (fieldMetadata, index) => { + return { + fieldMetadataId: fieldMetadata.id, + order: index, + }; + }, + ), + workspaceId, + objectMetadataId: objectMetadata.id, + ...(isDefined(indexType) ? { indexType: indexType } : {}), + isCustom: isCustom, + }, + { + conflictPaths: ['workspaceId', 'name', 'objectMetadataId'], + skipUpdateIfNoValuesChanged: true, + }, + ); } catch (error) { throw new Error( `Failed to create index ${indexName} on object metadata ${objectMetadata.nameSingular}`, ); } - if (!savedIndexMetadata) { + if (!result.identifiers.length) { throw new Error( `Failed to return saved index ${indexName} on object metadata ${objectMetadata.nameSingular}`, );