From 417cfb462568a932dc7c0bb3fedd2b0135a4d9b9 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Thu, 23 Jan 2025 11:37:32 +0100 Subject: [PATCH 01/15] WIP, make the whole saveEntity process transactional --- app/api/entities/entities.js | 34 ++++---- app/api/entities/entitySavingManager.ts | 81 +++++++++++-------- app/api/entities/managerFunctions.ts | 42 ++++++---- .../specs/entitySavingManager.spec.ts | 46 +++++++++++ app/api/files/S3Storage.ts | 32 ++++++++ app/api/files/files.ts | 16 ++-- app/api/files/specs/s3Storage.spec.ts | 75 +++++++++++++++++ app/api/odm/MultiTenantMongooseModel.ts | 20 ++--- app/api/odm/model.ts | 31 ++++--- 9 files changed, 285 insertions(+), 92 deletions(-) diff --git a/app/api/entities/entities.js b/app/api/entities/entities.js index 76272d95a9..83e26aa9d2 100644 --- a/app/api/entities/entities.js +++ b/app/api/entities/entities.js @@ -45,8 +45,8 @@ const FIELD_TYPES_TO_SYNC = [ propertyTypes.numeric, ]; -async function updateEntity(entity, _template, unrestricted = false) { - const docLanguages = await this.getAllLanguages(entity.sharedId); +async function updateEntity(entity, _template, unrestricted = false, session) { + const docLanguages = await this.getAllLanguages(entity.sharedId, { session }); if ( docLanguages[0].template && @@ -55,7 +55,7 @@ async function updateEntity(entity, _template, unrestricted = false) { ) { await Promise.all([ this.deleteRelatedEntityFromMetadata(docLanguages[0]), - relationships.delete({ entity: entity.sharedId }, null, false), + relationships.delete({ entity: entity.sharedId }, null, false, { session }), ]); } const template = _template || { properties: [] }; @@ -156,7 +156,7 @@ async function updateEntity(entity, _template, unrestricted = false) { return result; } -async function createEntity(doc, [currentLanguage, languages], sharedId, docTemplate) { +async function createEntity(doc, [currentLanguage, languages], sharedId, docTemplate, session) { if (!docTemplate) docTemplate = await templates.getById(doc.template); const thesauriByKey = await templates.getRelatedThesauri(docTemplate); @@ -331,11 +331,12 @@ const validateWritePermissions = (ids, entitiesToUpdate) => { } }; -const withDocuments = async (entities, documentsFullText) => { +const withDocuments = async (entities, documentsFullText, options = {}) => { const sharedIds = entities.map(entity => entity.sharedId); const allFiles = await files.get( { entity: { $in: sharedIds } }, - documentsFullText ? '+fullText ' : ' ' + documentsFullText ? '+fullText ' : ' ', + options ); const idFileMap = new Map(); allFiles.forEach(file => { @@ -389,7 +390,7 @@ export default { createEntity, getEntityTemplate, async save(_doc, { user, language }, options = {}) { - const { updateRelationships = true, index = true, includeDocuments = true } = options; + const { updateRelationships = true, index = true, includeDocuments = true, session } = options; await validateEntity(_doc); await saveSelections(_doc); const doc = _doc; @@ -405,7 +406,7 @@ export default { doc.editDate = date.currentUTC(); if (doc.sharedId) { - await this.updateEntity(this.sanitize(doc, template), template); + await this.updateEntity(this.sanitize(doc, template), template, false, session); } else { const [{ languages }, [defaultTemplate]] = await Promise.all([ settings.get(), @@ -421,12 +422,13 @@ export default { this.sanitize(doc, docTemplate), [language, languages], sharedId, - docTemplate + docTemplate, + session ); } const [entity] = includeDocuments - ? await this.getUnrestrictedWithDocuments({ sharedId, language }, '+permissions') + ? await this.getUnrestrictedWithDocuments({ sharedId, language }, '+permissions', { session }) : await this.getUnrestricted({ sharedId, language }, '+permissions'); if (updateRelationships) { @@ -503,7 +505,7 @@ export default { async getUnrestrictedWithDocuments(query, select, options = {}) { const { documentsFullText, ...restOfOptions } = options; const entities = await this.getUnrestricted(query, select, restOfOptions); - return withDocuments(entities, documentsFullText); + return withDocuments(entities, documentsFullText, restOfOptions); }, async get(query, select, options = {}) { @@ -511,7 +513,7 @@ export default { const extendedSelect = withoutDocuments ? select : extendSelect(select); const entities = await model.get(query, extendedSelect, restOfOptions); - return withoutDocuments ? entities : withDocuments(entities, documentsFullText); + return withoutDocuments ? entities : withDocuments(entities, documentsFullText, options); }, async getWithRelationships(query, select, pagination) { @@ -575,8 +577,8 @@ export default { return this.get({ sharedId: { $in: ids }, language: params.language }); }, - async getAllLanguages(sharedId) { - const entities = await model.get({ sharedId }); + async getAllLanguages(sharedId, options = {}) { + const entities = await model.get({ sharedId }, null, options); return entities; }, @@ -915,4 +917,8 @@ export default { }, count: model.count.bind(model), + + async startSession() { + return model.db.startSession(); + }, }; diff --git a/app/api/entities/entitySavingManager.ts b/app/api/entities/entitySavingManager.ts index 2d20a8ed3d..7925491405 100644 --- a/app/api/entities/entitySavingManager.ts +++ b/app/api/entities/entitySavingManager.ts @@ -13,46 +13,61 @@ const saveEntity = async ( socketEmiter, }: { user: UserSchema; language: string; socketEmiter?: Function; files?: FileAttachment[] } ) => { - const { attachments, documents } = (reqFiles || []).reduce( - (acum, file) => set(acum, file.fieldname, file), - { - attachments: [] as FileAttachment[], - documents: [] as FileAttachment[], - } - ); + const session = await entities.startSession(); - const entity = handleAttachmentInMetadataProperties(_entity, attachments); + try { + await session.startTransaction(); - const updatedEntity = await entities.save( - entity, - { user, language }, - { includeDocuments: false } - ); + const { attachments, documents } = (reqFiles || []).reduce( + (acum, file) => set(acum, file.fieldname, file), + { + attachments: [] as FileAttachment[], + documents: [] as FileAttachment[], + } + ); - const { proccessedAttachments, proccessedDocuments } = await processFiles( - entity, - updatedEntity, - attachments, - documents - ); + const entity = handleAttachmentInMetadataProperties(_entity, attachments); - const fileSaveErrors = await saveFiles( - proccessedAttachments, - proccessedDocuments, - updatedEntity, - socketEmiter - ); + const updatedEntity = await entities.save( + entity, + { user, language }, + { includeDocuments: false, session } + ); - const [entityWithAttachments]: EntityWithFilesSchema[] = - await entities.getUnrestrictedWithDocuments( - { - sharedId: updatedEntity.sharedId, - language: updatedEntity.language, - }, - '+permissions' + const { proccessedAttachments, proccessedDocuments } = await processFiles( + entity, + updatedEntity, + attachments, + documents, + session ); - return { entity: entityWithAttachments, errors: fileSaveErrors }; + const fileSaveErrors = await saveFiles( + proccessedAttachments, + proccessedDocuments, + updatedEntity, + socketEmiter, + session + ); + + const [entityWithAttachments]: EntityWithFilesSchema[] = + await entities.getUnrestrictedWithDocuments( + { + sharedId: updatedEntity.sharedId, + language: updatedEntity.language, + }, + '+permissions', + { session } + ); + + await session.commitTransaction(); + return { entity: entityWithAttachments, errors: fileSaveErrors }; + } catch (e) { + await session.abortTransaction(); + throw e; + } finally { + await session.endSession(); + } }; export type FileAttachment = { diff --git a/app/api/entities/managerFunctions.ts b/app/api/entities/managerFunctions.ts index 0bb2a8308e..03f2ec44cf 100644 --- a/app/api/entities/managerFunctions.ts +++ b/app/api/entities/managerFunctions.ts @@ -13,6 +13,7 @@ import { MetadataObjectSchema } from 'shared/types/commonTypes'; import { EntityWithFilesSchema } from 'shared/types/entityType'; import { TypeOfFile } from 'shared/types/fileSchema'; import { FileAttachment } from './entitySavingManager'; +import { ClientSession } from 'mongodb'; const prepareNewFiles = async ( entity: EntityWithFilesSchema, @@ -67,7 +68,8 @@ const prepareNewFiles = async ( const updateDeletedFiles = async ( entityFiles: WithId[], entity: EntityWithFilesSchema, - type: TypeOfFile.attachment | TypeOfFile.document + type: TypeOfFile.attachment | TypeOfFile.document, + session?: ClientSession ) => { const deletedFiles = entityFiles.filter( existingFile => @@ -79,9 +81,12 @@ const updateDeletedFiles = async ( ); const fileIdList = deletedFiles.map(file => file._id.toString()); const fileNameList = fileIdList.map(fileId => `${fileId}.jpg`); - await filesAPI.delete({ - $or: [{ _id: { $in: fileIdList } }, { filename: { $in: fileNameList } }], - }); + await filesAPI.delete( + { + $or: [{ _id: { $in: fileIdList } }, { filename: { $in: fileNameList } }], + }, + { session } + ); }; const filterRenamedFiles = (entity: EntityWithFilesSchema, entityFiles: WithId[]) => { @@ -111,32 +116,34 @@ const filterRenamedFiles = (entity: EntityWithFilesSchema, entityFiles: WithId { - const { attachments, documents } = await prepareNewFiles( + const { attachments: newAttachments, documents: newDocuments } = await prepareNewFiles( entity, updatedEntity, - fileAttachments, - documentAttachments + attachments, + documents ); if (entity._id && (entity.attachments || entity.documents)) { const entityFiles: WithId[] = await filesAPI.get( { entity: entity.sharedId, type: { $in: [TypeOfFile.attachment, TypeOfFile.document] } }, - '_id, originalname, type' + '_id, originalname, type', + { session } ); - await updateDeletedFiles(entityFiles, entity, TypeOfFile.attachment); - await updateDeletedFiles(entityFiles, entity, TypeOfFile.document); + await updateDeletedFiles(entityFiles, entity, TypeOfFile.attachment, session); + await updateDeletedFiles(entityFiles, entity, TypeOfFile.document, session); const { renamedAttachments, renamedDocuments } = filterRenamedFiles(entity, entityFiles); - attachments.push(...renamedAttachments); - documents.push(...renamedDocuments); + newAttachments.push(...renamedAttachments); + newDocuments.push(...renamedDocuments); } - return { proccessedAttachments: attachments, proccessedDocuments: documents }; + return { proccessedAttachments: newAttachments, proccessedDocuments: newDocuments }; }; const bindAttachmentToMetadataProperty = ( @@ -175,7 +182,8 @@ const saveFiles = async ( attachments: FileType[], documents: FileType[], entity: ClientEntitySchema, - socketEmiter?: Function + socketEmiter?: Function, + session?: ClientSession ) => { const saveResults: string[] = []; @@ -188,7 +196,7 @@ const saveFiles = async ( await Promise.all( filesToSave.map(async file => { try { - await filesAPI.save(file, false); + await filesAPI.save(file, false, session); } catch (e) { legacyLogger.error(prettifyError(e)); saveResults.push(`Could not save file/s: ${file.originalname}`); diff --git a/app/api/entities/specs/entitySavingManager.spec.ts b/app/api/entities/specs/entitySavingManager.spec.ts index 44bc1fe647..18dc23b1ee 100644 --- a/app/api/entities/specs/entitySavingManager.spec.ts +++ b/app/api/entities/specs/entitySavingManager.spec.ts @@ -614,4 +614,50 @@ describe('entitySavingManager', () => { }); }); }); + + describe('transactions', () => { + const reqData = { user: editorUser, language: 'en', socketEmiter: () => {} }; + + it('should rollback all operations if any operation fails', async () => { + // Setup initial entity with a document + const { entity: savedEntity } = await saveEntity( + { title: 'initial entity', template: template1Id }, + { + ...reqData, + files: [{ ...newMainPdfDocument, fieldname: 'documents[0]' }], + } + ); + + // Force files.save to fail + jest.spyOn(filesAPI, 'save').mockImplementationOnce(() => { + throw new Error('Forced file save error'); + }); + + // Attempt to update entity with new attachment that will fail + const updateOperation = saveEntity( + { + _id: savedEntity._id, + sharedId: savedEntity.sharedId, + title: 'updated title', + template: template1Id, + attachments: [{ originalname: 'will fail', url: 'https://fail.com' }], + }, + { + ...reqData, + files: [{ ...file, fieldname: 'attachments[0]' }], + } + ); + + // await expect(updateOperation).rejects.toThrow('Forced file save error'); + + // Verify entity was not updated + const [entityAfterFailure] = await entities.get({ _id: savedEntity._id }); + expect(entityAfterFailure.title).toBe('initial entity'); + + // Verify original document still exists and no new files were added + const entityFiles = await filesAPI.get({ entity: savedEntity.sharedId }); + expect(entityFiles).toHaveLength(1); + expect(entityFiles[0].originalname).toBe('myNewFile.pdf'); + }); + }); }); diff --git a/app/api/files/S3Storage.ts b/app/api/files/S3Storage.ts index a456ef3110..83d48addb5 100644 --- a/app/api/files/S3Storage.ts +++ b/app/api/files/S3Storage.ts @@ -90,6 +90,38 @@ export class S3Storage { ) ); } + + async uploadMany(files: { key: string; body: Buffer }[]) { + const uploadedKeys: string[] = []; + + try { + const uploadPromises = files.map(async ({ key, body }) => { + const result = await this.client.send( + new PutObjectCommand({ + Bucket: S3Storage.bucketName(), + Key: key, + Body: body, + }) + ); + uploadedKeys.push(key); + return result; + }); + + return await catchS3Errors(async () => Promise.all(uploadPromises)); + } catch (error) { + // If any upload fails, delete all successfully uploaded files + if (uploadedKeys.length > 0) { + const deletePromises = uploadedKeys.map(async key => this.delete(key)); + await Promise.all(deletePromises).catch(deleteError => { + // Enhance the original error with cleanup failure information + throw new Error( + `Upload failed and cleanup was incomplete. Original error: ${error.message}. Cleanup error: ${deleteError.message}` + ); + }); + } + throw error; + } + } } export { S3TimeoutError }; diff --git a/app/api/files/files.ts b/app/api/files/files.ts index 1c91d0c9ce..0a3a725d2c 100644 --- a/app/api/files/files.ts +++ b/app/api/files/files.ts @@ -5,15 +5,16 @@ import { DefaultLogger } from 'api/log.v2/infrastructure/StandardLogger'; import connections from 'api/relationships'; import { search } from 'api/search'; import { cleanupRecordsOfFiles } from 'api/services/ocr/ocrRecords'; +import { ClientSession } from 'mongodb'; import { validateFile } from 'shared/types/fileSchema'; import { FileType } from 'shared/types/fileType'; +import { inspect } from 'util'; import { FileCreatedEvent } from './events/FileCreatedEvent'; import { FilesDeletedEvent } from './events/FilesDeletedEvent'; import { FileUpdatedEvent } from './events/FileUpdatedEvent'; import { filesModel } from './filesModel'; import { storage } from './storage'; import { V2 } from './v2_support'; -import { inspect } from 'util'; const deduceMimeType = (_file: FileType) => { const file = { ..._file }; @@ -26,11 +27,11 @@ const deduceMimeType = (_file: FileType) => { }; export const files = { - async save(_file: FileType, index = true) { + async save(_file: FileType, index = true, session?: ClientSession) { const file = deduceMimeType(_file); const existingFile = file._id ? await filesModel.getById(file._id) : undefined; - const savedFile = await filesModel.save(await validateFile(file)); + const savedFile = await filesModel.save(await validateFile(file), undefined, session); if (index) { await search.indexEntities({ sharedId: savedFile.entity }, '+fullText'); } @@ -52,17 +53,18 @@ export const files = { return savedFile; }, - get: filesModel.get.bind(filesModel), + get: (query: any, select?: any, options?: { session?: ClientSession }) => + filesModel.get(query, select, options), - async delete(query: any = {}) { + async delete(query: any = {}, options: { session?: ClientSession } = {}) { const hasFileName = (file: FileType): file is FileType & { filename: string } => !!file.filename; const toDeleteFiles: FileType[] = await filesModel.get(query); - await filesModel.delete(query); + await filesModel.delete(query, options); if (toDeleteFiles.length > 0) { const idsToDelete = toDeleteFiles.map(f => f._id!.toString()); - await connections.delete({ file: { $in: idsToDelete } }); + await connections.delete({ file: { $in: idsToDelete } }, null, false, options); await V2.deleteTextReferencesToFiles(idsToDelete); await Promise.all( diff --git a/app/api/files/specs/s3Storage.spec.ts b/app/api/files/specs/s3Storage.spec.ts index 35c601fe97..6bc6423568 100644 --- a/app/api/files/specs/s3Storage.spec.ts +++ b/app/api/files/specs/s3Storage.spec.ts @@ -1,3 +1,4 @@ +import { DeleteObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3'; import { S3Storage, S3TimeoutError } from '../S3Storage'; let s3Storage: S3Storage; @@ -11,6 +12,30 @@ class S3TimeoutClient { } } +class MockS3Client { + uploadedFiles: { key: string; body: Buffer }[] = []; + deletedFiles: string[] = []; + shouldFailOnKey?: string; + + async send(command: any) { + if (command instanceof PutObjectCommand) { + if (command.input.Key === this.shouldFailOnKey) { + throw new Error(`Failed to upload ${command.input.Key}`); + } + this.uploadedFiles.push({ + key: command.input.Key, + body: command.input.Body + }); + return {}; + } + if (command instanceof DeleteObjectCommand) { + this.deletedFiles.push(command.input.Key); + return {}; + } + return {}; + } +} + describe('s3Storage', () => { beforeAll(async () => { // @ts-ignore @@ -42,4 +67,54 @@ describe('s3Storage', () => { await expect(s3Storage.list()).rejects.toBeInstanceOf(S3TimeoutError); }); }); + + describe('uploadMany', () => { + let mockS3Client: MockS3Client; + + beforeEach(() => { + mockS3Client = new MockS3Client(); + // @ts-ignore + s3Storage = new S3Storage(mockS3Client); + }); + + it('should upload multiple files successfully', async () => { + const files = [ + { key: 'file1.txt', body: Buffer.from('content1') }, + { key: 'file2.txt', body: Buffer.from('content2') }, + ]; + + await s3Storage.uploadMany(files); + + expect(mockS3Client.uploadedFiles).toHaveLength(2); + expect(mockS3Client.uploadedFiles[0]).toEqual(files[0]); + expect(mockS3Client.uploadedFiles[1]).toEqual(files[1]); + expect(mockS3Client.deletedFiles).toHaveLength(0); + }); + + it('should cleanup uploaded files if one upload fails', async () => { + mockS3Client.shouldFailOnKey = 'file2.txt'; + const files = [ + { key: 'file1.txt', body: Buffer.from('content1') }, + { key: 'file2.txt', body: Buffer.from('content2') }, + ]; + + await expect(s3Storage.uploadMany(files)).rejects.toThrow('Failed to upload file2.txt'); + + expect(mockS3Client.uploadedFiles).toHaveLength(1); + expect(mockS3Client.uploadedFiles[0]).toEqual(files[0]); + expect(mockS3Client.deletedFiles).toHaveLength(1); + expect(mockS3Client.deletedFiles[0]).toBe('file1.txt'); + }); + + it('should throw S3TimeoutError on timeout', async () => { + // @ts-ignore + s3Storage = new S3Storage(new S3TimeoutClient()); + + await expect( + s3Storage.uploadMany([ + { key: 'file1.txt', body: Buffer.from('content1') } + ]) + ).rejects.toBeInstanceOf(S3TimeoutError); + }); + }); }); diff --git a/app/api/odm/MultiTenantMongooseModel.ts b/app/api/odm/MultiTenantMongooseModel.ts index 1da4ebc474..86fae7075d 100644 --- a/app/api/odm/MultiTenantMongooseModel.ts +++ b/app/api/odm/MultiTenantMongooseModel.ts @@ -1,4 +1,4 @@ -import { BulkWriteOptions } from 'mongodb'; +import { BulkWriteOptions, ClientSession } from 'mongodb'; import mongoose, { Schema } from 'mongoose'; import { DataType, @@ -11,7 +11,7 @@ import { import { tenants } from '../tenants/tenantContext'; import { DB } from './DB'; -class MultiTenantMongooseModel { +export class MultiTenantMongooseModel { dbs: { [k: string]: mongoose.Model> }; collectionName: string; @@ -43,13 +43,13 @@ class MultiTenantMongooseModel { async findOneAndUpdate( query: UwaziFilterQuery>, update: UwaziUpdateQuery>, - options: UwaziQueryOptions + options: UwaziQueryOptions & { session?: ClientSession } ) { return this.dbForCurrentTenant().findOneAndUpdate(query, update, options); } - async create(data: Partial>) { - return this.dbForCurrentTenant().create(data); + async create(data: Partial>[], options?: any) { + return this.dbForCurrentTenant().create(data, options); } async createMany(dataArray: Partial>[]) { @@ -81,8 +81,8 @@ class MultiTenantMongooseModel { return this.dbForCurrentTenant().distinct(field, query); } - async deleteMany(query: UwaziFilterQuery>) { - return this.dbForCurrentTenant().deleteMany(query); + async deleteMany(query: UwaziFilterQuery>, options?: { session?: ClientSession }) { + return this.dbForCurrentTenant().deleteMany(query, options); } async aggregate(aggregations?: any[]) { @@ -108,6 +108,8 @@ class MultiTenantMongooseModel { async ensureIndexes() { return this.dbForCurrentTenant().ensureIndexes(); } -} -export { MultiTenantMongooseModel }; + async startSession() { + return this.dbForCurrentTenant().db.startSession(); + } +} diff --git a/app/api/odm/model.ts b/app/api/odm/model.ts index 26ace86cad..9f8ab5db85 100644 --- a/app/api/odm/model.ts +++ b/app/api/odm/model.ts @@ -13,6 +13,7 @@ import { inspect } from 'util'; import { MultiTenantMongooseModel } from './MultiTenantMongooseModel'; import { UpdateLogger, createUpdateLogHelper } from './logHelper'; import { ModelBulkWriteStream } from './modelBulkWriteStream'; +import { ClientSession } from 'mongodb'; /** Ideas! * T is the actual model-specific document Schema! @@ -84,18 +85,24 @@ export class OdmModel implements SyncDBDataSource { } } - async save(data: Partial>, _query?: any) { + async startSession(): Promise { + return this.db.startSession(); + } + + async save( + data: Partial>, + _query?: any, + session?: ClientSession + ) { if (await this.documentExists(data)) { - // @ts-ignore const { __v: version, ...toSaveData } = data; - const query = - _query && (await this.documentExistsByQuery(_query)) ? _query : { _id: data._id }; + const query = _query && (await this.documentExistsByQuery(_query)) ? _query : { _id: data._id }; await this.checkVersion(query, version, data); const saved = await this.db.findOneAndUpdate( query, { $set: toSaveData as UwaziUpdateQuery>, $inc: { __v: 1 } }, - { new: true } + { new: true, session } ); if (saved === null) { @@ -105,13 +112,13 @@ export class OdmModel implements SyncDBDataSource { await this.logHelper.upsertLogOne(saved); return saved.toObject>(); } - return this.create(data); + return this.create(data, session); } - async create(data: Partial>) { - const saved = await this.db.create(data); - await this.logHelper.upsertLogOne(saved); - return saved.toObject>(); + async create(data: Partial>, session?: ClientSession) { + const saved = await this.db.create([data], { session }); + await this.logHelper.upsertLogOne(saved[0]); + return saved[0].toObject>(); } async saveMultiple( @@ -208,13 +215,13 @@ export class OdmModel implements SyncDBDataSource { return results as EnforcedWithId | null; } - async delete(condition: any) { + async delete(condition: any, options: { session?: ClientSession } = {}) { let cond = condition; if (mongoose.Types.ObjectId.isValid(condition)) { cond = { _id: condition }; } await this.logHelper.upsertLogMany(cond, true); - return this.db.deleteMany(cond); + return this.db.deleteMany(cond, options); } async facet(aggregations: any[], pipelines: any, project: any) { From 943e693d077fbffb93d2e4fab5d8e64189f3d85e Mon Sep 17 00:00:00 2001 From: Daneryl Date: Fri, 24 Jan 2025 14:19:55 +0100 Subject: [PATCH 02/15] WIP, sharing mongo session via appContext --- app/api/entities/entities.js | 35 +++--- app/api/entities/entitySavingManager.ts | 20 +-- app/api/entities/managerFunctions.ts | 27 ++-- .../specs/entitySavingManager.spec.ts | 77 +++++++----- .../specs/entitySavingManagerFixtures.ts | 1 + app/api/entities/validateEntity.ts | 1 + .../entities/validation/validateEntityData.ts | 3 + app/api/files/files.ts | 6 +- app/api/odm/MultiTenantMongooseModel.ts | 23 ++-- app/api/odm/model.ts | 67 ++++++---- app/api/services/ocr/ocrRecords.ts | 50 ++++---- app/api/templates/templates.ts | 117 +++++++++++------- app/api/users/specs/users.spec.js | 18 +-- app/api/utils/testingEnvironment.ts | 16 +++ app/api/utils/testingTenants.ts | 9 ++ app/api/utils/testing_db.ts | 12 ++ 16 files changed, 293 insertions(+), 189 deletions(-) diff --git a/app/api/entities/entities.js b/app/api/entities/entities.js index 83e26aa9d2..6b7166b68d 100644 --- a/app/api/entities/entities.js +++ b/app/api/entities/entities.js @@ -47,14 +47,13 @@ const FIELD_TYPES_TO_SYNC = [ async function updateEntity(entity, _template, unrestricted = false, session) { const docLanguages = await this.getAllLanguages(entity.sharedId, { session }); - if ( docLanguages[0].template && entity.template && docLanguages[0].template.toString() !== entity.template.toString() ) { await Promise.all([ - this.deleteRelatedEntityFromMetadata(docLanguages[0]), + this.deleteRelatedEntityFromMetadata(docLanguages[0], session), relationships.delete({ entity: entity.sharedId }, null, false, { session }), ]); } @@ -100,9 +99,9 @@ async function updateEntity(entity, _template, unrestricted = false, session) { if (template._id) { await denormalizeRelated(fullEntity, template, currentDoc); } - const saveResult = await saveFunc(toSave); + const saveResult = await saveFunc(toSave, undefined, session); - await updateNewRelationships(v2RelationshipsUpdates); + await updateNewRelationships(v2RelationshipsUpdates, session); return saveResult; } @@ -139,16 +138,17 @@ async function updateEntity(entity, _template, unrestricted = false, session) { await denormalizeRelated(toSave, template, d); } - return saveFunc(toSave); + return saveFunc(toSave, undefined, session); }) ); - await denormalizeAfterEntityUpdate(entity); + await denormalizeAfterEntityUpdate(entity, session); + const afterEntities = await model.get({ sharedId: entity.sharedId }, null, { session }); await applicationEventsBus.emit( new EntityUpdatedEvent({ before: docLanguages, - after: await model.get({ sharedId: entity.sharedId }), + after: afterEntities, targetLanguageKey: entity.language, }) ); @@ -195,17 +195,18 @@ async function createEntity(doc, [currentLanguage, languages], sharedId, docTemp { thesauriByKey } ); - return model.save(langDoc); + return model.save(langDoc, undefined, session); }) ); - await updateNewRelationships(v2RelationshipsUpdates); + await updateNewRelationships(v2RelationshipsUpdates, session); - await Promise.all(result.map(r => denormalizeAfterEntityCreation(r))); + await Promise.all(result.map(r => denormalizeAfterEntityCreation(r, session))); + const createdEntities = await model.get({ sharedId }, null, { session }); await applicationEventsBus.emit( new EntityCreatedEvent({ - entities: await model.get({ sharedId }), + entities: createdEntities, targetLanguageKey: currentLanguage, }) ); @@ -390,7 +391,8 @@ export default { createEntity, getEntityTemplate, async save(_doc, { user, language }, options = {}) { - const { updateRelationships = true, index = true, includeDocuments = true, session } = options; + const { updateRelationships = true, index = true, includeDocuments = true } = options; + await validateEntity(_doc); await saveSelections(_doc); const doc = _doc; @@ -406,7 +408,7 @@ export default { doc.editDate = date.currentUTC(); if (doc.sharedId) { - await this.updateEntity(this.sanitize(doc, template), template, false, session); + await this.updateEntity(this.sanitize(doc, template), template); } else { const [{ languages }, [defaultTemplate]] = await Promise.all([ settings.get(), @@ -422,13 +424,12 @@ export default { this.sanitize(doc, docTemplate), [language, languages], sharedId, - docTemplate, - session + docTemplate ); } const [entity] = includeDocuments - ? await this.getUnrestrictedWithDocuments({ sharedId, language }, '+permissions', { session }) + ? await this.getUnrestrictedWithDocuments({ sharedId, language }, '+permissions') : await this.getUnrestricted({ sharedId, language }, '+permissions'); if (updateRelationships) { @@ -582,7 +583,7 @@ export default { return entities; }, - countByTemplate(template, language) { + async countByTemplate(template, language) { const query = language ? { template, language } : { template }; return model.count(query); }, diff --git a/app/api/entities/entitySavingManager.ts b/app/api/entities/entitySavingManager.ts index 7925491405..1a88ddc28e 100644 --- a/app/api/entities/entitySavingManager.ts +++ b/app/api/entities/entitySavingManager.ts @@ -1,8 +1,10 @@ -import { set } from 'lodash'; import entities from 'api/entities/entities'; +import { appContext } from 'api/utils/AppContext'; +import { set } from 'lodash'; import { EntityWithFilesSchema } from 'shared/types/entityType'; import { UserSchema } from 'shared/types/userType'; import { handleAttachmentInMetadataProperties, processFiles, saveFiles } from './managerFunctions'; +import templates from 'api/templates'; const saveEntity = async ( _entity: EntityWithFilesSchema, @@ -14,9 +16,11 @@ const saveEntity = async ( }: { user: UserSchema; language: string; socketEmiter?: Function; files?: FileAttachment[] } ) => { const session = await entities.startSession(); + appContext.set('mongoSession', undefined); // Clear any existing session first try { - await session.startTransaction(); + session.startTransaction(); + appContext.set('mongoSession', session); const { attachments, documents } = (reqFiles || []).reduce( (acum, file) => set(acum, file.fieldname, file), @@ -31,23 +35,21 @@ const saveEntity = async ( const updatedEntity = await entities.save( entity, { user, language }, - { includeDocuments: false, session } + { includeDocuments: false } ); const { proccessedAttachments, proccessedDocuments } = await processFiles( entity, updatedEntity, attachments, - documents, - session + documents ); const fileSaveErrors = await saveFiles( proccessedAttachments, proccessedDocuments, updatedEntity, - socketEmiter, - session + socketEmiter ); const [entityWithAttachments]: EntityWithFilesSchema[] = @@ -56,8 +58,7 @@ const saveEntity = async ( sharedId: updatedEntity.sharedId, language: updatedEntity.language, }, - '+permissions', - { session } + '+permissions' ); await session.commitTransaction(); @@ -66,6 +67,7 @@ const saveEntity = async ( await session.abortTransaction(); throw e; } finally { + appContext.set('mongoSession', undefined); await session.endSession(); } }; diff --git a/app/api/entities/managerFunctions.ts b/app/api/entities/managerFunctions.ts index 03f2ec44cf..db5e7cd9c1 100644 --- a/app/api/entities/managerFunctions.ts +++ b/app/api/entities/managerFunctions.ts @@ -68,8 +68,7 @@ const prepareNewFiles = async ( const updateDeletedFiles = async ( entityFiles: WithId[], entity: EntityWithFilesSchema, - type: TypeOfFile.attachment | TypeOfFile.document, - session?: ClientSession + type: TypeOfFile.attachment | TypeOfFile.document ) => { const deletedFiles = entityFiles.filter( existingFile => @@ -81,12 +80,9 @@ const updateDeletedFiles = async ( ); const fileIdList = deletedFiles.map(file => file._id.toString()); const fileNameList = fileIdList.map(fileId => `${fileId}.jpg`); - await filesAPI.delete( - { - $or: [{ _id: { $in: fileIdList } }, { filename: { $in: fileNameList } }], - }, - { session } - ); + await filesAPI.delete({ + $or: [{ _id: { $in: fileIdList } }, { filename: { $in: fileNameList } }], + }); }; const filterRenamedFiles = (entity: EntityWithFilesSchema, entityFiles: WithId[]) => { @@ -117,8 +113,7 @@ const processFiles = async ( entity: EntityWithFilesSchema, updatedEntity: EntityWithFilesSchema, attachments: FileAttachment[] = [], - documents: FileAttachment[] = [], - session?: ClientSession + documents: FileAttachment[] = [] ) => { const { attachments: newAttachments, documents: newDocuments } = await prepareNewFiles( entity, @@ -130,12 +125,11 @@ const processFiles = async ( if (entity._id && (entity.attachments || entity.documents)) { const entityFiles: WithId[] = await filesAPI.get( { entity: entity.sharedId, type: { $in: [TypeOfFile.attachment, TypeOfFile.document] } }, - '_id, originalname, type', - { session } + '_id, originalname, type' ); - await updateDeletedFiles(entityFiles, entity, TypeOfFile.attachment, session); - await updateDeletedFiles(entityFiles, entity, TypeOfFile.document, session); + await updateDeletedFiles(entityFiles, entity, TypeOfFile.attachment); + await updateDeletedFiles(entityFiles, entity, TypeOfFile.document); const { renamedAttachments, renamedDocuments } = filterRenamedFiles(entity, entityFiles); @@ -182,8 +176,7 @@ const saveFiles = async ( attachments: FileType[], documents: FileType[], entity: ClientEntitySchema, - socketEmiter?: Function, - session?: ClientSession + socketEmiter?: Function ) => { const saveResults: string[] = []; @@ -196,7 +189,7 @@ const saveFiles = async ( await Promise.all( filesToSave.map(async file => { try { - await filesAPI.save(file, false, session); + await filesAPI.save(file, false); } catch (e) { legacyLogger.error(prettifyError(e)); saveResults.push(`Could not save file/s: ${file.originalname}`); diff --git a/app/api/entities/specs/entitySavingManager.spec.ts b/app/api/entities/specs/entitySavingManager.spec.ts index 18dc23b1ee..9d7845c1e7 100644 --- a/app/api/entities/specs/entitySavingManager.spec.ts +++ b/app/api/entities/specs/entitySavingManager.spec.ts @@ -28,6 +28,10 @@ import { template2Id, textFile, } from './entitySavingManagerFixtures'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; +import { appContext } from 'api/utils/AppContext'; +import { string } from 'yargs'; +import templates from 'api/templates'; const validPdfString = ` %PDF-1.0 @@ -618,46 +622,51 @@ describe('entitySavingManager', () => { describe('transactions', () => { const reqData = { user: editorUser, language: 'en', socketEmiter: () => {} }; - it('should rollback all operations if any operation fails', async () => { - // Setup initial entity with a document - const { entity: savedEntity } = await saveEntity( - { title: 'initial entity', template: template1Id }, - { - ...reqData, - files: [{ ...newMainPdfDocument, fieldname: 'documents[0]' }], - } - ); - + xit('should rollback all operations if any operation fails', async () => { + testingEnvironment.unsetFakeContext(); // Force files.save to fail - jest.spyOn(filesAPI, 'save').mockImplementationOnce(() => { - throw new Error('Forced file save error'); - }); // Attempt to update entity with new attachment that will fail - const updateOperation = saveEntity( - { - _id: savedEntity._id, - sharedId: savedEntity.sharedId, - title: 'updated title', - template: template1Id, - attachments: [{ originalname: 'will fail', url: 'https://fail.com' }], - }, - { - ...reqData, - files: [{ ...file, fieldname: 'attachments[0]' }], - } - ); + await appContext.run(async () => { + // Setup initial entity with a document + const { entity: savedEntity } = await saveEntity( + { title: 'initial entity', template: template1Id }, + { + ...reqData, + files: [{ ...newMainPdfDocument, fieldname: 'documents[0]' }], + } + ); - // await expect(updateOperation).rejects.toThrow('Forced file save error'); + jest.spyOn(filesAPI, 'save').mockImplementationOnce(() => { + throw new Error('Forced file save error'); + }); + + const updateOperation = await saveEntity( + { + _id: savedEntity._id, + sharedId: savedEntity.sharedId, + title: 'updated title', + template: template1Id, + attachments: [{ originalname: 'will fail', url: 'https://fail.com' }], + }, + { + ...reqData, + files: [{ ...file, fieldname: 'attachments[0]' }], + } + ); - // Verify entity was not updated - const [entityAfterFailure] = await entities.get({ _id: savedEntity._id }); - expect(entityAfterFailure.title).toBe('initial entity'); + // Verify entity was not updated + const [entityAfterFailure] = await entities.get({ _id: savedEntity._id }); + expect(entityAfterFailure.title).toBe('initial entity'); + }); - // Verify original document still exists and no new files were added - const entityFiles = await filesAPI.get({ entity: savedEntity.sharedId }); - expect(entityFiles).toHaveLength(1); - expect(entityFiles[0].originalname).toBe('myNewFile.pdf'); + // await expect(updateOperation).rejects.toThrow('Forced file save error'); + // + // + // // Verify original document still exists and no new files were added + // const entityFiles = await filesAPI.get({ entity: savedEntity.sharedId }); + // expect(entityFiles).toHaveLength(1); + // expect(entityFiles[0].originalname).toBe('myNewFile.pdf'); }); }); }); diff --git a/app/api/entities/specs/entitySavingManagerFixtures.ts b/app/api/entities/specs/entitySavingManagerFixtures.ts index a1e277ca0c..e650fa194d 100644 --- a/app/api/entities/specs/entitySavingManagerFixtures.ts +++ b/app/api/entities/specs/entitySavingManagerFixtures.ts @@ -76,6 +76,7 @@ const entity3textFile: FileType = { }; const fixtures: DBFixture = { + ocr_records: [], entities: [ { _id: entity1Id, diff --git a/app/api/entities/validateEntity.ts b/app/api/entities/validateEntity.ts index 6c6aec9b91..bfd276f12b 100644 --- a/app/api/entities/validateEntity.ts +++ b/app/api/entities/validateEntity.ts @@ -1,5 +1,6 @@ import { validateEntitySchema } from './validation/validateEntitySchema'; import { validateEntityData } from './validation/validateEntityData'; +import templates from 'api/templates'; export const validateEntity = async (entity: any) => { await validateEntitySchema(entity); diff --git a/app/api/entities/validation/validateEntityData.ts b/app/api/entities/validation/validateEntityData.ts index afc2234781..ac4f09ffa1 100644 --- a/app/api/entities/validation/validateEntityData.ts +++ b/app/api/entities/validation/validateEntityData.ts @@ -8,6 +8,8 @@ import ValidationError from 'ajv/dist/runtime/validation_error'; import { validateMetadataField } from './validateMetadataField'; import { customErrorMessages, validators } from './metadataValidators'; +import { tenants } from 'api/tenants'; +import templates from 'api/templates'; const ajv = new Ajv({ allErrors: true }); ajv.addVocabulary(['tsType']); @@ -86,6 +88,7 @@ ajv.addKeyword({ if (!entity.template) { return true; } + const [template] = await templatesModel.get({ _id: entity.template }); if (!template) { throw new Ajv.ValidationError([ diff --git a/app/api/files/files.ts b/app/api/files/files.ts index 0a3a725d2c..8907482647 100644 --- a/app/api/files/files.ts +++ b/app/api/files/files.ts @@ -56,15 +56,15 @@ export const files = { get: (query: any, select?: any, options?: { session?: ClientSession }) => filesModel.get(query, select, options), - async delete(query: any = {}, options: { session?: ClientSession } = {}) { + async delete(query: any = {}) { const hasFileName = (file: FileType): file is FileType & { filename: string } => !!file.filename; const toDeleteFiles: FileType[] = await filesModel.get(query); - await filesModel.delete(query, options); + await filesModel.delete(query); if (toDeleteFiles.length > 0) { const idsToDelete = toDeleteFiles.map(f => f._id!.toString()); - await connections.delete({ file: { $in: idsToDelete } }, null, false, options); + await connections.delete({ file: { $in: idsToDelete } }, null, false); await V2.deleteTextReferencesToFiles(idsToDelete); await Promise.all( diff --git a/app/api/odm/MultiTenantMongooseModel.ts b/app/api/odm/MultiTenantMongooseModel.ts index 86fae7075d..9bed4f85f5 100644 --- a/app/api/odm/MultiTenantMongooseModel.ts +++ b/app/api/odm/MultiTenantMongooseModel.ts @@ -32,11 +32,15 @@ export class MultiTenantMongooseModel { ); } - findById(id: any, select?: any) { - return this.dbForCurrentTenant().findById(id, select, { lean: true }); + findById(id: any, select?: any, options: { session?: ClientSession } = {}) { + return this.dbForCurrentTenant().findById(id, select, { ...options, lean: true }); } - find(query: UwaziFilterQuery>, select = '', options = {}) { + find( + query: UwaziFilterQuery>, + select = '', + options: { session?: ClientSession } = {} + ) { return this.dbForCurrentTenant().find(query, select, options); } @@ -52,14 +56,14 @@ export class MultiTenantMongooseModel { return this.dbForCurrentTenant().create(data, options); } - async createMany(dataArray: Partial>[]) { - return this.dbForCurrentTenant().create(dataArray); + async createMany(dataArray: Partial>[], options: { session?: ClientSession } = {}) { + return this.dbForCurrentTenant().create(dataArray, options); } async _updateMany( conditions: UwaziFilterQuery>, doc: UwaziUpdateQuery>, - options?: UwaziUpdateOptions> + options: UwaziUpdateOptions> & { session?: ClientSession } = {} ) { return this.dbForCurrentTenant().updateMany(conditions, doc, options); } @@ -73,8 +77,11 @@ export class MultiTenantMongooseModel { return this.dbForCurrentTenant().replaceOne(conditions, replacement); } - async countDocuments(query: UwaziFilterQuery> = {}) { - return this.dbForCurrentTenant().countDocuments(query); + async countDocuments( + query: UwaziFilterQuery> = {}, + options: { session?: ClientSession } = {} + ) { + return this.dbForCurrentTenant().countDocuments(query, options); } async distinct(field: string, query: UwaziFilterQuery> = {}) { diff --git a/app/api/odm/model.ts b/app/api/odm/model.ts index 9f8ab5db85..0e1f8855a1 100644 --- a/app/api/odm/model.ts +++ b/app/api/odm/model.ts @@ -14,6 +14,7 @@ import { MultiTenantMongooseModel } from './MultiTenantMongooseModel'; import { UpdateLogger, createUpdateLogHelper } from './logHelper'; import { ModelBulkWriteStream } from './modelBulkWriteStream'; import { ClientSession } from 'mongodb'; +import { appContext } from 'api/utils/AppContext'; /** Ideas! * T is the actual model-specific document Schema! @@ -85,24 +86,26 @@ export class OdmModel implements SyncDBDataSource { } } + private getSession(): ClientSession | undefined { + return appContext.get('mongoSession') as ClientSession | undefined; + } + async startSession(): Promise { return this.db.startSession(); } - async save( - data: Partial>, - _query?: any, - session?: ClientSession - ) { + async save(data: Partial>, _query?: any) { + const session = this.getSession(); if (await this.documentExists(data)) { const { __v: version, ...toSaveData } = data; - const query = _query && (await this.documentExistsByQuery(_query)) ? _query : { _id: data._id }; + const query = + _query && (await this.documentExistsByQuery(_query)) ? _query : { _id: data._id }; await this.checkVersion(query, version, data); const saved = await this.db.findOneAndUpdate( query, { $set: toSaveData as UwaziUpdateQuery>, $inc: { __v: 1 } }, - { new: true, session } + { new: true, ...(session && { session }) } ); if (saved === null) { @@ -112,11 +115,12 @@ export class OdmModel implements SyncDBDataSource { await this.logHelper.upsertLogOne(saved); return saved.toObject>(); } - return this.create(data, session); + return this.create(data); } - async create(data: Partial>, session?: ClientSession) { - const saved = await this.db.create([data], { session }); + async create(data: Partial>) { + const session = this.getSession(); + const saved = await this.db.create([data], session && { session }); await this.logHelper.upsertLogOne(saved[0]); return saved[0].toObject>(); } @@ -143,8 +147,9 @@ export class OdmModel implements SyncDBDataSource { } private async saveNew(existingIds: Set, dataArray: Partial>[]) { + const session = this.getSession(); const newData = dataArray.filter(d => !d._id || !existingIds.has(d._id.toString())); - return (await this.db.createMany(newData)) || []; + return (await this.db.createMany(newData, session && { session })) || []; } private async saveExisting( @@ -152,6 +157,7 @@ export class OdmModel implements SyncDBDataSource { query?: any, updateExisting: boolean = true ) { + const session = this.getSession(); const ids: DataType['_id'][] = []; dataArray.forEach(d => { if (d._id) { @@ -162,6 +168,7 @@ export class OdmModel implements SyncDBDataSource { ( await this.db.find({ _id: { $in: ids } } as UwaziFilterQuery>, '_id', { lean: true, + ...(session && { session }), }) ).map(d => d._id.toString()) ); @@ -174,13 +181,18 @@ export class OdmModel implements SyncDBDataSource { filter: { ...query, _id: data._id }, update: data, }, - })) + })), + session && { session } ); - const updated = await this.db.find({ - ...query, - _id: { $in: Array.from(existingIds) }, - } as UwaziFilterQuery>); + const updated = await this.db.find( + { + ...query, + _id: { $in: Array.from(existingIds) }, + } as UwaziFilterQuery>, + undefined, + session && { session } + ); return { existingIds, existingData, updated }; } @@ -191,14 +203,16 @@ export class OdmModel implements SyncDBDataSource { async updateMany( conditions: UwaziFilterQuery>, doc: UwaziUpdateQuery, - options?: UwaziUpdateOptions> + options: UwaziUpdateOptions> = {} ) { + const session = this.getSession(); await this.logHelper.upsertLogMany(conditions); - return this.db._updateMany(conditions, doc, options); + return this.db._updateMany(conditions, doc, { ...options, ...(session && { session }) }); } async count(query: UwaziFilterQuery> = {}) { - return this.db.countDocuments(query); + const session = this.getSession(); + return this.db.countDocuments(query, session && { session }); } async get( @@ -206,22 +220,29 @@ export class OdmModel implements SyncDBDataSource { select: any = '', options: UwaziQueryOptions = {} ) { - const results = await this.db.find(query, select, { lean: true, ...options }); + const session = this.getSession(); + const results = await this.db.find(query, select, { + ...options, + ...(session && { session }), + lean: true, + }); return results as EnforcedWithId[]; } async getById(id: any, select?: any) { - const results = await this.db.findById(id, select); + const session = this.getSession(); + const results = await this.db.findById(id, select, { lean: true, ...(session && { session }) }); return results as EnforcedWithId | null; } - async delete(condition: any, options: { session?: ClientSession } = {}) { + async delete(condition: any) { + const session = this.getSession(); let cond = condition; if (mongoose.Types.ObjectId.isValid(condition)) { cond = { _id: condition }; } await this.logHelper.upsertLogMany(cond, true); - return this.db.deleteMany(cond, options); + return this.db.deleteMany(cond, session && { session }); } async facet(aggregations: any[], pipelines: any, project: any) { diff --git a/app/api/services/ocr/ocrRecords.ts b/app/api/services/ocr/ocrRecords.ts index 6d8643dc3a..a831650caf 100644 --- a/app/api/services/ocr/ocrRecords.ts +++ b/app/api/services/ocr/ocrRecords.ts @@ -19,32 +19,32 @@ const cleanupRecordsOfFiles = async (fileIds: (ObjectIdSchema | undefined)[]) => const records = await OcrModel.get({ $or: [{ sourceFile: { $in: idStrings } }, { resultFile: { $in: idStrings } }], }); - const idRecordMap = new Map(); - const recordsToNullSource: OcrRecord[] = []; - const recordIdsToDelete: string[] = []; + // const idRecordMap = new Map(); + // const recordsToNullSource: OcrRecord[] = []; + // const recordIdsToDelete: string[] = []; + // + // records.forEach(record => { + // if (record.sourceFile) { + // idRecordMap.set(record.sourceFile.toString(), record); + // } + // if (record.resultFile) { + // idRecordMap.set(record.resultFile.toString(), record); + // } + // }); + // + // idStrings.forEach(fileId => { + // if (idRecordMap.has(fileId)) { + // const record = idRecordMap.get(fileId); + // if (record.sourceFile?.toString() === fileId) { + // recordsToNullSource.push({ ...record, sourceFile: null }); + // } else if (record.resultFile?.toString() === fileId) { + // recordIdsToDelete.push(record._id.toString()); + // } + // } + // }); - records.forEach(record => { - if (record.sourceFile) { - idRecordMap.set(record.sourceFile.toString(), record); - } - if (record.resultFile) { - idRecordMap.set(record.resultFile.toString(), record); - } - }); - - idStrings.forEach(fileId => { - if (idRecordMap.has(fileId)) { - const record = idRecordMap.get(fileId); - if (record.sourceFile?.toString() === fileId) { - recordsToNullSource.push({ ...record, sourceFile: null }); - } else if (record.resultFile?.toString() === fileId) { - recordIdsToDelete.push(record._id.toString()); - } - } - }); - - await OcrModel.saveMultiple(recordsToNullSource); - await OcrModel.delete({ _id: { $in: recordIdsToDelete } }); + // await OcrModel.saveMultiple(recordsToNullSource); + // await OcrModel.delete({ _id: { $in: recordIdsToDelete } }); }; const markReady = async (record: OcrRecord, resultFile: EnforcedWithId) => diff --git a/app/api/templates/templates.ts b/app/api/templates/templates.ts index c28acf0805..ea90777c57 100644 --- a/app/api/templates/templates.ts +++ b/app/api/templates/templates.ts @@ -1,4 +1,5 @@ import { ObjectId } from 'mongodb'; +import { ClientSession } from 'mongodb'; import entities from 'api/entities'; import { populateGeneratedIdByTemplate } from 'api/entities/generatedIdPropertyAutoFiller'; @@ -137,16 +138,17 @@ const checkAndFillGeneratedIdProperties = async ( return newGeneratedIdProps.length > 0; }; -const _save = async (template: TemplateSchema) => { - const newTemplate = await model.save(template); +const _save = async (template: TemplateSchema, session?: ClientSession) => { + const newTemplate = await model.save(template, undefined, session); await addTemplateTranslation(newTemplate); - return newTemplate; }; -const getRelatedThesauri = async (template: TemplateSchema) => { +const getRelatedThesauri = async (template: TemplateSchema, session?: ClientSession) => { const thesauriIds = (template.properties || []).map(p => p.content).filter(p => p); - const thesauri = await dictionariesModel.get({ _id: { $in: thesauriIds } }); + const thesauri = await dictionariesModel.get({ _id: { $in: thesauriIds } }, undefined, { + session, + }); const thesauriByKey: Record = {}; thesauri.forEach(t => { thesauriByKey[t._id.toString()] = t; @@ -156,11 +158,9 @@ const getRelatedThesauri = async (template: TemplateSchema) => { export default { async save(template: TemplateSchema, language: string, reindex = true) { - /* eslint-disable no-param-reassign */ template.properties = template.properties || []; template.properties = await generateNames(template.properties); template.properties = await denormalizeInheritedProperties(template); - /* eslint-enable no-param-reassign */ await validateTemplate(template); const mappedTemplate = await v2.processNewRelationshipProperties(template); @@ -176,11 +176,11 @@ export default { : _save(mappedTemplate); }, - async swapNamesValidation(template: TemplateSchema) { + async swapNamesValidation(template: TemplateSchema, session?: ClientSession) { if (!template._id) { return; } - const current = await this.getById(ensure(template._id)); + const current = await this.getById(ensure(template._id), session); const currentTemplate = ensure(current); currentTemplate.properties = currentTemplate.properties || []; @@ -194,11 +194,16 @@ export default { }); }, - async _update(template: TemplateSchema, language: string, _reindex = true) { + async _update( + template: TemplateSchema, + language: string, + _reindex = true, + session?: ClientSession + ) { const reindex = _reindex && !template.synced; const templateStructureChanges = await checkIfReindex(template); const currentTemplate = ensure>( - await this.getById(ensure(template._id)) + await this.getById(ensure(template._id), session) ); if (templateStructureChanges || currentTemplate.name !== template.name) { await updateTranslation(currentTemplate, template); @@ -209,13 +214,14 @@ export default { } const generatedIdAdded = await checkAndFillGeneratedIdProperties(currentTemplate, template); - const savedTemplate = await model.save(template); + const savedTemplate = await model.save(template, undefined, session); if (templateStructureChanges) { await v2.processNewRelationshipPropertiesOnUpdate(currentTemplate, savedTemplate); await entities.updateMetadataProperties(template, currentTemplate, language, { reindex, generatedIdAdded, + session, }); } @@ -229,8 +235,12 @@ export default { return savedTemplate; }, - async canDeleteProperty(template: ObjectId, property: ObjectId | string | undefined) { - const tmps = await model.get(); + async canDeleteProperty( + template: ObjectId, + property: ObjectId | string | undefined, + session?: ClientSession + ) { + const tmps = await model.get({}, undefined, { session }); return tmps.every(iteratedTemplate => (iteratedTemplate.properties || []).every( iteratedProperty => @@ -253,14 +263,20 @@ export default { return property; }, - async getPropertiesByName(propertyNames: string[]): Promise { + async getPropertiesByName( + propertyNames: string[], + session?: ClientSession + ): Promise { const nameSet = new Set(propertyNames); - const templates = await this.get({ - $or: [ - { 'properties.name': { $in: propertyNames } }, - { 'commonProperties.name': { $in: propertyNames } }, - ], - }); + const templates = await this.get( + { + $or: [ + { 'properties.name': { $in: propertyNames } }, + { 'commonProperties.name': { $in: propertyNames } }, + ], + }, + session + ); const allProperties = templates .map(template => [template.properties || [], template.commonProperties || []]) .flat() @@ -278,30 +294,39 @@ export default { return Array.from(Object.values(propertiesByName)); }, - async setAsDefault(_id: string) { - const [templateToBeDefault] = await this.get({ _id }); - const [currentDefault] = await this.get({ _id: { $nin: [_id] }, default: true }); + async setAsDefault(_id: string, session?: ClientSession) { + const [templateToBeDefault] = await this.get({ _id }, session); + const [currentDefault] = await this.get({ _id: { $nin: [_id] }, default: true }, session); if (templateToBeDefault) { let saveCurrentDefault = Promise.resolve({}); if (currentDefault) { - saveCurrentDefault = model.save({ - _id: currentDefault._id, - default: false, - }); + saveCurrentDefault = model.save( + { + _id: currentDefault._id, + default: false, + }, + undefined, + session + ); } - return Promise.all([model.save({ _id, default: true }), saveCurrentDefault]); + return Promise.all([ + model.save({ _id, default: true }, undefined, session), + saveCurrentDefault, + ]); } throw createError('Invalid ID'); }, - async getById(templateId: ObjectId | string) { - return model.getById(templateId); + async getById(templateId: ObjectId | string, session?: ClientSession) { + return model.getById(templateId, undefined, { session }); }, - async removePropsWithNonexistentId(nonexistentId: string) { - const relatedTemplates = await model.get({ 'properties.content': nonexistentId }); + async removePropsWithNonexistentId(nonexistentId: string, session?: ClientSession) { + const relatedTemplates = await model.get({ 'properties.content': nonexistentId }, undefined, { + session, + }); const defaultLanguage = (await settings.getDefaultLanguage())?.key; if (!defaultLanguage) { throw Error('Missing default language.'); @@ -313,40 +338,42 @@ export default { ...t, properties: (t.properties || []).filter(prop => prop.content !== nonexistentId), }, - defaultLanguage + defaultLanguage, + false, + session ) ) ); }, - async delete(template: Partial) { - const count = await this.countByTemplate(ensure(template._id)); + async delete(template: Partial, session?: ClientSession) { + const count = await this.countByTemplate(ensure(template._id), session); if (count > 0) { - return Promise.reject({ key: 'documents_using_template', value: count }); // eslint-disable-line prefer-promise-reject-errors + return Promise.reject({ key: 'documents_using_template', value: count }); } await v2.processNewRelationshipPropertiesOnDelete(template._id); const _id = ensure(template._id); await translations.deleteContext(_id); - await this.removePropsWithNonexistentId(_id); - await model.delete(_id); + await this.removePropsWithNonexistentId(_id, session); + await model.delete(_id, { session }); await applicationEventsBus.emit(new TemplateDeletedEvent({ templateId: _id })); return template; }, - async countByTemplate(template: string) { - return entities.countByTemplate(template); + async countByTemplate(template: string, session?: ClientSession) { + return entities.countByTemplate(template, session); }, - async countByThesauri(thesauriId: string) { - return model.count({ 'properties.content': thesauriId }); + async countByThesauri(thesauriId: string, session?: ClientSession) { + return model.count({ 'properties.content': thesauriId }, { session }); }, - async findUsingRelationTypeInProp(relationTypeId: string) { - return model.get({ 'properties.relationType': relationTypeId }, 'name'); + async findUsingRelationTypeInProp(relationTypeId: string, session?: ClientSession) { + return model.get({ 'properties.relationType': relationTypeId }, 'name', { session }); }, getRelatedThesauri, diff --git a/app/api/users/specs/users.spec.js b/app/api/users/specs/users.spec.js index 45da5465a4..6a3b4a3ae8 100644 --- a/app/api/users/specs/users.spec.js +++ b/app/api/users/specs/users.spec.js @@ -6,24 +6,25 @@ import mailer from 'api/utils/mailer'; import db from 'api/utils/testing_db'; import * as random from 'shared/uniqueID'; -import { encryptPassword, comparePasswords } from 'api/auth/encryptPassword'; +import { comparePasswords, encryptPassword } from 'api/auth/encryptPassword'; import * as usersUtils from 'api/auth2fa/usersUtils'; import { settingsModel } from 'api/settings/settingsModel'; import userGroups from 'api/usergroups/userGroups'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; +import * as unlockCode from '../generateUnlockCode'; +import passwordRecoveriesModel from '../passwordRecoveriesModel'; +import users from '../users.js'; +import usersModel from '../usersModel'; import fixtures, { - userId, + blockedUserId, expectedKey, - recoveryUserId, group1Id, group2Id, + recoveryUserId, + userId, userToDelete, userToDelete2, - blockedUserId, } from './fixtures.js'; -import users from '../users.js'; -import passwordRecoveriesModel from '../passwordRecoveriesModel'; -import usersModel from '../usersModel'; -import * as unlockCode from '../generateUnlockCode'; jest.mock('api/users/generateUnlockCode.ts', () => ({ generateUnlockCode: () => 'hash', @@ -509,6 +510,7 @@ describe('Users', () => { jest.restoreAllMocks(); jest.spyOn(mailer, 'send').mockImplementation(async () => Promise.resolve('OK')); jest.spyOn(Date, 'now').mockReturnValue(1000); + testingEnvironment.setFakeContext(); }); it('should find the matching email create a recover password doc in the database and send an email', async () => { diff --git a/app/api/utils/testingEnvironment.ts b/app/api/utils/testingEnvironment.ts index f875d53c82..315e782afe 100644 --- a/app/api/utils/testingEnvironment.ts +++ b/app/api/utils/testingEnvironment.ts @@ -6,6 +6,8 @@ import { UserInContextMockFactory } from 'api/utils/testingUserInContext'; import { setupTestUploadedPaths } from 'api/files'; import { UserSchema } from 'shared/types/userType'; +const originalAppContextGet = appContext.get.bind(appContext); + const testingEnvironment = { userInContextMockFactory: new UserInContextMockFactory(), @@ -25,6 +27,20 @@ const testingEnvironment = { await setupTestUploadedPaths(); }, + setFakeContext() { + jest.spyOn(appContext, 'get').mockImplementation((key: string) => { + if (key === 'mongoSession') { + return undefined; + } + return originalAppContextGet(key); + }); + }, + + unsetFakeContext() { + appContext.get.mockRestore(); + appContext.set.mockRestore(); + }, + async setFixtures(fixtures?: DBFixture) { if (fixtures) { await testingDB.setupFixturesAndContext(fixtures); diff --git a/app/api/utils/testingTenants.ts b/app/api/utils/testingTenants.ts index ab02b28243..92521ca15a 100644 --- a/app/api/utils/testingTenants.ts +++ b/app/api/utils/testingTenants.ts @@ -1,15 +1,24 @@ import { config } from 'api/config'; import { Tenant, tenants } from 'api/tenants/tenantContext'; +import { appContext } from './AppContext'; const originalCurrentFN = tenants.current.bind(tenants); let mockedTenant: Partial; +const originalAppContextGet = appContext.get.bind(appContext); + const testingTenants = { mockCurrentTenant(tenant: Partial) { mockedTenant = tenant; mockedTenant.featureFlags = mockedTenant.featureFlags || config.defaultTenant.featureFlags; tenants.current = () => mockedTenant; + jest.spyOn(appContext, 'get').mockImplementation((key: string) => { + if (key === 'mongoSession') { + return undefined; + } + return originalAppContextGet(key); + }); }, changeCurrentTenant(changes: Partial) { diff --git a/app/api/utils/testing_db.ts b/app/api/utils/testing_db.ts index cfe6e55020..325b93cfd4 100644 --- a/app/api/utils/testing_db.ts +++ b/app/api/utils/testing_db.ts @@ -19,6 +19,9 @@ import { config } from 'api/config'; import { UserSchema } from '../../shared/types/userType'; import { elasticTesting } from './elastic_testing'; import { testingTenants } from './testingTenants'; +import { appContext } from './AppContext'; + +const originalAppContextGet = appContext.get.bind(appContext); mongoose.Promise = Promise; let connected = false; @@ -155,6 +158,15 @@ const testingDB: { this.UserInContextMockFactory.mockEditorUser(); + jest.spyOn(appContext, 'get').mockImplementation((key: string) => { + if (key === 'mongoSession') { + return undefined; + } + return originalAppContextGet(key); + }); + + jest.spyOn(appContext, 'set').mockImplementation(() => { }); + if (elasticIndex) { testingTenants.changeCurrentTenant({ indexName: elasticIndex }); await elasticTesting.reindex(); From 1c1a96f6e63af7bd8a48a49490064930cd282d81 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Mon, 27 Jan 2025 12:19:14 +0100 Subject: [PATCH 03/15] WIP, proof of concept test of a transactional saveEntity --- app/api/csv/specs/csvLoader.spec.js | 25 +++++---- app/api/entities/entitySavingManager.ts | 3 +- app/api/entities/managerFunctions.ts | 2 +- .../specs/entitySavingManager.spec.ts | 51 ++++++++----------- .../driving/specs/ATServiceListener.spec.ts | 1 + app/api/search/specs/index.spec.ts | 3 +- app/api/services/ocr/ocrRecords.ts | 50 +++++++++--------- .../preserve/specs/preserveSync.spec.ts | 1 + app/api/settings/specs/settings.spec.ts | 7 +-- .../toc_generation/specs/tocService.spec.ts | 1 + app/api/utils/testingEnvironment.ts | 2 + app/api/utils/testing_db.ts | 12 ----- 12 files changed, 73 insertions(+), 85 deletions(-) diff --git a/app/api/csv/specs/csvLoader.spec.js b/app/api/csv/specs/csvLoader.spec.js index 4007ea71af..2c8218fa7c 100644 --- a/app/api/csv/specs/csvLoader.spec.js +++ b/app/api/csv/specs/csvLoader.spec.js @@ -8,18 +8,19 @@ import entities from 'api/entities'; import translations from 'api/i18n'; import { search } from 'api/search'; import settings from 'api/settings'; -import db from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import moment from 'moment'; import typeParsers from '../typeParsers'; import fixtures, { template1Id } from './csvLoaderFixtures'; import { mockCsvFileReadStream } from './helpers'; +import testingDB from 'api/utils/testing_db'; describe('csvLoader', () => { const csvFile = path.join(__dirname, '/test.csv'); const loader = new CSVLoader(); beforeAll(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); beforeEach(() => { @@ -28,7 +29,7 @@ describe('csvLoader', () => { jest.spyOn(entities, 'save').mockImplementation(async e => e); }); - afterAll(async () => db.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); describe('user', () => { it('should use the passed user', async () => { @@ -41,7 +42,7 @@ describe('csvLoader', () => { let csv; let readStreamMock; beforeEach(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); const nonExistent = 'Russian'; @@ -128,7 +129,7 @@ describe('csvLoader', () => { beforeAll(async () => { jest.restoreAllMocks(); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); loader.on('entityLoaded', entity => { events.push(entity.title); }); @@ -219,7 +220,7 @@ describe('csvLoader', () => { it('should stop processing on the first error', async () => { const testingLoader = new CSVLoader(); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); jest.spyOn(entities, 'save').mockImplementation(entity => { throw new Error(`error-${entity.title}`); }); @@ -234,7 +235,7 @@ describe('csvLoader', () => { it('should throw the error that occurred even if it was not the first row', async () => { const testingLoader = new CSVLoader(); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); jest .spyOn(entities, 'save') .mockImplementationOnce(({ title }) => Promise.resolve({ title })) @@ -257,7 +258,7 @@ describe('csvLoader', () => { } return entity; }); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); it('should emit an error', async () => { @@ -317,7 +318,7 @@ describe('csvLoader', () => { describe('when sharedId is provided', () => { beforeEach(async () => { jest.restoreAllMocks(); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); it('should update the entity', async () => { @@ -344,7 +345,7 @@ describe('csvLoader', () => { describe('when the title is not provided', () => { beforeEach(async () => { jest.restoreAllMocks(); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); describe('title not marked with generated Id option', () => { @@ -418,7 +419,9 @@ describe('csvLoader', () => { dateFormat, }, ]; - await db.setupFixturesAndContext(_fixtures); + + await testingDB.setupFixturesAndContext(_fixtures); + testingEnvironment.setFakeContext(); }; it('should correctly parse MM/dd/yyyy', async () => { diff --git a/app/api/entities/entitySavingManager.ts b/app/api/entities/entitySavingManager.ts index 1a88ddc28e..3b1264bb08 100644 --- a/app/api/entities/entitySavingManager.ts +++ b/app/api/entities/entitySavingManager.ts @@ -4,7 +4,6 @@ import { set } from 'lodash'; import { EntityWithFilesSchema } from 'shared/types/entityType'; import { UserSchema } from 'shared/types/userType'; import { handleAttachmentInMetadataProperties, processFiles, saveFiles } from './managerFunctions'; -import templates from 'api/templates'; const saveEntity = async ( _entity: EntityWithFilesSchema, @@ -65,7 +64,7 @@ const saveEntity = async ( return { entity: entityWithAttachments, errors: fileSaveErrors }; } catch (e) { await session.abortTransaction(); - throw e; + return { errors: [e.message] }; } finally { appContext.set('mongoSession', undefined); await session.endSession(); diff --git a/app/api/entities/managerFunctions.ts b/app/api/entities/managerFunctions.ts index db5e7cd9c1..a67e1a9a03 100644 --- a/app/api/entities/managerFunctions.ts +++ b/app/api/entities/managerFunctions.ts @@ -192,7 +192,7 @@ const saveFiles = async ( await filesAPI.save(file, false); } catch (e) { legacyLogger.error(prettifyError(e)); - saveResults.push(`Could not save file/s: ${file.originalname}`); + throw new Error(`Could not save file/s: ${file.originalname}`, { cause: e }); } }) ); diff --git a/app/api/entities/specs/entitySavingManager.spec.ts b/app/api/entities/specs/entitySavingManager.spec.ts index 9d7845c1e7..e5c7b1460b 100644 --- a/app/api/entities/specs/entitySavingManager.spec.ts +++ b/app/api/entities/specs/entitySavingManager.spec.ts @@ -1,13 +1,15 @@ /* eslint-disable max-lines */ import { saveEntity } from 'api/entities/entitySavingManager'; -import * as os from 'os'; import { attachmentsPath, fileExistsOnPath, files as filesAPI, uploadsPath } from 'api/files'; import * as processDocumentApi from 'api/files/processDocument'; import { search } from 'api/search'; import db from 'api/utils/testing_db'; import { advancedSort } from 'app/utils/advancedSort'; +import * as os from 'os'; // eslint-disable-next-line node/no-restricted-import import { writeFile } from 'fs/promises'; +import { appContext } from 'api/utils/AppContext'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { ObjectId } from 'mongodb'; import path from 'path'; import { EntityWithFilesSchema } from 'shared/types/entityType'; @@ -28,10 +30,6 @@ import { template2Id, textFile, } from './entitySavingManagerFixtures'; -import { testingEnvironment } from 'api/utils/testingEnvironment'; -import { appContext } from 'api/utils/AppContext'; -import { string } from 'yargs'; -import templates from 'api/templates'; const validPdfString = ` %PDF-1.0 @@ -67,12 +65,12 @@ describe('entitySavingManager', () => { }); beforeEach(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); jest.spyOn(search, 'indexEntities').mockImplementation(async () => Promise.resolve()); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); afterEach(() => { @@ -222,10 +220,10 @@ describe('entitySavingManager', () => { }; }); - it('should continue saving if a file fails to save', async () => { - const { entity: savedEntity } = await saveEntity(entity, { ...reqData }); - expect(savedEntity.attachments).toEqual([textFile]); - }); + // it('should continue saving if a file fails to save', async () => { + // const { entity: savedEntity } = await saveEntity(entity, { ...reqData }); + // expect(savedEntity.attachments).toEqual([textFile]); + // }); it('should return an error', async () => { const { errors } = await saveEntity(entity, { ...reqData }); @@ -595,6 +593,7 @@ describe('entitySavingManager', () => { _id: mainPdfFileId.toString(), originalname: 'Renamed main pdf.pdf', }); + processDocumentApi.processDocument.mockRestore(); }); it('should return an error if an existing main document cannot be saved', async () => { @@ -614,6 +613,7 @@ describe('entitySavingManager', () => { } ); expect(errors[0]).toBe('Could not save file/s: changed.pdf'); + filesAPI.save.mockRestore(); }); }); }); @@ -622,7 +622,7 @@ describe('entitySavingManager', () => { describe('transactions', () => { const reqData = { user: editorUser, language: 'en', socketEmiter: () => {} }; - xit('should rollback all operations if any operation fails', async () => { + it('should rollback all operations if any operation fails', async () => { testingEnvironment.unsetFakeContext(); // Force files.save to fail @@ -637,36 +637,27 @@ describe('entitySavingManager', () => { } ); - jest.spyOn(filesAPI, 'save').mockImplementationOnce(() => { + // const filesOnDb = await filesAPI.get({ entity: savedEntity.sharedId }); + // console.log(JSON.stringify(filesOnDb, null, ' ')); + + jest.spyOn(filesAPI, 'save').mockImplementation(() => { throw new Error('Forced file save error'); }); - const updateOperation = await saveEntity( - { - _id: savedEntity._id, - sharedId: savedEntity.sharedId, - title: 'updated title', - template: template1Id, - attachments: [{ originalname: 'will fail', url: 'https://fail.com' }], - }, - { - ...reqData, - files: [{ ...file, fieldname: 'attachments[0]' }], - } - ); // Verify entity was not updated const [entityAfterFailure] = await entities.get({ _id: savedEntity._id }); expect(entityAfterFailure.title).toBe('initial entity'); + + // Verify original document still exists and no new files were added + const entityFiles = await filesAPI.get({ entity: savedEntity.sharedId }); + expect(entityFiles).toHaveLength(1); + expect(entityFiles[0].originalname).toBe('myNewFile.pdf'); }); // await expect(updateOperation).rejects.toThrow('Forced file save error'); // // - // // Verify original document still exists and no new files were added - // const entityFiles = await filesAPI.get({ entity: savedEntity.sharedId }); - // expect(entityFiles).toHaveLength(1); - // expect(entityFiles[0].originalname).toBe('myNewFile.pdf'); }); }); }); diff --git a/app/api/externalIntegrations.v2/automaticTranslation/adapters/driving/specs/ATServiceListener.spec.ts b/app/api/externalIntegrations.v2/automaticTranslation/adapters/driving/specs/ATServiceListener.spec.ts index 0e9a3361b4..5653ed2719 100644 --- a/app/api/externalIntegrations.v2/automaticTranslation/adapters/driving/specs/ATServiceListener.spec.ts +++ b/app/api/externalIntegrations.v2/automaticTranslation/adapters/driving/specs/ATServiceListener.spec.ts @@ -33,6 +33,7 @@ describe('ATServiceListener', () => { settings: [{ features: { automaticTranslation: { active: true } } }], }); testingEnvironment.resetPermissions(); + testingEnvironment.unsetFakeContext(); await testingEnvironment.setTenant('tenant'); executeSpy = jest.fn().mockImplementation(() => { diff --git a/app/api/search/specs/index.spec.ts b/app/api/search/specs/index.spec.ts index e0a1e9151a..f8f50cedf1 100644 --- a/app/api/search/specs/index.spec.ts +++ b/app/api/search/specs/index.spec.ts @@ -3,11 +3,12 @@ import { elasticTesting } from 'api/utils/elastic_testing'; import { search } from '../search'; import { elastic } from '../elastic'; import { fixturesTimeOut } from './fixtures_elastic'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; describe('index (search)', () => { beforeEach(async () => { const elasticIndex = 'index_for_index_testing'; - await db.setupFixturesAndContext({}, elasticIndex); + await testingEnvironment.setUp({}, elasticIndex); }, fixturesTimeOut); afterAll(async () => { diff --git a/app/api/services/ocr/ocrRecords.ts b/app/api/services/ocr/ocrRecords.ts index a831650caf..6d8643dc3a 100644 --- a/app/api/services/ocr/ocrRecords.ts +++ b/app/api/services/ocr/ocrRecords.ts @@ -19,32 +19,32 @@ const cleanupRecordsOfFiles = async (fileIds: (ObjectIdSchema | undefined)[]) => const records = await OcrModel.get({ $or: [{ sourceFile: { $in: idStrings } }, { resultFile: { $in: idStrings } }], }); - // const idRecordMap = new Map(); - // const recordsToNullSource: OcrRecord[] = []; - // const recordIdsToDelete: string[] = []; - // - // records.forEach(record => { - // if (record.sourceFile) { - // idRecordMap.set(record.sourceFile.toString(), record); - // } - // if (record.resultFile) { - // idRecordMap.set(record.resultFile.toString(), record); - // } - // }); - // - // idStrings.forEach(fileId => { - // if (idRecordMap.has(fileId)) { - // const record = idRecordMap.get(fileId); - // if (record.sourceFile?.toString() === fileId) { - // recordsToNullSource.push({ ...record, sourceFile: null }); - // } else if (record.resultFile?.toString() === fileId) { - // recordIdsToDelete.push(record._id.toString()); - // } - // } - // }); + const idRecordMap = new Map(); + const recordsToNullSource: OcrRecord[] = []; + const recordIdsToDelete: string[] = []; - // await OcrModel.saveMultiple(recordsToNullSource); - // await OcrModel.delete({ _id: { $in: recordIdsToDelete } }); + records.forEach(record => { + if (record.sourceFile) { + idRecordMap.set(record.sourceFile.toString(), record); + } + if (record.resultFile) { + idRecordMap.set(record.resultFile.toString(), record); + } + }); + + idStrings.forEach(fileId => { + if (idRecordMap.has(fileId)) { + const record = idRecordMap.get(fileId); + if (record.sourceFile?.toString() === fileId) { + recordsToNullSource.push({ ...record, sourceFile: null }); + } else if (record.resultFile?.toString() === fileId) { + recordIdsToDelete.push(record._id.toString()); + } + } + }); + + await OcrModel.saveMultiple(recordsToNullSource); + await OcrModel.delete({ _id: { $in: recordIdsToDelete } }); }; const markReady = async (record: OcrRecord, resultFile: EnforcedWithId) => diff --git a/app/api/services/preserve/specs/preserveSync.spec.ts b/app/api/services/preserve/specs/preserveSync.spec.ts index c044e5f94d..90ae5a3d56 100644 --- a/app/api/services/preserve/specs/preserveSync.spec.ts +++ b/app/api/services/preserve/specs/preserveSync.spec.ts @@ -22,6 +22,7 @@ import { config } from 'api/config'; import { preserveSync } from '../preserveSync'; import { preserveSyncModel } from '../preserveSyncModel'; import { anotherTemplateId, fixtures, templateId, thesauri1Id, user } from './fixtures'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; const mockVault = async (evidences: any[], token: string = '', isoDate = '') => { const host = 'http://preserve-testing.org'; diff --git a/app/api/settings/specs/settings.spec.ts b/app/api/settings/specs/settings.spec.ts index 7612fd9d59..68476b1ea2 100644 --- a/app/api/settings/specs/settings.spec.ts +++ b/app/api/settings/specs/settings.spec.ts @@ -1,6 +1,7 @@ +import translations from 'api/i18n/translations'; import { WithId } from 'api/odm'; import db from 'api/utils/testing_db'; -import translations from 'api/i18n/translations'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { Settings } from 'shared/types/settingsType'; import settings from '../settings'; import fixtures, { linkFixtures, newLinks } from './fixtures'; @@ -9,11 +10,11 @@ describe('settings', () => { beforeEach(async () => { jest.restoreAllMocks(); jest.spyOn(translations, 'updateContext').mockImplementation(async () => Promise.resolve('ok')); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('save()', () => { diff --git a/app/api/toc_generation/specs/tocService.spec.ts b/app/api/toc_generation/specs/tocService.spec.ts index 2afef453d4..708dc83483 100644 --- a/app/api/toc_generation/specs/tocService.spec.ts +++ b/app/api/toc_generation/specs/tocService.spec.ts @@ -4,6 +4,7 @@ import { testingDB } from 'api/utils/testing_db'; import request from 'shared/JSONRequest'; import { tocService } from '../tocService'; import { fixtures } from './fixtures'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; describe('tocService', () => { let requestMock: jest.SpyInstance; diff --git a/app/api/utils/testingEnvironment.ts b/app/api/utils/testingEnvironment.ts index 315e782afe..ccf9d7a6dd 100644 --- a/app/api/utils/testingEnvironment.ts +++ b/app/api/utils/testingEnvironment.ts @@ -14,6 +14,7 @@ const testingEnvironment = { async setUp(fixtures?: DBFixture, elasticIndex?: string) { await this.setTenant(); this.setPermissions(); + this.setFakeContext(); await this.setFixtures(fixtures); await this.setElastic(elasticIndex); }, @@ -34,6 +35,7 @@ const testingEnvironment = { } return originalAppContextGet(key); }); + jest.spyOn(appContext, 'set').mockImplementation(() => {}); }, unsetFakeContext() { diff --git a/app/api/utils/testing_db.ts b/app/api/utils/testing_db.ts index 325b93cfd4..cfe6e55020 100644 --- a/app/api/utils/testing_db.ts +++ b/app/api/utils/testing_db.ts @@ -19,9 +19,6 @@ import { config } from 'api/config'; import { UserSchema } from '../../shared/types/userType'; import { elasticTesting } from './elastic_testing'; import { testingTenants } from './testingTenants'; -import { appContext } from './AppContext'; - -const originalAppContextGet = appContext.get.bind(appContext); mongoose.Promise = Promise; let connected = false; @@ -158,15 +155,6 @@ const testingDB: { this.UserInContextMockFactory.mockEditorUser(); - jest.spyOn(appContext, 'get').mockImplementation((key: string) => { - if (key === 'mongoSession') { - return undefined; - } - return originalAppContextGet(key); - }); - - jest.spyOn(appContext, 'set').mockImplementation(() => { }); - if (elasticIndex) { testingTenants.changeCurrentTenant({ indexName: elasticIndex }); await elasticTesting.reindex(); From c579138707097741180625f084fda7f1ef8a3b38 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Mon, 27 Jan 2025 16:06:29 +0100 Subject: [PATCH 04/15] Deprecated fixture setup from testingDb, testingEnvironment should be used --- .../specs/activitylogParser.spec.js | 5 +-- app/api/auth/specs/captchaMiddleware.spec.ts | 5 +-- .../auth/specs/publicAPIMiddleware.spec.ts | 8 ++--- app/api/auth/specs/routes.spec.js | 5 +-- app/api/auth2fa/specs/usersUtils.spec.ts | 5 +-- app/api/contact/specs/contact.spec.js | 10 +++--- app/api/csv/specs/csvLoaderLanguages.spec.ts | 6 ++-- app/api/csv/specs/csvLoaderThesauri.spec.ts | 5 +-- app/api/csv/specs/csvLoaderZip.spec.ts | 5 +-- .../csv/typeParsers/specs/multiselect.spec.js | 6 ++-- .../typeParsers/specs/relationship.spec.js | 5 +-- app/api/csv/typeParsers/specs/select.spec.js | 6 ++-- app/api/documents/specs/documents.spec.ts | 5 +-- .../specs/saveSelections.spec.ts | 5 +-- .../entities/specs/denormalization.spec.ts | 11 +++--- .../entities/specs/deprecatedRoutes.spec.js | 14 ++++---- app/api/entities/specs/entities.spec.js | 25 ++++++------- .../specs/entitiesDeleteMultiple.spec.ts | 5 +-- app/api/entities/specs/entitiesModel.spec.js | 3 +- app/api/entities/specs/routes.spec.ts | 5 +-- app/api/entities/specs/validateEntity.spec.ts | 5 +-- app/api/files/specs/downloadRoute.spec.ts | 4 --- app/api/files/specs/jsRoutes.spec.js | 5 +-- app/api/files/specs/publicRoutes.spec.ts | 6 ++-- app/api/i18n/specs/translations.spec.ts | 5 +-- .../odm/specs/ModelWithPermissions.spec.ts | 2 ++ app/api/odm/specs/model.spec.ts | 5 +-- .../odm/specs/modelBulkWriteStream.spec.ts | 7 ++-- app/api/pages/specs/pages.spec.ts | 5 +-- .../permissions/specs/collaborators.spec.ts | 8 ++--- .../specs/entitiesPermissions.spec.ts | 12 +++---- .../relationships/specs/relationships.spec.js | 19 +++++----- app/api/search.v2/specs/permissions.spec.ts | 6 ++-- .../search.v2/specs/snippetsSearch.spec.ts | 16 ++++----- app/api/search/specs/analyzers.spec.ts | 6 ++-- app/api/search/specs/entitiesIndex.spec.ts | 7 ++-- app/api/search/specs/filterByAssignee.spec.ts | 10 +++--- .../search/specs/permissionsFilters.spec.ts | 14 ++++---- app/api/search/specs/routes.spec.ts | 12 +++---- .../specs/search.searchGeolocations.spec.ts | 5 +-- app/api/search/specs/search.spec.js | 8 ++--- .../specs/convertToPdfWorker.spec.ts | 5 +-- .../specs/ixmodels.spec.ts | 8 ++--- .../specs/eventListeners.spec.ts | 5 +-- app/api/settings/specs/settings.spec.ts | 4 +-- .../suggestions/specs/customFilters.spec.ts | 5 +-- .../suggestions/specs/eventListeners.spec.ts | 5 +-- app/api/suggestions/specs/suggestions.spec.ts | 33 ++++++++--------- .../specs/extractedMetadataFunctions.spec.ts | 6 ++-- .../generatedIdPropertyAutoFiller.spec.ts | 6 ++-- app/api/templates/specs/reindex.spec.js | 8 ++--- .../templates/specs/templateSchema.spec.ts | 11 +++--- app/api/templates/specs/templates.spec.js | 35 ++++++++++--------- app/api/templates/specs/utils.spec.ts | 11 +++--- app/api/thesauri/specs/routes.old.spec.js | 12 +++---- app/api/thesauri/specs/routes.spec.ts | 3 +- app/api/thesauri/specs/thesauri.spec.js | 5 +-- .../toc_generation/specs/tocService.spec.ts | 1 - .../topicclassification/specs/common.spec.ts | 6 ++-- .../topicclassification/specs/routes.spec.ts | 5 +-- .../topicclassification/specs/sync.spec.ts | 5 +-- app/api/usergroups/specs/userGroups.spec.ts | 8 ++--- .../specs/userGroupsMembers.spec.ts | 5 +-- app/api/users/specs/users.spec.js | 4 +-- .../utils/specs/languageMiddleware.spec.ts | 8 ++--- app/api/utils/testingEnvironment.ts | 4 +-- app/api/utils/testingTenants.ts | 9 ----- app/api/utils/testing_db.ts | 6 ++++ 68 files changed, 281 insertions(+), 253 deletions(-) diff --git a/app/api/activitylog/specs/activitylogParser.spec.js b/app/api/activitylog/specs/activitylogParser.spec.js index 98245cf3a8..edf10e120e 100644 --- a/app/api/activitylog/specs/activitylogParser.spec.js +++ b/app/api/activitylog/specs/activitylogParser.spec.js @@ -1,4 +1,5 @@ /* eslint-disable max-lines */ +import { testingEnvironment } from 'api/utils/testingEnvironment'; /* eslint-disable max-statements */ import db from 'api/utils/testing_db'; @@ -27,11 +28,11 @@ describe('Activitylog Parser', () => { action: 'MIGRATE', description: 'Dummy log', }); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); async function testBeautified(log, expected) { diff --git a/app/api/auth/specs/captchaMiddleware.spec.ts b/app/api/auth/specs/captchaMiddleware.spec.ts index ba41a20227..68e41fec37 100644 --- a/app/api/auth/specs/captchaMiddleware.spec.ts +++ b/app/api/auth/specs/captchaMiddleware.spec.ts @@ -1,4 +1,5 @@ import db from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { NextFunction } from 'express'; import captchaMiddleware from '../captchaMiddleware'; import { CaptchaModel } from '../CaptchaModel'; @@ -26,10 +27,10 @@ describe('captchaMiddleware', () => { captchas: [{ _id: captchaId, text: 'k0n2170' }], }; - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); - afterAll(async () => db.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); it('should return an error when there is no captcha in the request', async () => { const middleWare = captchaMiddleware(); diff --git a/app/api/auth/specs/publicAPIMiddleware.spec.ts b/app/api/auth/specs/publicAPIMiddleware.spec.ts index 5f40d8cb45..bd15f8e1f3 100644 --- a/app/api/auth/specs/publicAPIMiddleware.spec.ts +++ b/app/api/auth/specs/publicAPIMiddleware.spec.ts @@ -1,6 +1,6 @@ -import { testingDB } from 'api/utils/testing_db'; -import { publicAPIMiddleware } from '../publicAPIMiddleware'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import * as auth from '../index'; +import { publicAPIMiddleware } from '../publicAPIMiddleware'; jest.mock('../index', () => ({ captchaAuthorization: jest.fn(), @@ -10,7 +10,7 @@ describe('publicAPIMiddleware', () => { let captchaMock: jest.Mock; const setUpSettings = async (open: boolean) => - testingDB.clearAllAndLoad({ + testingEnvironment.setUp({ settings: [ { openPublicEndpoint: open, @@ -24,7 +24,7 @@ describe('publicAPIMiddleware', () => { captchaMock.mockReset(); }); - afterAll(async () => testingDB.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); it('should bypass captcha if enabled on settings and request has corresponding header', async () => { await setUpSettings(true); diff --git a/app/api/auth/specs/routes.spec.js b/app/api/auth/specs/routes.spec.js index 153393a1ee..cca177eca0 100644 --- a/app/api/auth/specs/routes.spec.js +++ b/app/api/auth/specs/routes.spec.js @@ -1,4 +1,5 @@ import express from 'express'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import bodyParser from 'body-parser'; import request from 'supertest'; @@ -18,12 +19,12 @@ describe('Auth Routes', () => { let app; beforeEach(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); routes = instrumentRoutes(authRoutes); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('/login', () => { diff --git a/app/api/auth2fa/specs/usersUtils.spec.ts b/app/api/auth2fa/specs/usersUtils.spec.ts index b7f7ca27f8..22a62c1a06 100644 --- a/app/api/auth2fa/specs/usersUtils.spec.ts +++ b/app/api/auth2fa/specs/usersUtils.spec.ts @@ -1,4 +1,5 @@ /** @format */ +import { testingEnvironment } from 'api/utils/testingEnvironment'; import * as otplib from 'otplib'; @@ -18,11 +19,11 @@ type Error = { code: number; message: string }; describe('auth2fa userUtils', () => { beforeEach(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); const expectError = async (method: string, _id: any, token: string, err: Error) => { diff --git a/app/api/contact/specs/contact.spec.js b/app/api/contact/specs/contact.spec.js index af0baebab6..729128be40 100644 --- a/app/api/contact/specs/contact.spec.js +++ b/app/api/contact/specs/contact.spec.js @@ -1,18 +1,18 @@ /* eslint-disable max-nested-callbacks */ -import mailer from 'api/utils/mailer'; -import db from 'api/utils/testing_db'; import settings from 'api/settings'; -import fixtures from './fixtures.js'; +import mailer from 'api/utils/mailer'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import contact from '../contact'; +import fixtures from './fixtures.js'; describe('contact', () => { beforeEach(async () => { jest.spyOn(mailer, 'send').mockImplementation(async () => Promise.resolve()); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(done => { - db.disconnect().then(done); + testingEnvironment.tearDown().then(done); }); describe('sendMessage', () => { diff --git a/app/api/csv/specs/csvLoaderLanguages.spec.ts b/app/api/csv/specs/csvLoaderLanguages.spec.ts index 206c97713f..680fc2dea0 100644 --- a/app/api/csv/specs/csvLoaderLanguages.spec.ts +++ b/app/api/csv/specs/csvLoaderLanguages.spec.ts @@ -4,7 +4,7 @@ import * as filesystem from 'api/files/filesystem'; import { uploadsPath } from 'api/files/filesystem'; import { search } from 'api/search'; import settings from 'api/settings'; -import db from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import path from 'path'; import { EntitySchema } from 'shared/types/entityType'; @@ -21,7 +21,7 @@ describe('csvLoader languages', () => { const loader = new CSVLoader(); beforeAll(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); await filesystem.setupTestUploadedPaths('csvLoader'); jest.spyOn(translations, 'updateContext').mockImplementation(async () => 'ok'); jest.spyOn(search, 'indexEntities').mockImplementation(async () => Promise.resolve()); @@ -62,7 +62,7 @@ describe('csvLoader languages', () => { ]); await removeTestingZip(); - await db.disconnect(); + await testingEnvironment.tearDown(); }); it('should import entities in the diferent languages', async () => { diff --git a/app/api/csv/specs/csvLoaderThesauri.spec.ts b/app/api/csv/specs/csvLoaderThesauri.spec.ts index 5bdb739ee3..4ad494f150 100644 --- a/app/api/csv/specs/csvLoaderThesauri.spec.ts +++ b/app/api/csv/specs/csvLoaderThesauri.spec.ts @@ -1,4 +1,5 @@ import db from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import thesauri from 'api/thesauri'; import translations from 'api/i18n'; import settings from 'api/settings'; @@ -19,13 +20,13 @@ const getTranslation = async (lang: string, id: ObjectId) => describe('csvLoader thesauri', () => { const loader = new CSVLoader(); - afterAll(async () => db.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); let thesauriId: ObjectId; let result: WithId; describe('load thesauri', () => { beforeAll(async () => { - await db.clearAllAndLoad(fixtures); + await testingEnvironment.setUp(fixtures); await settings.addLanguage({ key: 'es', label: 'spanish' }); await translations.addLanguage('es'); diff --git a/app/api/csv/specs/csvLoaderZip.spec.ts b/app/api/csv/specs/csvLoaderZip.spec.ts index c44731e17a..9c38a3e031 100644 --- a/app/api/csv/specs/csvLoaderZip.spec.ts +++ b/app/api/csv/specs/csvLoaderZip.spec.ts @@ -1,4 +1,5 @@ import db from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { files } from 'api/files/files'; import { search } from 'api/search'; import path from 'path'; @@ -20,7 +21,7 @@ describe('csvLoader zip file', () => { beforeAll(async () => { const zip = path.join(__dirname, '/zipData/test.zip'); const loader = new CSVLoader(); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); await filesystem.setupTestUploadedPaths('csvLoaderZip'); await createTestingZip( [ @@ -54,7 +55,7 @@ describe('csvLoader zip file', () => { filesystem.attachmentsPath('generatedatt2.doc'), ]); await removeTestingZip(); - await db.disconnect(); + await testingEnvironment.tearDown(); }); it('should save files into uploaded_documents', async () => { diff --git a/app/api/csv/typeParsers/specs/multiselect.spec.js b/app/api/csv/typeParsers/specs/multiselect.spec.js index da783bd245..7acf11dfcc 100644 --- a/app/api/csv/typeParsers/specs/multiselect.spec.js +++ b/app/api/csv/typeParsers/specs/multiselect.spec.js @@ -1,7 +1,7 @@ /** @format */ +import { testingEnvironment } from 'api/utils/testingEnvironment'; import thesauri from 'api/thesauri'; -import db from 'api/utils/testing_db'; import { fixtures, thesauri1Id } from '../../specs/fixtures'; import typeParsers from '../../typeParsers'; @@ -17,9 +17,9 @@ describe('multiselect', () => { const templateProp = { name: 'multiselect_prop', content: thesauri1Id }; - afterAll(async () => db.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); beforeAll(async () => { - await db.clearAllAndLoad(fixtures); + await testingEnvironment.setUp(fixtures); thesauri1 = await thesauri.getById(thesauri1Id); }); diff --git a/app/api/csv/typeParsers/specs/relationship.spec.js b/app/api/csv/typeParsers/specs/relationship.spec.js index a98a8240a0..386d287534 100644 --- a/app/api/csv/typeParsers/specs/relationship.spec.js +++ b/app/api/csv/typeParsers/specs/relationship.spec.js @@ -1,6 +1,7 @@ import entities, { model } from 'api/entities'; import { search } from 'api/search'; import db from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { fixtures, templateToRelateId } from '../../specs/fixtures'; import typeParsers from '../../typeParsers'; @@ -72,7 +73,7 @@ describe('relationship', () => { }; beforeAll(async () => { - await db.clearAllAndLoad(fixtures); + await testingEnvironment.setUp(fixtures); jest.spyOn(search, 'indexEntities').mockImplementation(async () => Promise.resolve()); await prepareExtraFixtures(); @@ -81,7 +82,7 @@ describe('relationship', () => { entitiesRelated = await entities.get({ template: templateToRelateId, language: 'en' }); }); - afterAll(async () => db.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); it('should create entities and return the ids', async () => { expect(entitiesRelated[0].title).toBe('value1'); diff --git a/app/api/csv/typeParsers/specs/select.spec.js b/app/api/csv/typeParsers/specs/select.spec.js index 5907fdda4f..88f1da9446 100644 --- a/app/api/csv/typeParsers/specs/select.spec.js +++ b/app/api/csv/typeParsers/specs/select.spec.js @@ -1,7 +1,7 @@ /** @format */ +import { testingEnvironment } from 'api/utils/testingEnvironment'; import thesauri from 'api/thesauri'; -import db from 'api/utils/testing_db'; import { fixtures, thesauri1Id } from '../../specs/fixtures'; import typeParsers from '../../typeParsers'; @@ -13,8 +13,8 @@ const rawEntityWithSelectValue = val => ({ }); describe('select', () => { - beforeEach(async () => db.clearAllAndLoad(fixtures)); - afterAll(async () => db.disconnect()); + beforeEach(async () => testingEnvironment.setUp(fixtures)); + afterAll(async () => testingEnvironment.tearDown()); it('should find thesauri value and return the id and value', async () => { const templateProp = { name: 'select_prop', content: thesauri1Id }; diff --git a/app/api/documents/specs/documents.spec.ts b/app/api/documents/specs/documents.spec.ts index 93631ea971..7e05df0872 100644 --- a/app/api/documents/specs/documents.spec.ts +++ b/app/api/documents/specs/documents.spec.ts @@ -1,4 +1,5 @@ import entities from 'api/entities'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { fileExistsOnPath, uploadsPath } from 'api/files'; import relationships from 'api/relationships'; import { search } from 'api/search'; @@ -20,10 +21,10 @@ describe('documents', () => { // @ts-ignore jest.spyOn(search, 'bulkIndex').mockImplementation(async () => Promise.resolve()); mockID(); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); - afterAll(async () => db.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); describe('get', () => { describe('when passing query', () => { diff --git a/app/api/entities/metadataExtraction/specs/saveSelections.spec.ts b/app/api/entities/metadataExtraction/specs/saveSelections.spec.ts index bfbb180a40..b465119751 100644 --- a/app/api/entities/metadataExtraction/specs/saveSelections.spec.ts +++ b/app/api/entities/metadataExtraction/specs/saveSelections.spec.ts @@ -1,4 +1,5 @@ import { files } from 'api/files'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { DBFixture, testingDB } from 'api/utils/testing_db'; import { saveSelections } from '../saveSelections'; @@ -50,7 +51,7 @@ const fixture: DBFixture = { describe('saveSelections', () => { beforeEach(async () => { jest.spyOn(files, 'save'); - await testingDB.setupFixturesAndContext(fixture); + await testingEnvironment.setUp(fixture); }); afterEach(() => { @@ -58,7 +59,7 @@ describe('saveSelections', () => { }); afterAll(async () => { - await testingDB.disconnect(); + await testingEnvironment.tearDown(); }); it('should not call save if entity has no main file', async () => { diff --git a/app/api/entities/specs/denormalization.spec.ts b/app/api/entities/specs/denormalization.spec.ts index 87faa11a74..f421f7e6ad 100644 --- a/app/api/entities/specs/denormalization.spec.ts +++ b/app/api/entities/specs/denormalization.spec.ts @@ -1,15 +1,16 @@ /* eslint-disable max-lines */ -import db, { DBFixture } from 'api/utils/testing_db'; import entities from 'api/entities'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; +import db, { DBFixture } from 'api/utils/testing_db'; -import { EntitySchema } from 'shared/types/entityType'; +import translations from 'api/i18n/translations'; import thesauris from 'api/thesauri'; import { elasticTesting } from 'api/utils/elastic_testing'; -import translations from 'api/i18n/translations'; +import { EntitySchema } from 'shared/types/entityType'; import { getFixturesFactory } from '../../utils/fixturesFactory'; const load = async (data: DBFixture, index?: string) => - db.setupFixturesAndContext( + testingEnvironment.setUp( { ...data, settings: [ @@ -37,7 +38,7 @@ describe('Denormalize relationships', () => { ); }; - afterAll(async () => db.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); describe('title and basic property (text)', () => { it('should update denormalized title and icon', async () => { diff --git a/app/api/entities/specs/deprecatedRoutes.spec.js b/app/api/entities/specs/deprecatedRoutes.spec.js index 3e17ae700b..2cb6873bfe 100644 --- a/app/api/entities/specs/deprecatedRoutes.spec.js +++ b/app/api/entities/specs/deprecatedRoutes.spec.js @@ -1,13 +1,13 @@ import { search } from 'api/search'; import 'api/utils/jasmineHelpers'; -import db from 'api/utils/testing_db'; -import documentRoutes from '../routes.js'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; +import templates from '../../templates/templates'; +import thesauri from '../../thesauri'; import instrumentRoutes from '../../utils/instrumentRoutes'; import entities from '../entities'; import * as entitiesSavingManager from '../entitySavingManager'; -import templates from '../../templates/templates'; -import thesauri from '../../thesauri'; -import fixtures, { templateId, unpublishedDocId, batmanFinishesId } from './fixtures.js'; +import documentRoutes from '../routes.js'; +import fixtures, { batmanFinishesId, templateId, unpublishedDocId } from './fixtures.js'; describe('entities', () => { let routes; @@ -17,11 +17,11 @@ describe('entities', () => { jest .spyOn(search, 'countPerTemplate') .mockImplementation(async () => Promise.resolve({ templateCount: 0 })); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('POST', () => { diff --git a/app/api/entities/specs/entities.spec.js b/app/api/entities/specs/entities.spec.js index 6e28939018..f896fc0b9d 100644 --- a/app/api/entities/specs/entities.spec.js +++ b/app/api/entities/specs/entities.spec.js @@ -7,7 +7,7 @@ import fs from 'fs/promises'; import entitiesModel from 'api/entities/entitiesModel'; import { spyOnEmit } from 'api/eventsbus/eventTesting'; -import { uploadsPath, storage } from 'api/files'; +import { storage, uploadsPath } from 'api/files'; import relationships from 'api/relationships'; import { search } from 'api/search'; import date from 'api/utils/date.js'; @@ -16,24 +16,25 @@ import { UserInContextMockFactory } from 'api/utils/testingUserInContext'; import { UserRole } from 'shared/types/userSchema'; import { applicationEventsBus } from 'api/eventsbus'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; +import entities from '../entities.js'; +import { EntityCreatedEvent } from '../events/EntityCreatedEvent'; +import { EntityDeletedEvent } from '../events/EntityDeletedEvent'; +import { EntityUpdatedEvent } from '../events/EntityUpdatedEvent'; import fixtures, { adminId, batmanFinishesId, - templateId, + docId1, + entityGetTestTemplateId, + syncPropertiesEntityId, templateChangingNames, templateChangingNamesProps, - syncPropertiesEntityId, + templateId, templateWithEntityAsThesauri, - docId1, + unpublishedDocId, uploadId1, uploadId2, - unpublishedDocId, - entityGetTestTemplateId, } from './fixtures.js'; -import entities from '../entities.js'; -import { EntityUpdatedEvent } from '../events/EntityUpdatedEvent'; -import { EntityDeletedEvent } from '../events/EntityDeletedEvent'; -import { EntityCreatedEvent } from '../events/EntityCreatedEvent'; describe('entities', () => { const userFactory = new UserInContextMockFactory(); @@ -43,11 +44,11 @@ describe('entities', () => { jest.spyOn(search, 'indexEntities').mockImplementation(async () => Promise.resolve()); jest.spyOn(search, 'bulkIndex').mockImplementation(async () => Promise.resolve()); jest.spyOn(search, 'bulkDelete').mockImplementation(async () => Promise.resolve()); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('save', () => { diff --git a/app/api/entities/specs/entitiesDeleteMultiple.spec.ts b/app/api/entities/specs/entitiesDeleteMultiple.spec.ts index c899e204c6..099b25b6bd 100644 --- a/app/api/entities/specs/entitiesDeleteMultiple.spec.ts +++ b/app/api/entities/specs/entitiesDeleteMultiple.spec.ts @@ -1,12 +1,13 @@ import entities from 'api/entities'; import { elasticTesting } from 'api/utils/elastic_testing'; import db from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { getFixturesFactory } from '../../utils/fixturesFactory'; import entitiesModel from '../entitiesModel'; const factory = getFixturesFactory(); const loadFixtures = async () => - db.setupFixturesAndContext( + testingEnvironment.setUp( { templates: [factory.template('templateA', [])], entities: [ @@ -29,7 +30,7 @@ const loadFixtures = async () => ); describe('Entities deleteMultiple', () => { - afterAll(async () => db.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); describe('without errors', () => { beforeAll(async () => { diff --git a/app/api/entities/specs/entitiesModel.spec.js b/app/api/entities/specs/entitiesModel.spec.js index 48f698987f..cef76394d3 100644 --- a/app/api/entities/specs/entitiesModel.spec.js +++ b/app/api/entities/specs/entitiesModel.spec.js @@ -1,9 +1,10 @@ +import { testingEnvironment } from 'api/utils/testingEnvironment'; import entitiesModel from '../entitiesModel'; import testingDB from '../../utils/testing_db'; describe('entitiesModel', () => { beforeEach(async () => { - await testingDB.setupFixturesAndContext({}); + await testingEnvironment.setUp({}); }); afterAll(async () => { diff --git a/app/api/entities/specs/routes.spec.ts b/app/api/entities/specs/routes.spec.ts index bb7abae840..7db1b1ecc1 100644 --- a/app/api/entities/specs/routes.spec.ts +++ b/app/api/entities/specs/routes.spec.ts @@ -1,4 +1,5 @@ import { Application, NextFunction, Request, Response } from 'express'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import request, { Response as SuperTestResponse } from 'supertest'; import { setUpApp } from 'api/utils/testingRoutes'; @@ -37,10 +38,10 @@ describe('entities routes', () => { beforeEach(async () => { // @ts-ignore - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); - afterAll(async () => db.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); describe('GET', () => { it('return asked entities with permissions', async () => { diff --git a/app/api/entities/specs/validateEntity.spec.ts b/app/api/entities/specs/validateEntity.spec.ts index 992124e77a..c6e02bf494 100644 --- a/app/api/entities/specs/validateEntity.spec.ts +++ b/app/api/entities/specs/validateEntity.spec.ts @@ -1,4 +1,5 @@ /* eslint-disable max-lines,max-statements */ +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { ErrorObject } from 'ajv'; import ValidationError from 'ajv/dist/runtime/validation_error'; @@ -17,11 +18,11 @@ describe('validateEntity', () => { beforeEach(async () => { jest.spyOn(entitiesIndex, 'updateMapping').mockImplementation(async () => Promise.resolve()); //@ts-ignore - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('validateEntity', () => { diff --git a/app/api/files/specs/downloadRoute.spec.ts b/app/api/files/specs/downloadRoute.spec.ts index 8294e0a708..d7546e8467 100644 --- a/app/api/files/specs/downloadRoute.spec.ts +++ b/app/api/files/specs/downloadRoute.spec.ts @@ -103,15 +103,11 @@ describe('files routes download', () => { describe('when there is no user logged in', () => { it('should serve custom files', async () => { - testingEnvironment.resetPermissions(); const response = await request(app).get(`/api/files/${customPdfFileName}`); - expect(response.status).toBe(200); }); it('should serve files that are related to public entities', async () => { - testingEnvironment.resetPermissions(); const response = await request(app).get(`/api/files/${fileOnPublicEntity}`); - expect(response.status).toBe(200); }); }); diff --git a/app/api/files/specs/jsRoutes.spec.js b/app/api/files/specs/jsRoutes.spec.js index 76092caafb..51a2222e74 100644 --- a/app/api/files/specs/jsRoutes.spec.js +++ b/app/api/files/specs/jsRoutes.spec.js @@ -1,4 +1,5 @@ /*eslint-disable max-lines*/ +import { testingEnvironment } from 'api/utils/testingEnvironment'; import db from 'api/utils/testing_db'; import entities from 'api/entities'; import { settingsModel } from 'api/settings/settingsModel'; @@ -67,7 +68,7 @@ describe('upload routes', () => { files: [file], }; }); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); jest.spyOn(legacyLogger, 'error'); //just to avoid annoying console outpu.mockImplementation(() => {}); }); @@ -190,6 +191,6 @@ describe('upload routes', () => { afterAll(async () => { await deleteAllFiles(() => {}); - await db.disconnect(); + await testingEnvironment.tearDown(); }); }); diff --git a/app/api/files/specs/publicRoutes.spec.ts b/app/api/files/specs/publicRoutes.spec.ts index 45313dedc5..007418c683 100644 --- a/app/api/files/specs/publicRoutes.spec.ts +++ b/app/api/files/specs/publicRoutes.spec.ts @@ -1,3 +1,4 @@ +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { Application, NextFunction, Request, Response } from 'express'; import os from 'os'; import path from 'path'; @@ -9,7 +10,6 @@ import { setupTestUploadedPaths, storage } from 'api/files'; import { search } from 'api/search'; import mailer from 'api/utils/mailer'; import { setUpApp, socketEmit } from 'api/utils/testingRoutes'; -import db from 'api/utils/testing_db'; // eslint-disable-next-line node/no-restricted-import import fs from 'fs/promises'; import { routes } from '../jsRoutes'; @@ -35,11 +35,11 @@ describe('public routes', () => { beforeEach(async () => { jest.spyOn(search, 'indexEntities').mockImplementation(async () => Promise.resolve()); jest.spyOn(Date, 'now').mockReturnValue(1000); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); await setupTestUploadedPaths(); }); - afterAll(async () => db.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); describe('POST /api/public', () => { it('should create the entity and store the files', async () => { diff --git a/app/api/i18n/specs/translations.spec.ts b/app/api/i18n/specs/translations.spec.ts index 473c3a8beb..c5aa500e85 100644 --- a/app/api/i18n/specs/translations.spec.ts +++ b/app/api/i18n/specs/translations.spec.ts @@ -1,4 +1,5 @@ import db from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import entities from 'api/entities'; import pages from 'api/pages'; @@ -15,11 +16,11 @@ import { addLanguage } from '../routes'; describe('translations', () => { beforeEach(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('get()', () => { diff --git a/app/api/odm/specs/ModelWithPermissions.spec.ts b/app/api/odm/specs/ModelWithPermissions.spec.ts index ea09dd1245..b05bdece7a 100644 --- a/app/api/odm/specs/ModelWithPermissions.spec.ts +++ b/app/api/odm/specs/ModelWithPermissions.spec.ts @@ -4,6 +4,7 @@ import { instanceModelWithPermissions, ModelWithPermissions } from 'api/odm/Mode import { permissionsContext } from 'api/permissions/permissionsContext'; import testingDB from 'api/utils/testing_db'; import * as mongoose from 'mongoose'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; describe('ModelWithPermissions', () => { let model: ModelWithPermissions; @@ -91,6 +92,7 @@ describe('ModelWithPermissions', () => { ]; beforeAll(async () => { connection = await testingDB.connect(); + testingEnvironment.setFakeContext(); model = instanceModelWithPermissions('docs', testSchema); await connection.collection('docs').insertMany(testdocs); }); diff --git a/app/api/odm/specs/model.spec.ts b/app/api/odm/specs/model.spec.ts index 41d0704209..0ef5cfabe9 100644 --- a/app/api/odm/specs/model.spec.ts +++ b/app/api/odm/specs/model.spec.ts @@ -1,4 +1,5 @@ import { legacyLogger } from 'api/log'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { UpdateLogHelper } from 'api/odm/logHelper'; import { tenants } from 'api/tenants'; import { model as updatelogsModel } from 'api/updatelogs'; @@ -23,12 +24,12 @@ describe('ODM Model', () => { const originalDatenow = Date.now; beforeEach(async () => { - await testingDB.setupFixturesAndContext({}); + await testingEnvironment.setUp({}); }); afterAll(async () => { Date.now = originalDatenow; - await testingDB.disconnect(); + await testingEnvironment.tearDown(); }); const instanceTestingModel = (collectionName: string, schema: Schema) => { diff --git a/app/api/odm/specs/modelBulkWriteStream.spec.ts b/app/api/odm/specs/modelBulkWriteStream.spec.ts index 740650df99..0894492295 100644 --- a/app/api/odm/specs/modelBulkWriteStream.spec.ts +++ b/app/api/odm/specs/modelBulkWriteStream.spec.ts @@ -1,4 +1,5 @@ import userModel from 'api/users/usersModel'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import db from 'api/utils/testing_db'; import { UserRole } from 'shared/types/userSchema'; import { ModelBulkWriteStream } from '../modelBulkWriteStream'; @@ -29,7 +30,7 @@ const fixtures = { const newUsers = Array(11) .fill(0) // eslint-disable-next-line @typescript-eslint/no-unused-vars - .map((value: any, index: number) => ({ + .map((_value: any, index: number) => ({ username: `new_user_${index}`, role: UserRole.COLLABORATOR, email: `new_user_${index}@tenant.xy`, @@ -49,12 +50,12 @@ describe('modelBulkWriteStream', () => { let stream: ModelBulkWriteStream; beforeEach(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); stream = new ModelBulkWriteStream(userModel, stackLimit); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); it('should be able to insert', async () => { await stream.insert(newUsers[0]); diff --git a/app/api/pages/specs/pages.spec.ts b/app/api/pages/specs/pages.spec.ts index 4d0ae9fc13..ddd462e75f 100644 --- a/app/api/pages/specs/pages.spec.ts +++ b/app/api/pages/specs/pages.spec.ts @@ -1,4 +1,5 @@ import { mockID } from 'shared/uniqueID'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import date from 'api/utils/date.js'; import db from 'api/utils/testing_db'; @@ -7,11 +8,11 @@ import pages from '../pages'; describe('pages', () => { beforeEach(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('save', () => { diff --git a/app/api/permissions/specs/collaborators.spec.ts b/app/api/permissions/specs/collaborators.spec.ts index 7b229fd0c9..578619216f 100644 --- a/app/api/permissions/specs/collaborators.spec.ts +++ b/app/api/permissions/specs/collaborators.spec.ts @@ -1,17 +1,17 @@ -import { testingDB } from 'api/utils/testing_db'; -import { fixtures, groupA, groupB, userA, userB } from 'api/permissions/specs/fixtures'; import { collaborators } from 'api/permissions/collaborators'; +import { fixtures, groupA, groupB, userA, userB } from 'api/permissions/specs/fixtures'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { PermissionType } from 'shared/types/permissionSchema'; import { UserInContextMockFactory } from '../../utils/testingUserInContext'; import { PUBLIC_PERMISSION } from '../publicPermission'; describe('collaborators', () => { beforeEach(async () => { - await testingDB.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await testingDB.disconnect(); + await testingEnvironment.tearDown(); }); describe('search', () => { diff --git a/app/api/permissions/specs/entitiesPermissions.spec.ts b/app/api/permissions/specs/entitiesPermissions.spec.ts index cc68bb6ead..b551dfdeeb 100644 --- a/app/api/permissions/specs/entitiesPermissions.spec.ts +++ b/app/api/permissions/specs/entitiesPermissions.spec.ts @@ -1,13 +1,13 @@ /* eslint-disable max-lines */ -import { testingDB } from 'api/utils/testing_db'; import entities from 'api/entities/entities'; import { entitiesPermissions } from 'api/permissions/entitiesPermissions'; -import { AccessLevels, PermissionType, MixedAccess } from 'shared/types/permissionSchema'; -import { search } from 'api/search'; import { fixtures, groupA, userA, userB } from 'api/permissions/specs/fixtures'; +import { search } from 'api/search'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; +import { UserInContextMockFactory } from 'api/utils/testingUserInContext'; import { EntitySchema, EntityWithFilesSchema } from 'shared/types/entityType'; +import { AccessLevels, MixedAccess, PermissionType } from 'shared/types/permissionSchema'; import { PermissionsDataSchema } from 'shared/types/permissionType'; -import { UserInContextMockFactory } from 'api/utils/testingUserInContext'; import { PUBLIC_PERMISSION } from '../publicPermission'; const publicPermission = { @@ -25,11 +25,11 @@ const mockCollab = () => describe('permissions', () => { beforeEach(async () => { - await testingDB.clearAllAndLoad(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await testingDB.disconnect(); + await testingEnvironment.tearDown(); }); describe('set entities permissions', () => { diff --git a/app/api/relationships/specs/relationships.spec.js b/app/api/relationships/specs/relationships.spec.js index 086cc6c034..3ed3a4202b 100644 --- a/app/api/relationships/specs/relationships.spec.js +++ b/app/api/relationships/specs/relationships.spec.js @@ -1,11 +1,14 @@ /* eslint-disable max-lines */ +import { testingEnvironment } from 'api/utils/testingEnvironment'; /* eslint-disable max-statements */ /* eslint-disable max-nested-callbacks */ -import db from 'api/utils/testing_db'; import entities from 'api/entities/entities'; +import db from 'api/utils/testing_db'; import { UserInContextMockFactory } from 'api/utils/testingUserInContext'; +import { search } from '../../search'; +import relationships from '../relationships'; import fixtures, { connectionID1, connectionID2, @@ -16,22 +19,20 @@ import fixtures, { connectionID8, connectionID9, entity3, + family, + friend, hub1, + hub11, + hub12, hub2, hub5, hub7, hub8, hub9, - hub11, - hub12, - friend, - family, relation1, relation2, template, } from './fixtures'; -import relationships from '../relationships'; -import { search } from '../../search'; describe('relationships', () => { beforeEach(async () => { @@ -39,11 +40,11 @@ describe('relationships', () => { jest .spyOn(entities, 'updateMetdataFromRelationships') .mockImplementation(async () => Promise.resolve()); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('getByDocument()', () => { diff --git a/app/api/search.v2/specs/permissions.spec.ts b/app/api/search.v2/specs/permissions.spec.ts index 593209f71f..82227cda6b 100644 --- a/app/api/search.v2/specs/permissions.spec.ts +++ b/app/api/search.v2/specs/permissions.spec.ts @@ -1,8 +1,8 @@ +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { Application } from 'express'; import request from 'supertest'; import { setUpApp } from 'api/utils/testingRoutes'; -import { testingDB } from 'api/utils/testing_db'; import { UserInContextMockFactory } from 'api/utils/testingUserInContext'; import { searchRoutes } from '../routes'; @@ -13,10 +13,10 @@ describe('entities GET permissions + published filter', () => { const userFactory = new UserInContextMockFactory(); beforeAll(async () => { - await testingDB.setupFixturesAndContext(permissionsLevelFixtures, 'entities.v2.permissions'); + await testingEnvironment.setUp(permissionsLevelFixtures, 'entities.v2.permissions'); }); - afterAll(async () => testingDB.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); describe('GET/public entities', () => { describe('when user is not logged in', () => { diff --git a/app/api/search.v2/specs/snippetsSearch.spec.ts b/app/api/search.v2/specs/snippetsSearch.spec.ts index 4527f874a1..644382f46a 100644 --- a/app/api/search.v2/specs/snippetsSearch.spec.ts +++ b/app/api/search.v2/specs/snippetsSearch.spec.ts @@ -1,21 +1,21 @@ -import qs from 'qs'; -import request from 'supertest'; -import { Application } from 'express'; -import { setUpApp } from 'api/utils/testingRoutes'; import { searchRoutes } from 'api/search.v2/routes'; -import { testingDB } from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; +import { setUpApp } from 'api/utils/testingRoutes'; import { advancedSort } from 'app/utils/advancedSort'; +import { Application } from 'express'; +import qs from 'qs'; +import request from 'supertest'; import entities from 'api/entities'; import { elasticTesting } from 'api/utils/elastic_testing'; import { SearchQuery } from 'shared/types/SearchQueryType'; -import { fixturesSnippetsSearch, entity1enId, entity2enId } from './fixturesSnippetsSearch'; +import { entity1enId, entity2enId, fixturesSnippetsSearch } from './fixturesSnippetsSearch'; describe('searchSnippets', () => { const app: Application = setUpApp(searchRoutes); beforeAll(async () => { - await testingDB.setupFixturesAndContext(fixturesSnippetsSearch, 'entities.v2.snippets_search'); + await testingEnvironment.setUp(fixturesSnippetsSearch, 'entities.v2.snippets_search'); await entities.save(await entities.getById({ _id: entity2enId.toString() }), { user: {}, language: 'en', @@ -23,7 +23,7 @@ describe('searchSnippets', () => { await elasticTesting.refresh(); }); - afterAll(async () => testingDB.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); async function search(filter: SearchQuery['filter'], fields = ['snippets', 'title']) { return request(app).get('/api/v2/search').query(qs.stringify({ filter, fields })).expect(200); diff --git a/app/api/search/specs/analyzers.spec.ts b/app/api/search/specs/analyzers.spec.ts index 027e5d4c9c..71353ccefd 100644 --- a/app/api/search/specs/analyzers.spec.ts +++ b/app/api/search/specs/analyzers.spec.ts @@ -1,15 +1,15 @@ -import db from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { elasticClient } from '../elastic'; import { fixturesTimeOut } from './fixtures_elastic'; describe('custom language analyzers', () => { const elasticIndex = 'analyzers_index_test'; beforeAll(async () => { - await db.clearAllAndLoad({}, elasticIndex); + await testingEnvironment.setUp({}, elasticIndex); }, fixturesTimeOut); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('persian', () => { diff --git a/app/api/search/specs/entitiesIndex.spec.ts b/app/api/search/specs/entitiesIndex.spec.ts index 3442788543..d99fd93737 100644 --- a/app/api/search/specs/entitiesIndex.spec.ts +++ b/app/api/search/specs/entitiesIndex.spec.ts @@ -1,4 +1,5 @@ import { legacyLogger } from 'api/log'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { elasticTesting } from 'api/utils/elastic_testing'; import { UserInContextMockFactory } from 'api/utils/testingUserInContext'; import db from 'api/utils/testing_db'; @@ -20,16 +21,16 @@ describe('entitiesIndex', () => { const userFactory = new UserInContextMockFactory(); beforeEach(async () => { - await db.setupFixturesAndContext({}, elasticIndex); + await testingEnvironment.setUp({}, elasticIndex); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('indexEntities', () => { const loadFailingFixtures = async () => { - await db.setupFixturesAndContext(fixturesForIndexErrors); + await testingEnvironment.setUp(fixturesForIndexErrors); await elasticTesting.resetIndex(); // force indexing will ensure that all exceptions are mapper_parsing. Otherwise you get different kinds of exceptions await forceIndexingOfNumberBasedProperty(); diff --git a/app/api/search/specs/filterByAssignee.spec.ts b/app/api/search/specs/filterByAssignee.spec.ts index afcacb5cf1..037fa1ed8c 100644 --- a/app/api/search/specs/filterByAssignee.spec.ts +++ b/app/api/search/specs/filterByAssignee.spec.ts @@ -1,23 +1,23 @@ -import db from 'api/utils/testing_db'; import { search } from 'api/search/search'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { UserInContextMockFactory } from 'api/utils/testingUserInContext'; import { AggregationBucket, Aggregations } from 'shared/types/aggregations'; import { UserSchema } from 'shared/types/userType'; -import { fixturesTimeOut } from './fixtures_elastic'; -import { permissionsLevelFixtures, users, group1 } from './permissionsFiltersFixtures'; import { EntitySchema } from '../../../shared/types/entityType'; +import { fixturesTimeOut } from './fixtures_elastic'; +import { group1, permissionsLevelFixtures, users } from './permissionsFiltersFixtures'; describe('Permissions filters', () => { const userFactory = new UserInContextMockFactory(); const user3WithGroups = { ...users.user3, groups: [{ _id: group1.toString(), name: 'Group1' }] }; beforeAll(async () => { - await db.setupFixturesAndContext(permissionsLevelFixtures, 'permissionsadminfilters'); + await testingEnvironment.setUp(permissionsLevelFixtures, 'permissionsadminfilters'); userFactory.restore(); }, fixturesTimeOut); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); userFactory.restore(); }); diff --git a/app/api/search/specs/permissionsFilters.spec.ts b/app/api/search/specs/permissionsFilters.spec.ts index f510ed9c0f..4960ff68c9 100644 --- a/app/api/search/specs/permissionsFilters.spec.ts +++ b/app/api/search/specs/permissionsFilters.spec.ts @@ -1,18 +1,18 @@ -import db from 'api/utils/testing_db'; import { search } from 'api/search/search'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { UserInContextMockFactory } from 'api/utils/testingUserInContext'; -import { Aggregations, AggregationBucket } from 'shared/types/aggregations'; -import { UserSchema } from 'shared/types/userType'; +import { AggregationBucket, Aggregations } from 'shared/types/aggregations'; import { ObjectIdSchema } from 'shared/types/commonTypes'; +import { UserSchema } from 'shared/types/userType'; import { fixturesTimeOut } from './fixtures_elastic'; import { - permissionsLevelFixtures, - users, group1, + permissionsLevelFixtures, template1Id, template2Id, template3Id, + users, } from './permissionsFiltersFixtures'; function getAggregationCountByType(typesBuckets: AggregationBucket[], templateId: ObjectIdSchema) { @@ -26,12 +26,12 @@ describe('Permissions filters', () => { const userFactory = new UserInContextMockFactory(); beforeAll(async () => { - await db.setupFixturesAndContext(permissionsLevelFixtures, 'permissionslevelfixtures'); + await testingEnvironment.setUp(permissionsLevelFixtures, 'permissionslevelfixtures'); userFactory.restore(); }, fixturesTimeOut); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); userFactory.restore(); }); diff --git a/app/api/search/specs/routes.spec.ts b/app/api/search/specs/routes.spec.ts index 5e04e038e2..5e94555a86 100644 --- a/app/api/search/specs/routes.spec.ts +++ b/app/api/search/specs/routes.spec.ts @@ -1,14 +1,14 @@ -import request, { Response as SuperTestResponse } from 'supertest'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { Application } from 'express'; +import request, { Response as SuperTestResponse } from 'supertest'; -import db from 'api/utils/testing_db'; -import { setUpApp } from 'api/utils/testingRoutes'; import searchRoutes from 'api/search/routes'; +import { setUpApp } from 'api/utils/testingRoutes'; import { UserRole } from 'shared/types/userSchema'; -import { fixtures, ids, fixturesTimeOut } from './fixtures_elastic'; import { UserInContextMockFactory } from '../../utils/testingUserInContext'; +import { fixtures, fixturesTimeOut, ids } from './fixtures_elastic'; describe('Search routes', () => { const app: Application = setUpApp(searchRoutes); @@ -16,10 +16,10 @@ describe('Search routes', () => { beforeAll(async () => { //@ts-ignore - await db.clearAllAndLoad(fixtures, elasticIndex); + await testingEnvironment.setUp(fixtures, elasticIndex); }, fixturesTimeOut); - afterAll(async () => db.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); describe('GET /search/lookup', () => { it('should return a list of entity options', async () => { diff --git a/app/api/search/specs/search.searchGeolocations.spec.ts b/app/api/search/specs/search.searchGeolocations.spec.ts index b683425459..03f46ba02b 100644 --- a/app/api/search/specs/search.searchGeolocations.spec.ts +++ b/app/api/search/specs/search.searchGeolocations.spec.ts @@ -4,17 +4,18 @@ import { EntitySchema } from 'shared/types/entityType'; import inheritanceFixtures, { ids } from './fixturesInheritance'; import { fixturesTimeOut } from './fixtures_elastic'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; describe('search.searchGeolocations', () => { const user = { _id: 'u1' }; beforeAll(async () => { const elasticIndex = 'search.geolocation_index_test'; - await db.clearAllAndLoad(inheritanceFixtures, elasticIndex); + await testingEnvironment.setUp(inheritanceFixtures, elasticIndex); }, fixturesTimeOut); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); it('should include all geolocation finds, inheriting metadata', async () => { diff --git a/app/api/search/specs/search.spec.js b/app/api/search/specs/search.spec.js index 21c93535b7..1fd71c0e0c 100644 --- a/app/api/search/specs/search.spec.js +++ b/app/api/search/specs/search.spec.js @@ -1,12 +1,12 @@ import { elastic } from 'api/search'; import { search } from 'api/search/search'; import date from 'api/utils/date'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { UserInContextMockFactory } from 'api/utils/testingUserInContext'; -import db from 'api/utils/testing_db'; import * as searchLimitsConfig from 'shared/config'; import { UserRole } from 'shared/types/userSchema'; import elasticResult from './elasticResult'; -import { fixtures as elasticFixtures, ids, fixturesTimeOut } from './fixtures_elastic'; +import { fixtures as elasticFixtures, fixturesTimeOut, ids } from './fixtures_elastic'; const editorUser = { _id: 'userId', role: 'editor' }; @@ -19,7 +19,7 @@ describe('search', () => { beforeAll(async () => { result = elasticResult().toObject(); const elasticIndex = 'search_index_test'; - await db.setupFixturesAndContext(elasticFixtures, elasticIndex); + await testingEnvironment.setUp(elasticFixtures, elasticIndex); }, fixturesTimeOut); beforeEach(async () => { @@ -35,7 +35,7 @@ describe('search', () => { }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('searchSnippets', () => { diff --git a/app/api/services/convertToPDF/specs/convertToPdfWorker.spec.ts b/app/api/services/convertToPDF/specs/convertToPdfWorker.spec.ts index 19b1d2ef08..f696e0a130 100644 --- a/app/api/services/convertToPDF/specs/convertToPdfWorker.spec.ts +++ b/app/api/services/convertToPDF/specs/convertToPdfWorker.spec.ts @@ -2,6 +2,7 @@ import { createReadStream } from 'fs'; import { config } from 'api/config'; import { files, storage, testingUploadPaths } from 'api/files'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { tenants } from 'api/tenants'; import testingDB from 'api/utils/testing_db'; import { permissionsContext } from 'api/permissions/permissionsContext'; @@ -34,7 +35,7 @@ describe('convertToPdfWorker', () => { beforeAll(async () => { await testingDB.connect({ defaultTenant: false }); jest.spyOn(setupSockets, 'emitToTenant').mockImplementation(() => {}); - await testingDB.setupFixturesAndContext({ + await testingEnvironment.setUp({ settings: [ { features: { convertToPdf: { active: true, url: 'http://localhost:5060' } }, @@ -73,7 +74,7 @@ describe('convertToPdfWorker', () => { afterAll(async () => { redisClient.end(true); - await testingDB.disconnect(); + await testingEnvironment.tearDown(); await worker.stop(); }); diff --git a/app/api/services/informationextraction/specs/ixmodels.spec.ts b/app/api/services/informationextraction/specs/ixmodels.spec.ts index 1c9949a3fc..eabb9a018f 100644 --- a/app/api/services/informationextraction/specs/ixmodels.spec.ts +++ b/app/api/services/informationextraction/specs/ixmodels.spec.ts @@ -1,8 +1,8 @@ import { Suggestions } from 'api/suggestions/suggestions'; import { getFixturesFactory } from 'api/utils/fixturesFactory'; -import db from 'api/utils/testing_db'; -import { ModelStatus } from 'shared/types/IXModelSchema'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { toHaveBeenCalledBefore } from 'jest-extended'; +import { ModelStatus } from 'shared/types/IXModelSchema'; import ixmodels from '../ixmodels'; expect.extend({ toHaveBeenCalledBefore }); @@ -11,13 +11,13 @@ const fixtureFactory = getFixturesFactory(); describe('save()', () => { beforeAll(async () => { - await db.clearAllAndLoad({ + await testingEnvironment.setUp({ settings: [{ languages: [{ default: true, label: 'English', key: 'en' }] }], }); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); it('should set suggestions obsolete on saving a ready model', async () => { diff --git a/app/api/services/pdfsegmentation/specs/eventListeners.spec.ts b/app/api/services/pdfsegmentation/specs/eventListeners.spec.ts index d181321fd8..1444300536 100644 --- a/app/api/services/pdfsegmentation/specs/eventListeners.spec.ts +++ b/app/api/services/pdfsegmentation/specs/eventListeners.spec.ts @@ -1,4 +1,5 @@ import { applicationEventsBus } from 'api/eventsbus'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { FilesDeletedEvent } from 'api/files/events/FilesDeletedEvent'; import db from 'api/utils/testing_db'; import { registerEventListeners } from '../eventListeners'; @@ -6,11 +7,11 @@ import { SegmentationModel } from '../segmentationModel'; beforeAll(async () => { registerEventListeners(applicationEventsBus); - await db.setupFixturesAndContext({}); + await testingEnvironment.setUp({}); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe(`On ${FilesDeletedEvent.name}`, () => { diff --git a/app/api/settings/specs/settings.spec.ts b/app/api/settings/specs/settings.spec.ts index 68476b1ea2..31f4cb1240 100644 --- a/app/api/settings/specs/settings.spec.ts +++ b/app/api/settings/specs/settings.spec.ts @@ -367,7 +367,7 @@ describe('settings', () => { describe('getLinks', () => { it('should return the links', async () => { - await db.setupFixturesAndContext(linkFixtures); + await testingEnvironment.setUp(linkFixtures); const result = await settings.getLinks(); expect(result).toEqual(linkFixtures.settings?.[0].links); }); @@ -375,7 +375,7 @@ describe('settings', () => { describe('saveLinks', () => { it('should save the links', async () => { - await db.setupFixturesAndContext(linkFixtures); + await testingEnvironment.setUp(fixtures); await settings.saveLinks(newLinks); const result = await settings.getLinks(); expect(result).toEqual(newLinks); diff --git a/app/api/suggestions/specs/customFilters.spec.ts b/app/api/suggestions/specs/customFilters.spec.ts index 045d17352a..5d9b9bfaca 100644 --- a/app/api/suggestions/specs/customFilters.spec.ts +++ b/app/api/suggestions/specs/customFilters.spec.ts @@ -1,4 +1,5 @@ import { testingDB } from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { SuggestionCustomFilter } from 'shared/types/suggestionType'; import { factory as f, stateFilterFixtures } from './fixtures'; import { Suggestions } from '../suggestions'; @@ -13,11 +14,11 @@ const blankCustomFilter: SuggestionCustomFilter = { }; beforeAll(async () => { - await testingDB.setupFixturesAndContext(stateFilterFixtures); + await testingEnvironment.setUp(stateFilterFixtures); await Suggestions.updateStates({}); }); -afterAll(async () => testingDB.disconnect()); +afterAll(async () => testingEnvironment.tearDown()); describe('suggestions with CustomFilters', () => { describe('get()', () => { diff --git a/app/api/suggestions/specs/eventListeners.spec.ts b/app/api/suggestions/specs/eventListeners.spec.ts index 14406dd055..054ae6af71 100644 --- a/app/api/suggestions/specs/eventListeners.spec.ts +++ b/app/api/suggestions/specs/eventListeners.spec.ts @@ -1,4 +1,5 @@ import entities from 'api/entities'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { EntityDeletedEvent } from 'api/entities/events/EntityDeletedEvent'; import { EntityUpdatedEvent } from 'api/entities/events/EntityUpdatedEvent'; import { applicationEventsBus } from 'api/eventsbus'; @@ -159,11 +160,11 @@ beforeAll(() => { beforeEach(async () => { jest.spyOn(search, 'indexEntities').mockReturnValue(Promise.resolve()); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe(`On ${EntityUpdatedEvent.name}`, () => { diff --git a/app/api/suggestions/specs/suggestions.spec.ts b/app/api/suggestions/specs/suggestions.spec.ts index b88b8954e4..ba792b8629 100644 --- a/app/api/suggestions/specs/suggestions.spec.ts +++ b/app/api/suggestions/specs/suggestions.spec.ts @@ -1,13 +1,14 @@ /* eslint-disable max-statements */ +import { testingEnvironment } from 'api/utils/testingEnvironment'; import db from 'api/utils/testing_db'; +import { ObjectId } from 'mongodb'; import { EntitySuggestionType, IXSuggestionStateType, IXSuggestionType, IXSuggestionsFilter, } from 'shared/types/suggestionType'; -import { ObjectId } from 'mongodb'; import { Suggestions } from '../suggestions'; import { factory, @@ -15,12 +16,12 @@ import { file3Id, fixtures, personTemplateId, + relationshipAcceptanceFixtureBase, + selectAcceptanceFixtureBase, + shared2AgeSuggestionId, shared2enId, shared2esId, suggestionId, - shared2AgeSuggestionId, - selectAcceptanceFixtureBase, - relationshipAcceptanceFixtureBase, } from './fixtures'; const getSuggestions = async (filter: IXSuggestionsFilter, size = 5) => @@ -457,12 +458,12 @@ const prepareAndAcceptRelationshipSuggestion = async ( describe('suggestions', () => { afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('get()', () => { beforeEach(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); await Suggestions.updateStates({}); }); @@ -781,7 +782,7 @@ describe('suggestions', () => { describe('accept()', () => { describe('general', () => { beforeAll(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); await Suggestions.updateStates({}); }); @@ -867,7 +868,7 @@ describe('suggestions', () => { describe('numeric/date', () => { beforeAll(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); await Suggestions.updateStates({}); }); @@ -909,7 +910,7 @@ describe('suggestions', () => { describe('select', () => { beforeEach(async () => { - await db.setupFixturesAndContext(selectAcceptanceFixtureBase); + await testingEnvironment.setUp(selectAcceptanceFixtureBase); }); it('should validate that the id exists in the dictionary', async () => { @@ -944,7 +945,7 @@ describe('suggestions', () => { describe('multiselect', () => { beforeEach(async () => { - await db.setupFixturesAndContext(selectAcceptanceFixtureBase); + await testingEnvironment.setUp(selectAcceptanceFixtureBase); }); it('should validate that the ids exist in the dictionary', async () => { @@ -1146,7 +1147,7 @@ describe('suggestions', () => { describe('relationship', () => { beforeEach(async () => { - await db.setupFixturesAndContext(relationshipAcceptanceFixtureBase); + await testingEnvironment.setUp(relationshipAcceptanceFixtureBase); }); it('should validate that the entities in the suggestion exist', async () => { @@ -1469,7 +1470,7 @@ describe('suggestions', () => { describe('save()', () => { beforeEach(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); describe('on suggestion status error', () => { @@ -1517,7 +1518,7 @@ describe('suggestions', () => { describe('updateStates()', () => { beforeAll(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); it.each(stateUpdateCases)('should mark $reason', async ({ state, suggestionQuery }) => { @@ -1534,7 +1535,7 @@ describe('suggestions', () => { describe('setObsolete()', () => { beforeEach(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); it('should set the queried suggestions to obsolete state', async () => { @@ -1548,7 +1549,7 @@ describe('suggestions', () => { describe('markSuggestionsWithoutSegmentation()', () => { beforeEach(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); it('should mark the suggestions without segmentation to error state', async () => { @@ -1578,7 +1579,7 @@ describe('suggestions', () => { describe('saveMultiple()', () => { beforeEach(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); it('should handle everything at once', async () => { diff --git a/app/api/templates/specs/extractedMetadataFunctions.spec.ts b/app/api/templates/specs/extractedMetadataFunctions.spec.ts index e67ae0996b..fccbf7476e 100644 --- a/app/api/templates/specs/extractedMetadataFunctions.spec.ts +++ b/app/api/templates/specs/extractedMetadataFunctions.spec.ts @@ -1,6 +1,6 @@ import { files } from 'api/files'; import translations from 'api/i18n/translations'; -import db from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { TemplateSchema } from 'shared/types/templateType'; import templates from '../templates'; import fixtures, { @@ -14,7 +14,7 @@ import fixtures, { describe('updateExtractedMetadataProperties()', () => { beforeEach(async () => { jest.spyOn(translations, 'updateContext').mockImplementation(async () => 'ok'); - await db.clearAllAndLoad(fixtures, 'uwazi_test_index'); + await testingEnvironment.setUp(fixtures, 'uwazi_test_index'); }); it('should remove deleted template properties from extracted metadata on files', async () => { @@ -135,5 +135,5 @@ describe('updateExtractedMetadataProperties()', () => { }); }); - afterAll(async () => db.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); }); diff --git a/app/api/templates/specs/generatedIdPropertyAutoFiller.spec.ts b/app/api/templates/specs/generatedIdPropertyAutoFiller.spec.ts index 96c9e3cfdf..df8f5d2d07 100644 --- a/app/api/templates/specs/generatedIdPropertyAutoFiller.spec.ts +++ b/app/api/templates/specs/generatedIdPropertyAutoFiller.spec.ts @@ -8,7 +8,7 @@ import { } from 'api/templates/specs/generatedIdPropertyAutoFillerFixtures'; import { elasticTesting } from 'api/utils/elastic_testing'; import { unique } from 'api/utils/filters'; -import db from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { propertyTypes } from 'shared/propertyTypes'; import { EntitySchema } from 'shared/types/entityType'; import { TemplateSchema } from 'shared/types/templateType'; @@ -17,11 +17,11 @@ import templates from '../templates'; describe('generatedId property auto filler', () => { beforeAll(async () => { jest.spyOn(translations, 'updateContext').mockImplementation(async () => 'ok'); - await db.setupFixturesAndContext(fixtures, 'generated_id_auto_filler_index'); + await testingEnvironment.setUp(fixtures, 'generated_id_auto_filler_index'); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('fill generated id fields for entities of a specified template', () => { diff --git a/app/api/templates/specs/reindex.spec.js b/app/api/templates/specs/reindex.spec.js index 4a0a7052da..f9b8c10850 100644 --- a/app/api/templates/specs/reindex.spec.js +++ b/app/api/templates/specs/reindex.spec.js @@ -1,6 +1,6 @@ import translations from 'api/i18n'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { search } from 'api/search'; -import db from 'api/utils/testing_db'; import { propertyTypes } from 'shared/propertyTypes'; import { checkIfReindex } from '../reindex'; import templates from '../templates'; @@ -20,7 +20,7 @@ const expectReindex = async (template, reindex) => { describe('reindex', () => { beforeAll(async () => { jest.spyOn(translations, 'updateContext').mockImplementation(async () => 'ok'); - await db.setupFixturesAndContext(fixtures, 'reindex'); + await testingEnvironment.setUp(fixtures, 'reindex'); jest.spyOn(search, 'indexEntities').mockReturnValue({}); }); @@ -29,7 +29,7 @@ describe('reindex', () => { }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('Not Reindex', () => { @@ -89,7 +89,7 @@ describe('reindex', () => { describe('Reindex', () => { beforeAll(async () => { - await db.setupFixturesAndContext(fixtures, 'reindex'); + await testingEnvironment.setUp(fixtures, 'reindex'); jest.spyOn(search, 'indexEntities').mockReturnValue({}); }); diff --git a/app/api/templates/specs/templateSchema.spec.ts b/app/api/templates/specs/templateSchema.spec.ts index 2c7b19605d..2251a87033 100644 --- a/app/api/templates/specs/templateSchema.spec.ts +++ b/app/api/templates/specs/templateSchema.spec.ts @@ -1,28 +1,29 @@ /* eslint-disable max-lines */ import Ajv from 'ajv'; import db from 'api/utils/testing_db'; -import { TemplateSchema } from 'shared/types/templateType'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { PropertySchema } from 'shared/types/commonTypes'; +import { TemplateSchema } from 'shared/types/templateType'; import fixtures, { + propertyToBeInherited, templateId, templateToBeInherited, - propertyToBeInherited, thesauriId1, thesauriId2, thesauriId4, } from './validatorFixtures'; -import { safeName } from '../utils'; import { validateTemplate } from '../../../shared/types/templateSchema'; +import { safeName } from '../utils'; describe('template schema', () => { beforeEach(async () => { - await db.clearAllAndLoad(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('validateTemplate', () => { diff --git a/app/api/templates/specs/templates.spec.js b/app/api/templates/specs/templates.spec.js index d18ba12f6c..080ff5ff58 100644 --- a/app/api/templates/specs/templates.spec.js +++ b/app/api/templates/specs/templates.spec.js @@ -1,33 +1,34 @@ import Ajv from 'ajv'; -import db from 'api/utils/testing_db'; import documents from 'api/documents/documents.js'; import entities from 'api/entities/entities.js'; +import * as generatedIdPropertyAutoFiller from 'api/entities/generatedIdPropertyAutoFiller'; import translations from 'api/i18n/translations'; import { elasticClient } from 'api/search/elastic'; +import db from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { propertyTypes } from 'shared/propertyTypes'; -import * as generatedIdPropertyAutoFiller from 'api/entities/generatedIdPropertyAutoFiller'; import { spyOnEmit } from 'api/eventsbus/eventTesting'; import templates from '../templates'; +import { TemplateDeletedEvent } from '../events/TemplateDeletedEvent'; +import { TemplateUpdatedEvent } from '../events/TemplateUpdatedEvent'; import fixtures, { - templateToBeEditedId, - templateToBeDeleted, - thesaurusTemplateId, - thesaurusTemplate2Id, - thesaurusTemplate3Id, - templateWithContents, - swapTemplate, - templateToBeInherited, propertyToBeInherited, relatedTo, - thesauriId1, - thesauriId2, select3id, select4id, + swapTemplate, + templateToBeDeleted, + templateToBeEditedId, + templateToBeInherited, + templateWithContents, + thesauriId1, + thesauriId2, + thesaurusTemplate2Id, + thesaurusTemplate3Id, + thesaurusTemplateId, } from './fixtures/fixtures'; -import { TemplateUpdatedEvent } from '../events/TemplateUpdatedEvent'; -import { TemplateDeletedEvent } from '../events/TemplateDeletedEvent'; describe('templates', () => { const elasticIndex = 'templates_spec_index'; @@ -43,11 +44,11 @@ describe('templates', () => { beforeAll(async () => { jest.spyOn(translations, 'addContext').mockImplementation(async () => Promise.resolve()); jest.spyOn(translations, 'updateContext').mockImplementation(() => {}); - await db.setupFixturesAndContext(fixtures, elasticIndex); + await testingEnvironment.setUp(fixtures, elasticIndex); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('save', () => { @@ -321,7 +322,7 @@ describe('templates', () => { describe('when passing _id', () => { beforeAll(async () => { - await db.setupFixturesAndContext(fixtures, elasticIndex); + await testingEnvironment.setUp(fixtures, elasticIndex); jest .spyOn(entities, 'updateMetadataProperties') .mockImplementation(async () => Promise.resolve()); diff --git a/app/api/templates/specs/utils.spec.ts b/app/api/templates/specs/utils.spec.ts index 71cc3de2d4..51ff63e477 100644 --- a/app/api/templates/specs/utils.spec.ts +++ b/app/api/templates/specs/utils.spec.ts @@ -1,18 +1,19 @@ +import settings from 'api/settings/settings'; import db from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { PropertySchema } from 'shared/types/commonTypes'; -import settings from 'api/settings/settings'; import { generateIds, + generateNames, + getDeletedProperties, getUpdatedIds, getUpdatedNames, - getDeletedProperties, - generateNames, PropertyOrThesaurusSchema, } from '../utils'; describe('templates utils', () => { beforeEach(async () => { - await db.clearAllAndLoad({}); + await testingEnvironment.setUp({}); }); describe('name generation', () => { @@ -418,4 +419,4 @@ describe('templates utils', () => { }); }); -afterAll(async () => db.disconnect()); +afterAll(async () => testingEnvironment.tearDown()); diff --git a/app/api/thesauri/specs/routes.old.spec.js b/app/api/thesauri/specs/routes.old.spec.js index 83d2006a25..408b0370e5 100644 --- a/app/api/thesauri/specs/routes.old.spec.js +++ b/app/api/thesauri/specs/routes.old.spec.js @@ -1,22 +1,22 @@ -import 'api/utils/jasmineHelpers'; -import db from 'api/utils/testing_db'; import translations from 'api/i18n/translations'; +import 'api/utils/jasmineHelpers'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; -import { fixtures } from './fixtures'; import instrumentRoutes from '../../utils/instrumentRoutes'; -import thesauri from '../thesauri'; import thesauriRoute from '../routes.js'; +import thesauri from '../thesauri'; +import { fixtures } from './fixtures'; describe('thesauri routes', () => { let routes; beforeEach(async () => { routes = instrumentRoutes(thesauriRoute); - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('GET', () => { diff --git a/app/api/thesauri/specs/routes.spec.ts b/app/api/thesauri/specs/routes.spec.ts index b833cad144..120a778343 100644 --- a/app/api/thesauri/specs/routes.spec.ts +++ b/app/api/thesauri/specs/routes.spec.ts @@ -24,8 +24,7 @@ describe('Thesauri routes', () => { beforeEach(async () => { jest.spyOn(search, 'indexEntities').mockImplementation(async () => Promise.resolve()); - await testingEnvironment.setTenant(); - await testingEnvironment.setFixtures(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => testingEnvironment.tearDown()); diff --git a/app/api/thesauri/specs/thesauri.spec.js b/app/api/thesauri/specs/thesauri.spec.js index a366776bf5..2c80d395c2 100644 --- a/app/api/thesauri/specs/thesauri.spec.js +++ b/app/api/thesauri/specs/thesauri.spec.js @@ -1,4 +1,5 @@ /* eslint-disable max-lines */ +import { testingEnvironment } from 'api/utils/testingEnvironment'; import _ from 'lodash'; import translations from 'api/i18n/translations'; @@ -21,11 +22,11 @@ const factory = getFixturesFactory(); describe('thesauri', () => { beforeEach(async () => { jest.spyOn(search, 'indexEntities').mockImplementation(async () => Promise.resolve()); - await testingDB.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await testingDB.disconnect(); + await testingEnvironment.tearDown(); search.indexEntities.mockRestore(); }); diff --git a/app/api/toc_generation/specs/tocService.spec.ts b/app/api/toc_generation/specs/tocService.spec.ts index 708dc83483..2afef453d4 100644 --- a/app/api/toc_generation/specs/tocService.spec.ts +++ b/app/api/toc_generation/specs/tocService.spec.ts @@ -4,7 +4,6 @@ import { testingDB } from 'api/utils/testing_db'; import request from 'shared/JSONRequest'; import { tocService } from '../tocService'; import { fixtures } from './fixtures'; -import { testingEnvironment } from 'api/utils/testingEnvironment'; describe('tocService', () => { let requestMock: jest.SpyInstance; diff --git a/app/api/topicclassification/specs/common.spec.ts b/app/api/topicclassification/specs/common.spec.ts index c5dd89975d..9aa147ca43 100644 --- a/app/api/topicclassification/specs/common.spec.ts +++ b/app/api/topicclassification/specs/common.spec.ts @@ -1,6 +1,6 @@ import templates from 'api/templates'; import { extractSequence } from 'api/topicclassification'; -import db from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { buildFullModelName, getThesaurusPropertyNames } from 'shared/commonTopicClassification'; import { EntitySchema } from 'shared/types/entityType'; import { TemplateSchema } from '../../../shared/types/templateType'; @@ -8,11 +8,11 @@ import fixtures, { moviesId, template1 } from './fixtures'; describe('templates utils', () => { beforeEach(async () => { - await db.clearAllAndLoad(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('buildModelName', () => { diff --git a/app/api/topicclassification/specs/routes.spec.ts b/app/api/topicclassification/specs/routes.spec.ts index 7db5baae83..b648b24476 100644 --- a/app/api/topicclassification/specs/routes.spec.ts +++ b/app/api/topicclassification/specs/routes.spec.ts @@ -1,4 +1,5 @@ import * as topicClassification from 'api/config/topicClassification'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { setUpApp } from 'api/utils/testingRoutes'; import db from 'api/utils/testing_db'; import { NextFunction } from 'express'; @@ -102,7 +103,7 @@ describe('topic classification routes', () => { beforeEach(async () => { const elasticIndex = 'tc_routes_test'; - await db.setupFixturesAndContext(fixtures, elasticIndex); + await testingEnvironment.setUp(fixtures, elasticIndex); jest.spyOn(JSONRequest, 'post').mockImplementation(fakePost); jest.spyOn(JSONRequest, 'get').mockImplementation(fakeGet); jest @@ -111,7 +112,7 @@ describe('topic classification routes', () => { }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('GET', () => { diff --git a/app/api/topicclassification/specs/sync.spec.ts b/app/api/topicclassification/specs/sync.spec.ts index 32daee53a7..27961cdeaa 100644 --- a/app/api/topicclassification/specs/sync.spec.ts +++ b/app/api/topicclassification/specs/sync.spec.ts @@ -1,4 +1,5 @@ import * as topicClassification from 'api/config/topicClassification'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import entities from 'api/entities'; import { search } from 'api/search'; import db from 'api/utils/testing_db'; @@ -71,7 +72,7 @@ async function fakeTopicClassification(url: string) { describe('templates utils', () => { beforeEach(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); jest.spyOn(search, 'indexEntities').mockImplementation(async () => Promise.resolve()); jest.spyOn(JSONRequest, 'post').mockImplementation(fakeTopicClassification); jest.spyOn(JSONRequest, 'get').mockImplementation(fakeTopicClassification); @@ -80,7 +81,7 @@ describe('templates utils', () => { .mockReturnValue(Promise.resolve(true)); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('sync one', () => { diff --git a/app/api/usergroups/specs/userGroups.spec.ts b/app/api/usergroups/specs/userGroups.spec.ts index a0c2eab069..eae3f0dff7 100644 --- a/app/api/usergroups/specs/userGroups.spec.ts +++ b/app/api/usergroups/specs/userGroups.spec.ts @@ -1,17 +1,17 @@ import Ajv from 'ajv'; -import userGroups from 'api/usergroups/userGroups'; -import db from 'api/utils/testing_db'; import { models } from 'api/odm'; +import userGroups from 'api/usergroups/userGroups'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { UserGroupSchema } from 'shared/types/userGroupType'; import { UserSchema } from 'shared/types/userType'; import { fixtures, group1Id, group2Id, user1Id, user2Id } from './fixtures'; describe('userGroups', () => { beforeEach(async () => { - await db.clearAllAndLoad(fixtures); + await testingEnvironment.setUp(fixtures); }); - afterAll(async () => db.disconnect()); + afterAll(async () => testingEnvironment.tearDown()); describe('get', () => { it('should return populated user groups from model', async () => { diff --git a/app/api/usergroups/specs/userGroupsMembers.spec.ts b/app/api/usergroups/specs/userGroupsMembers.spec.ts index 8a6dd5ce6d..78ef8aeea0 100644 --- a/app/api/usergroups/specs/userGroupsMembers.spec.ts +++ b/app/api/usergroups/specs/userGroupsMembers.spec.ts @@ -1,4 +1,5 @@ import { testingDB } from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { getByMemberIdList, updateUserMemberships, @@ -10,11 +11,11 @@ import { fixtures, group1Id, group2Id, user1Id, user2Id, user3Id } from './fixtu describe('userGroupsMembers', () => { beforeEach(async () => { - await testingDB.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await testingDB.disconnect(); + await testingEnvironment.tearDown(); }); describe('getByMemberIdList', () => { diff --git a/app/api/users/specs/users.spec.js b/app/api/users/specs/users.spec.js index 6a3b4a3ae8..75f3331998 100644 --- a/app/api/users/specs/users.spec.js +++ b/app/api/users/specs/users.spec.js @@ -32,11 +32,11 @@ jest.mock('api/users/generateUnlockCode.ts', () => ({ describe('Users', () => { beforeEach(async () => { - await db.setupFixturesAndContext(fixtures); + await testingEnvironment.setUp(fixtures); }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('save', () => { diff --git a/app/api/utils/specs/languageMiddleware.spec.ts b/app/api/utils/specs/languageMiddleware.spec.ts index bbafc3c0ba..b3dd0e8eee 100644 --- a/app/api/utils/specs/languageMiddleware.spec.ts +++ b/app/api/utils/specs/languageMiddleware.spec.ts @@ -1,5 +1,5 @@ -import db from 'api/utils/testing_db'; -import { Request, NextFunction, Response } from 'express'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; +import { NextFunction, Request, Response } from 'express'; import middleware from '../languageMiddleware'; import fixtures from './languageFixtures'; @@ -11,7 +11,7 @@ describe('languageMiddleware', () => { const createRequest = (request: Partial) => { ...request }; beforeEach(async () => { - await db.clearAllAndLoad(fixtures); + await testingEnvironment.setUp(fixtures); req = { get: (headerName: string) => //@ts-ignore @@ -21,7 +21,7 @@ describe('languageMiddleware', () => { }); afterAll(async () => { - await db.disconnect(); + await testingEnvironment.tearDown(); }); describe('when there is an error', () => { diff --git a/app/api/utils/testingEnvironment.ts b/app/api/utils/testingEnvironment.ts index ccf9d7a6dd..4c8e8dc181 100644 --- a/app/api/utils/testingEnvironment.ts +++ b/app/api/utils/testingEnvironment.ts @@ -6,8 +6,6 @@ import { UserInContextMockFactory } from 'api/utils/testingUserInContext'; import { setupTestUploadedPaths } from 'api/files'; import { UserSchema } from 'shared/types/userType'; -const originalAppContextGet = appContext.get.bind(appContext); - const testingEnvironment = { userInContextMockFactory: new UserInContextMockFactory(), @@ -29,6 +27,8 @@ const testingEnvironment = { }, setFakeContext() { + const originalAppContextGet = appContext.get.bind(appContext); + jest.spyOn(appContext, 'get').mockImplementation((key: string) => { if (key === 'mongoSession') { return undefined; diff --git a/app/api/utils/testingTenants.ts b/app/api/utils/testingTenants.ts index 92521ca15a..ab02b28243 100644 --- a/app/api/utils/testingTenants.ts +++ b/app/api/utils/testingTenants.ts @@ -1,24 +1,15 @@ import { config } from 'api/config'; import { Tenant, tenants } from 'api/tenants/tenantContext'; -import { appContext } from './AppContext'; const originalCurrentFN = tenants.current.bind(tenants); let mockedTenant: Partial; -const originalAppContextGet = appContext.get.bind(appContext); - const testingTenants = { mockCurrentTenant(tenant: Partial) { mockedTenant = tenant; mockedTenant.featureFlags = mockedTenant.featureFlags || config.defaultTenant.featureFlags; tenants.current = () => mockedTenant; - jest.spyOn(appContext, 'get').mockImplementation((key: string) => { - if (key === 'mongoSession') { - return undefined; - } - return originalAppContextGet(key); - }); }, changeCurrentTenant(changes: Partial) { diff --git a/app/api/utils/testing_db.ts b/app/api/utils/testing_db.ts index cfe6e55020..6281865912 100644 --- a/app/api/utils/testing_db.ts +++ b/app/api/utils/testing_db.ts @@ -144,6 +144,9 @@ const testingDB: { await fixturer.clear(mongodb, collections); }, + /** + * @deprecated + */ async setupFixturesAndContext(fixtures: DBFixture, elasticIndex?: string, dbName?: string) { await this.connect(); let optionalMongo: Db | null = null; @@ -180,6 +183,9 @@ const testingDB: { await this.setupFixturesAndContext(fixtures, elasticIndex); }, + /** + * @deprecated + */ async clearAllAndLoadFixtures(fixtures: DBFixture) { await fixturer.clearAllAndLoad(mongodb, fixtures); }, From 2de5875c7dd5527eaa99059ec5e0c4700ae3edbf Mon Sep 17 00:00:00 2001 From: Daneryl Date: Wed, 29 Jan 2025 10:55:32 +0100 Subject: [PATCH 05/15] WIP, moved mongo session management to multitenantMongoose model --- app/api/entities/entities.js | 4 - app/api/entities/entitySavingManager.ts | 84 ++-- app/api/entities/managerFunctions.ts | 19 +- app/api/entities/routes.js | 26 +- .../specs/entitySavingManager.spec.ts | 44 +- app/api/odm/MultiTenantMongooseModel.ts | 125 +++-- app/api/odm/model.ts | 64 +-- app/api/odm/modelBulkWriteStream.ts | 4 +- app/api/odm/sessionsContext.ts | 22 + .../specs/multiTenantMongooseModel.spec.ts | 450 ++++++++++++++++++ app/api/sync/syncsModel.ts | 4 +- app/api/updatelogs/updatelogsModel.ts | 4 +- app/api/utils/specs/withTransaction.spec.ts | 148 ++++++ app/api/utils/withTransaction.ts | 35 ++ 14 files changed, 828 insertions(+), 205 deletions(-) create mode 100644 app/api/odm/sessionsContext.ts create mode 100644 app/api/odm/specs/multiTenantMongooseModel.spec.ts create mode 100644 app/api/utils/specs/withTransaction.spec.ts create mode 100644 app/api/utils/withTransaction.ts diff --git a/app/api/entities/entities.js b/app/api/entities/entities.js index 6b7166b68d..472bac65fc 100644 --- a/app/api/entities/entities.js +++ b/app/api/entities/entities.js @@ -918,8 +918,4 @@ export default { }, count: model.count.bind(model), - - async startSession() { - return model.db.startSession(); - }, }; diff --git a/app/api/entities/entitySavingManager.ts b/app/api/entities/entitySavingManager.ts index 3b1264bb08..2d20a8ed3d 100644 --- a/app/api/entities/entitySavingManager.ts +++ b/app/api/entities/entitySavingManager.ts @@ -1,6 +1,5 @@ -import entities from 'api/entities/entities'; -import { appContext } from 'api/utils/AppContext'; import { set } from 'lodash'; +import entities from 'api/entities/entities'; import { EntityWithFilesSchema } from 'shared/types/entityType'; import { UserSchema } from 'shared/types/userType'; import { handleAttachmentInMetadataProperties, processFiles, saveFiles } from './managerFunctions'; @@ -14,61 +13,46 @@ const saveEntity = async ( socketEmiter, }: { user: UserSchema; language: string; socketEmiter?: Function; files?: FileAttachment[] } ) => { - const session = await entities.startSession(); - appContext.set('mongoSession', undefined); // Clear any existing session first + const { attachments, documents } = (reqFiles || []).reduce( + (acum, file) => set(acum, file.fieldname, file), + { + attachments: [] as FileAttachment[], + documents: [] as FileAttachment[], + } + ); - try { - session.startTransaction(); - appContext.set('mongoSession', session); + const entity = handleAttachmentInMetadataProperties(_entity, attachments); - const { attachments, documents } = (reqFiles || []).reduce( - (acum, file) => set(acum, file.fieldname, file), - { - attachments: [] as FileAttachment[], - documents: [] as FileAttachment[], - } - ); + const updatedEntity = await entities.save( + entity, + { user, language }, + { includeDocuments: false } + ); - const entity = handleAttachmentInMetadataProperties(_entity, attachments); + const { proccessedAttachments, proccessedDocuments } = await processFiles( + entity, + updatedEntity, + attachments, + documents + ); - const updatedEntity = await entities.save( - entity, - { user, language }, - { includeDocuments: false } - ); + const fileSaveErrors = await saveFiles( + proccessedAttachments, + proccessedDocuments, + updatedEntity, + socketEmiter + ); - const { proccessedAttachments, proccessedDocuments } = await processFiles( - entity, - updatedEntity, - attachments, - documents - ); - - const fileSaveErrors = await saveFiles( - proccessedAttachments, - proccessedDocuments, - updatedEntity, - socketEmiter + const [entityWithAttachments]: EntityWithFilesSchema[] = + await entities.getUnrestrictedWithDocuments( + { + sharedId: updatedEntity.sharedId, + language: updatedEntity.language, + }, + '+permissions' ); - const [entityWithAttachments]: EntityWithFilesSchema[] = - await entities.getUnrestrictedWithDocuments( - { - sharedId: updatedEntity.sharedId, - language: updatedEntity.language, - }, - '+permissions' - ); - - await session.commitTransaction(); - return { entity: entityWithAttachments, errors: fileSaveErrors }; - } catch (e) { - await session.abortTransaction(); - return { errors: [e.message] }; - } finally { - appContext.set('mongoSession', undefined); - await session.endSession(); - } + return { entity: entityWithAttachments, errors: fileSaveErrors }; }; export type FileAttachment = { diff --git a/app/api/entities/managerFunctions.ts b/app/api/entities/managerFunctions.ts index a67e1a9a03..0bb2a8308e 100644 --- a/app/api/entities/managerFunctions.ts +++ b/app/api/entities/managerFunctions.ts @@ -13,7 +13,6 @@ import { MetadataObjectSchema } from 'shared/types/commonTypes'; import { EntityWithFilesSchema } from 'shared/types/entityType'; import { TypeOfFile } from 'shared/types/fileSchema'; import { FileAttachment } from './entitySavingManager'; -import { ClientSession } from 'mongodb'; const prepareNewFiles = async ( entity: EntityWithFilesSchema, @@ -112,14 +111,14 @@ const filterRenamedFiles = (entity: EntityWithFilesSchema, entityFiles: WithId { - const { attachments: newAttachments, documents: newDocuments } = await prepareNewFiles( + const { attachments, documents } = await prepareNewFiles( entity, updatedEntity, - attachments, - documents + fileAttachments, + documentAttachments ); if (entity._id && (entity.attachments || entity.documents)) { @@ -133,11 +132,11 @@ const processFiles = async ( const { renamedAttachments, renamedDocuments } = filterRenamedFiles(entity, entityFiles); - newAttachments.push(...renamedAttachments); - newDocuments.push(...renamedDocuments); + attachments.push(...renamedAttachments); + documents.push(...renamedDocuments); } - return { proccessedAttachments: newAttachments, proccessedDocuments: newDocuments }; + return { proccessedAttachments: attachments, proccessedDocuments: documents }; }; const bindAttachmentToMetadataProperty = ( @@ -192,7 +191,7 @@ const saveFiles = async ( await filesAPI.save(file, false); } catch (e) { legacyLogger.error(prettifyError(e)); - throw new Error(`Could not save file/s: ${file.originalname}`, { cause: e }); + saveResults.push(`Could not save file/s: ${file.originalname}`); } }) ); diff --git a/app/api/entities/routes.js b/app/api/entities/routes.js index 6748921e87..1f70a719a8 100644 --- a/app/api/entities/routes.js +++ b/app/api/entities/routes.js @@ -8,6 +8,7 @@ import thesauri from '../thesauri/thesauri'; import date from '../utils/date'; import needsAuthorization from '../auth/authMiddleware'; import { parseQuery, validation } from '../utils'; +import { withTransaction } from 'api/utils/withTransaction'; async function updateThesauriWithEntity(entity, req) { const template = await templates.getById(entity.template); @@ -80,18 +81,23 @@ export default app => { activitylogMiddleware, async (req, res, next) => { try { - const entityToSave = req.body.entity ? JSON.parse(req.body.entity) : req.body; - const result = await saveEntity(entityToSave, { - user: req.user, - language: req.language, - socketEmiter: req.emitToSessionSocket, - files: req.files, + await withTransaction(async () => { + const entityToSave = req.body.entity ? JSON.parse(req.body.entity) : req.body; + const result = await saveEntity(entityToSave, { + user: req.user, + language: req.language, + socketEmiter: req.emitToSessionSocket, + files: req.files, + }); + const { entity, errors } = result; + await updateThesauriWithEntity(entity, req); + // if (errors.length) { + // throw new Error('Errors during saveEntity', { cause: errors }); + // } + res.json(req.body.entity ? result : entity); }); - const { entity } = result; - await updateThesauriWithEntity(entity, req); - return res.json(req.body.entity ? result : entity); } catch (e) { - return next(e); + next(e); } } ); diff --git a/app/api/entities/specs/entitySavingManager.spec.ts b/app/api/entities/specs/entitySavingManager.spec.ts index e5c7b1460b..cb89628ea9 100644 --- a/app/api/entities/specs/entitySavingManager.spec.ts +++ b/app/api/entities/specs/entitySavingManager.spec.ts @@ -596,7 +596,7 @@ describe('entitySavingManager', () => { processDocumentApi.processDocument.mockRestore(); }); - it('should return an error if an existing main document cannot be saved', async () => { + it('should throw an error if a document cannot be saved', async () => { jest.spyOn(filesAPI, 'save').mockRejectedValueOnce({ error: { name: 'failed' } }); const { errors } = await saveEntity( @@ -618,46 +618,4 @@ describe('entitySavingManager', () => { }); }); }); - - describe('transactions', () => { - const reqData = { user: editorUser, language: 'en', socketEmiter: () => {} }; - - it('should rollback all operations if any operation fails', async () => { - testingEnvironment.unsetFakeContext(); - // Force files.save to fail - - // Attempt to update entity with new attachment that will fail - await appContext.run(async () => { - // Setup initial entity with a document - const { entity: savedEntity } = await saveEntity( - { title: 'initial entity', template: template1Id }, - { - ...reqData, - files: [{ ...newMainPdfDocument, fieldname: 'documents[0]' }], - } - ); - - // const filesOnDb = await filesAPI.get({ entity: savedEntity.sharedId }); - // console.log(JSON.stringify(filesOnDb, null, ' ')); - - jest.spyOn(filesAPI, 'save').mockImplementation(() => { - throw new Error('Forced file save error'); - }); - - - // Verify entity was not updated - const [entityAfterFailure] = await entities.get({ _id: savedEntity._id }); - expect(entityAfterFailure.title).toBe('initial entity'); - - // Verify original document still exists and no new files were added - const entityFiles = await filesAPI.get({ entity: savedEntity.sharedId }); - expect(entityFiles).toHaveLength(1); - expect(entityFiles[0].originalname).toBe('myNewFile.pdf'); - }); - - // await expect(updateOperation).rejects.toThrow('Forced file save error'); - // - // - }); - }); }); diff --git a/app/api/odm/MultiTenantMongooseModel.ts b/app/api/odm/MultiTenantMongooseModel.ts index 9bed4f85f5..0562b2aecb 100644 --- a/app/api/odm/MultiTenantMongooseModel.ts +++ b/app/api/odm/MultiTenantMongooseModel.ts @@ -1,17 +1,18 @@ -import { BulkWriteOptions, ClientSession } from 'mongodb'; +import { BulkWriteOptions } from 'mongodb'; import mongoose, { Schema } from 'mongoose'; +import { tenants } from '../tenants/tenantContext'; +import { DB } from './DB'; import { DataType, + EnforcedWithId, UwaziFilterQuery, - UwaziUpdateQuery, UwaziQueryOptions, - EnforcedWithId, UwaziUpdateOptions, + UwaziUpdateQuery, } from './model'; -import { tenants } from '../tenants/tenantContext'; -import { DB } from './DB'; +import { dbSessionContext } from './sessionsContext'; -export class MultiTenantMongooseModel { +export class MongooseModelWrapper { dbs: { [k: string]: mongoose.Model> }; collectionName: string; @@ -32,84 +33,136 @@ export class MultiTenantMongooseModel { ); } - findById(id: any, select?: any, options: { session?: ClientSession } = {}) { - return this.dbForCurrentTenant().findById(id, select, { ...options, lean: true }); + findById(id: any, select?: any) { + const session = dbSessionContext.getSession(); + return this.dbForCurrentTenant().findById(id, select, { + lean: true, + ...(session && { session }), + }); } - find( - query: UwaziFilterQuery>, - select = '', - options: { session?: ClientSession } = {} - ) { - return this.dbForCurrentTenant().find(query, select, options); + find(query: UwaziFilterQuery>, select = '', options: any = {}) { + const session = dbSessionContext.getSession(); + return this.dbForCurrentTenant().find(query, select, { + ...options, + ...(session && { session }), + }); } async findOneAndUpdate( query: UwaziFilterQuery>, update: UwaziUpdateQuery>, - options: UwaziQueryOptions & { session?: ClientSession } + options: UwaziQueryOptions = {} ) { - return this.dbForCurrentTenant().findOneAndUpdate(query, update, options); + const session = dbSessionContext.getSession(); + return this.dbForCurrentTenant().findOneAndUpdate(query, update, { + ...options, + ...(session && { session }), + }); } - async create(data: Partial>[], options?: any) { - return this.dbForCurrentTenant().create(data, options); + async create(data: Partial>[], options: any = {}) { + const session = dbSessionContext.getSession(); + return this.dbForCurrentTenant().create(data, { + ...options, + ...(session && { session }), + }); } - async createMany(dataArray: Partial>[], options: { session?: ClientSession } = {}) { - return this.dbForCurrentTenant().create(dataArray, options); + async createMany(dataArray: Partial>[], options: any = {}) { + const session = dbSessionContext.getSession(); + return this.dbForCurrentTenant().create(dataArray, { + ...options, + ...(session && { session }), + }); } async _updateMany( conditions: UwaziFilterQuery>, doc: UwaziUpdateQuery>, - options: UwaziUpdateOptions> & { session?: ClientSession } = {} + options: UwaziUpdateOptions> = {} ) { - return this.dbForCurrentTenant().updateMany(conditions, doc, options); + const session = dbSessionContext.getSession(); + return this.dbForCurrentTenant().updateMany(conditions, doc, { + ...options, + ...(session && { session }), + }); } async findOne(conditions: UwaziFilterQuery>, projection: any) { - const result = await this.dbForCurrentTenant().findOne(conditions, projection, { lean: true }); + const session = dbSessionContext.getSession(); + const result = await this.dbForCurrentTenant().findOne(conditions, projection, { + lean: true, + ...(session && { session }), + }); return result as EnforcedWithId | null; } async replaceOne(conditions: UwaziFilterQuery>, replacement: any) { - return this.dbForCurrentTenant().replaceOne(conditions, replacement); + const session = dbSessionContext.getSession(); + return this.dbForCurrentTenant().replaceOne(conditions, replacement, { + ...(session && { session }), + }); } - async countDocuments( - query: UwaziFilterQuery> = {}, - options: { session?: ClientSession } = {} - ) { - return this.dbForCurrentTenant().countDocuments(query, options); + async countDocuments(query: UwaziFilterQuery> = {}, options: any = {}) { + const session = dbSessionContext.getSession(); + return this.dbForCurrentTenant().countDocuments(query, { + ...options, + ...(session && { session }), + }); } async distinct(field: string, query: UwaziFilterQuery> = {}) { + const session = dbSessionContext.getSession(); + if (session) { + return this.dbForCurrentTenant().distinct(field, query).session(session); + } return this.dbForCurrentTenant().distinct(field, query); } - async deleteMany(query: UwaziFilterQuery>, options?: { session?: ClientSession }) { - return this.dbForCurrentTenant().deleteMany(query, options); + async deleteMany(query: UwaziFilterQuery>, options: any = {}) { + const session = dbSessionContext.getSession(); + return this.dbForCurrentTenant().deleteMany(query, { + ...options, + ...(session && { session }), + }); } async aggregate(aggregations?: any[]) { - return this.dbForCurrentTenant().aggregate(aggregations); + const session = dbSessionContext.getSession(); + return this.dbForCurrentTenant().aggregate(aggregations, session && { session }); } aggregateCursor(aggregations?: any[]) { - return this.dbForCurrentTenant().aggregate(aggregations) as mongoose.Aggregate; + const session = dbSessionContext.getSession(); + return this.dbForCurrentTenant().aggregate( + aggregations, + session && { session } + ) as mongoose.Aggregate; } async facet(aggregations: any[], pipelines: any, project: any) { - return this.dbForCurrentTenant().aggregate(aggregations).facet(pipelines).project(project); + const session = dbSessionContext.getSession(); + return this.dbForCurrentTenant() + .aggregate(aggregations, session && { session }) + .facet(pipelines) + .project(project); } async updateOne(conditions: UwaziFilterQuery>, doc: UwaziUpdateQuery) { - return this.dbForCurrentTenant().updateOne(conditions, doc); + const session = dbSessionContext.getSession(); + return this.dbForCurrentTenant().updateOne(conditions, doc, { + ...(session && { session }), + }); } async bulkWrite(writes: Array, options?: BulkWriteOptions) { - return this.dbForCurrentTenant().bulkWrite(writes, options); + const session = dbSessionContext.getSession(); + return this.dbForCurrentTenant().bulkWrite(writes, { + ...options, + ...(session && { session }), + }); } async ensureIndexes() { diff --git a/app/api/odm/model.ts b/app/api/odm/model.ts index 0e1f8855a1..4d2a98e2d8 100644 --- a/app/api/odm/model.ts +++ b/app/api/odm/model.ts @@ -10,11 +10,9 @@ import mongoose, { } from 'mongoose'; import { ObjectIdSchema } from 'shared/types/commonTypes'; import { inspect } from 'util'; -import { MultiTenantMongooseModel } from './MultiTenantMongooseModel'; +import { MongooseModelWrapper } from './MultiTenantMongooseModel'; import { UpdateLogger, createUpdateLogHelper } from './logHelper'; import { ModelBulkWriteStream } from './modelBulkWriteStream'; -import { ClientSession } from 'mongodb'; -import { appContext } from 'api/utils/AppContext'; /** Ideas! * T is the actual model-specific document Schema! @@ -37,7 +35,7 @@ export type UwaziUpdateOptions = (UpdateOptions & Omit implements SyncDBDataSource { private collectionName: string; - db: MultiTenantMongooseModel; + db: MongooseModelWrapper; logHelper: UpdateLogger; @@ -50,7 +48,7 @@ export class OdmModel implements SyncDBDataSource { options: { optimisticLock: boolean } = { optimisticLock: false } ) { this.collectionName = collectionName; - this.db = new MultiTenantMongooseModel(collectionName, schema); + this.db = new MongooseModelWrapper(collectionName, schema); this.logHelper = logHelper; this.options = options; } @@ -86,17 +84,9 @@ export class OdmModel implements SyncDBDataSource { } } - private getSession(): ClientSession | undefined { - return appContext.get('mongoSession') as ClientSession | undefined; - } - - async startSession(): Promise { - return this.db.startSession(); - } - async save(data: Partial>, _query?: any) { - const session = this.getSession(); if (await this.documentExists(data)) { + // @ts-ignore const { __v: version, ...toSaveData } = data; const query = _query && (await this.documentExistsByQuery(_query)) ? _query : { _id: data._id }; @@ -105,7 +95,7 @@ export class OdmModel implements SyncDBDataSource { const saved = await this.db.findOneAndUpdate( query, { $set: toSaveData as UwaziUpdateQuery>, $inc: { __v: 1 } }, - { new: true, ...(session && { session }) } + { new: true } ); if (saved === null) { @@ -119,8 +109,7 @@ export class OdmModel implements SyncDBDataSource { } async create(data: Partial>) { - const session = this.getSession(); - const saved = await this.db.create([data], session && { session }); + const saved = await this.db.create([data]); await this.logHelper.upsertLogOne(saved[0]); return saved[0].toObject>(); } @@ -147,9 +136,8 @@ export class OdmModel implements SyncDBDataSource { } private async saveNew(existingIds: Set, dataArray: Partial>[]) { - const session = this.getSession(); const newData = dataArray.filter(d => !d._id || !existingIds.has(d._id.toString())); - return (await this.db.createMany(newData, session && { session })) || []; + return (await this.db.createMany(newData)) || []; } private async saveExisting( @@ -157,7 +145,6 @@ export class OdmModel implements SyncDBDataSource { query?: any, updateExisting: boolean = true ) { - const session = this.getSession(); const ids: DataType['_id'][] = []; dataArray.forEach(d => { if (d._id) { @@ -168,7 +155,6 @@ export class OdmModel implements SyncDBDataSource { ( await this.db.find({ _id: { $in: ids } } as UwaziFilterQuery>, '_id', { lean: true, - ...(session && { session }), }) ).map(d => d._id.toString()) ); @@ -181,18 +167,13 @@ export class OdmModel implements SyncDBDataSource { filter: { ...query, _id: data._id }, update: data, }, - })), - session && { session } + })) ); - const updated = await this.db.find( - { - ...query, - _id: { $in: Array.from(existingIds) }, - } as UwaziFilterQuery>, - undefined, - session && { session } - ); + const updated = await this.db.find({ + ...query, + _id: { $in: Array.from(existingIds) }, + } as UwaziFilterQuery>); return { existingIds, existingData, updated }; } @@ -203,16 +184,14 @@ export class OdmModel implements SyncDBDataSource { async updateMany( conditions: UwaziFilterQuery>, doc: UwaziUpdateQuery, - options: UwaziUpdateOptions> = {} + options?: UwaziUpdateOptions> ) { - const session = this.getSession(); await this.logHelper.upsertLogMany(conditions); - return this.db._updateMany(conditions, doc, { ...options, ...(session && { session }) }); + return this.db._updateMany(conditions, doc, options); } async count(query: UwaziFilterQuery> = {}) { - const session = this.getSession(); - return this.db.countDocuments(query, session && { session }); + return this.db.countDocuments(query); } async get( @@ -220,29 +199,22 @@ export class OdmModel implements SyncDBDataSource { select: any = '', options: UwaziQueryOptions = {} ) { - const session = this.getSession(); - const results = await this.db.find(query, select, { - ...options, - ...(session && { session }), - lean: true, - }); + const results = await this.db.find(query, select, { lean: true, ...options }); return results as EnforcedWithId[]; } async getById(id: any, select?: any) { - const session = this.getSession(); - const results = await this.db.findById(id, select, { lean: true, ...(session && { session }) }); + const results = await this.db.findById(id, select); return results as EnforcedWithId | null; } async delete(condition: any) { - const session = this.getSession(); let cond = condition; if (mongoose.Types.ObjectId.isValid(condition)) { cond = { _id: condition }; } await this.logHelper.upsertLogMany(cond, true); - return this.db.deleteMany(cond, session && { session }); + return this.db.deleteMany(cond); } async facet(aggregations: any[], pipelines: any, project: any) { diff --git a/app/api/odm/modelBulkWriteStream.ts b/app/api/odm/modelBulkWriteStream.ts index 732ba3a0ad..bd9d4c560c 100644 --- a/app/api/odm/modelBulkWriteStream.ts +++ b/app/api/odm/modelBulkWriteStream.ts @@ -1,8 +1,8 @@ import { OdmModel } from 'api/odm/model'; -import { MultiTenantMongooseModel } from 'api/odm/MultiTenantMongooseModel'; +import { MongooseModelWrapper } from 'api/odm/MultiTenantMongooseModel'; class ModelBulkWriteStream { - db: MultiTenantMongooseModel; + db: MongooseModelWrapper; stackLimit: number; diff --git a/app/api/odm/sessionsContext.ts b/app/api/odm/sessionsContext.ts new file mode 100644 index 0000000000..fa74e4b845 --- /dev/null +++ b/app/api/odm/sessionsContext.ts @@ -0,0 +1,22 @@ +import { tenants } from 'api/tenants'; +import { appContext } from 'api/utils/AppContext'; +import { ClientSession } from 'mongoose'; +import { DB } from './DB'; + +export const dbSessionContext = { + getSession() { + return appContext.get('mongoSession') as ClientSession | undefined; + }, + + clearSession() { + appContext.set('mongoSession', undefined); + }, + + async startSession() { + const currentTenant = tenants.current(); + const connection = DB.connectionForDB(currentTenant.dbName); + const session = await connection.startSession(); + appContext.set('mongoSession', session); + return session; + }, +}; diff --git a/app/api/odm/specs/multiTenantMongooseModel.spec.ts b/app/api/odm/specs/multiTenantMongooseModel.spec.ts new file mode 100644 index 0000000000..5f1f395d71 --- /dev/null +++ b/app/api/odm/specs/multiTenantMongooseModel.spec.ts @@ -0,0 +1,450 @@ +/*eslint-disable max-statements*/ + +import { appContext } from 'api/utils/AppContext'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; +import { ClientSession } from 'mongodb'; +import { Schema } from 'mongoose'; +import { MongooseModelWrapper } from '../MultiTenantMongooseModel'; + +interface TestDoc { + title: string; + value?: number; +} + +describe('MultiTenantMongooseModel Session operations', () => { + let model: MongooseModelWrapper; + let session: ClientSession; + + beforeAll(async () => { + const schema = new Schema({ + title: String, + value: Number, + }); + model = new MongooseModelWrapper('sessiontest', schema); + }); + + beforeEach(async () => { + await testingEnvironment.setUp({}); + testingEnvironment.unsetFakeContext(); + session = await model.startSession(); + }); + + afterEach(async () => { + await session.endSession(); + }); + + afterAll(async () => { + await testingEnvironment.tearDown(); + }); + + describe('create()', () => { + it('should use session when creating documents', async () => { + await appContext.run(async () => { + session.startTransaction(); + appContext.set('mongoSession', session); + + const doc = await model.create([{ title: 'test1' }]); + + await session.abortTransaction(); + const notFound = await model.findById(doc[0]._id); + expect(notFound).toBeNull(); + + session.startTransaction(); + const saved = await model.create([{ title: 'test1' }]); + await session.commitTransaction(); + const found = await model.findById(saved[0]._id); + expect(found?.title).toBe('test1'); + }); + }); + }); + + describe('findOneAndUpdate()', () => { + it('should use session when updating documents', async () => { + await appContext.run(async () => { + session.startTransaction(); + appContext.set('mongoSession', session); + const doc = await model.create([{ title: 'test2' }]); + await session.commitTransaction(); + + session.startTransaction(); + await model.findOneAndUpdate( + { _id: doc[0]._id }, + { $set: { title: 'test2-updated' } }, + { new: true } + ); + + await session.abortTransaction(); + const notUpdated = await model.findById(doc[0]._id); + expect(notUpdated?.title).toBe('test2'); + + session.startTransaction(); + await model.findOneAndUpdate( + { _id: doc[0]._id }, + { $set: { title: 'test2-updated' } }, + { new: true } + ); + await session.commitTransaction(); + const afterCommit = await model.findById(doc[0]._id); + expect(afterCommit?.title).toBe('test2-updated'); + }); + }); + }); + + describe('find() and countDocuments()', () => { + it('should use session for queries', async () => { + await appContext.run(async () => { + session.startTransaction(); + appContext.set('mongoSession', session); + await model.create([ + { title: 'test3', value: 1 }, + { title: 'test4', value: 2 }, + ]); + await session.commitTransaction(); + + session.startTransaction(); + + await model.create([{ title: 'test5', value: 3 }]); + + const docsInTransaction = await model.find({ value: { $gt: 0 } }); + expect(docsInTransaction).toHaveLength(3); + const countInTransaction = await model.countDocuments({ value: { $gt: 0 } }); + expect(countInTransaction).toBe(3); + + await session.abortTransaction(); + + const docsAfterAbort = await model.find({ value: { $gt: 0 } }); + expect(docsAfterAbort).toHaveLength(2); + const countAfterAbort = await model.countDocuments({ value: { $gt: 0 } }); + expect(countAfterAbort).toBe(2); + }); + }); + }); + + describe('deleteMany()', () => { + it('should use session for deletions', async () => { + await appContext.run(async () => { + session.startTransaction(); + appContext.set('mongoSession', session); + const doc = await model.create([{ title: 'to-delete' }]); + await session.commitTransaction(); + + session.startTransaction(); + await model.deleteMany({ _id: doc[0]._id }); + + await session.abortTransaction(); + const stillExists = await model.findById(doc[0]._id); + expect(stillExists).not.toBeNull(); + + session.startTransaction(); + await model.deleteMany({ _id: doc[0]._id }); + await session.commitTransaction(); + const deleted = await model.findById(doc[0]._id); + expect(deleted).toBeNull(); + }); + }); + }); + + describe('_updateMany()', () => { + it('should use session for bulk updates', async () => { + await appContext.run(async () => { + session.startTransaction(); + appContext.set('mongoSession', session); + await model.create([ + { title: 'bulk1', value: 1 }, + { title: 'bulk2', value: 1 }, + ]); + await session.commitTransaction(); + + session.startTransaction(); + await model._updateMany({ value: 1 }, { $set: { value: 2 } }); + + await session.abortTransaction(); + const unchanged = await model.countDocuments({ value: 1 }); + expect(unchanged).toBe(2); + + session.startTransaction(); + await model._updateMany({ value: 1 }, { $set: { value: 2 } }); + await session.commitTransaction(); + const updated = await model.countDocuments({ value: 2 }); + expect(updated).toBe(2); + }); + }); + }); + + describe('findOne()', () => { + it('should use session for single document queries', async () => { + await appContext.run(async () => { + session.startTransaction(); + appContext.set('mongoSession', session); + const doc = await model.create([{ title: 'findOne', value: 1 }]); + await session.commitTransaction(); + + session.startTransaction(); + await model._updateMany({ _id: doc[0]._id }, { $set: { value: 2 } }); + + const foundInTransaction = await model.findOne({ _id: doc[0]._id }, {}); + expect(foundInTransaction?.value).toBe(2); + + await session.abortTransaction(); + + const foundAfterAbort = await model.findOne({ _id: doc[0]._id }, {}); + expect(foundAfterAbort?.value).toBe(1); + }); + }); + }); + + describe('replaceOne()', () => { + it('should use session for document replacement', async () => { + await appContext.run(async () => { + session.startTransaction(); + appContext.set('mongoSession', session); + const doc = await model.create([{ title: 'original', value: 1 }]); + await session.commitTransaction(); + + session.startTransaction(); + await model.replaceOne({ _id: doc[0]._id }, { title: 'replaced', value: 2 }); + + await session.abortTransaction(); + const notReplaced = await model.findOne({ _id: doc[0]._id }, {}); + expect(notReplaced?.title).toBe('original'); + expect(notReplaced?.value).toBe(1); + + session.startTransaction(); + await model.replaceOne({ _id: doc[0]._id }, { title: 'replaced', value: 2 }); + await session.commitTransaction(); + const replaced = await model.findOne({ _id: doc[0]._id }, {}); + expect(replaced?.title).toBe('replaced'); + expect(replaced?.value).toBe(2); + }); + }); + }); + + describe('distinct()', () => { + it('should use session for distinct queries', async () => { + await appContext.run(async () => { + session.startTransaction(); + appContext.set('mongoSession', session); + await model.create([ + { title: 'distinct1', value: 1 }, + { title: 'distinct2', value: 1 }, + { title: 'distinct3', value: 2 }, + ]); + await session.commitTransaction(); + + session.startTransaction(); + await model.create([{ title: 'distinct4', value: 3 }]); + + const valuesInTransaction = await model.distinct('value', {}); + expect(valuesInTransaction.sort()).toEqual([1, 2, 3]); + + await session.abortTransaction(); + + const valuesAfterAbort = await model.distinct('value', {}); + expect(valuesAfterAbort.sort()).toEqual([1, 2]); + }); + }); + }); + + describe('aggregate() and aggregateCursor()', () => { + it('should use session for aggregation queries', async () => { + await appContext.run(async () => { + session.startTransaction(); + appContext.set('mongoSession', session); + await model.create([ + { title: 'agg1', value: 1 }, + { title: 'agg2', value: 1 }, + { title: 'agg3', value: 2 }, + ]); + await session.commitTransaction(); + + session.startTransaction(); + await model.create([{ title: 'agg4', value: 2 }]); + + const pipeline = [{ $group: { _id: '$value', count: { $sum: 1 } } }, { $sort: { _id: 1 } }]; + + const aggInTransaction = await model.aggregate(pipeline); + expect(aggInTransaction).toEqual([ + { _id: 1, count: 2 }, + { _id: 2, count: 2 }, + ]); + + const cursorInTransaction = await model + .aggregateCursor<{ _id: number; count: number }>(pipeline) + .exec(); + expect(cursorInTransaction).toEqual([ + { _id: 1, count: 2 }, + { _id: 2, count: 2 }, + ]); + + await session.abortTransaction(); + + const aggAfterAbort = await model.aggregate(pipeline); + expect(aggAfterAbort).toEqual([ + { _id: 1, count: 2 }, + { _id: 2, count: 1 }, + ]); + + const cursorAfterAbort = await model + .aggregateCursor<{ _id: number; count: number }>(pipeline) + .exec(); + expect(cursorAfterAbort).toEqual([ + { _id: 1, count: 2 }, + { _id: 2, count: 1 }, + ]); + }); + }); + }); + + describe('facet()', () => { + it('should use session for faceted aggregation', async () => { + await appContext.run(async () => { + session.startTransaction(); + appContext.set('mongoSession', session); + await model.create([ + { title: 'facet1', value: 1 }, + { title: 'facet2', value: 1 }, + { title: 'facet3', value: 2 }, + ]); + await session.commitTransaction(); + + session.startTransaction(); + await model.create([{ title: 'facet4', value: 2 }]); + + const resultsInTransaction = await model.facet( + [{ $match: {} }], + { + byValue: [{ $group: { _id: '$value', count: { $sum: 1 } } }, { $sort: { _id: 1 } }], + }, + { byValue: 1 } + ); + + expect(resultsInTransaction[0].byValue).toEqual([ + { _id: 1, count: 2 }, + { _id: 2, count: 2 }, + ]); + + await session.abortTransaction(); + + const resultsAfterAbort = await model.facet( + [{ $match: {} }], + { + byValue: [{ $group: { _id: '$value', count: { $sum: 1 } } }, { $sort: { _id: 1 } }], + }, + { byValue: 1 } + ); + + expect(resultsAfterAbort[0].byValue).toEqual([ + { _id: 1, count: 2 }, + { _id: 2, count: 1 }, + ]); + }); + }); + }); + + describe('updateOne()', () => { + it('should use session for single document updates', async () => { + await appContext.run(async () => { + session.startTransaction(); + appContext.set('mongoSession', session); + const doc = await model.create([{ title: 'updateOne', value: 1 }]); + await session.commitTransaction(); + + session.startTransaction(); + await model.updateOne({ _id: doc[0]._id }, { $set: { value: 2 } }); + + await session.abortTransaction(); + const notUpdated = await model.findOne({ _id: doc[0]._id }, {}); + expect(notUpdated?.value).toBe(1); + + session.startTransaction(); + await model.updateOne({ _id: doc[0]._id }, { $set: { value: 2 } }); + await session.commitTransaction(); + const updated = await model.findOne({ _id: doc[0]._id }, {}); + expect(updated?.value).toBe(2); + }); + }); + }); + + describe('bulkWrite()', () => { + it('should use session for bulk operations', async () => { + await appContext.run(async () => { + session.startTransaction(); + appContext.set('mongoSession', session); + const docs = await model.create([ + { title: 'bulk1', value: 1 }, + { title: 'bulk2', value: 1 }, + ]); + await session.commitTransaction(); + + session.startTransaction(); + await model.bulkWrite([ + { + updateOne: { + filter: { _id: docs[0]._id }, + update: { $set: { value: 2 } }, + }, + }, + { + updateOne: { + filter: { _id: docs[1]._id }, + update: { $set: { value: 3 } }, + }, + }, + ]); + + await session.abortTransaction(); + const unchanged = await model.find({ _id: { $in: docs.map(d => d._id) } }); + expect(unchanged.map(d => d.value)).toEqual([1, 1]); + + session.startTransaction(); + await model.bulkWrite([ + { + updateOne: { + filter: { _id: docs[0]._id }, + update: { $set: { value: 2 } }, + }, + }, + { + updateOne: { + filter: { _id: docs[1]._id }, + update: { $set: { value: 3 } }, + }, + }, + ]); + await session.commitTransaction(); + const updated = await model.find({ _id: { $in: docs.map(d => d._id) } }); + expect(updated.map(d => d.value).sort()).toEqual([2, 3]); + }); + }); + }); + + describe('createMany()', () => { + it('should use session when creating multiple documents', async () => { + await appContext.run(async () => { + session.startTransaction(); + appContext.set('mongoSession', session); + + const docs = await model.createMany([ + { title: 'many1', value: 1 }, + { title: 'many2', value: 2 }, + ]); + + await session.abortTransaction(); + const notFound = await model.find({ _id: { $in: docs.map(d => d._id) } }); + expect(notFound).toHaveLength(0); + + session.startTransaction(); + await model.createMany([ + { title: 'many1', value: 1 }, + { title: 'many2', value: 2 }, + ]); + await session.commitTransaction(); + + const found = await model.find({ title: /^many/ }).sort({ value: 1 }); + expect(found).toHaveLength(2); + expect(found[0].value).toBe(1); + expect(found[1].value).toBe(2); + }); + }); + }); +}); diff --git a/app/api/sync/syncsModel.ts b/app/api/sync/syncsModel.ts index affbdeeadb..bcff4d2105 100644 --- a/app/api/sync/syncsModel.ts +++ b/app/api/sync/syncsModel.ts @@ -1,5 +1,5 @@ import mongoose from 'mongoose'; -import { MultiTenantMongooseModel } from 'api/odm/MultiTenantMongooseModel'; +import { MongooseModelWrapper } from 'api/odm/MultiTenantMongooseModel'; const syncSchema = new mongoose.Schema({ lastSyncs: { type: mongoose.Schema.Types.Mixed, default: {} }, @@ -10,4 +10,4 @@ export interface Sync extends mongoose.Document { name: string; } -export default new MultiTenantMongooseModel('syncs', syncSchema); +export default new MongooseModelWrapper('syncs', syncSchema); diff --git a/app/api/updatelogs/updatelogsModel.ts b/app/api/updatelogs/updatelogsModel.ts index 29c579b2ef..6a41208d58 100644 --- a/app/api/updatelogs/updatelogsModel.ts +++ b/app/api/updatelogs/updatelogsModel.ts @@ -1,5 +1,5 @@ import mongoose from 'mongoose'; -import { MultiTenantMongooseModel } from 'api/odm/MultiTenantMongooseModel'; +import { MongooseModelWrapper } from 'api/odm/MultiTenantMongooseModel'; import { ObjectIdSchema } from 'shared/types/commonTypes'; const updateLogSchema = new mongoose.Schema({ @@ -17,4 +17,4 @@ export interface UpdateLog extends mongoose.Document { deleted: boolean; } -export const model = new MultiTenantMongooseModel('updatelogs', updateLogSchema); +export const model = new MongooseModelWrapper('updatelogs', updateLogSchema); diff --git a/app/api/utils/specs/withTransaction.spec.ts b/app/api/utils/specs/withTransaction.spec.ts new file mode 100644 index 0000000000..9e170c5d0b --- /dev/null +++ b/app/api/utils/specs/withTransaction.spec.ts @@ -0,0 +1,148 @@ +import { instanceModel } from 'api/odm/model'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; +import { withTransaction } from 'api/utils/withTransaction'; +import { ClientSession } from 'mongodb'; +import { Schema } from 'mongoose'; +import { appContext } from '../AppContext'; + +interface TestDoc { + title: string; + value?: number; +} + +describe('withTransaction utility', () => { + let model: any; + + beforeAll(async () => { + const schema = new Schema({ + title: String, + value: Number, + }); + model = instanceModel('transactiontest', schema); + }); + + beforeEach(async () => { + await testingEnvironment.setUp({ transactiontests: [] }); + testingEnvironment.unsetFakeContext(); + }); + + afterAll(async () => { + await testingEnvironment.tearDown(); + }); + + it('should commit transaction when operation succeeds', async () => { + await appContext.run(async () => { + await withTransaction(async () => { + await model.save({ title: 'test1', value: 1 }); + }); + + const docs = await model.get({ title: 'test1' }); + expect(docs[0]).toBeTruthy(); + expect(docs[0].value).toBe(1); + }); + }); + + it('should rollback transaction when operation fails', async () => { + await appContext.run(async () => { + let errorThrown; + try { + await withTransaction(async () => { + await model.save({ title: 'test2', value: 2 }); + throw new Error('Intentional error'); + }); + } catch (error) { + errorThrown = error; + } + + expect(errorThrown.message).toBe('Intentional error'); + + const docs = await model.get({ title: 'test2' }); + expect(docs).toHaveLength(0); + }); + }); + + it('should handle nested operations in transaction', async () => { + await appContext.run(async () => { + await withTransaction(async () => { + await model.save({ title: 'doc1', value: 1 }); + await model.save({ title: 'doc2', value: 2 }); + await model.updateMany({ value: 1 }, { $set: { value: 3 } }); + }); + + const docs = await model.get({}, '', { sort: { title: 1 } }); + expect(docs).toHaveLength(2); + expect(docs[0].value).toBe(3); + expect(docs[1].value).toBe(2); + }); + }); + + it('should properly clean up session after transaction', async () => { + await appContext.run(async () => { + await withTransaction(async () => { + await model.save({ title: 'test3' }); + }); + + const session = appContext.get('mongoSession'); + expect(session).toBeUndefined(); + }); + }); + + it('should maintain session context during transaction', async () => { + await appContext.run(async () => { + await withTransaction(async () => { + const session = appContext.get('mongoSession') as ClientSession; + expect(session).toBeTruthy(); + expect(session.inTransaction()).toBe(true); + + await model.save({ title: 'test4' }); + expect(appContext.get('mongoSession')).toBe(session); + }); + }); + }); + + it('should handle concurrent transactions', async () => { + await appContext.run(async () => { + const transaction1 = withTransaction(async () => { + await model.save({ title: 'concurrent1', value: 1 }); + return 'tx1'; + }); + + const transaction2 = withTransaction(async () => { + await model.save({ title: 'concurrent2', value: 2 }); + return 'tx2'; + }); + + const [result1, result2] = await Promise.all([transaction1, transaction2]); + expect(result1).toBe('tx1'); + expect(result2).toBe('tx2'); + + const docs = await model.get({}, '', { sort: { title: 1 } }); + expect(docs).toHaveLength(2); + expect(docs[0].title).toBe('concurrent1'); + expect(docs[1].title).toBe('concurrent2'); + }); + }); + + it('should properly abort concurrent transactions', async () => { + await appContext.run(async () => { + await withTransaction(async () => { + await model.save({ title: 'concurrent', value: 2 }); + }); + + let error; + try { + await withTransaction(async () => { + await model.save({ title: 'abort1', value: 1 }); + throw new Error('Abort transaction 1'); + }); + } catch (e) { + error = e; + } + + expect(error?.message).toBe('Abort transaction 1'); + + const docs = await model.get({}); + expect(docs).toMatchObject([{ title: 'concurrent' }]); + }); + }); +}); diff --git a/app/api/utils/withTransaction.ts b/app/api/utils/withTransaction.ts new file mode 100644 index 0000000000..f1bcf7d2e9 --- /dev/null +++ b/app/api/utils/withTransaction.ts @@ -0,0 +1,35 @@ +import entities from 'api/entities/entities'; +import { dbSessionContext } from 'api/odm/sessionsContext'; +import { search } from 'api/search'; +import { appContext } from 'api/utils/AppContext'; + +// const indexEntities = search.indexEntities.bind(search); +const indexCalls = []; +// search.indexEntities = async (...args) => { +// if (appContext.get('mongoSession')) { +// indexCalls.push(args); +// return; +// } +// await indexEntities(...args); +// }; + +const withTransaction = async (operation: () => Promise): Promise => { + const session = await dbSessionContext.startSession(); + session.startTransaction(); + + try { + const result = await operation(); + + await session.commitTransaction(); + // await Promise.all(indexCalls.map(indexArgs => indexEntities(...indexArgs))); + return result; + } catch (e) { + await session.abortTransaction(); + throw e; + } finally { + dbSessionContext.clearSession(); + await session.endSession(); + } +}; + +export { withTransaction }; From 66d2ad1e627a9756c273fa8e7125aff9b4c21e3c Mon Sep 17 00:00:00 2001 From: Daneryl Date: Thu, 30 Jan 2025 10:35:15 +0100 Subject: [PATCH 06/15] withTransacion now supports manual abort --- app/api/entities/routes.js | 14 ++-- ... => mongooseModelWrapper_sessions.spec.ts} | 0 ...s => mongooseModelWrapper_tenants.spec.ts} | 0 app/api/utils/specs/withTransaction.spec.ts | 74 ++++++++++++++++++- app/api/utils/withTransaction.ts | 41 +++++----- 5 files changed, 100 insertions(+), 29 deletions(-) rename app/api/odm/specs/{multiTenantMongooseModel.spec.ts => mongooseModelWrapper_sessions.spec.ts} (100%) rename app/api/odm/specs/{model_multi_tenant.spec.ts => mongooseModelWrapper_tenants.spec.ts} (100%) diff --git a/app/api/entities/routes.js b/app/api/entities/routes.js index 1f70a719a8..4801e2642a 100644 --- a/app/api/entities/routes.js +++ b/app/api/entities/routes.js @@ -1,14 +1,14 @@ -import { search } from 'api/search'; -import { uploadMiddleware } from 'api/files'; -import { saveEntity } from 'api/entities/entitySavingManager'; import activitylogMiddleware from 'api/activitylog/activitylogMiddleware'; -import entities from './entities'; +import { saveEntity } from 'api/entities/entitySavingManager'; +import { uploadMiddleware } from 'api/files'; +import { search } from 'api/search'; +import { withTransaction } from 'api/utils/withTransaction'; +import needsAuthorization from '../auth/authMiddleware'; import templates from '../templates/templates'; import thesauri from '../thesauri/thesauri'; -import date from '../utils/date'; -import needsAuthorization from '../auth/authMiddleware'; import { parseQuery, validation } from '../utils'; -import { withTransaction } from 'api/utils/withTransaction'; +import date from '../utils/date'; +import entities from './entities'; async function updateThesauriWithEntity(entity, req) { const template = await templates.getById(entity.template); diff --git a/app/api/odm/specs/multiTenantMongooseModel.spec.ts b/app/api/odm/specs/mongooseModelWrapper_sessions.spec.ts similarity index 100% rename from app/api/odm/specs/multiTenantMongooseModel.spec.ts rename to app/api/odm/specs/mongooseModelWrapper_sessions.spec.ts diff --git a/app/api/odm/specs/model_multi_tenant.spec.ts b/app/api/odm/specs/mongooseModelWrapper_tenants.spec.ts similarity index 100% rename from app/api/odm/specs/model_multi_tenant.spec.ts rename to app/api/odm/specs/mongooseModelWrapper_tenants.spec.ts diff --git a/app/api/utils/specs/withTransaction.spec.ts b/app/api/utils/specs/withTransaction.spec.ts index 9e170c5d0b..2b5faaa57e 100644 --- a/app/api/utils/specs/withTransaction.spec.ts +++ b/app/api/utils/specs/withTransaction.spec.ts @@ -1,4 +1,5 @@ import { instanceModel } from 'api/odm/model'; +import { dbSessionContext } from 'api/odm/sessionsContext'; import { testingEnvironment } from 'api/utils/testingEnvironment'; import { withTransaction } from 'api/utils/withTransaction'; import { ClientSession } from 'mongodb'; @@ -82,7 +83,7 @@ describe('withTransaction utility', () => { await model.save({ title: 'test3' }); }); - const session = appContext.get('mongoSession'); + const session = dbSessionContext.getSession(); expect(session).toBeUndefined(); }); }); @@ -90,12 +91,12 @@ describe('withTransaction utility', () => { it('should maintain session context during transaction', async () => { await appContext.run(async () => { await withTransaction(async () => { - const session = appContext.get('mongoSession') as ClientSession; + const session = dbSessionContext.getSession(); expect(session).toBeTruthy(); - expect(session.inTransaction()).toBe(true); + expect(session?.inTransaction()).toBe(true); await model.save({ title: 'test4' }); - expect(appContext.get('mongoSession')).toBe(session); + expect(dbSessionContext.getSession()).toBe(session); }); }); }); @@ -145,4 +146,69 @@ describe('withTransaction utility', () => { expect(docs).toMatchObject([{ title: 'concurrent' }]); }); }); + + describe('manual abort', () => { + it('should allow manual abort without throwing error', async () => { + await appContext.run(async () => { + await withTransaction(async ({ abort }) => { + await model.save({ title: 'manual-abort', value: 1 }); + await abort(); + }); + + const session = dbSessionContext.getSession(); + expect(session).toBeUndefined(); + const docs = await model.get({ title: 'manual-abort' }); + expect(docs).toHaveLength(0); + }); + }); + + it('should clean up session after manual abort', async () => { + await appContext.run(async () => { + await withTransaction(async ({ abort }) => { + const sessionBeforeAbort = dbSessionContext.getSession(); + expect(sessionBeforeAbort).toBeTruthy(); + expect(sessionBeforeAbort?.inTransaction()).toBe(true); + + await model.save({ title: 'session-cleanup', value: 1 }); + await abort(); + }); + + expect(dbSessionContext.getSession()).toBeUndefined(); + const docs = await model.get({ title: 'session-cleanup' }); + expect(docs).toHaveLength(0); + }); + }); + + it('should abort transaction even if subsequent operations fail', async () => { + await appContext.run(async () => { + let error; + try { + await withTransaction(async ({ abort }) => { + await model.save({ title: 'abort-then-error', value: 1 }); + await abort(); + throw new Error('Subsequent error'); + }); + } catch (e) { + error = e; + } + + expect(error?.message).toBe('Subsequent error'); + const docs = await model.get({ title: 'abort-then-error' }); + expect(docs).toHaveLength(0); + }); + }); + + it('should end session after abort', async () => { + await appContext.run(async () => { + let sessionToTest: ClientSession | undefined; + await withTransaction(async ({ abort }) => { + sessionToTest = dbSessionContext.getSession(); + await model.save({ title: 'session-ended', value: 1 }); + await abort(); + }); + + expect(sessionToTest?.hasEnded).toBe(true); + }); + }); + }); }); diff --git a/app/api/utils/withTransaction.ts b/app/api/utils/withTransaction.ts index f1bcf7d2e9..d4235876f6 100644 --- a/app/api/utils/withTransaction.ts +++ b/app/api/utils/withTransaction.ts @@ -1,30 +1,35 @@ -import entities from 'api/entities/entities'; import { dbSessionContext } from 'api/odm/sessionsContext'; -import { search } from 'api/search'; -import { appContext } from 'api/utils/AppContext'; -// const indexEntities = search.indexEntities.bind(search); -const indexCalls = []; -// search.indexEntities = async (...args) => { -// if (appContext.get('mongoSession')) { -// indexCalls.push(args); -// return; -// } -// await indexEntities(...args); -// }; +interface TransactionOperation { + abort: () => Promise; +} -const withTransaction = async (operation: () => Promise): Promise => { +const withTransaction = async ( + operation: (context: TransactionOperation) => Promise +): Promise => { const session = await dbSessionContext.startSession(); session.startTransaction(); + let wasManuallyAborted = false; - try { - const result = await operation(); + const context: TransactionOperation = { + abort: async () => { + if (session.inTransaction()) { + await session.abortTransaction(); + } + wasManuallyAborted = true; + }, + }; - await session.commitTransaction(); - // await Promise.all(indexCalls.map(indexArgs => indexEntities(...indexArgs))); + try { + const result = await operation(context); + if (!wasManuallyAborted) { + await session.commitTransaction(); + } return result; } catch (e) { - await session.abortTransaction(); + if (!wasManuallyAborted) { + await session.abortTransaction(); + } throw e; } finally { dbSessionContext.clearSession(); From c058ffb38dc4bc0b5770e9ce6f6b719ee18d12f7 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Thu, 30 Jan 2025 14:11:18 +0100 Subject: [PATCH 07/15] fix types --- .../specs/entitySavingManager.spec.ts | 17 ++--- app/api/files/S3Storage.ts | 32 -------- app/api/files/files.ts | 7 +- app/api/files/specs/s3Storage.spec.ts | 75 ------------------- app/api/odm/MultiTenantMongooseModel.ts | 2 +- app/api/odm/model.ts | 8 +- app/api/sync/syncWorker.ts | 2 +- app/api/templates/templates.ts | 64 +++++++--------- app/api/utils/testingEnvironment.ts | 11 ++- 9 files changed, 46 insertions(+), 172 deletions(-) diff --git a/app/api/entities/specs/entitySavingManager.spec.ts b/app/api/entities/specs/entitySavingManager.spec.ts index cb89628ea9..03a962a865 100644 --- a/app/api/entities/specs/entitySavingManager.spec.ts +++ b/app/api/entities/specs/entitySavingManager.spec.ts @@ -1,15 +1,13 @@ /* eslint-disable max-lines */ import { saveEntity } from 'api/entities/entitySavingManager'; +import * as os from 'os'; import { attachmentsPath, fileExistsOnPath, files as filesAPI, uploadsPath } from 'api/files'; import * as processDocumentApi from 'api/files/processDocument'; import { search } from 'api/search'; import db from 'api/utils/testing_db'; import { advancedSort } from 'app/utils/advancedSort'; -import * as os from 'os'; // eslint-disable-next-line node/no-restricted-import import { writeFile } from 'fs/promises'; -import { appContext } from 'api/utils/AppContext'; -import { testingEnvironment } from 'api/utils/testingEnvironment'; import { ObjectId } from 'mongodb'; import path from 'path'; import { EntityWithFilesSchema } from 'shared/types/entityType'; @@ -30,6 +28,7 @@ import { template2Id, textFile, } from './entitySavingManagerFixtures'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; const validPdfString = ` %PDF-1.0 @@ -220,10 +219,10 @@ describe('entitySavingManager', () => { }; }); - // it('should continue saving if a file fails to save', async () => { - // const { entity: savedEntity } = await saveEntity(entity, { ...reqData }); - // expect(savedEntity.attachments).toEqual([textFile]); - // }); + it('should continue saving if a file fails to save', async () => { + const { entity: savedEntity } = await saveEntity(entity, { ...reqData }); + expect(savedEntity.attachments).toEqual([textFile]); + }); it('should return an error', async () => { const { errors } = await saveEntity(entity, { ...reqData }); @@ -593,10 +592,9 @@ describe('entitySavingManager', () => { _id: mainPdfFileId.toString(), originalname: 'Renamed main pdf.pdf', }); - processDocumentApi.processDocument.mockRestore(); }); - it('should throw an error if a document cannot be saved', async () => { + it('should return an error if an existing main document cannot be saved', async () => { jest.spyOn(filesAPI, 'save').mockRejectedValueOnce({ error: { name: 'failed' } }); const { errors } = await saveEntity( @@ -613,7 +611,6 @@ describe('entitySavingManager', () => { } ); expect(errors[0]).toBe('Could not save file/s: changed.pdf'); - filesAPI.save.mockRestore(); }); }); }); diff --git a/app/api/files/S3Storage.ts b/app/api/files/S3Storage.ts index 83d48addb5..a456ef3110 100644 --- a/app/api/files/S3Storage.ts +++ b/app/api/files/S3Storage.ts @@ -90,38 +90,6 @@ export class S3Storage { ) ); } - - async uploadMany(files: { key: string; body: Buffer }[]) { - const uploadedKeys: string[] = []; - - try { - const uploadPromises = files.map(async ({ key, body }) => { - const result = await this.client.send( - new PutObjectCommand({ - Bucket: S3Storage.bucketName(), - Key: key, - Body: body, - }) - ); - uploadedKeys.push(key); - return result; - }); - - return await catchS3Errors(async () => Promise.all(uploadPromises)); - } catch (error) { - // If any upload fails, delete all successfully uploaded files - if (uploadedKeys.length > 0) { - const deletePromises = uploadedKeys.map(async key => this.delete(key)); - await Promise.all(deletePromises).catch(deleteError => { - // Enhance the original error with cleanup failure information - throw new Error( - `Upload failed and cleanup was incomplete. Original error: ${error.message}. Cleanup error: ${deleteError.message}` - ); - }); - } - throw error; - } - } } export { S3TimeoutError }; diff --git a/app/api/files/files.ts b/app/api/files/files.ts index 8907482647..4e43956a09 100644 --- a/app/api/files/files.ts +++ b/app/api/files/files.ts @@ -27,11 +27,11 @@ const deduceMimeType = (_file: FileType) => { }; export const files = { - async save(_file: FileType, index = true, session?: ClientSession) { + async save(_file: FileType, index = true) { const file = deduceMimeType(_file); const existingFile = file._id ? await filesModel.getById(file._id) : undefined; - const savedFile = await filesModel.save(await validateFile(file), undefined, session); + const savedFile = await filesModel.save(await validateFile(file), undefined); if (index) { await search.indexEntities({ sharedId: savedFile.entity }, '+fullText'); } @@ -53,8 +53,7 @@ export const files = { return savedFile; }, - get: (query: any, select?: any, options?: { session?: ClientSession }) => - filesModel.get(query, select, options), + get: filesModel.get.bind(filesModel), async delete(query: any = {}) { const hasFileName = (file: FileType): file is FileType & { filename: string } => diff --git a/app/api/files/specs/s3Storage.spec.ts b/app/api/files/specs/s3Storage.spec.ts index 6bc6423568..35c601fe97 100644 --- a/app/api/files/specs/s3Storage.spec.ts +++ b/app/api/files/specs/s3Storage.spec.ts @@ -1,4 +1,3 @@ -import { DeleteObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3'; import { S3Storage, S3TimeoutError } from '../S3Storage'; let s3Storage: S3Storage; @@ -12,30 +11,6 @@ class S3TimeoutClient { } } -class MockS3Client { - uploadedFiles: { key: string; body: Buffer }[] = []; - deletedFiles: string[] = []; - shouldFailOnKey?: string; - - async send(command: any) { - if (command instanceof PutObjectCommand) { - if (command.input.Key === this.shouldFailOnKey) { - throw new Error(`Failed to upload ${command.input.Key}`); - } - this.uploadedFiles.push({ - key: command.input.Key, - body: command.input.Body - }); - return {}; - } - if (command instanceof DeleteObjectCommand) { - this.deletedFiles.push(command.input.Key); - return {}; - } - return {}; - } -} - describe('s3Storage', () => { beforeAll(async () => { // @ts-ignore @@ -67,54 +42,4 @@ describe('s3Storage', () => { await expect(s3Storage.list()).rejects.toBeInstanceOf(S3TimeoutError); }); }); - - describe('uploadMany', () => { - let mockS3Client: MockS3Client; - - beforeEach(() => { - mockS3Client = new MockS3Client(); - // @ts-ignore - s3Storage = new S3Storage(mockS3Client); - }); - - it('should upload multiple files successfully', async () => { - const files = [ - { key: 'file1.txt', body: Buffer.from('content1') }, - { key: 'file2.txt', body: Buffer.from('content2') }, - ]; - - await s3Storage.uploadMany(files); - - expect(mockS3Client.uploadedFiles).toHaveLength(2); - expect(mockS3Client.uploadedFiles[0]).toEqual(files[0]); - expect(mockS3Client.uploadedFiles[1]).toEqual(files[1]); - expect(mockS3Client.deletedFiles).toHaveLength(0); - }); - - it('should cleanup uploaded files if one upload fails', async () => { - mockS3Client.shouldFailOnKey = 'file2.txt'; - const files = [ - { key: 'file1.txt', body: Buffer.from('content1') }, - { key: 'file2.txt', body: Buffer.from('content2') }, - ]; - - await expect(s3Storage.uploadMany(files)).rejects.toThrow('Failed to upload file2.txt'); - - expect(mockS3Client.uploadedFiles).toHaveLength(1); - expect(mockS3Client.uploadedFiles[0]).toEqual(files[0]); - expect(mockS3Client.deletedFiles).toHaveLength(1); - expect(mockS3Client.deletedFiles[0]).toBe('file1.txt'); - }); - - it('should throw S3TimeoutError on timeout', async () => { - // @ts-ignore - s3Storage = new S3Storage(new S3TimeoutClient()); - - await expect( - s3Storage.uploadMany([ - { key: 'file1.txt', body: Buffer.from('content1') } - ]) - ).rejects.toBeInstanceOf(S3TimeoutError); - }); - }); }); diff --git a/app/api/odm/MultiTenantMongooseModel.ts b/app/api/odm/MultiTenantMongooseModel.ts index 0562b2aecb..18db8e84af 100644 --- a/app/api/odm/MultiTenantMongooseModel.ts +++ b/app/api/odm/MultiTenantMongooseModel.ts @@ -41,7 +41,7 @@ export class MongooseModelWrapper { }); } - find(query: UwaziFilterQuery>, select = '', options: any = {}) { + find(query: UwaziFilterQuery>, select = '', options = {}) { const session = dbSessionContext.getSession(); return this.dbForCurrentTenant().find(query, select, { ...options, diff --git a/app/api/odm/model.ts b/app/api/odm/model.ts index 4d2a98e2d8..b8f1f970ca 100644 --- a/app/api/odm/model.ts +++ b/app/api/odm/model.ts @@ -130,7 +130,7 @@ export class OdmModel implements SyncDBDataSource { throw Error('A document was not updated!'); } - const saved = updated.concat(created); + const saved = created.concat(updated); await Promise.all(saved.map(async s => this.logHelper.upsertLogOne(s))); return saved.map(s => s.toObject>()); } @@ -152,11 +152,7 @@ export class OdmModel implements SyncDBDataSource { } }); const existingIds = new Set( - ( - await this.db.find({ _id: { $in: ids } } as UwaziFilterQuery>, '_id', { - lean: true, - }) - ).map(d => d._id.toString()) + (await this.db.find({ _id: { $in: ids } }, '_id', { lean: true })).map(d => d._id.toString()) ); const existingData = dataArray.filter(d => d._id && existingIds.has(d._id.toString())); diff --git a/app/api/sync/syncWorker.ts b/app/api/sync/syncWorker.ts index e459303d3f..f6ec5167af 100644 --- a/app/api/sync/syncWorker.ts +++ b/app/api/sync/syncWorker.ts @@ -15,7 +15,7 @@ const updateSyncs = async (name: string, collection: string, lastSync: number) = async function createSyncIfNotExists(config: SettingsSyncSchema) { const syncs = await syncsModel.find({ name: config.name }); if (syncs.length === 0) { - await syncsModel.create({ lastSyncs: {}, name: config.name }); + await syncsModel.create([{ lastSyncs: {}, name: config.name }]); } } diff --git a/app/api/templates/templates.ts b/app/api/templates/templates.ts index ea90777c57..81f88e2f85 100644 --- a/app/api/templates/templates.ts +++ b/app/api/templates/templates.ts @@ -139,7 +139,7 @@ const checkAndFillGeneratedIdProperties = async ( }; const _save = async (template: TemplateSchema, session?: ClientSession) => { - const newTemplate = await model.save(template, undefined, session); + const newTemplate = await model.save(template, undefined); await addTemplateTranslation(newTemplate); return newTemplate; }; @@ -176,11 +176,11 @@ export default { : _save(mappedTemplate); }, - async swapNamesValidation(template: TemplateSchema, session?: ClientSession) { + async swapNamesValidation(template: TemplateSchema) { if (!template._id) { return; } - const current = await this.getById(ensure(template._id), session); + const current = await this.getById(ensure(template._id)); const currentTemplate = ensure(current); currentTemplate.properties = currentTemplate.properties || []; @@ -194,16 +194,11 @@ export default { }); }, - async _update( - template: TemplateSchema, - language: string, - _reindex = true, - session?: ClientSession - ) { + async _update(template: TemplateSchema, language: string, _reindex = true) { const reindex = _reindex && !template.synced; const templateStructureChanges = await checkIfReindex(template); const currentTemplate = ensure>( - await this.getById(ensure(template._id), session) + await this.getById(ensure(template._id)) ); if (templateStructureChanges || currentTemplate.name !== template.name) { await updateTranslation(currentTemplate, template); @@ -214,14 +209,13 @@ export default { } const generatedIdAdded = await checkAndFillGeneratedIdProperties(currentTemplate, template); - const savedTemplate = await model.save(template, undefined, session); + const savedTemplate = await model.save(template, undefined); if (templateStructureChanges) { await v2.processNewRelationshipPropertiesOnUpdate(currentTemplate, savedTemplate); await entities.updateMetadataProperties(template, currentTemplate, language, { reindex, generatedIdAdded, - session, }); } @@ -268,15 +262,12 @@ export default { session?: ClientSession ): Promise { const nameSet = new Set(propertyNames); - const templates = await this.get( - { - $or: [ - { 'properties.name': { $in: propertyNames } }, - { 'commonProperties.name': { $in: propertyNames } }, - ], - }, - session - ); + const templates = await this.get({ + $or: [ + { 'properties.name': { $in: propertyNames } }, + { 'commonProperties.name': { $in: propertyNames } }, + ], + }); const allProperties = templates .map(template => [template.properties || [], template.commonProperties || []]) .flat() @@ -295,8 +286,8 @@ export default { }, async setAsDefault(_id: string, session?: ClientSession) { - const [templateToBeDefault] = await this.get({ _id }, session); - const [currentDefault] = await this.get({ _id: { $nin: [_id] }, default: true }, session); + const [templateToBeDefault] = await this.get({ _id }); + const [currentDefault] = await this.get({ _id: { $nin: [_id] }, default: true }); if (templateToBeDefault) { let saveCurrentDefault = Promise.resolve({}); @@ -306,21 +297,17 @@ export default { _id: currentDefault._id, default: false, }, - undefined, - session + undefined ); } - return Promise.all([ - model.save({ _id, default: true }, undefined, session), - saveCurrentDefault, - ]); + return Promise.all([model.save({ _id, default: true }, undefined), saveCurrentDefault]); } throw createError('Invalid ID'); }, - async getById(templateId: ObjectId | string, session?: ClientSession) { - return model.getById(templateId, undefined, { session }); + async getById(templateId: ObjectId | string) { + return model.getById(templateId, undefined); }, async removePropsWithNonexistentId(nonexistentId: string, session?: ClientSession) { @@ -339,15 +326,14 @@ export default { properties: (t.properties || []).filter(prop => prop.content !== nonexistentId), }, defaultLanguage, - false, - session + false ) ) ); }, - async delete(template: Partial, session?: ClientSession) { - const count = await this.countByTemplate(ensure(template._id), session); + async delete(template: Partial) { + const count = await this.countByTemplate(ensure(template._id)); if (count > 0) { return Promise.reject({ key: 'documents_using_template', value: count }); } @@ -356,8 +342,8 @@ export default { const _id = ensure(template._id); await translations.deleteContext(_id); - await this.removePropsWithNonexistentId(_id, session); - await model.delete(_id, { session }); + await this.removePropsWithNonexistentId(_id); + await model.delete(_id); await applicationEventsBus.emit(new TemplateDeletedEvent({ templateId: _id })); @@ -368,8 +354,8 @@ export default { return entities.countByTemplate(template, session); }, - async countByThesauri(thesauriId: string, session?: ClientSession) { - return model.count({ 'properties.content': thesauriId }, { session }); + async countByThesauri(thesauriId: string) { + return model.count({ 'properties.content': thesauriId }); }, async findUsingRelationTypeInProp(relationTypeId: string, session?: ClientSession) { diff --git a/app/api/utils/testingEnvironment.ts b/app/api/utils/testingEnvironment.ts index 4c8e8dc181..e678be0d38 100644 --- a/app/api/utils/testingEnvironment.ts +++ b/app/api/utils/testingEnvironment.ts @@ -6,6 +6,9 @@ import { UserInContextMockFactory } from 'api/utils/testingUserInContext'; import { setupTestUploadedPaths } from 'api/files'; import { UserSchema } from 'shared/types/userType'; +let appContextGetMock: jest.SpyInstance; +let appContextSetMock: jest.SpyInstance; + const testingEnvironment = { userInContextMockFactory: new UserInContextMockFactory(), @@ -29,18 +32,18 @@ const testingEnvironment = { setFakeContext() { const originalAppContextGet = appContext.get.bind(appContext); - jest.spyOn(appContext, 'get').mockImplementation((key: string) => { + appContextGetMock = jest.spyOn(appContext, 'get').mockImplementation((key: string) => { if (key === 'mongoSession') { return undefined; } return originalAppContextGet(key); }); - jest.spyOn(appContext, 'set').mockImplementation(() => {}); + appContextSetMock = jest.spyOn(appContext, 'set').mockImplementation(() => {}); }, unsetFakeContext() { - appContext.get.mockRestore(); - appContext.set.mockRestore(); + appContextGetMock.mockRestore(); + appContextSetMock.mockRestore(); }, async setFixtures(fixtures?: DBFixture) { From 477f9e9f96dd32d27555ed6e5fb8aab21322b2cf Mon Sep 17 00:00:00 2001 From: Daneryl Date: Fri, 31 Jan 2025 06:20:08 +0100 Subject: [PATCH 08/15] Fix eslint errors --- app/api/auth/specs/routes.spec.js | 10 +++---- app/api/csv/specs/csvLoaderThesauri.spec.ts | 9 +++--- app/api/csv/specs/csvLoaderZip.spec.ts | 1 - app/api/entities/entities.js | 30 +++++++++---------- app/api/entities/routes.js | 6 ++-- app/api/entities/validateEntity.ts | 1 - .../entities/validation/validateEntityData.ts | 2 -- app/api/files/files.ts | 1 - app/api/files/specs/jsRoutes.spec.js | 13 ++++---- app/api/i18n/specs/translations.spec.ts | 3 +- app/api/search/specs/routes.spec.ts | 1 - .../preserve/specs/preserveSync.spec.ts | 7 ++--- app/api/templates/templates.ts | 12 +++----- .../topicclassification/specs/routes.spec.ts | 1 - .../topicclassification/specs/sync.spec.ts | 3 +- .../specs/userGroupsMembers.spec.ts | 7 ++--- 16 files changed, 44 insertions(+), 63 deletions(-) diff --git a/app/api/auth/specs/routes.spec.js b/app/api/auth/specs/routes.spec.js index cca177eca0..e99d8354e6 100644 --- a/app/api/auth/specs/routes.spec.js +++ b/app/api/auth/specs/routes.spec.js @@ -1,18 +1,16 @@ -import express from 'express'; import { testingEnvironment } from 'api/utils/testingEnvironment'; import bodyParser from 'body-parser'; +import express from 'express'; import request from 'supertest'; -import db from 'api/utils/testing_db'; - import users from 'api/users/users'; -import svgCaptcha from 'svg-captcha'; import backend from 'fetch-mock'; +import svgCaptcha from 'svg-captcha'; +import instrumentRoutes from '../../utils/instrumentRoutes'; import { CaptchaModel } from '../CaptchaModel'; +import { comparePasswords } from '../encryptPassword'; import authRoutes from '../routes'; import fixtures from './fixtures.js'; -import instrumentRoutes from '../../utils/instrumentRoutes'; -import { comparePasswords } from '../encryptPassword'; describe('Auth Routes', () => { let routes; diff --git a/app/api/csv/specs/csvLoaderThesauri.spec.ts b/app/api/csv/specs/csvLoaderThesauri.spec.ts index 4ad494f150..322aec5940 100644 --- a/app/api/csv/specs/csvLoaderThesauri.spec.ts +++ b/app/api/csv/specs/csvLoaderThesauri.spec.ts @@ -1,13 +1,12 @@ -import db from 'api/utils/testing_db'; -import { testingEnvironment } from 'api/utils/testingEnvironment'; -import thesauri from 'api/thesauri'; import translations from 'api/i18n'; import settings from 'api/settings'; +import thesauri from 'api/thesauri'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; +import { IndexedContextValues } from 'api/i18n/translations'; +import { WithId } from 'api/odm'; import { ObjectId } from 'mongodb'; import { ThesaurusSchema } from 'shared/types/thesaurusType'; -import { WithId } from 'api/odm'; -import { IndexedContextValues } from 'api/i18n/translations'; import { CSVLoader } from '../csvLoader'; import { fixtures, thesauri1Id } from './fixtures'; import { mockCsvFileReadStream } from './helpers'; diff --git a/app/api/csv/specs/csvLoaderZip.spec.ts b/app/api/csv/specs/csvLoaderZip.spec.ts index 9c38a3e031..ae8d5d77f0 100644 --- a/app/api/csv/specs/csvLoaderZip.spec.ts +++ b/app/api/csv/specs/csvLoaderZip.spec.ts @@ -1,4 +1,3 @@ -import db from 'api/utils/testing_db'; import { testingEnvironment } from 'api/utils/testingEnvironment'; import { files } from 'api/files/files'; import { search } from 'api/search'; diff --git a/app/api/entities/entities.js b/app/api/entities/entities.js index 472bac65fc..89d45ec525 100644 --- a/app/api/entities/entities.js +++ b/app/api/entities/entities.js @@ -45,16 +45,16 @@ const FIELD_TYPES_TO_SYNC = [ propertyTypes.numeric, ]; -async function updateEntity(entity, _template, unrestricted = false, session) { - const docLanguages = await this.getAllLanguages(entity.sharedId, { session }); +async function updateEntity(entity, _template, unrestricted = false) { + const docLanguages = await this.getAllLanguages(entity.sharedId); if ( docLanguages[0].template && entity.template && docLanguages[0].template.toString() !== entity.template.toString() ) { await Promise.all([ - this.deleteRelatedEntityFromMetadata(docLanguages[0], session), - relationships.delete({ entity: entity.sharedId }, null, false, { session }), + this.deleteRelatedEntityFromMetadata(docLanguages[0]), + relationships.delete({ entity: entity.sharedId }, null, false), ]); } const template = _template || { properties: [] }; @@ -99,9 +99,9 @@ async function updateEntity(entity, _template, unrestricted = false, session) { if (template._id) { await denormalizeRelated(fullEntity, template, currentDoc); } - const saveResult = await saveFunc(toSave, undefined, session); + const saveResult = await saveFunc(toSave, undefined); - await updateNewRelationships(v2RelationshipsUpdates, session); + await updateNewRelationships(v2RelationshipsUpdates); return saveResult; } @@ -138,13 +138,13 @@ async function updateEntity(entity, _template, unrestricted = false, session) { await denormalizeRelated(toSave, template, d); } - return saveFunc(toSave, undefined, session); + return saveFunc(toSave, undefined); }) ); - await denormalizeAfterEntityUpdate(entity, session); + await denormalizeAfterEntityUpdate(entity); - const afterEntities = await model.get({ sharedId: entity.sharedId }, null, { session }); + const afterEntities = await model.get({ sharedId: entity.sharedId }); await applicationEventsBus.emit( new EntityUpdatedEvent({ before: docLanguages, @@ -156,7 +156,7 @@ async function updateEntity(entity, _template, unrestricted = false, session) { return result; } -async function createEntity(doc, [currentLanguage, languages], sharedId, docTemplate, session) { +async function createEntity(doc, [currentLanguage, languages], sharedId, docTemplate) { if (!docTemplate) docTemplate = await templates.getById(doc.template); const thesauriByKey = await templates.getRelatedThesauri(docTemplate); @@ -195,15 +195,15 @@ async function createEntity(doc, [currentLanguage, languages], sharedId, docTemp { thesauriByKey } ); - return model.save(langDoc, undefined, session); + return model.save(langDoc); }) ); - await updateNewRelationships(v2RelationshipsUpdates, session); + await updateNewRelationships(v2RelationshipsUpdates); - await Promise.all(result.map(r => denormalizeAfterEntityCreation(r, session))); + await Promise.all(result.map(r => denormalizeAfterEntityCreation(r))); - const createdEntities = await model.get({ sharedId }, null, { session }); + const createdEntities = await model.get({ sharedId }); await applicationEventsBus.emit( new EntityCreatedEvent({ entities: createdEntities, @@ -551,7 +551,7 @@ export default { validateWritePermissions(ids, entitiesToUpdate); await Promise.all( ids.map(async id => { - const entity = await entitiesToUpdate.find( + const entity = entitiesToUpdate.find( e => e.sharedId === id && e.language === params.language ); diff --git a/app/api/entities/routes.js b/app/api/entities/routes.js index 4801e2642a..0a8f35b14b 100644 --- a/app/api/entities/routes.js +++ b/app/api/entities/routes.js @@ -91,9 +91,9 @@ export default app => { }); const { entity, errors } = result; await updateThesauriWithEntity(entity, req); - // if (errors.length) { - // throw new Error('Errors during saveEntity', { cause: errors }); - // } + if (errors.length) { + console.log(errors); + } res.json(req.body.entity ? result : entity); }); } catch (e) { diff --git a/app/api/entities/validateEntity.ts b/app/api/entities/validateEntity.ts index bfd276f12b..6c6aec9b91 100644 --- a/app/api/entities/validateEntity.ts +++ b/app/api/entities/validateEntity.ts @@ -1,6 +1,5 @@ import { validateEntitySchema } from './validation/validateEntitySchema'; import { validateEntityData } from './validation/validateEntityData'; -import templates from 'api/templates'; export const validateEntity = async (entity: any) => { await validateEntitySchema(entity); diff --git a/app/api/entities/validation/validateEntityData.ts b/app/api/entities/validation/validateEntityData.ts index ac4f09ffa1..4e9bc5d939 100644 --- a/app/api/entities/validation/validateEntityData.ts +++ b/app/api/entities/validation/validateEntityData.ts @@ -8,8 +8,6 @@ import ValidationError from 'ajv/dist/runtime/validation_error'; import { validateMetadataField } from './validateMetadataField'; import { customErrorMessages, validators } from './metadataValidators'; -import { tenants } from 'api/tenants'; -import templates from 'api/templates'; const ajv = new Ajv({ allErrors: true }); ajv.addVocabulary(['tsType']); diff --git a/app/api/files/files.ts b/app/api/files/files.ts index 4e43956a09..f6174fcde2 100644 --- a/app/api/files/files.ts +++ b/app/api/files/files.ts @@ -5,7 +5,6 @@ import { DefaultLogger } from 'api/log.v2/infrastructure/StandardLogger'; import connections from 'api/relationships'; import { search } from 'api/search'; import { cleanupRecordsOfFiles } from 'api/services/ocr/ocrRecords'; -import { ClientSession } from 'mongodb'; import { validateFile } from 'shared/types/fileSchema'; import { FileType } from 'shared/types/fileType'; import { inspect } from 'util'; diff --git a/app/api/files/specs/jsRoutes.spec.js b/app/api/files/specs/jsRoutes.spec.js index 51a2222e74..0ed03f9174 100644 --- a/app/api/files/specs/jsRoutes.spec.js +++ b/app/api/files/specs/jsRoutes.spec.js @@ -1,20 +1,19 @@ /*eslint-disable max-lines*/ -import { testingEnvironment } from 'api/utils/testingEnvironment'; -import db from 'api/utils/testing_db'; import entities from 'api/entities'; -import { settingsModel } from 'api/settings/settingsModel'; import { search } from 'api/search'; -import request from 'supertest'; +import { settingsModel } from 'api/settings/settingsModel'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import express from 'express'; +import request from 'supertest'; import mailer from 'api/utils/mailer'; // eslint-disable-next-line node/no-restricted-import import fs from 'fs/promises'; -import { allowedPublicTemplate, fixtures, templateId } from './fixtures'; -import instrumentRoutes from '../../utils/instrumentRoutes'; -import uploadRoutes from '../jsRoutes.js'; import { legacyLogger } from '../../log'; +import instrumentRoutes from '../../utils/instrumentRoutes'; import { createDirIfNotExists, deleteFiles } from '../filesystem'; +import uploadRoutes from '../jsRoutes.js'; +import { allowedPublicTemplate, fixtures, templateId } from './fixtures'; const mockExport = jest.fn(); jest.mock('api/csv/csvExporter', () => diff --git a/app/api/i18n/specs/translations.spec.ts b/app/api/i18n/specs/translations.spec.ts index c5aa500e85..a6b1872ad1 100644 --- a/app/api/i18n/specs/translations.spec.ts +++ b/app/api/i18n/specs/translations.spec.ts @@ -1,4 +1,3 @@ -import db from 'api/utils/testing_db'; import { testingEnvironment } from 'api/utils/testingEnvironment'; import entities from 'api/entities'; @@ -9,10 +8,10 @@ import { ContextType } from 'shared/translationSchema'; // eslint-disable-next-line node/no-restricted-import import * as fs from 'fs'; import { UITranslationNotAvailable } from '../defaultTranslations'; +import { addLanguage } from '../routes'; import translations from '../translations'; import fixtures, { dictionaryId } from './fixtures'; import { sortByLocale } from './sortByLocale'; -import { addLanguage } from '../routes'; describe('translations', () => { beforeEach(async () => { diff --git a/app/api/search/specs/routes.spec.ts b/app/api/search/specs/routes.spec.ts index 5e94555a86..f13b3bb1e6 100644 --- a/app/api/search/specs/routes.spec.ts +++ b/app/api/search/specs/routes.spec.ts @@ -2,7 +2,6 @@ import { testingEnvironment } from 'api/utils/testingEnvironment'; import { Application } from 'express'; import request, { Response as SuperTestResponse } from 'supertest'; - import searchRoutes from 'api/search/routes'; import { setUpApp } from 'api/utils/testingRoutes'; diff --git a/app/api/services/preserve/specs/preserveSync.spec.ts b/app/api/services/preserve/specs/preserveSync.spec.ts index 90ae5a3d56..39820de8f1 100644 --- a/app/api/services/preserve/specs/preserveSync.spec.ts +++ b/app/api/services/preserve/specs/preserveSync.spec.ts @@ -1,4 +1,3 @@ -import backend from 'fetch-mock'; import entities from 'api/entities'; import { generateFileName, testingUploadPaths } from 'api/files/filesystem'; import { storage } from 'api/files/storage'; @@ -8,6 +7,7 @@ import { search } from 'api/search'; import { tenants } from 'api/tenants'; import thesauri from 'api/thesauri'; import db from 'api/utils/testing_db'; +import backend from 'fetch-mock'; import path from 'path'; import qs from 'qs'; import { EntitySchema, EntityWithFilesSchema } from 'shared/types/entityType'; @@ -15,14 +15,13 @@ import { FileType } from 'shared/types/fileType'; import { URL } from 'url'; // eslint-disable-next-line node/no-restricted-import import fs from 'fs/promises'; +import { config } from 'api/config'; +import { Tenant } from 'api/tenants/tenantContext'; // eslint-disable-next-line node/no-restricted-import import { createReadStream } from 'fs'; -import { Tenant } from 'api/tenants/tenantContext'; -import { config } from 'api/config'; import { preserveSync } from '../preserveSync'; import { preserveSyncModel } from '../preserveSyncModel'; import { anotherTemplateId, fixtures, templateId, thesauri1Id, user } from './fixtures'; -import { testingEnvironment } from 'api/utils/testingEnvironment'; const mockVault = async (evidences: any[], token: string = '', isoDate = '') => { const host = 'http://preserve-testing.org'; diff --git a/app/api/templates/templates.ts b/app/api/templates/templates.ts index 81f88e2f85..3ac8008234 100644 --- a/app/api/templates/templates.ts +++ b/app/api/templates/templates.ts @@ -1,5 +1,4 @@ -import { ObjectId } from 'mongodb'; -import { ClientSession } from 'mongodb'; +import { ClientSession, ObjectId } from 'mongodb'; import entities from 'api/entities'; import { populateGeneratedIdByTemplate } from 'api/entities/generatedIdPropertyAutoFiller'; @@ -138,7 +137,7 @@ const checkAndFillGeneratedIdProperties = async ( return newGeneratedIdProps.length > 0; }; -const _save = async (template: TemplateSchema, session?: ClientSession) => { +const _save = async (template: TemplateSchema) => { const newTemplate = await model.save(template, undefined); await addTemplateTranslation(newTemplate); return newTemplate; @@ -257,10 +256,7 @@ export default { return property; }, - async getPropertiesByName( - propertyNames: string[], - session?: ClientSession - ): Promise { + async getPropertiesByName(propertyNames: string[]): Promise { const nameSet = new Set(propertyNames); const templates = await this.get({ $or: [ @@ -285,7 +281,7 @@ export default { return Array.from(Object.values(propertiesByName)); }, - async setAsDefault(_id: string, session?: ClientSession) { + async setAsDefault(_id: string) { const [templateToBeDefault] = await this.get({ _id }); const [currentDefault] = await this.get({ _id: { $nin: [_id] }, default: true }); diff --git a/app/api/topicclassification/specs/routes.spec.ts b/app/api/topicclassification/specs/routes.spec.ts index b648b24476..c6feb3bc68 100644 --- a/app/api/topicclassification/specs/routes.spec.ts +++ b/app/api/topicclassification/specs/routes.spec.ts @@ -1,7 +1,6 @@ import * as topicClassification from 'api/config/topicClassification'; import { testingEnvironment } from 'api/utils/testingEnvironment'; import { setUpApp } from 'api/utils/testingRoutes'; -import db from 'api/utils/testing_db'; import { NextFunction } from 'express'; import JSONRequest from 'shared/JSONRequest'; import request from 'supertest'; diff --git a/app/api/topicclassification/specs/sync.spec.ts b/app/api/topicclassification/specs/sync.spec.ts index 27961cdeaa..73336caca7 100644 --- a/app/api/topicclassification/specs/sync.spec.ts +++ b/app/api/topicclassification/specs/sync.spec.ts @@ -1,8 +1,7 @@ import * as topicClassification from 'api/config/topicClassification'; -import { testingEnvironment } from 'api/utils/testingEnvironment'; import entities from 'api/entities'; import { search } from 'api/search'; -import db from 'api/utils/testing_db'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import JSONRequest from 'shared/JSONRequest'; import { provenanceTypes } from 'shared/provenanceTypes'; import { TaskProvider } from 'shared/tasks/tasks'; diff --git a/app/api/usergroups/specs/userGroupsMembers.spec.ts b/app/api/usergroups/specs/userGroupsMembers.spec.ts index 78ef8aeea0..731f6bd4df 100644 --- a/app/api/usergroups/specs/userGroupsMembers.spec.ts +++ b/app/api/usergroups/specs/userGroupsMembers.spec.ts @@ -1,12 +1,11 @@ -import { testingDB } from 'api/utils/testing_db'; -import { testingEnvironment } from 'api/utils/testingEnvironment'; +import userGroups from 'api/usergroups/userGroups'; import { getByMemberIdList, - updateUserMemberships, removeUsersFromAllGroups, + updateUserMemberships, } from 'api/usergroups/userGroupsMembers'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; import { UserRole } from 'shared/types/userSchema'; -import userGroups from 'api/usergroups/userGroups'; import { fixtures, group1Id, group2Id, user1Id, user2Id, user3Id } from './fixtures'; describe('userGroupsMembers', () => { From a6f4bd82bd27147ad2cade1c799724aba5efd3da Mon Sep 17 00:00:00 2001 From: Daneryl Date: Fri, 31 Jan 2025 11:29:18 +0100 Subject: [PATCH 09/15] include elasticsearch entities index into the transaction process --- app/api/entities/routes.js | 6 +- app/api/odm/sessionsContext.ts | 18 +++ app/api/utils/specs/withTransaction.spec.ts | 115 ++++++++++++++++++-- app/api/utils/testing_db.ts | 1 + app/api/utils/withTransaction.ts | 19 ++++ 5 files changed, 148 insertions(+), 11 deletions(-) diff --git a/app/api/entities/routes.js b/app/api/entities/routes.js index 0a8f35b14b..fb005cb472 100644 --- a/app/api/entities/routes.js +++ b/app/api/entities/routes.js @@ -5,7 +5,7 @@ import { search } from 'api/search'; import { withTransaction } from 'api/utils/withTransaction'; import needsAuthorization from '../auth/authMiddleware'; import templates from '../templates/templates'; -import thesauri from '../thesauri/thesauri'; +import { thesauri } from '../thesauri/thesauri'; import { parseQuery, validation } from '../utils'; import date from '../utils/date'; import entities from './entities'; @@ -81,7 +81,7 @@ export default app => { activitylogMiddleware, async (req, res, next) => { try { - await withTransaction(async () => { + await withTransaction(async ({ abort }) => { const entityToSave = req.body.entity ? JSON.parse(req.body.entity) : req.body; const result = await saveEntity(entityToSave, { user: req.user, @@ -92,7 +92,7 @@ export default app => { const { entity, errors } = result; await updateThesauriWithEntity(entity, req); if (errors.length) { - console.log(errors); + await abort(); } res.json(req.body.entity ? result : entity); }); diff --git a/app/api/odm/sessionsContext.ts b/app/api/odm/sessionsContext.ts index fa74e4b845..d3d2dbf427 100644 --- a/app/api/odm/sessionsContext.ts +++ b/app/api/odm/sessionsContext.ts @@ -8,10 +8,22 @@ export const dbSessionContext = { return appContext.get('mongoSession') as ClientSession | undefined; }, + getReindexOperations() { + return ( + (appContext.get('reindexOperations') as [query?: any, select?: string, limit?: number][]) || + [] + ); + }, + clearSession() { appContext.set('mongoSession', undefined); }, + clearContext() { + appContext.set('mongoSession', undefined); + appContext.set('reindexOperations', undefined); + }, + async startSession() { const currentTenant = tenants.current(); const connection = DB.connectionForDB(currentTenant.dbName); @@ -19,4 +31,10 @@ export const dbSessionContext = { appContext.set('mongoSession', session); return session; }, + + registerESIndexOperation(args: [query?: any, select?: string, limit?: number]) { + const reindexOperations = dbSessionContext.getReindexOperations(); + reindexOperations.push(args); + appContext.set('reindexOperations', reindexOperations); + }, }; diff --git a/app/api/utils/specs/withTransaction.spec.ts b/app/api/utils/specs/withTransaction.spec.ts index 2b5faaa57e..6b82d33eca 100644 --- a/app/api/utils/specs/withTransaction.spec.ts +++ b/app/api/utils/specs/withTransaction.spec.ts @@ -1,16 +1,38 @@ -import { instanceModel } from 'api/odm/model'; -import { dbSessionContext } from 'api/odm/sessionsContext'; -import { testingEnvironment } from 'api/utils/testingEnvironment'; -import { withTransaction } from 'api/utils/withTransaction'; import { ClientSession } from 'mongodb'; import { Schema } from 'mongoose'; + +import entities from 'api/entities'; +import { instanceModel } from 'api/odm/model'; +import { dbSessionContext } from 'api/odm/sessionsContext'; + import { appContext } from '../AppContext'; +import { elasticTesting } from '../elastic_testing'; +import { getFixturesFactory } from '../fixturesFactory'; +import { testingEnvironment } from '../testingEnvironment'; +import { withTransaction } from '../withTransaction'; +import { EntitySchema } from 'shared/types/entityType'; + +const factory = getFixturesFactory(); interface TestDoc { title: string; value?: number; } +afterAll(async () => { + await testingEnvironment.tearDown(); +}); + +const saveEntity = async (entity: EntitySchema) => + entities.save(entity, { user: {}, language: 'es' }, { updateRelationships: false }); + +const createEntity = async (entity: EntitySchema) => + entities.save( + { ...entity, _id: undefined, sharedId: undefined }, + { user: {}, language: 'es' }, + { updateRelationships: false } + ); + describe('withTransaction utility', () => { let model: any; @@ -27,10 +49,6 @@ describe('withTransaction utility', () => { testingEnvironment.unsetFakeContext(); }); - afterAll(async () => { - await testingEnvironment.tearDown(); - }); - it('should commit transaction when operation succeeds', async () => { await appContext.run(async () => { await withTransaction(async () => { @@ -212,3 +230,84 @@ describe('withTransaction utility', () => { }); }); }); + +describe('Entities elasticsearch index', () => { beforeEach(async () => { + await testingEnvironment.setUp( + { + transactiontests: [], + templates: [factory.template('template1')], + entities: [ + factory.entity('existing1', 'template1'), + factory.entity('existing2', 'template1'), + ], + settings: [{ languages: [{ label: 'English', key: 'en', default: true }] }], + }, + 'with_transaction_index' + ); + testingEnvironment.unsetFakeContext(); + }); + + it('should handle delayed reindexing after a successful transaction', async () => { + await appContext.run(async () => { + await withTransaction(async () => { + await entities.save( + { ...factory.entity('test1', 'template1'), _id: undefined, sharedId: undefined }, + { user: {}, language: 'es' }, + { updateRelationships: false } + ); + await entities.save( + { ...factory.entity('test2', 'template1'), _id: undefined, sharedId: undefined }, + { user: {}, language: 'es' }, + { updateRelationships: false } + ); + }); + + await elasticTesting.refresh(); + const indexedEntities = await elasticTesting.getIndexedEntities(); + expect(indexedEntities).toHaveLength(4); + expect(indexedEntities).toEqual( + expect.arrayContaining([ + expect.objectContaining({ title: 'test1' }), + expect.objectContaining({ title: 'test2' }), + expect.objectContaining({ title: 'existing1' }), + expect.objectContaining({ title: 'existing2' }), + ]) + ); + }); + }); + + it('should not index changes to elasticsearch if transaction is aborted manually', async () => { + await appContext.run(async () => { + await withTransaction(async ({ abort }) => { + await saveEntity({ ...factory.entity('existing1', 'template1'), title: 'update1' }); + await saveEntity({ ...factory.entity('existing2', 'template1'), title: 'update2' }); + await createEntity(factory.entity('new', 'template1')); + await abort(); + }); + + const indexedEntities = await elasticTesting.getIndexedEntities(); + expect(indexedEntities).toMatchObject([{ title: 'existing1' }, { title: 'existing2' }]); + }); + }); + + it('should not index changes to elasticsearch if transaction is aborted by an error', async () => { + await appContext.run(async () => { + let error; + try { + await withTransaction(async () => { + await saveEntity({ ...factory.entity('existing1', 'template1'), title: 'update1' }); + await saveEntity({ ...factory.entity('existing2', 'template1'), title: 'update2' }); + await createEntity(factory.entity('new', 'template1')); + throw new Error('Testing error'); + }); + } catch (e) { + error = e; + } + + expect(error.message).toBe('Testing error'); + + const indexedEntities = await elasticTesting.getIndexedEntities(); + expect(indexedEntities).toMatchObject([{ title: 'existing1' }, { title: 'existing2' }]); + }); + }); +}); diff --git a/app/api/utils/testing_db.ts b/app/api/utils/testing_db.ts index 6281865912..f6a27484b0 100644 --- a/app/api/utils/testing_db.ts +++ b/app/api/utils/testing_db.ts @@ -126,6 +126,7 @@ const testingDB: { async tearDown() { await this.disconnect(); + connected = false; }, async disconnect() { diff --git a/app/api/utils/withTransaction.ts b/app/api/utils/withTransaction.ts index d4235876f6..58c9a41e0e 100644 --- a/app/api/utils/withTransaction.ts +++ b/app/api/utils/withTransaction.ts @@ -1,9 +1,26 @@ import { dbSessionContext } from 'api/odm/sessionsContext'; +import { search } from 'api/search'; interface TransactionOperation { abort: () => Promise; } +const originalIndexEntities = search.indexEntities.bind(search); +search.indexEntities = async (query, select, limit) => { + if (dbSessionContext.getSession()) { + return dbSessionContext.registerESIndexOperation([query, select, limit]); + } + return originalIndexEntities(query, select, limit); +}; + +const performDelayedReindexes = async () => { + await Promise.all( + dbSessionContext + .getReindexOperations() + .map(async reindexArgs => originalIndexEntities(...reindexArgs)) + ); +}; + const withTransaction = async ( operation: (context: TransactionOperation) => Promise ): Promise => { @@ -24,6 +41,8 @@ const withTransaction = async ( const result = await operation(context); if (!wasManuallyAborted) { await session.commitTransaction(); + dbSessionContext.clearSession(); + await performDelayedReindexes(); } return result; } catch (e) { From ae3052f70c21d510f8434ad63ec22ae0ed04e602 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Mon, 3 Feb 2025 06:46:03 +0100 Subject: [PATCH 10/15] test to ensure entities POST runs as a transaction --- app/api/entities/specs/routes.spec.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/app/api/entities/specs/routes.spec.ts b/app/api/entities/specs/routes.spec.ts index 7db1b1ecc1..16072c260f 100644 --- a/app/api/entities/specs/routes.spec.ts +++ b/app/api/entities/specs/routes.spec.ts @@ -16,6 +16,9 @@ import { AccessLevels, PermissionType } from 'shared/types/permissionSchema'; import { UserRole } from 'shared/types/userSchema'; import { ObjectId } from 'mongodb'; import fixtures, { permissions } from './fixtures'; +import { storage } from 'api/files'; +import entities from '../entities'; +import { appContext } from 'api/utils/AppContext'; jest.mock( '../../auth/authMiddleware.ts', @@ -244,5 +247,28 @@ describe('entities routes', () => { expect(legacyLogger.error).toHaveBeenCalledWith(expect.stringContaining('Deprecation')); }); + + it('should run saveEntity process as a transaction', async () => { + jest.restoreAllMocks(); + jest.spyOn(entities, 'getUnrestrictedWithDocuments').mockImplementationOnce(() => { + throw new Error('error at the end of the saveEntity'); + }); + new UserInContextMockFactory().mock(user); + const response: SuperTestResponse = await request(app) + .post('/api/entities') + .field('entity', JSON.stringify(entityToSave)) + .attach('documents[0]', path.join(__dirname, 'Hello, World.pdf'), 'Nombre en español') + .field('documents_originalname[0]', 'Nombre en español') + .expect(500); + + expect(response.body).toMatchObject({ + error: expect.any(String), + }); + + await appContext.run(async () => { + const myEntity = await entities.get({ title: 'my entity' }); + expect(myEntity.length).toBe(0); + }); + }); }); }); From 5eff286c5d0dad4f6c83145db5d5a3ef6a5c9cad Mon Sep 17 00:00:00 2001 From: Daneryl Date: Mon, 3 Feb 2025 09:27:14 +0100 Subject: [PATCH 11/15] fix migrator.spec --- app/api/migrations/specs/migrator.spec.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/app/api/migrations/specs/migrator.spec.js b/app/api/migrations/specs/migrator.spec.js index 46ca86e37b..d662391140 100644 --- a/app/api/migrations/specs/migrator.spec.js +++ b/app/api/migrations/specs/migrator.spec.js @@ -1,23 +1,26 @@ import { toHaveBeenCalledBefore } from 'jest-extended'; import path from 'path'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; + +import testingDB from '../../utils/testing_db'; +import migrationsModel from '../migrationsModel'; +import { migrator } from '../migrator'; import migration1 from './testMigrations/1-migrationTest'; import migration10 from './testMigrations/10-migrationTest'; import migration2 from './testMigrations/2-migrationTest'; -import migrationsModel from '../migrationsModel'; -import { migrator } from '../migrator'; -import testingDB from '../../utils/testing_db'; expect.extend({ toHaveBeenCalledBefore }); describe('migrator', () => { let connection; beforeAll(async () => { - connection = await testingDB.connect(); + await testingEnvironment.setUp({}); + connection = testingDB.mongodb; }); afterAll(async () => { - await testingDB.disconnect(); + await testingEnvironment.tearDown(); }); it('should have migrations directory configured', () => { From 5e46fed0ff526e145465ade5c644ea2bacb7b205 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Mon, 3 Feb 2025 09:31:37 +0100 Subject: [PATCH 12/15] fix esling errors --- app/api/entities/specs/routes.spec.ts | 9 ++++----- app/api/utils/specs/withTransaction.spec.ts | 5 +++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/api/entities/specs/routes.spec.ts b/app/api/entities/specs/routes.spec.ts index 16072c260f..97bdb31a40 100644 --- a/app/api/entities/specs/routes.spec.ts +++ b/app/api/entities/specs/routes.spec.ts @@ -1,5 +1,5 @@ -import { Application, NextFunction, Request, Response } from 'express'; import { testingEnvironment } from 'api/utils/testingEnvironment'; +import { Application, NextFunction, Request, Response } from 'express'; import request, { Response as SuperTestResponse } from 'supertest'; import { setUpApp } from 'api/utils/testingRoutes'; @@ -10,15 +10,14 @@ import routes from 'api/entities/routes'; import { legacyLogger } from 'api/log'; import templates from 'api/templates'; import thesauri from 'api/thesauri'; +import { appContext } from 'api/utils/AppContext'; import { UserInContextMockFactory } from 'api/utils/testingUserInContext'; +import { ObjectId } from 'mongodb'; import path from 'path'; import { AccessLevels, PermissionType } from 'shared/types/permissionSchema'; import { UserRole } from 'shared/types/userSchema'; -import { ObjectId } from 'mongodb'; -import fixtures, { permissions } from './fixtures'; -import { storage } from 'api/files'; import entities from '../entities'; -import { appContext } from 'api/utils/AppContext'; +import fixtures, { permissions } from './fixtures'; jest.mock( '../../auth/authMiddleware.ts', diff --git a/app/api/utils/specs/withTransaction.spec.ts b/app/api/utils/specs/withTransaction.spec.ts index 6b82d33eca..4094f64c4c 100644 --- a/app/api/utils/specs/withTransaction.spec.ts +++ b/app/api/utils/specs/withTransaction.spec.ts @@ -4,13 +4,13 @@ import { Schema } from 'mongoose'; import entities from 'api/entities'; import { instanceModel } from 'api/odm/model'; import { dbSessionContext } from 'api/odm/sessionsContext'; +import { EntitySchema } from 'shared/types/entityType'; import { appContext } from '../AppContext'; import { elasticTesting } from '../elastic_testing'; import { getFixturesFactory } from '../fixturesFactory'; import { testingEnvironment } from '../testingEnvironment'; import { withTransaction } from '../withTransaction'; -import { EntitySchema } from 'shared/types/entityType'; const factory = getFixturesFactory(); @@ -231,7 +231,8 @@ describe('withTransaction utility', () => { }); }); -describe('Entities elasticsearch index', () => { beforeEach(async () => { +describe('Entities elasticsearch index', () => { + beforeEach(async () => { await testingEnvironment.setUp( { transactiontests: [], From 6d311c25fed19dd7cd837ca200eefcad4fb6f4ee Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 4 Feb 2025 06:11:18 +0100 Subject: [PATCH 13/15] WIP, included storage.storeFile as part of the transaction --- app/api/files/specs/storage.spec.ts | 38 ++++ app/api/files/storage.ts | 33 ++- app/api/odm/sessionsContext.ts | 19 +- .../mongooseModelWrapper_sessions.spec.ts | 1 + app/api/utils/specs/withTransaction.spec.ts | 189 ++++++++++++------ app/api/utils/withTransaction.ts | 17 ++ 6 files changed, 228 insertions(+), 69 deletions(-) diff --git a/app/api/files/specs/storage.spec.ts b/app/api/files/specs/storage.spec.ts index ea99a0e254..5f19cac679 100644 --- a/app/api/files/specs/storage.spec.ts +++ b/app/api/files/specs/storage.spec.ts @@ -412,6 +412,44 @@ describe('storage', () => { }); }); + describe('storeMultipleFiles', () => { + beforeEach(async () => { + testingTenants.changeCurrentTenant({ featureFlags: { s3Storage: false } }); + }); + + afterEach(async () => { + await storage.removeFile('file1.txt', 'document'); + await storage.removeFile('file2.txt', 'document'); + await storage.removeFile('file3.txt', 'document'); + jest.restoreAllMocks(); + }); + + it('should rollback already uploaded files if an error occurs', async () => { + const files = [ + { filename: 'file1.txt', file: Readable.from(['content1']), type: 'document' }, + { filename: 'file2.txt', file: Readable.from(['content2']), type: 'document' }, + { filename: 'file3.txt', file: Readable.from(['content3']), type: 'document' }, + ]; + + const originalStoreFile = storage.storeFile.bind(storage); + jest.spyOn(storage, 'storeFile').mockImplementation(async (filename, file, type) => { + if (filename === 'file2.txt') { + throw new Error('Upload error'); + } + return originalStoreFile(filename, file, type); + }); + + await expect(storage.storeMultipleFiles(files)).rejects.toThrow('Upload error'); + + const file1Exists = await storage.fileExists('file1.txt', 'document'); + const file2Exists = await storage.fileExists('file2.txt', 'document'); + const file3Exists = await storage.fileExists('file3.txt', 'document'); + + expect(file1Exists).toBe(false); + expect(file2Exists).toBe(false); + expect(file3Exists).toBe(false); + }); + }); describe('createDirectory', () => { afterEach(async () => { try { diff --git a/app/api/files/storage.ts b/app/api/files/storage.ts index ba43d25b75..a9ad693464 100644 --- a/app/api/files/storage.ts +++ b/app/api/files/storage.ts @@ -1,15 +1,19 @@ import { NoSuchKey, S3Client } from '@aws-sdk/client-s3'; -import { config } from 'api/config'; -import { tenants } from 'api/tenants'; import { NodeHttpHandler } from '@smithy/node-http-handler'; +import { inspect } from 'util'; // eslint-disable-next-line node/no-restricted-import import { createReadStream, createWriteStream } from 'fs'; // eslint-disable-next-line node/no-restricted-import import { access, readdir } from 'fs/promises'; import path from 'path'; + +import { config } from 'api/config'; +import { legacyLogger } from 'api/log'; +import { tenants } from 'api/tenants'; import { FileType } from 'shared/types/fileType'; import { Readable } from 'stream'; import { pipeline } from 'stream/promises'; + import { FileNotFound } from './FileNotFound'; import { activityLogPath, @@ -173,4 +177,29 @@ export const storage = { return paths[type](filename); }, + + async storeMultipleFiles(files: { filename: string; file: Readable; type: FileTypes }[]) { + const uploadedFiles: { filename: string; type: FileTypes }[] = []; + + try { + await files.reduce(async (promise, { filename, file, type }) => { + await promise; + await this.storeFile(filename, file, type); + uploadedFiles.push({ filename, type }); + }, Promise.resolve()); + } catch (error) { + await Promise.all( + uploadedFiles.map(async ({ filename, type }) => { + try { + await this.removeFile(filename, type); + } catch (rollbackError) { + legacyLogger.error( + inspect(new Error('Failed to rollback file', { cause: rollbackError })) + ); + } + }) + ); + throw error; + } + }, }; diff --git a/app/api/odm/sessionsContext.ts b/app/api/odm/sessionsContext.ts index d3d2dbf427..b77c76df8e 100644 --- a/app/api/odm/sessionsContext.ts +++ b/app/api/odm/sessionsContext.ts @@ -1,6 +1,9 @@ +import { ClientSession } from 'mongoose'; +import { Readable } from 'stream'; + import { tenants } from 'api/tenants'; import { appContext } from 'api/utils/AppContext'; -import { ClientSession } from 'mongoose'; + import { DB } from './DB'; export const dbSessionContext = { @@ -15,6 +18,13 @@ export const dbSessionContext = { ); }, + getFileOperations() { + return ( + (appContext.get('fileOperations') as { filename: string; file: Readable; type: string }[]) || + [] + ); + }, + clearSession() { appContext.set('mongoSession', undefined); }, @@ -22,6 +32,7 @@ export const dbSessionContext = { clearContext() { appContext.set('mongoSession', undefined); appContext.set('reindexOperations', undefined); + appContext.set('fileOperations', undefined); }, async startSession() { @@ -37,4 +48,10 @@ export const dbSessionContext = { reindexOperations.push(args); appContext.set('reindexOperations', reindexOperations); }, + + registerFileOperation(args: { filename: string; file: Readable; type: string }) { + const fileOperations = dbSessionContext.getFileOperations(); + fileOperations.push(args); + appContext.set('fileOperations', fileOperations); + }, }; diff --git a/app/api/odm/specs/mongooseModelWrapper_sessions.spec.ts b/app/api/odm/specs/mongooseModelWrapper_sessions.spec.ts index 5f1f395d71..31f570bc1a 100644 --- a/app/api/odm/specs/mongooseModelWrapper_sessions.spec.ts +++ b/app/api/odm/specs/mongooseModelWrapper_sessions.spec.ts @@ -35,6 +35,7 @@ describe('MultiTenantMongooseModel Session operations', () => { afterAll(async () => { await testingEnvironment.tearDown(); + await session.endSession(); }); describe('create()', () => { diff --git a/app/api/utils/specs/withTransaction.spec.ts b/app/api/utils/specs/withTransaction.spec.ts index 4094f64c4c..0a6a481e25 100644 --- a/app/api/utils/specs/withTransaction.spec.ts +++ b/app/api/utils/specs/withTransaction.spec.ts @@ -1,5 +1,5 @@ import { ClientSession } from 'mongodb'; -import { Schema } from 'mongoose'; +import { model, Schema } from 'mongoose'; import entities from 'api/entities'; import { instanceModel } from 'api/odm/model'; @@ -11,6 +11,8 @@ import { elasticTesting } from '../elastic_testing'; import { getFixturesFactory } from '../fixturesFactory'; import { testingEnvironment } from '../testingEnvironment'; import { withTransaction } from '../withTransaction'; +import { storage } from 'api/files'; +import { Readable } from 'stream'; const factory = getFixturesFactory(); @@ -229,86 +231,141 @@ describe('withTransaction utility', () => { }); }); }); -}); + describe('Entities elasticsearch index', () => { + beforeEach(async () => { + await testingEnvironment.setUp( + { + transactiontests: [], + templates: [factory.template('template1')], + entities: [ + factory.entity('existing1', 'template1'), + factory.entity('existing2', 'template1'), + ], + settings: [{ languages: [{ label: 'English', key: 'en', default: true }] }], + }, + 'with_transaction_index' + ); + testingEnvironment.unsetFakeContext(); + }); -describe('Entities elasticsearch index', () => { - beforeEach(async () => { - await testingEnvironment.setUp( - { - transactiontests: [], - templates: [factory.template('template1')], - entities: [ - factory.entity('existing1', 'template1'), - factory.entity('existing2', 'template1'), - ], - settings: [{ languages: [{ label: 'English', key: 'en', default: true }] }], - }, - 'with_transaction_index' - ); - testingEnvironment.unsetFakeContext(); - }); + it('should handle delayed reindexing after a successful transaction', async () => { + await appContext.run(async () => { + await withTransaction(async () => { + await entities.save( + { ...factory.entity('test1', 'template1'), _id: undefined, sharedId: undefined }, + { user: {}, language: 'es' }, + { updateRelationships: false } + ); + await entities.save( + { ...factory.entity('test2', 'template1'), _id: undefined, sharedId: undefined }, + { user: {}, language: 'es' }, + { updateRelationships: false } + ); + }); - it('should handle delayed reindexing after a successful transaction', async () => { - await appContext.run(async () => { - await withTransaction(async () => { - await entities.save( - { ...factory.entity('test1', 'template1'), _id: undefined, sharedId: undefined }, - { user: {}, language: 'es' }, - { updateRelationships: false } - ); - await entities.save( - { ...factory.entity('test2', 'template1'), _id: undefined, sharedId: undefined }, - { user: {}, language: 'es' }, - { updateRelationships: false } + await elasticTesting.refresh(); + const indexedEntities = await elasticTesting.getIndexedEntities(); + expect(indexedEntities).toHaveLength(4); + expect(indexedEntities).toEqual( + expect.arrayContaining([ + expect.objectContaining({ title: 'test1' }), + expect.objectContaining({ title: 'test2' }), + expect.objectContaining({ title: 'existing1' }), + expect.objectContaining({ title: 'existing2' }), + ]) ); }); - - await elasticTesting.refresh(); - const indexedEntities = await elasticTesting.getIndexedEntities(); - expect(indexedEntities).toHaveLength(4); - expect(indexedEntities).toEqual( - expect.arrayContaining([ - expect.objectContaining({ title: 'test1' }), - expect.objectContaining({ title: 'test2' }), - expect.objectContaining({ title: 'existing1' }), - expect.objectContaining({ title: 'existing2' }), - ]) - ); }); - }); - it('should not index changes to elasticsearch if transaction is aborted manually', async () => { - await appContext.run(async () => { - await withTransaction(async ({ abort }) => { - await saveEntity({ ...factory.entity('existing1', 'template1'), title: 'update1' }); - await saveEntity({ ...factory.entity('existing2', 'template1'), title: 'update2' }); - await createEntity(factory.entity('new', 'template1')); - await abort(); + it('should not index changes to elasticsearch if transaction is aborted manually', async () => { + await appContext.run(async () => { + await withTransaction(async ({ abort }) => { + await saveEntity({ ...factory.entity('existing1', 'template1'), title: 'update1' }); + await saveEntity({ ...factory.entity('existing2', 'template1'), title: 'update2' }); + await createEntity(factory.entity('new', 'template1')); + await abort(); + }); + + const indexedEntities = await elasticTesting.getIndexedEntities(); + expect(indexedEntities).toMatchObject([{ title: 'existing1' }, { title: 'existing2' }]); }); + }); + + it('should not index changes to elasticsearch if transaction is aborted by an error', async () => { + await appContext.run(async () => { + let error; + try { + await withTransaction(async () => { + await saveEntity({ ...factory.entity('existing1', 'template1'), title: 'update1' }); + await saveEntity({ ...factory.entity('existing2', 'template1'), title: 'update2' }); + await createEntity(factory.entity('new', 'template1')); + throw new Error('Testing error'); + }); + } catch (e) { + error = e; + } + + expect(error.message).toBe('Testing error'); - const indexedEntities = await elasticTesting.getIndexedEntities(); - expect(indexedEntities).toMatchObject([{ title: 'existing1' }, { title: 'existing2' }]); + const indexedEntities = await elasticTesting.getIndexedEntities(); + expect(indexedEntities).toMatchObject([{ title: 'existing1' }, { title: 'existing2' }]); + }); }); }); - it('should not index changes to elasticsearch if transaction is aborted by an error', async () => { - await appContext.run(async () => { - let error; - try { + describe('storeFile', () => { + it('should store file after transaction is committed', async () => { + await appContext.run(async () => { await withTransaction(async () => { - await saveEntity({ ...factory.entity('existing1', 'template1'), title: 'update1' }); - await saveEntity({ ...factory.entity('existing2', 'template1'), title: 'update2' }); - await createEntity(factory.entity('new', 'template1')); - throw new Error('Testing error'); + await model.save({ title: 'test-file', value: 1 }); + await storage.storeFile('file_to_commit.txt', Readable.from(['content']), 'document'); }); - } catch (e) { - error = e; - } - expect(error.message).toBe('Testing error'); + const docs = await model.get({ title: 'test-file' }); + expect(docs[0]).toBeTruthy(); + expect(docs[0].value).toBe(1); + + expect(await storage.fileExists('file_to_commit.txt', 'document')).toBe(true); + }); + }); + + it('should rollback transaction when storeFile operation fails', async () => { + await appContext.run(async () => { + let errorThrown; + jest.spyOn(storage, 'storeMultipleFiles').mockImplementation(async () => { + throw new Error('Intentional storeFile error'); + }); + + try { + await withTransaction(async () => { + await model.save({ title: 'test-file-fail', value: 1 }); + await storage.storeFile('file_to_fail.txt', Readable.from(['content']), 'document'); + }); + } catch (error) { + errorThrown = error; + } + + expect(errorThrown.message).toBe('Intentional storeFile error'); + + const docs = await model.get({ title: 'test-file-fail' }); + expect(docs).toHaveLength(0); + }); + }); - const indexedEntities = await elasticTesting.getIndexedEntities(); - expect(indexedEntities).toMatchObject([{ title: 'existing1' }, { title: 'existing2' }]); + it('should rollback transaction when manually aborted after storeFile operation', async () => { + await appContext.run(async () => { + jest.spyOn(storage, 'storeMultipleFiles').mockImplementation(async () => { + throw new Error('Intentional storeFile error'); + }); + await withTransaction(async ({ abort }) => { + await model.save({ title: 'test-file-abort', value: 1 }); + await storage.storeFile('file_to_abort.txt', Readable.from(['content']), 'document'); + await abort(); + }); + + const docs = await model.get({ title: 'test-file-abort' }); + expect(docs).toHaveLength(0); + }); }); }); }); diff --git a/app/api/utils/withTransaction.ts b/app/api/utils/withTransaction.ts index 58c9a41e0e..2f5d1e1b27 100644 --- a/app/api/utils/withTransaction.ts +++ b/app/api/utils/withTransaction.ts @@ -1,5 +1,7 @@ +import { storage } from 'api/files/storage'; import { dbSessionContext } from 'api/odm/sessionsContext'; import { search } from 'api/search'; +import { appContext } from './AppContext'; interface TransactionOperation { abort: () => Promise; @@ -13,6 +15,19 @@ search.indexEntities = async (query, select, limit) => { return originalIndexEntities(query, select, limit); }; +const originalStoreFile = storage.storeFile.bind(storage); +storage.storeFile = async (filename, file, type) => { + if (dbSessionContext.getSession() && !appContext.get('fileOperationsNow')) { + return dbSessionContext.registerFileOperation({ filename, file, type }); + } + return originalStoreFile(filename, file, type); +}; + +const performDelayedFileStores = async () => { + appContext.set('fileOperationsNow', true); + await storage.storeMultipleFiles(dbSessionContext.getFileOperations()); +}; + const performDelayedReindexes = async () => { await Promise.all( dbSessionContext @@ -40,6 +55,7 @@ const withTransaction = async ( try { const result = await operation(context); if (!wasManuallyAborted) { + await performDelayedFileStores(); await session.commitTransaction(); dbSessionContext.clearSession(); await performDelayedReindexes(); @@ -51,6 +67,7 @@ const withTransaction = async ( } throw e; } finally { + appContext.set('fileOperationsNow', false); dbSessionContext.clearSession(); await session.endSession(); } From ebd8e58a648b6f279ceb0e09ff5838a810369627 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 4 Feb 2025 11:01:51 +0100 Subject: [PATCH 14/15] fix eslint and ts errors --- app/api/entities/entities.js | 5 ++--- app/api/entities/routes.js | 6 +++--- app/api/files/specs/storage.spec.ts | 6 +++--- app/api/files/storage.ts | 2 +- app/api/odm/sessionsContext.ts | 10 +++++++--- app/api/utils/specs/withTransaction.spec.ts | 6 +++--- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/app/api/entities/entities.js b/app/api/entities/entities.js index f7653f5560..1158119f18 100644 --- a/app/api/entities/entities.js +++ b/app/api/entities/entities.js @@ -332,12 +332,11 @@ const validateWritePermissions = (ids, entitiesToUpdate) => { } }; -const withDocuments = async (entities, documentsFullText, options = {}) => { +const withDocuments = async (entities, documentsFullText) => { const sharedIds = entities.map(entity => entity.sharedId); const allFiles = await files.get( { entity: { $in: sharedIds } }, - documentsFullText ? '+fullText ' : ' ', - options + documentsFullText ? '+fullText ' : ' ' ); const idFileMap = new Map(); allFiles.forEach(file => { diff --git a/app/api/entities/routes.js b/app/api/entities/routes.js index 8910aeea75..c716c86a2f 100644 --- a/app/api/entities/routes.js +++ b/app/api/entities/routes.js @@ -83,18 +83,18 @@ export default app => { try { const result = await withTransaction(async ({ abort }) => { const entityToSave = req.body.entity ? JSON.parse(req.body.entity) : req.body; - const result = await saveEntity(entityToSave, { + const saveResult = await saveEntity(entityToSave, { user: req.user, language: req.language, socketEmiter: req.emitToSessionSocket, files: req.files, }); - const { entity, errors } = result; + const { entity, errors } = saveResult; await updateThesauriWithEntity(entity, req); if (errors.length) { await abort(); } - return req.body.entity ? result : entity; + return req.body.entity ? saveResult : entity; }); res.json(result); } catch (e) { diff --git a/app/api/files/specs/storage.spec.ts b/app/api/files/specs/storage.spec.ts index 5f19cac679..bf7688a06d 100644 --- a/app/api/files/specs/storage.spec.ts +++ b/app/api/files/specs/storage.spec.ts @@ -426,9 +426,9 @@ describe('storage', () => { it('should rollback already uploaded files if an error occurs', async () => { const files = [ - { filename: 'file1.txt', file: Readable.from(['content1']), type: 'document' }, - { filename: 'file2.txt', file: Readable.from(['content2']), type: 'document' }, - { filename: 'file3.txt', file: Readable.from(['content3']), type: 'document' }, + { filename: 'file1.txt', file: Readable.from(['content1']), type: 'document' as const }, + { filename: 'file2.txt', file: Readable.from(['content2']), type: 'document' as const }, + { filename: 'file3.txt', file: Readable.from(['content3']), type: 'document' as const }, ]; const originalStoreFile = storage.storeFile.bind(storage); diff --git a/app/api/files/storage.ts b/app/api/files/storage.ts index a9ad693464..b282539345 100644 --- a/app/api/files/storage.ts +++ b/app/api/files/storage.ts @@ -25,7 +25,7 @@ import { } from './filesystem'; import { S3Storage } from './S3Storage'; -type FileTypes = NonNullable | 'activitylog' | 'segmentation'; +export type FileTypes = NonNullable | 'activitylog' | 'segmentation'; let s3Instance: S3Storage; const s3 = () => { diff --git a/app/api/odm/sessionsContext.ts b/app/api/odm/sessionsContext.ts index b77c76df8e..6d2dc60fef 100644 --- a/app/api/odm/sessionsContext.ts +++ b/app/api/odm/sessionsContext.ts @@ -5,6 +5,7 @@ import { tenants } from 'api/tenants'; import { appContext } from 'api/utils/AppContext'; import { DB } from './DB'; +import { FileTypes } from 'api/files/storage'; export const dbSessionContext = { getSession() { @@ -20,8 +21,11 @@ export const dbSessionContext = { getFileOperations() { return ( - (appContext.get('fileOperations') as { filename: string; file: Readable; type: string }[]) || - [] + (appContext.get('fileOperations') as { + filename: string; + file: Readable; + type: FileTypes; + }[]) || [] ); }, @@ -49,7 +53,7 @@ export const dbSessionContext = { appContext.set('reindexOperations', reindexOperations); }, - registerFileOperation(args: { filename: string; file: Readable; type: string }) { + registerFileOperation(args: { filename: string; file: Readable; type: FileTypes }) { const fileOperations = dbSessionContext.getFileOperations(); fileOperations.push(args); appContext.set('fileOperations', fileOperations); diff --git a/app/api/utils/specs/withTransaction.spec.ts b/app/api/utils/specs/withTransaction.spec.ts index 0a6a481e25..882cc14e84 100644 --- a/app/api/utils/specs/withTransaction.spec.ts +++ b/app/api/utils/specs/withTransaction.spec.ts @@ -1,18 +1,18 @@ import { ClientSession } from 'mongodb'; -import { model, Schema } from 'mongoose'; +import { Schema } from 'mongoose'; import entities from 'api/entities'; import { instanceModel } from 'api/odm/model'; import { dbSessionContext } from 'api/odm/sessionsContext'; import { EntitySchema } from 'shared/types/entityType'; +import { storage } from 'api/files'; +import { Readable } from 'stream'; import { appContext } from '../AppContext'; import { elasticTesting } from '../elastic_testing'; import { getFixturesFactory } from '../fixturesFactory'; import { testingEnvironment } from '../testingEnvironment'; import { withTransaction } from '../withTransaction'; -import { storage } from 'api/files'; -import { Readable } from 'stream'; const factory = getFixturesFactory(); From 03f6dc0f238aed9a4a10db8652ec2740ee1e82c8 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 4 Feb 2025 11:48:55 +0100 Subject: [PATCH 15/15] Added feature flag for the v1 withTransaction --- app/api/entities/specs/routes.spec.ts | 2 ++ app/api/tenants/tenantContext.ts | 1 + app/api/utils/specs/withTransaction.spec.ts | 35 +++++++++++++++++++++ app/api/utils/withTransaction.ts | 4 +++ 4 files changed, 42 insertions(+) diff --git a/app/api/entities/specs/routes.spec.ts b/app/api/entities/specs/routes.spec.ts index 97bdb31a40..955b7cd319 100644 --- a/app/api/entities/specs/routes.spec.ts +++ b/app/api/entities/specs/routes.spec.ts @@ -11,6 +11,7 @@ import { legacyLogger } from 'api/log'; import templates from 'api/templates'; import thesauri from 'api/thesauri'; import { appContext } from 'api/utils/AppContext'; +import { testingTenants } from 'api/utils/testingTenants'; import { UserInContextMockFactory } from 'api/utils/testingUserInContext'; import { ObjectId } from 'mongodb'; import path from 'path'; @@ -248,6 +249,7 @@ describe('entities routes', () => { }); it('should run saveEntity process as a transaction', async () => { + testingTenants.changeCurrentTenant({ featureFlags: { v1_transactions: true } }); jest.restoreAllMocks(); jest.spyOn(entities, 'getUnrestrictedWithDocuments').mockImplementationOnce(() => { throw new Error('error at the end of the saveEntity'); diff --git a/app/api/tenants/tenantContext.ts b/app/api/tenants/tenantContext.ts index 1b0ad037cb..619b8146e7 100644 --- a/app/api/tenants/tenantContext.ts +++ b/app/api/tenants/tenantContext.ts @@ -16,6 +16,7 @@ type Tenant = { featureFlags?: { s3Storage?: boolean; sync?: boolean; + v1_transactions?: boolean; }; globalMatomo?: { id: string; url: string }; ciMatomoActive?: boolean; diff --git a/app/api/utils/specs/withTransaction.spec.ts b/app/api/utils/specs/withTransaction.spec.ts index 882cc14e84..5b1022888d 100644 --- a/app/api/utils/specs/withTransaction.spec.ts +++ b/app/api/utils/specs/withTransaction.spec.ts @@ -13,6 +13,7 @@ import { elasticTesting } from '../elastic_testing'; import { getFixturesFactory } from '../fixturesFactory'; import { testingEnvironment } from '../testingEnvironment'; import { withTransaction } from '../withTransaction'; +import { testingTenants } from '../testingTenants'; const factory = getFixturesFactory(); @@ -48,6 +49,7 @@ describe('withTransaction utility', () => { beforeEach(async () => { await testingEnvironment.setUp({ transactiontests: [] }); + testingTenants.changeCurrentTenant({ featureFlags: { v1_transactions: true } }); testingEnvironment.unsetFakeContext(); }); @@ -167,6 +169,26 @@ describe('withTransaction utility', () => { }); }); + it('should do nothing when the feature flag is off when there is an error', async () => { + testingTenants.changeCurrentTenant({ featureFlags: { v1_transactions: false } }); + + await appContext.run(async () => { + let error: Error | undefined; + try { + await withTransaction(async () => { + await model.save({ title: 'test-flag-off', value: 1 }); + throw new Error('Testing error'); + }); + } catch (e) { + error = e; + } + expect(error?.message).toBe('Testing error'); + + const docs = await model.get({ title: 'test-flag-off' }); + expect(docs).toHaveLength(1); + }); + }); + describe('manual abort', () => { it('should allow manual abort without throwing error', async () => { await appContext.run(async () => { @@ -230,6 +252,19 @@ describe('withTransaction utility', () => { expect(sessionToTest?.hasEnded).toBe(true); }); }); + it('should do nothing when the feature flag is off', async () => { + testingTenants.changeCurrentTenant({ featureFlags: { v1_transactions: false } }); + + await appContext.run(async () => { + await withTransaction(async ({ abort }) => { + await model.save({ title: 'test-flag-off-abort' }); + await abort(); + }); + + const docs = await model.get({ title: 'test-flag-off-abort' }); + expect(docs).toHaveLength(1); + }); + }); }); describe('Entities elasticsearch index', () => { beforeEach(async () => { diff --git a/app/api/utils/withTransaction.ts b/app/api/utils/withTransaction.ts index 2f5d1e1b27..381b36094a 100644 --- a/app/api/utils/withTransaction.ts +++ b/app/api/utils/withTransaction.ts @@ -2,6 +2,7 @@ import { storage } from 'api/files/storage'; import { dbSessionContext } from 'api/odm/sessionsContext'; import { search } from 'api/search'; import { appContext } from './AppContext'; +import { tenants } from 'api/tenants'; interface TransactionOperation { abort: () => Promise; @@ -39,6 +40,9 @@ const performDelayedReindexes = async () => { const withTransaction = async ( operation: (context: TransactionOperation) => Promise ): Promise => { + if (!tenants.current().featureFlags?.v1_transactions) { + return operation({ abort: async () => {} }); + } const session = await dbSessionContext.startSession(); session.startTransaction(); let wasManuallyAborted = false;