From 505909e8d49747c65362d95464386f405d1b8f2a Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Wed, 19 Jun 2024 11:22:54 +0200 Subject: [PATCH 1/5] correct blank suggestions for extractor creation and update --- .../informationextraction/getFiles.ts | 7 ++ .../specs/ixextractors.spec.ts | 119 ++++++++++++++++++ app/api/suggestions/blankSuggestions.ts | 10 +- 3 files changed, 134 insertions(+), 2 deletions(-) diff --git a/app/api/services/informationextraction/getFiles.ts b/app/api/services/informationextraction/getFiles.ts index 9318260a7a..d0ef637b72 100644 --- a/app/api/services/informationextraction/getFiles.ts +++ b/app/api/services/informationextraction/getFiles.ts @@ -49,12 +49,18 @@ const propertiesWithoutExtractedMetadata: Set = new Set([ ...Array.from(selectProperties), propertyTypes.relationship, ]); +const multiValuedProperties: Set = new Set([ + propertyTypes.multiselect, + propertyTypes.relationship, +]); const propertyTypeIsSelectOrMultiSelect = (type: string) => selectProperties.has(type); const propertyTypeIsWithoutExtractedMetadata = (type: string) => propertiesWithoutExtractedMetadata.has(type); +const propertyTypeIsMultiValued = (type: string) => multiValuedProperties.has(type); + async function getFilesWithAggregations(files: (FileType & FileEnforcedNotUndefined)[]) { const filesNames = files.filter(x => x.filename).map(x => x.filename); @@ -231,5 +237,6 @@ export { getSegmentedFilesIds, propertyTypeIsSelectOrMultiSelect, propertyTypeIsWithoutExtractedMetadata, + propertyTypeIsMultiValued, }; export type { FileWithAggregation }; diff --git a/app/api/services/informationextraction/specs/ixextractors.spec.ts b/app/api/services/informationextraction/specs/ixextractors.spec.ts index c6e387ef1d..eb6313b7a6 100644 --- a/app/api/services/informationextraction/specs/ixextractors.spec.ts +++ b/app/api/services/informationextraction/specs/ixextractors.spec.ts @@ -30,11 +30,20 @@ const fixtures: DBFixture = { }, }, ], + relationtypes: [fixtureFactory.relationType('owns')], templates: [ fixtureFactory.template('personTemplate', [ fixtureFactory.property('age', 'numeric'), fixtureFactory.property('enemy', 'text'), fixtureFactory.property('location', 'geolocation'), + fixtureFactory.property('occupation', 'select', { content: fixtureFactory.idString('Jobs') }), + fixtureFactory.property('spoken_languages', 'multiselect', { + content: fixtureFactory.idString('Languages'), + }), + fixtureFactory.property('pets', 'relationship', { + content: fixtureFactory.idString('animalTemplate'), + relationType: fixtureFactory.idString('owns'), + }), ]), fixtureFactory.template('animalTemplate', [fixtureFactory.property('kind', 'text')]), fixtureFactory.template('plantTemplate', [fixtureFactory.property('kind', 'text')]), @@ -127,6 +136,10 @@ const fixtures: DBFixture = { { language: 'es' } ), ], + dictionaries: [ + fixtureFactory.thesauri('Jobs', ['Developer', 'Tester']), + fixtureFactory.thesauri('Languages', ['English', 'Spanish']), + ], }; const emptyState: IXSuggestionStateType = { @@ -149,6 +162,16 @@ const expectedStates: Record = { ...emptyState, withValue: true, }, + onlyContext: { + ...emptyState, + hasContext: true, + }, + defaultForMultiValued: { + ...emptyState, + hasContext: true, + withSuggestion: true, + match: true, + }, }; describe('ixextractors', () => { @@ -263,6 +286,102 @@ describe('ixextractors', () => { }, ], }, + { + case: 'selects', + name: 'occupation_test', + property: 'occupation', + templates: [fixtureFactory.id('personTemplate').toString()], + expectedSuggestions: [ + { + status: 'ready', + entityId: 'shared2', + language: 'en', + fileId: fixtureFactory.id('F1'), + propertyName: 'occupation', + error: '', + segment: '', + suggestedValue: '', + state: expectedStates.onlyContext, + entityTemplate: fixtureFactory.id('personTemplate').toString(), + }, + { + status: 'ready', + entityId: 'shared2', + language: 'es', + fileId: fixtureFactory.id('F2'), + propertyName: 'occupation', + error: '', + segment: '', + suggestedValue: '', + state: expectedStates.onlyContext, + entityTemplate: fixtureFactory.id('personTemplate').toString(), + }, + ], + }, + { + case: 'multiselects', + name: 'spoken_languages_test', + property: 'spoken_languages', + templates: [fixtureFactory.id('personTemplate').toString()], + expectedSuggestions: [ + { + status: 'ready', + entityId: 'shared2', + language: 'en', + fileId: fixtureFactory.id('F1'), + propertyName: 'spoken_languages', + error: '', + segment: '', + suggestedValue: [], + state: expectedStates.defaultForMultiValued, + entityTemplate: fixtureFactory.id('personTemplate').toString(), + }, + { + status: 'ready', + entityId: 'shared2', + language: 'es', + fileId: fixtureFactory.id('F2'), + propertyName: 'spoken_languages', + error: '', + segment: '', + suggestedValue: [], + state: expectedStates.defaultForMultiValued, + entityTemplate: fixtureFactory.id('personTemplate').toString(), + }, + ], + }, + { + case: 'relationships', + name: 'pets_test', + property: 'pets', + templates: [fixtureFactory.id('personTemplate').toString()], + expectedSuggestions: [ + { + status: 'ready', + entityId: 'shared2', + language: 'en', + fileId: fixtureFactory.id('F1'), + propertyName: 'pets', + error: '', + segment: '', + suggestedValue: [], + state: expectedStates.defaultForMultiValued, + entityTemplate: fixtureFactory.id('personTemplate').toString(), + }, + { + status: 'ready', + entityId: 'shared2', + language: 'es', + fileId: fixtureFactory.id('F2'), + propertyName: 'pets', + error: '', + segment: '', + suggestedValue: [], + state: expectedStates.defaultForMultiValued, + entityTemplate: fixtureFactory.id('personTemplate').toString(), + }, + ], + }, ])( 'should create empty suggestions for $case', async ({ name, property, templates, expectedSuggestions }) => { diff --git a/app/api/suggestions/blankSuggestions.ts b/app/api/suggestions/blankSuggestions.ts index 7963eb9380..50d069fe58 100644 --- a/app/api/suggestions/blankSuggestions.ts +++ b/app/api/suggestions/blankSuggestions.ts @@ -2,12 +2,14 @@ import entitiesModel from 'api/entities/entitiesModel'; import { files } from 'api/files'; import { EnforcedWithId } from 'api/odm'; import settings from 'api/settings'; +import { propertyTypeIsMultiValued } from 'api/services/informationextraction/getFiles'; import languages from 'shared/languages'; import { ObjectIdSchema } from 'shared/types/commonTypes'; import { IXExtractorType } from 'shared/types/extractorType'; import { FileType } from 'shared/types/fileType'; import { IXSuggestionType } from 'shared/types/suggestionType'; import { Suggestions } from './suggestions'; +import templates from 'api/templates'; const fetchEntitiesBatch = async (query: any, limit: number = 100) => entitiesModel.db.find(query).select('sharedId').limit(limit).sort({ _id: 1 }).lean(); @@ -43,6 +45,7 @@ export const getBlankSuggestion = ( file: EnforcedWithId, { _id: extractorId, property: propertyName }: { _id: ObjectIdSchema; property: string }, template: ObjectIdSchema, + propertyType: string, defaultLanguage: string ) => ({ language: file.language @@ -56,7 +59,7 @@ export const getBlankSuggestion = ( status: 'ready' as 'ready', error: '', segment: '', - suggestedValue: '', + suggestedValue: propertyTypeIsMultiValued(propertyType) ? [] : '', date: new Date().getTime(), }); @@ -67,6 +70,7 @@ export const createBlankSuggestionsForPartialExtractor = async ( ) => { const defaultLanguage = (await settings.getDefaultLanguage()).key; const extractorTemplates = new Set(extractor.templates.map(t => t.toString())); + const exampleProperty = await templates.getPropertyByName(extractor.property); const templatesPromises = selectedTemplates .filter(template => extractorTemplates.has(template.toString())) @@ -80,7 +84,9 @@ export const createBlankSuggestionsForPartialExtractor = async ( const suggestionsToSave: IXSuggestionType[] = fetchedFiles .filter(file => file.entity) - .map(file => getBlankSuggestion(file, extractor, template, defaultLanguage)); + .map(file => + getBlankSuggestion(file, extractor, template, exampleProperty.type, defaultLanguage) + ); await Suggestions.saveMultiple(suggestionsToSave); }); From 7eb755a04fdc3ab1149ab5b030e7d4d5a0046fa4 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Wed, 19 Jun 2024 11:52:16 +0200 Subject: [PATCH 2/5] correct default suggestions for eventListeners --- app/api/suggestions/eventListeners.ts | 14 +- .../suggestions/specs/eventListeners.spec.ts | 167 ++++++++++++++++++ 2 files changed, 180 insertions(+), 1 deletion(-) diff --git a/app/api/suggestions/eventListeners.ts b/app/api/suggestions/eventListeners.ts index c2c13c889e..4e08f1efab 100644 --- a/app/api/suggestions/eventListeners.ts +++ b/app/api/suggestions/eventListeners.ts @@ -10,6 +10,7 @@ import { FilesDeletedEvent } from 'api/files/events/FilesDeletedEvent'; import { FileUpdatedEvent } from 'api/files/events/FileUpdatedEvent'; import { Extractors } from 'api/services/informationextraction/ixextractors'; import settings from 'api/settings'; +import templates from 'api/templates'; import { TemplateDeletedEvent } from 'api/templates/events/TemplateDeletedEvent'; import { TemplateUpdatedEvent } from 'api/templates/events/TemplateUpdatedEvent'; import { objectIndex } from 'shared/data_utils/objectIndex'; @@ -62,11 +63,22 @@ const createDefaultSuggestionsForFiles = async ( ) => { const blankSuggestions: IXSuggestionType[] = []; + const template = await templates.getById(entityTemplateId); + const extractorPropertySet = new Set(extractorsInvolved.map(e => e.property)); + const involvedProperties = + template!.properties?.filter(p => extractorPropertySet.has(p.name)) || []; + const involvedPropertiesByName = objectIndex( + involvedProperties, + p => p.name, + p => p + ); + fileList.forEach(file => { extractorsInvolved.forEach(extractor => { + const propertyType = involvedPropertiesByName[extractor.property]?.type; if (file.entity) { blankSuggestions.push( - getBlankSuggestion(file, extractor, entityTemplateId, defaultLanguage) + getBlankSuggestion(file, extractor, entityTemplateId, propertyType, defaultLanguage) ); } }); diff --git a/app/api/suggestions/specs/eventListeners.spec.ts b/app/api/suggestions/specs/eventListeners.spec.ts index 7c2f244a8e..b26821f102 100644 --- a/app/api/suggestions/specs/eventListeners.spec.ts +++ b/app/api/suggestions/specs/eventListeners.spec.ts @@ -38,6 +38,9 @@ const fixtures: DBFixture = { fixturesFactory.property('not_extracted_property_2', propertyTypes.numeric), fixturesFactory.property('extracted_property_1', propertyTypes.text), fixturesFactory.property('extracted_property_2', propertyTypes.numeric), + fixturesFactory.property('select_property', propertyTypes.select), + fixturesFactory.property('multiselect_property', propertyTypes.multiselect), + fixturesFactory.property('relationship_property', propertyTypes.relationship), ]), fixturesFactory.template(otherExtractedTemplateName, [ fixturesFactory.property('extracted_property_1', propertyTypes.text), @@ -91,6 +94,9 @@ const fixtures: DBFixture = { fixturesFactory.ixExtractor('extractor5', 'extracted_property_2_2', [ otherExtractedTemplateName, ]), + fixturesFactory.ixExtractor('extractor6', 'select_property', [extractedTemplateName]), + fixturesFactory.ixExtractor('extractor7', 'multiselect_property', [extractedTemplateName]), + fixturesFactory.ixExtractor('extractor8', 'relationship_property', [extractedTemplateName]), ], ixsuggestions: [ fixturesFactory.ixSuggestion( @@ -117,6 +123,30 @@ const fixtures: DBFixture = { 'entfile', 'title' ), + fixturesFactory.ixSuggestion( + 'new_select_suggestion', + 'extractor6', + 'entity for new file', + extractedTemplateName, + 'entfile', + 'select_property' + ), + fixturesFactory.ixSuggestion( + 'new_multiselect_suggestion', + 'extractor7', + 'entity for new file', + extractedTemplateName, + 'entfile', + 'multiselect_property' + ), + fixturesFactory.ixSuggestion( + 'new_relationship_suggestion', + 'extractor8', + 'entity for new file', + extractedTemplateName, + 'entfile', + 'relationship_property' + ), ], }; @@ -277,6 +307,24 @@ describe(`On ${EntityUpdatedEvent.name}`, () => { propertyName: 'extracted_property_2', fileId: fixturesFactory.id('entfile'), }, + { + entityId: 'entity for new file', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + propertyName: 'multiselect_property', + fileId: fixturesFactory.id('entfile'), + }, + { + entityId: 'entity for new file', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + propertyName: 'relationship_property', + fileId: fixturesFactory.id('entfile'), + }, + { + entityId: 'entity for new file', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + propertyName: 'select_property', + fileId: fixturesFactory.id('entfile'), + }, { entityId: 'entity for new file', entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), @@ -320,6 +368,42 @@ describe(`On ${EntityUpdatedEvent.name}`, () => { propertyName: 'extracted_property_2', fileId: fixturesFactory.id('entfile2'), }, + { + entityId: 'entity for new file', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + propertyName: 'multiselect_property', + fileId: fixturesFactory.id('entfile'), + }, + { + entityId: 'entity with template not in config', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + propertyName: 'multiselect_property', + fileId: fixturesFactory.id('entfile2'), + }, + { + entityId: 'entity for new file', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + propertyName: 'relationship_property', + fileId: fixturesFactory.id('entfile'), + }, + { + entityId: 'entity with template not in config', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + propertyName: 'relationship_property', + fileId: fixturesFactory.id('entfile2'), + }, + { + entityId: 'entity for new file', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + propertyName: 'select_property', + fileId: fixturesFactory.id('entfile'), + }, + { + entityId: 'entity with template not in config', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + propertyName: 'select_property', + fileId: fixturesFactory.id('entfile2'), + }, { entityId: 'entity for new file', entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), @@ -502,6 +586,45 @@ describe(`On ${FileCreatedEvent.name}`, () => { status: 'ready', suggestedValue: '', }, + { + date: expect.any(Number), + entityId: 'entity for new file', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + error: '', + fileId: fixturesFactory.id('new file'), + language: 'en', + propertyName: 'select_property', + extractorId: fixturesFactory.id('extractor6'), + segment: '', + status: 'ready', + suggestedValue: '', + }, + { + date: expect.any(Number), + entityId: 'entity for new file', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + error: '', + fileId: fixturesFactory.id('new file'), + language: 'en', + propertyName: 'multiselect_property', + extractorId: fixturesFactory.id('extractor7'), + segment: '', + status: 'ready', + suggestedValue: [], + }, + { + date: expect.any(Number), + entityId: 'entity for new file', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + error: '', + fileId: fixturesFactory.id('new file'), + language: 'en', + propertyName: 'relationship_property', + extractorId: fixturesFactory.id('extractor8'), + segment: '', + status: 'ready', + suggestedValue: [], + }, ]); saveSpy.mockRestore(); @@ -731,6 +854,9 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { fixturesFactory.property('not_extracted_property_2', propertyTypes.numeric), fixturesFactory.property('extracted_property_1', propertyTypes.text), fixturesFactory.property('extracted_property_2', propertyTypes.numeric), + fixturesFactory.property('select_property', propertyTypes.select), + fixturesFactory.property('multiselect_property', propertyTypes.multiselect), + fixturesFactory.property('relationship_property', propertyTypes.relationship), ], }, after: { @@ -740,6 +866,8 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { fixturesFactory.property('not_extracted_property_1', propertyTypes.text), fixturesFactory.property('not_extracted_property_2', propertyTypes.numeric), fixturesFactory.property('extracted_property_1', propertyTypes.text), + fixturesFactory.property('select_property', propertyTypes.select), + fixturesFactory.property('multiselect_property', propertyTypes.multiselect), ], }, }) @@ -764,6 +892,9 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { fixturesFactory.ixExtractor('extractor5', 'extracted_property_2_2', [ otherExtractedTemplateName, ]), + fixturesFactory.ixExtractor('extractor6', 'select_property', [extractedTemplateName]), + fixturesFactory.ixExtractor('extractor7', 'multiselect_property', [extractedTemplateName]), + fixturesFactory.ixExtractor('extractor8', 'relationship_property', []), ]); const suggestions = await testingDB.mongodb?.collection('ixsuggestions').find({}).toArray(); @@ -781,6 +912,18 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { propertyName: 'title', extractorId: fixturesFactory.id('title_extractor'), }, + { + entityId: 'entity for new file', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + propertyName: 'select_property', + extractorId: fixturesFactory.id('extractor6'), + }, + { + entityId: 'entity for new file', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + propertyName: 'multiselect_property', + extractorId: fixturesFactory.id('extractor7'), + }, ]); }); @@ -795,6 +938,9 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { fixturesFactory.property('not_extracted_property_2', propertyTypes.numeric), fixturesFactory.property('extracted_property_1', propertyTypes.text), fixturesFactory.property('extracted_property_2', propertyTypes.numeric), + fixturesFactory.property('select_property', propertyTypes.select), + fixturesFactory.property('multiselect_property', propertyTypes.multiselect), + fixturesFactory.property('relationship_property', propertyTypes.relationship), ], }, after: { @@ -805,6 +951,9 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { fixturesFactory.property('not_extracted_property_2', propertyTypes.numeric), fixturesFactory.property('extracted_property_1', propertyTypes.text), fixturesFactory.property('extracted_property_2_renamed', propertyTypes.numeric), + fixturesFactory.property('select_property', propertyTypes.select), + fixturesFactory.property('multiselect_property_renamed', propertyTypes.multiselect), + fixturesFactory.property('relationship_property', propertyTypes.relationship), ], }, }) @@ -829,6 +978,9 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { fixturesFactory.ixExtractor('extractor5', 'extracted_property_2_2', [ otherExtractedTemplateName, ]), + fixturesFactory.ixExtractor('extractor6', 'select_property', [extractedTemplateName]), + fixturesFactory.ixExtractor('extractor7', 'multiselect_property', []), + fixturesFactory.ixExtractor('extractor8', 'relationship_property', [extractedTemplateName]), ]); const suggestions = await testingDB.mongodb?.collection('ixsuggestions').find({}).toArray(); @@ -846,6 +998,18 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { propertyName: 'title', extractorId: fixturesFactory.id('title_extractor'), }, + { + entityId: 'entity for new file', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + propertyName: 'select_property', + extractorId: fixturesFactory.id('extractor6'), + }, + { + entityId: 'entity for new file', + entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), + propertyName: 'relationship_property', + extractorId: fixturesFactory.id('extractor8'), + }, ]); }); }); @@ -873,6 +1037,9 @@ describe(`On ${TemplateDeletedEvent.name}`, () => { fixturesFactory.ixExtractor('extractor5', 'extracted_property_2_2', [ otherExtractedTemplateName, ]), + fixturesFactory.ixExtractor('extractor6', 'select_property', []), + fixturesFactory.ixExtractor('extractor7', 'multiselect_property', []), + fixturesFactory.ixExtractor('extractor8', 'relationship_property', []), ]); }); From 21ff31f8506b714530da87256c32800517dbd16a Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Wed, 19 Jun 2024 11:56:17 +0200 Subject: [PATCH 3/5] Revert "correct default suggestions for eventListeners" This reverts commit 7eb755a04fdc3ab1149ab5b030e7d4d5a0046fa4. --- app/api/suggestions/eventListeners.ts | 14 +- .../suggestions/specs/eventListeners.spec.ts | 167 ------------------ 2 files changed, 1 insertion(+), 180 deletions(-) diff --git a/app/api/suggestions/eventListeners.ts b/app/api/suggestions/eventListeners.ts index 4e08f1efab..c2c13c889e 100644 --- a/app/api/suggestions/eventListeners.ts +++ b/app/api/suggestions/eventListeners.ts @@ -10,7 +10,6 @@ import { FilesDeletedEvent } from 'api/files/events/FilesDeletedEvent'; import { FileUpdatedEvent } from 'api/files/events/FileUpdatedEvent'; import { Extractors } from 'api/services/informationextraction/ixextractors'; import settings from 'api/settings'; -import templates from 'api/templates'; import { TemplateDeletedEvent } from 'api/templates/events/TemplateDeletedEvent'; import { TemplateUpdatedEvent } from 'api/templates/events/TemplateUpdatedEvent'; import { objectIndex } from 'shared/data_utils/objectIndex'; @@ -63,22 +62,11 @@ const createDefaultSuggestionsForFiles = async ( ) => { const blankSuggestions: IXSuggestionType[] = []; - const template = await templates.getById(entityTemplateId); - const extractorPropertySet = new Set(extractorsInvolved.map(e => e.property)); - const involvedProperties = - template!.properties?.filter(p => extractorPropertySet.has(p.name)) || []; - const involvedPropertiesByName = objectIndex( - involvedProperties, - p => p.name, - p => p - ); - fileList.forEach(file => { extractorsInvolved.forEach(extractor => { - const propertyType = involvedPropertiesByName[extractor.property]?.type; if (file.entity) { blankSuggestions.push( - getBlankSuggestion(file, extractor, entityTemplateId, propertyType, defaultLanguage) + getBlankSuggestion(file, extractor, entityTemplateId, defaultLanguage) ); } }); diff --git a/app/api/suggestions/specs/eventListeners.spec.ts b/app/api/suggestions/specs/eventListeners.spec.ts index b26821f102..7c2f244a8e 100644 --- a/app/api/suggestions/specs/eventListeners.spec.ts +++ b/app/api/suggestions/specs/eventListeners.spec.ts @@ -38,9 +38,6 @@ const fixtures: DBFixture = { fixturesFactory.property('not_extracted_property_2', propertyTypes.numeric), fixturesFactory.property('extracted_property_1', propertyTypes.text), fixturesFactory.property('extracted_property_2', propertyTypes.numeric), - fixturesFactory.property('select_property', propertyTypes.select), - fixturesFactory.property('multiselect_property', propertyTypes.multiselect), - fixturesFactory.property('relationship_property', propertyTypes.relationship), ]), fixturesFactory.template(otherExtractedTemplateName, [ fixturesFactory.property('extracted_property_1', propertyTypes.text), @@ -94,9 +91,6 @@ const fixtures: DBFixture = { fixturesFactory.ixExtractor('extractor5', 'extracted_property_2_2', [ otherExtractedTemplateName, ]), - fixturesFactory.ixExtractor('extractor6', 'select_property', [extractedTemplateName]), - fixturesFactory.ixExtractor('extractor7', 'multiselect_property', [extractedTemplateName]), - fixturesFactory.ixExtractor('extractor8', 'relationship_property', [extractedTemplateName]), ], ixsuggestions: [ fixturesFactory.ixSuggestion( @@ -123,30 +117,6 @@ const fixtures: DBFixture = { 'entfile', 'title' ), - fixturesFactory.ixSuggestion( - 'new_select_suggestion', - 'extractor6', - 'entity for new file', - extractedTemplateName, - 'entfile', - 'select_property' - ), - fixturesFactory.ixSuggestion( - 'new_multiselect_suggestion', - 'extractor7', - 'entity for new file', - extractedTemplateName, - 'entfile', - 'multiselect_property' - ), - fixturesFactory.ixSuggestion( - 'new_relationship_suggestion', - 'extractor8', - 'entity for new file', - extractedTemplateName, - 'entfile', - 'relationship_property' - ), ], }; @@ -307,24 +277,6 @@ describe(`On ${EntityUpdatedEvent.name}`, () => { propertyName: 'extracted_property_2', fileId: fixturesFactory.id('entfile'), }, - { - entityId: 'entity for new file', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - propertyName: 'multiselect_property', - fileId: fixturesFactory.id('entfile'), - }, - { - entityId: 'entity for new file', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - propertyName: 'relationship_property', - fileId: fixturesFactory.id('entfile'), - }, - { - entityId: 'entity for new file', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - propertyName: 'select_property', - fileId: fixturesFactory.id('entfile'), - }, { entityId: 'entity for new file', entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), @@ -368,42 +320,6 @@ describe(`On ${EntityUpdatedEvent.name}`, () => { propertyName: 'extracted_property_2', fileId: fixturesFactory.id('entfile2'), }, - { - entityId: 'entity for new file', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - propertyName: 'multiselect_property', - fileId: fixturesFactory.id('entfile'), - }, - { - entityId: 'entity with template not in config', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - propertyName: 'multiselect_property', - fileId: fixturesFactory.id('entfile2'), - }, - { - entityId: 'entity for new file', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - propertyName: 'relationship_property', - fileId: fixturesFactory.id('entfile'), - }, - { - entityId: 'entity with template not in config', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - propertyName: 'relationship_property', - fileId: fixturesFactory.id('entfile2'), - }, - { - entityId: 'entity for new file', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - propertyName: 'select_property', - fileId: fixturesFactory.id('entfile'), - }, - { - entityId: 'entity with template not in config', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - propertyName: 'select_property', - fileId: fixturesFactory.id('entfile2'), - }, { entityId: 'entity for new file', entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), @@ -586,45 +502,6 @@ describe(`On ${FileCreatedEvent.name}`, () => { status: 'ready', suggestedValue: '', }, - { - date: expect.any(Number), - entityId: 'entity for new file', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - error: '', - fileId: fixturesFactory.id('new file'), - language: 'en', - propertyName: 'select_property', - extractorId: fixturesFactory.id('extractor6'), - segment: '', - status: 'ready', - suggestedValue: '', - }, - { - date: expect.any(Number), - entityId: 'entity for new file', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - error: '', - fileId: fixturesFactory.id('new file'), - language: 'en', - propertyName: 'multiselect_property', - extractorId: fixturesFactory.id('extractor7'), - segment: '', - status: 'ready', - suggestedValue: [], - }, - { - date: expect.any(Number), - entityId: 'entity for new file', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - error: '', - fileId: fixturesFactory.id('new file'), - language: 'en', - propertyName: 'relationship_property', - extractorId: fixturesFactory.id('extractor8'), - segment: '', - status: 'ready', - suggestedValue: [], - }, ]); saveSpy.mockRestore(); @@ -854,9 +731,6 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { fixturesFactory.property('not_extracted_property_2', propertyTypes.numeric), fixturesFactory.property('extracted_property_1', propertyTypes.text), fixturesFactory.property('extracted_property_2', propertyTypes.numeric), - fixturesFactory.property('select_property', propertyTypes.select), - fixturesFactory.property('multiselect_property', propertyTypes.multiselect), - fixturesFactory.property('relationship_property', propertyTypes.relationship), ], }, after: { @@ -866,8 +740,6 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { fixturesFactory.property('not_extracted_property_1', propertyTypes.text), fixturesFactory.property('not_extracted_property_2', propertyTypes.numeric), fixturesFactory.property('extracted_property_1', propertyTypes.text), - fixturesFactory.property('select_property', propertyTypes.select), - fixturesFactory.property('multiselect_property', propertyTypes.multiselect), ], }, }) @@ -892,9 +764,6 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { fixturesFactory.ixExtractor('extractor5', 'extracted_property_2_2', [ otherExtractedTemplateName, ]), - fixturesFactory.ixExtractor('extractor6', 'select_property', [extractedTemplateName]), - fixturesFactory.ixExtractor('extractor7', 'multiselect_property', [extractedTemplateName]), - fixturesFactory.ixExtractor('extractor8', 'relationship_property', []), ]); const suggestions = await testingDB.mongodb?.collection('ixsuggestions').find({}).toArray(); @@ -912,18 +781,6 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { propertyName: 'title', extractorId: fixturesFactory.id('title_extractor'), }, - { - entityId: 'entity for new file', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - propertyName: 'select_property', - extractorId: fixturesFactory.id('extractor6'), - }, - { - entityId: 'entity for new file', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - propertyName: 'multiselect_property', - extractorId: fixturesFactory.id('extractor7'), - }, ]); }); @@ -938,9 +795,6 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { fixturesFactory.property('not_extracted_property_2', propertyTypes.numeric), fixturesFactory.property('extracted_property_1', propertyTypes.text), fixturesFactory.property('extracted_property_2', propertyTypes.numeric), - fixturesFactory.property('select_property', propertyTypes.select), - fixturesFactory.property('multiselect_property', propertyTypes.multiselect), - fixturesFactory.property('relationship_property', propertyTypes.relationship), ], }, after: { @@ -951,9 +805,6 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { fixturesFactory.property('not_extracted_property_2', propertyTypes.numeric), fixturesFactory.property('extracted_property_1', propertyTypes.text), fixturesFactory.property('extracted_property_2_renamed', propertyTypes.numeric), - fixturesFactory.property('select_property', propertyTypes.select), - fixturesFactory.property('multiselect_property_renamed', propertyTypes.multiselect), - fixturesFactory.property('relationship_property', propertyTypes.relationship), ], }, }) @@ -978,9 +829,6 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { fixturesFactory.ixExtractor('extractor5', 'extracted_property_2_2', [ otherExtractedTemplateName, ]), - fixturesFactory.ixExtractor('extractor6', 'select_property', [extractedTemplateName]), - fixturesFactory.ixExtractor('extractor7', 'multiselect_property', []), - fixturesFactory.ixExtractor('extractor8', 'relationship_property', [extractedTemplateName]), ]); const suggestions = await testingDB.mongodb?.collection('ixsuggestions').find({}).toArray(); @@ -998,18 +846,6 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => { propertyName: 'title', extractorId: fixturesFactory.id('title_extractor'), }, - { - entityId: 'entity for new file', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - propertyName: 'select_property', - extractorId: fixturesFactory.id('extractor6'), - }, - { - entityId: 'entity for new file', - entityTemplate: fixturesFactory.id(extractedTemplateName).toString(), - propertyName: 'relationship_property', - extractorId: fixturesFactory.id('extractor8'), - }, ]); }); }); @@ -1037,9 +873,6 @@ describe(`On ${TemplateDeletedEvent.name}`, () => { fixturesFactory.ixExtractor('extractor5', 'extracted_property_2_2', [ otherExtractedTemplateName, ]), - fixturesFactory.ixExtractor('extractor6', 'select_property', []), - fixturesFactory.ixExtractor('extractor7', 'multiselect_property', []), - fixturesFactory.ixExtractor('extractor8', 'relationship_property', []), ]); }); From 1694551d06397896c25c2502523db04026afac62 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Wed, 19 Jun 2024 11:56:47 +0200 Subject: [PATCH 4/5] Revert "correct blank suggestions for extractor creation and update" This reverts commit 505909e8d49747c65362d95464386f405d1b8f2a. --- .../informationextraction/getFiles.ts | 7 -- .../specs/ixextractors.spec.ts | 119 ------------------ app/api/suggestions/blankSuggestions.ts | 10 +- 3 files changed, 2 insertions(+), 134 deletions(-) diff --git a/app/api/services/informationextraction/getFiles.ts b/app/api/services/informationextraction/getFiles.ts index d0ef637b72..9318260a7a 100644 --- a/app/api/services/informationextraction/getFiles.ts +++ b/app/api/services/informationextraction/getFiles.ts @@ -49,18 +49,12 @@ const propertiesWithoutExtractedMetadata: Set = new Set([ ...Array.from(selectProperties), propertyTypes.relationship, ]); -const multiValuedProperties: Set = new Set([ - propertyTypes.multiselect, - propertyTypes.relationship, -]); const propertyTypeIsSelectOrMultiSelect = (type: string) => selectProperties.has(type); const propertyTypeIsWithoutExtractedMetadata = (type: string) => propertiesWithoutExtractedMetadata.has(type); -const propertyTypeIsMultiValued = (type: string) => multiValuedProperties.has(type); - async function getFilesWithAggregations(files: (FileType & FileEnforcedNotUndefined)[]) { const filesNames = files.filter(x => x.filename).map(x => x.filename); @@ -237,6 +231,5 @@ export { getSegmentedFilesIds, propertyTypeIsSelectOrMultiSelect, propertyTypeIsWithoutExtractedMetadata, - propertyTypeIsMultiValued, }; export type { FileWithAggregation }; diff --git a/app/api/services/informationextraction/specs/ixextractors.spec.ts b/app/api/services/informationextraction/specs/ixextractors.spec.ts index eb6313b7a6..c6e387ef1d 100644 --- a/app/api/services/informationextraction/specs/ixextractors.spec.ts +++ b/app/api/services/informationextraction/specs/ixextractors.spec.ts @@ -30,20 +30,11 @@ const fixtures: DBFixture = { }, }, ], - relationtypes: [fixtureFactory.relationType('owns')], templates: [ fixtureFactory.template('personTemplate', [ fixtureFactory.property('age', 'numeric'), fixtureFactory.property('enemy', 'text'), fixtureFactory.property('location', 'geolocation'), - fixtureFactory.property('occupation', 'select', { content: fixtureFactory.idString('Jobs') }), - fixtureFactory.property('spoken_languages', 'multiselect', { - content: fixtureFactory.idString('Languages'), - }), - fixtureFactory.property('pets', 'relationship', { - content: fixtureFactory.idString('animalTemplate'), - relationType: fixtureFactory.idString('owns'), - }), ]), fixtureFactory.template('animalTemplate', [fixtureFactory.property('kind', 'text')]), fixtureFactory.template('plantTemplate', [fixtureFactory.property('kind', 'text')]), @@ -136,10 +127,6 @@ const fixtures: DBFixture = { { language: 'es' } ), ], - dictionaries: [ - fixtureFactory.thesauri('Jobs', ['Developer', 'Tester']), - fixtureFactory.thesauri('Languages', ['English', 'Spanish']), - ], }; const emptyState: IXSuggestionStateType = { @@ -162,16 +149,6 @@ const expectedStates: Record = { ...emptyState, withValue: true, }, - onlyContext: { - ...emptyState, - hasContext: true, - }, - defaultForMultiValued: { - ...emptyState, - hasContext: true, - withSuggestion: true, - match: true, - }, }; describe('ixextractors', () => { @@ -286,102 +263,6 @@ describe('ixextractors', () => { }, ], }, - { - case: 'selects', - name: 'occupation_test', - property: 'occupation', - templates: [fixtureFactory.id('personTemplate').toString()], - expectedSuggestions: [ - { - status: 'ready', - entityId: 'shared2', - language: 'en', - fileId: fixtureFactory.id('F1'), - propertyName: 'occupation', - error: '', - segment: '', - suggestedValue: '', - state: expectedStates.onlyContext, - entityTemplate: fixtureFactory.id('personTemplate').toString(), - }, - { - status: 'ready', - entityId: 'shared2', - language: 'es', - fileId: fixtureFactory.id('F2'), - propertyName: 'occupation', - error: '', - segment: '', - suggestedValue: '', - state: expectedStates.onlyContext, - entityTemplate: fixtureFactory.id('personTemplate').toString(), - }, - ], - }, - { - case: 'multiselects', - name: 'spoken_languages_test', - property: 'spoken_languages', - templates: [fixtureFactory.id('personTemplate').toString()], - expectedSuggestions: [ - { - status: 'ready', - entityId: 'shared2', - language: 'en', - fileId: fixtureFactory.id('F1'), - propertyName: 'spoken_languages', - error: '', - segment: '', - suggestedValue: [], - state: expectedStates.defaultForMultiValued, - entityTemplate: fixtureFactory.id('personTemplate').toString(), - }, - { - status: 'ready', - entityId: 'shared2', - language: 'es', - fileId: fixtureFactory.id('F2'), - propertyName: 'spoken_languages', - error: '', - segment: '', - suggestedValue: [], - state: expectedStates.defaultForMultiValued, - entityTemplate: fixtureFactory.id('personTemplate').toString(), - }, - ], - }, - { - case: 'relationships', - name: 'pets_test', - property: 'pets', - templates: [fixtureFactory.id('personTemplate').toString()], - expectedSuggestions: [ - { - status: 'ready', - entityId: 'shared2', - language: 'en', - fileId: fixtureFactory.id('F1'), - propertyName: 'pets', - error: '', - segment: '', - suggestedValue: [], - state: expectedStates.defaultForMultiValued, - entityTemplate: fixtureFactory.id('personTemplate').toString(), - }, - { - status: 'ready', - entityId: 'shared2', - language: 'es', - fileId: fixtureFactory.id('F2'), - propertyName: 'pets', - error: '', - segment: '', - suggestedValue: [], - state: expectedStates.defaultForMultiValued, - entityTemplate: fixtureFactory.id('personTemplate').toString(), - }, - ], - }, ])( 'should create empty suggestions for $case', async ({ name, property, templates, expectedSuggestions }) => { diff --git a/app/api/suggestions/blankSuggestions.ts b/app/api/suggestions/blankSuggestions.ts index 50d069fe58..7963eb9380 100644 --- a/app/api/suggestions/blankSuggestions.ts +++ b/app/api/suggestions/blankSuggestions.ts @@ -2,14 +2,12 @@ import entitiesModel from 'api/entities/entitiesModel'; import { files } from 'api/files'; import { EnforcedWithId } from 'api/odm'; import settings from 'api/settings'; -import { propertyTypeIsMultiValued } from 'api/services/informationextraction/getFiles'; import languages from 'shared/languages'; import { ObjectIdSchema } from 'shared/types/commonTypes'; import { IXExtractorType } from 'shared/types/extractorType'; import { FileType } from 'shared/types/fileType'; import { IXSuggestionType } from 'shared/types/suggestionType'; import { Suggestions } from './suggestions'; -import templates from 'api/templates'; const fetchEntitiesBatch = async (query: any, limit: number = 100) => entitiesModel.db.find(query).select('sharedId').limit(limit).sort({ _id: 1 }).lean(); @@ -45,7 +43,6 @@ export const getBlankSuggestion = ( file: EnforcedWithId, { _id: extractorId, property: propertyName }: { _id: ObjectIdSchema; property: string }, template: ObjectIdSchema, - propertyType: string, defaultLanguage: string ) => ({ language: file.language @@ -59,7 +56,7 @@ export const getBlankSuggestion = ( status: 'ready' as 'ready', error: '', segment: '', - suggestedValue: propertyTypeIsMultiValued(propertyType) ? [] : '', + suggestedValue: '', date: new Date().getTime(), }); @@ -70,7 +67,6 @@ export const createBlankSuggestionsForPartialExtractor = async ( ) => { const defaultLanguage = (await settings.getDefaultLanguage()).key; const extractorTemplates = new Set(extractor.templates.map(t => t.toString())); - const exampleProperty = await templates.getPropertyByName(extractor.property); const templatesPromises = selectedTemplates .filter(template => extractorTemplates.has(template.toString())) @@ -84,9 +80,7 @@ export const createBlankSuggestionsForPartialExtractor = async ( const suggestionsToSave: IXSuggestionType[] = fetchedFiles .filter(file => file.entity) - .map(file => - getBlankSuggestion(file, extractor, template, exampleProperty.type, defaultLanguage) - ); + .map(file => getBlankSuggestion(file, extractor, template, defaultLanguage)); await Suggestions.saveMultiple(suggestionsToSave); }); From c7b6dae3238a6ca68973725b172cc34ea23ac0a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20P=C3=B3lit?= Date: Wed, 19 Jun 2024 15:40:03 -0500 Subject: [PATCH 5/5] 6873 - Moved login inside changes array (#6886) * Moved login insde changes array * Fixed double await --------- Co-authored-by: Txau --- app/api/sync/specs/syncWorker.spec.ts | 8 ++-- app/api/sync/syncWorker.ts | 55 +++++++++++++++------------ 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/app/api/sync/specs/syncWorker.spec.ts b/app/api/sync/specs/syncWorker.spec.ts index e8447cb3ba..147d468604 100644 --- a/app/api/sync/specs/syncWorker.spec.ts +++ b/app/api/sync/specs/syncWorker.spec.ts @@ -179,7 +179,7 @@ describe('syncWorker', () => { await db.disconnect(); }); - it('should sync the configured templates and its defined properties', async () => { + it('should sync the whitelisted templates and properties', async () => { await runAllTenants(); await tenants.run(async () => { const syncedTemplates = await templates.get(); @@ -459,7 +459,7 @@ describe('syncWorker', () => { }, 'target1'); }); - describe('when a template that is configured has been deleted', () => { + describe('when a template that is whitelisted has been deleted', () => { it('should not throw an error', async () => { await tenants.run(async () => { await entitiesModel.delete({ template: template1 }); @@ -467,7 +467,7 @@ describe('syncWorker', () => { await templates.delete({ _id: template1 }); }, 'host1'); - await expect(syncWorker.runAllTenants()).resolves.not.toThrowError(); + await expect(syncWorker.runAllTenants()).resolves.not.toThrow(); }); }); @@ -575,7 +575,7 @@ describe('syncWorker', () => { fixtures.settings[0].sync[0].config.pages = []; await applyFixtures(fixtures, {}); - await expect(runAllTenants).rejects.toThrowError( + await expect(runAllTenants).rejects.toThrow( new Error('Invalid elements found in ordering - pages') ); diff --git a/app/api/sync/syncWorker.ts b/app/api/sync/syncWorker.ts index 540a330cb2..e459303d3f 100644 --- a/app/api/sync/syncWorker.ts +++ b/app/api/sync/syncWorker.ts @@ -26,7 +26,7 @@ class InvalidSyncConfig extends Error { } } -export interface SyncConfig { +interface SyncConfig { url: string; active?: boolean; username: string; @@ -71,39 +71,42 @@ export const syncWorker = { await previousSync; const syncConfig = validateConfig(config); if (syncConfig.active) { - const cookie = await this.login(syncConfig); - await this.syncronizeConfig(syncConfig, cookie); + await this.syncronizeConfig(syncConfig); } }, Promise.resolve()); }, - async syncronizeConfig(config: SyncConfig, cookie: string) { + async syncronizeConfig(config: SyncConfig) { await createSyncIfNotExists(config); const syncConfig = await createSyncConfig(config, config.name, this.UPDATE_LOG_TARGET_COUNT); - await ( - await syncConfig.lastChanges() - ).reduce(async (previousChange, change) => { - await previousChange; - const shouldSync: { skip?: boolean; data?: any } = await syncConfig.shouldSync(change); - if (shouldSync.skip) { - await synchronizer.syncDelete(change, config.url, cookie); - } + const lastChanges = await syncConfig.lastChanges(); - if (shouldSync.data) { - await synchronizer.syncData( - { - url: config.url, - change, - data: shouldSync.data, - cookie, - }, - 'post' - ); - } - await updateSyncs(config.name, change.namespace, change.timestamp); - }, Promise.resolve()); + if (lastChanges.length) { + const cookie = await this.login(config); + + await lastChanges.reduce(async (previousChange, change) => { + await previousChange; + const shouldSync: { skip?: boolean; data?: any } = await syncConfig.shouldSync(change); + if (shouldSync.skip) { + await synchronizer.syncDelete(change, config.url, cookie); + } + + if (shouldSync.data) { + await synchronizer.syncData( + { + url: config.url, + change, + data: shouldSync.data, + cookie, + }, + 'post' + ); + } + await updateSyncs(config.name, change.namespace, change.timestamp); + }, Promise.resolve()); + } }, async login({ url, username, password }: SyncConfig) { @@ -112,3 +115,5 @@ export const syncWorker = { return response.cookie || ''; }, }; + +export type { SyncConfig };