diff --git a/app/api/services/informationextraction/InformationExtraction.ts b/app/api/services/informationextraction/InformationExtraction.ts index 3a3e58a3ca..b7bd04f7fa 100644 --- a/app/api/services/informationextraction/InformationExtraction.ts +++ b/app/api/services/informationextraction/InformationExtraction.ts @@ -29,6 +29,7 @@ import { FileWithAggregation, getFilesForTraining, getFilesForSuggestions, + propertyTypeIsWithoutExtractedMetadata, propertyTypeIsSelectOrMultiSelect, } from 'api/services/informationextraction/getFiles'; import { Suggestions } from 'api/suggestions/suggestions'; @@ -95,6 +96,16 @@ type MaterialsData = | TextSelectionMaterialsData | ValuesSelectionMaterialsData; +async function fetchCandidates(property: PropertySchema) { + const defaultLanguageKey = (await settings.getDefaultLanguage()).key; + const query: { template?: ObjectId; language: string } = { + language: defaultLanguageKey, + }; + if (property.content !== '') query.template = new ObjectId(property.content); + const candidates = await entities.getUnrestricted(query, ['title', 'sharedId']); + return candidates; +} + class InformationExtraction { static SERVICE_NAME = 'information_extraction'; @@ -142,9 +153,9 @@ class InformationExtraction { let data: MaterialsData = { ..._data, language_iso }; - const isSelect = propertyTypeIsSelectOrMultiSelect(propertyType); + const noExtractedData = propertyTypeIsWithoutExtractedMetadata(propertyType); - if (!isSelect && propertyLabeledData) { + if (!noExtractedData && propertyLabeledData) { data = { ...data, label_text: propertyValue || propertyLabeledData?.selection?.text, @@ -155,7 +166,7 @@ class InformationExtraction { }; } - if (isSelect) { + if (noExtractedData) { if (!Array.isArray(propertyValue)) { throw new Error('Property value should be an array'); } @@ -184,7 +195,7 @@ class InformationExtraction { ); const { propertyValue, propertyType } = file; - const missingData = propertyTypeIsSelectOrMultiSelect(propertyType) + const missingData = propertyTypeIsWithoutExtractedMetadata(propertyType) ? !propertyValue : type === 'labeled_data' && !propertyLabeledData; @@ -383,15 +394,22 @@ class InformationExtraction { const params: TaskParameters = { id: extractorId.toString(), - multi_value: property.type === 'multiselect', + multi_value: property.type === 'multiselect' || property.type === 'relationship', }; - if (property.type === 'select' || property.type === 'multiselect') { + if (propertyTypeIsSelectOrMultiSelect(property.type)) { const thesauri = await dictionatiesModel.getById(property.content); params.options = thesauri?.values?.map(value => ({ label: value.label, id: value.id as string })) || []; } + if (property.type === 'relationship') { + const candidates = await fetchCandidates(property); + params.options = candidates.map(candidate => ({ + label: candidate.title || '', + id: candidate.sharedId || '', + })); + } await this.taskManager.startTask({ task: 'create_model', diff --git a/app/api/services/informationextraction/getFiles.ts b/app/api/services/informationextraction/getFiles.ts index 9c66ab0270..9318260a7a 100644 --- a/app/api/services/informationextraction/getFiles.ts +++ b/app/api/services/informationextraction/getFiles.ts @@ -45,9 +45,16 @@ type FileEnforcedNotUndefined = { }; const selectProperties: Set = new Set([propertyTypes.select, propertyTypes.multiselect]); +const propertiesWithoutExtractedMetadata: Set = new Set([ + ...Array.from(selectProperties), + propertyTypes.relationship, +]); const propertyTypeIsSelectOrMultiSelect = (type: string) => selectProperties.has(type); +const propertyTypeIsWithoutExtractedMetadata = (type: string) => + propertiesWithoutExtractedMetadata.has(type); + async function getFilesWithAggregations(files: (FileType & FileEnforcedNotUndefined)[]) { const filesNames = files.filter(x => x.filename).map(x => x.filename); @@ -98,7 +105,7 @@ async function fileQuery( propertyType: string, entitiesFromTrainingTemplatesIds: string[] ) { - const needsExtractedMetadata = !propertyTypeIsSelectOrMultiSelect(propertyType); + const needsExtractedMetadata = !propertyTypeIsWithoutExtractedMetadata(propertyType); const query: { type: string; filename: { $exists: Boolean }; @@ -125,7 +132,7 @@ function entityForTrainingQuery( const query: { [key: string]: { $in?: ObjectIdSchema[]; $exists?: Boolean; $ne?: any[] }; } = { template: { $in: templates } }; - if (propertyTypeIsSelectOrMultiSelect(propertyType)) { + if (propertyTypeIsWithoutExtractedMetadata(propertyType)) { query[`metadata.${property}`] = { $exists: true, $ne: [] }; } return query; @@ -162,8 +169,8 @@ async function getFilesForTraining(templates: ObjectIdSchema[], property: string return { ...file, propertyType }; } - if (propertyTypeIsSelectOrMultiSelect(propertyType)) { - const propertyValue = (entity.metadata[property] || []).map(({ value, label }) => ({ + if (propertyTypeIsWithoutExtractedMetadata(propertyType)) { + const propertyValue = (entity.metadata?.[property] || []).map(({ value, label }) => ({ value: ensure(value), label: ensure(label), })); @@ -223,5 +230,6 @@ export { getFilesForSuggestions, getSegmentedFilesIds, propertyTypeIsSelectOrMultiSelect, + propertyTypeIsWithoutExtractedMetadata, }; export type { FileWithAggregation }; diff --git a/app/api/services/informationextraction/ixextractors.ts b/app/api/services/informationextraction/ixextractors.ts index e420b6d2b9..1b6032e33f 100644 --- a/app/api/services/informationextraction/ixextractors.ts +++ b/app/api/services/informationextraction/ixextractors.ts @@ -8,9 +8,16 @@ import { createBlankSuggestionsForExtractor, createBlankSuggestionsForPartialExtractor, } from 'api/suggestions/blankSuggestions'; +import { Subset } from 'shared/tsUtils'; +import { PropertyTypeSchema } from 'shared/types/commonTypes'; import { IXExtractorModel as model } from './IXExtractorModel'; -type AllowedPropertyTypes = 'title' | 'text' | 'numeric' | 'date' | 'select' | 'multiselect'; +type AllowedPropertyTypes = + | Subset< + PropertyTypeSchema, + 'text' | 'numeric' | 'date' | 'select' | 'multiselect' | 'relationship' + > + | 'title'; const ALLOWED_PROPERTY_TYPES: AllowedPropertyTypes[] = [ 'title', @@ -19,6 +26,7 @@ const ALLOWED_PROPERTY_TYPES: AllowedPropertyTypes[] = [ 'date', 'select', 'multiselect', + 'relationship', ]; const allowedTypeSet = new Set(ALLOWED_PROPERTY_TYPES); diff --git a/app/api/services/informationextraction/specs/InformationExtraction.spec.ts b/app/api/services/informationextraction/specs/InformationExtraction.spec.ts index 6dcb28a90e..513079e23c 100644 --- a/app/api/services/informationextraction/specs/InformationExtraction.spec.ts +++ b/app/api/services/informationextraction/specs/InformationExtraction.spec.ts @@ -188,6 +188,25 @@ describe('InformationExtraction', () => { ); }); + it('should send xmls (relationship)', async () => { + await informationExtraction.trainModel(factory.id('extractorWithRelationship')); + + const xmlK = await readDocument('K'); + const xmlL = await readDocument('L'); + + expect(IXExternalService.materialsFileParams).toEqual({ + 0: `/xml_to_train/tenant1/${factory.id('extractorWithRelationship')}`, + id: factory.id('extractorWithRelationship').toString(), + tenant: 'tenant1', + }); + + expect(IXExternalService.files.length).toBe(2); + expect(IXExternalService.files).toEqual(expect.arrayContaining([xmlK, xmlL])); + expect(IXExternalService.filesNames.sort()).toEqual( + ['documentK.xml', 'documentL.xml'].sort() + ); + }); + it('should send labeled data', async () => { await informationExtraction.trainModel(factory.id('prop1extractor')); @@ -244,6 +263,48 @@ describe('InformationExtraction', () => { }); }); + it('should send labeled data (relationship)', async () => { + await informationExtraction.trainModel(factory.id('extractorWithRelationship')); + + expect(IXExternalService.materials.length).toBe(2); + expect(IXExternalService.materials.find(m => m.xml_file_name === 'documentL.xml')).toEqual({ + xml_file_name: 'documentL.xml', + id: factory.id('extractorWithRelationship').toString(), + tenant: 'tenant1', + xml_segments_boxes: [ + { + left: 1, + top: 1, + width: 1, + height: 1, + page_number: 1, + text: 'P1', + }, + { + left: 1, + top: 1, + width: 1, + height: 1, + page_number: 1, + text: 'P2', + }, + ], + page_width: 13, + page_height: 13, + language_iso: 'en', + values: [ + { + id: 'P1sharedId', + label: 'P1', + }, + { + id: 'P3sharedId', + label: 'P3', + }, + ], + }); + }); + it('should sanitize dates before sending', async () => { await informationExtraction.trainModel(factory.id('prop2extractor')); @@ -277,7 +338,9 @@ describe('InformationExtraction', () => { tenant: 'tenant1', task: 'create_model', }); + }); + it('should start the task to train the model (multiselect)', async () => { await informationExtraction.trainModel(factory.id('extractorWithMultiselect')); expect(informationExtraction.taskManager?.startTask).toHaveBeenCalledWith({ @@ -304,10 +367,68 @@ describe('InformationExtraction', () => { }); }); + it('should start the task to train the model (relationship)', async () => { + await informationExtraction.trainModel(factory.id('extractorWithRelationship')); + + expect(informationExtraction.taskManager?.startTask).toHaveBeenCalledWith({ + params: { + id: factory.id('extractorWithRelationship').toString(), + multi_value: true, + options: [ + { + id: 'P1sharedId', + label: 'P1', + }, + { + id: 'P2sharedId', + label: 'P2', + }, + { + id: 'P3sharedId', + label: 'P3', + }, + ], + }, + tenant: 'tenant1', + task: 'create_model', + }); + }); + + it('should start the task to train the model (relationship to any template)', async () => { + await informationExtraction.trainModel(factory.id('extractorWithRelationshipToAny')); + + expect(informationExtraction.taskManager?.startTask).toHaveBeenCalledWith({ + params: { + id: factory.id('extractorWithRelationshipToAny').toString(), + multi_value: true, + options: [ + { + id: 'P1sharedId', + label: 'P1', + }, + { + id: 'P2sharedId', + label: 'P2', + }, + { + id: 'P3sharedId', + label: 'P3', + }, + ...Array.from({ length: 23 }, (_, i) => ({ + id: `A${i + 1}`, + label: `A${i + 1}`, + })), + ], + }, + tenant: 'tenant1', + task: 'create_model', + }); + }); + it('should return error status and stop finding suggestions, when there is no labaled data', async () => { const expectedError = { status: 'error', message: 'No labeled data' }; - const result = await informationExtraction.trainModel(factory.id('prop3extractor')); + const result = await informationExtraction.trainModel(factory.id('prop3extractor')); expect(result).toMatchObject(expectedError); const [model] = await IXModelsModel.get({ extractorId: factory.id('prop3extractor') }); expect(model.findingSuggestions).toBe(false); @@ -320,6 +441,15 @@ describe('InformationExtraction', () => { extractorId: factory.id('extractorWithMultiselectWithoutTrainingData'), }); expect(multiSelectModel.findingSuggestions).toBe(false); + + const relationshipResult = await informationExtraction.trainModel( + factory.id('extractorWithEmptyRelationship') + ); + expect(relationshipResult).toMatchObject(expectedError); + const [relationshipModel] = await IXModelsModel.get({ + extractorId: factory.id('extractorWithEmptyRelationship'), + }); + expect(relationshipModel.findingSuggestions).toBe(false); }); }); @@ -470,6 +600,90 @@ describe('InformationExtraction', () => { ]); }); + it('should send the materials for the suggestions (relationship)', async () => { + await informationExtraction.getSuggestions(factory.id('extractorWithRelationship')); + + const [xmlK, xmlL, xmlM] = await Promise.all(['K', 'L', 'M'].map(readDocument)); + + expect(IXExternalService.materialsFileParams).toEqual({ + 0: `/xml_to_predict/tenant1/${factory.id('extractorWithRelationship')}`, + id: factory.id('extractorWithRelationship').toString(), + tenant: 'tenant1', + }); + + expect(IXExternalService.filesNames.sort()).toEqual( + ['documentK.xml', 'documentL.xml', 'documentM.xml'].sort() + ); + expect(IXExternalService.files.length).toBe(3); + expect(IXExternalService.files).toEqual(expect.arrayContaining([xmlK, xmlL, xmlM])); + + expect(IXExternalService.materials.length).toBe(3); + const sortedMaterials = sortByStrings(IXExternalService.materials, [ + (m: any) => m.xml_file_name, + ]); + expect(sortedMaterials).toEqual([ + { + xml_file_name: 'documentK.xml', + id: factory.id('extractorWithRelationship').toString(), + tenant: 'tenant1', + page_height: 13, + page_width: 13, + xml_segments_boxes: [ + { + height: 1, + left: 1, + page_number: 1, + text: 'P1', + top: 1, + width: 1, + }, + ], + }, + { + xml_file_name: 'documentL.xml', + id: factory.id('extractorWithRelationship').toString(), + tenant: 'tenant1', + page_height: 13, + page_width: 13, + xml_segments_boxes: [ + { + height: 1, + left: 1, + page_number: 1, + text: 'P1', + top: 1, + width: 1, + }, + { + height: 1, + left: 1, + page_number: 1, + text: 'P2', + top: 1, + width: 1, + }, + ], + }, + { + xml_file_name: 'documentM.xml', + id: factory.id('extractorWithRelationship').toString(), + tenant: 'tenant1', + page_height: 13, + page_width: 13, + xml_segments_boxes: [ + { + height: 1, + left: 1, + page_number: 1, + text: 'P3', + top: 1, + width: 1, + }, + ], + }, + ]); + }); + it('should avoid sending materials for failed suggestions because no segmentation for instance', async () => { await informationExtraction.getSuggestions(factory.id('extractorWithOneFailedSegmentation')); @@ -1064,5 +1278,111 @@ describe('InformationExtraction', () => { ]); }); }); + + describe('relationship', () => { + it('should request and store the suggestions (relationship)', async () => { + setIXServiceResults( + [ + { + id: factory.id('extractorWithRelationship').toString(), + xml_file_name: 'documentK.xml', + values: [{ id: 'P1sharedId', label: 'P1' }], + segment_text: 'it is P1', + }, + { + id: factory.id('extractorWithRelationship').toString(), + xml_file_name: 'documentL.xml', + values: [ + { id: 'P1sharedId', label: 'P1' }, + { id: 'P2sharedId', label: 'P2' }, + ], + segment_text: 'it is P1 or P2', + }, + { + id: factory.id('extractorWithRelationship').toString(), + xml_file_name: 'documentM.xml', + values: [{ id: 'P3sharedId', label: 'P3' }], + segment_text: 'it is P3', + }, + ], + 'value' + ); + + await saveSuggestionProcess('SUG21', 'A21', 'eng', 'extractorWithRelationship'); + await saveSuggestionProcess('SUG22', 'A22', 'eng', 'extractorWithRelationship'); + await saveSuggestionProcess('SUG23', 'A23', 'eng', 'extractorWithRelationship'); + + await informationExtraction.processResults({ + params: { id: factory.id('extractorWithRelationship').toString() }, + tenant: 'tenant1', + task: 'suggestions', + success: true, + data_url: 'http://localhost:1234/suggestions_results', + }); + + const suggestions = await IXSuggestionsModel.get({ + status: 'ready', + extractorId: factory.id('extractorWithRelationship'), + }); + + const sorted = sortByStrings(suggestions, [(s: any) => s.entityId]); + + const expectedBase = { + _id: expect.any(ObjectId), + entityTemplate: factory.id('templateToSegmentF').toString(), + language: 'en', + propertyName: 'property_relationship', + extractorId: factory.id('extractorWithRelationship'), + status: 'ready', + page: 1, + date: expect.any(Number), + error: '', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: true, + hasContext: true, + processing: false, + obsolete: false, + error: false, + }, + }; + + expect(sorted).toEqual([ + { + ...expectedBase, + fileId: factory.id('F21'), + entityId: 'A21', + suggestedValue: ['P1sharedId'], + segment: 'it is P1', + }, + { + ...expectedBase, + fileId: factory.id('F22'), + entityId: 'A22', + suggestedValue: ['P1sharedId', 'P2sharedId'], + segment: 'it is P1 or P2', + state: { + ...expectedBase.state, + match: false, + }, + }, + { + ...expectedBase, + fileId: factory.id('F23'), + entityId: 'A23', + suggestedValue: ['P3sharedId'], + segment: 'it is P3', + state: { + ...expectedBase.state, + withValue: false, + labeled: false, + match: false, + }, + }, + ]); + }); + }); }); }); diff --git a/app/api/services/informationextraction/specs/fixtures.ts b/app/api/services/informationextraction/specs/fixtures.ts index dc7dc2909d..fd87182dee 100644 --- a/app/api/services/informationextraction/specs/fixtures.ts +++ b/app/api/services/informationextraction/specs/fixtures.ts @@ -13,6 +13,9 @@ const fixturesPdfNameG = 'documentG.pdf'; const fixturesPdfNameH = 'documentH.pdf'; const fixturesPdfNameI = 'documentI.pdf'; const ficturesPdfNameJ = 'documentJ.pdf'; +const fixturesPdfNameK = 'documentK.pdf'; +const fixturesPdfNameL = 'documentL.pdf'; +const fixturesPdfNameM = 'documentM.pdf'; const fixtures: DBFixture = { settings: [ @@ -42,8 +45,20 @@ const fixtures: DBFixture = { factory.ixExtractor('extractorWithMultiselectWithoutTrainingData', 'property_multiselect', [ 'templateToSegmentE', ]), + factory.ixExtractor('extractorWithRelationship', 'property_relationship', [ + 'templateToSegmentF', + ]), + factory.ixExtractor('extractorWithEmptyRelationship', 'property_empty_relationship', [ + 'templateToSegmentF', + ]), + factory.ixExtractor('extractorWithRelationshipToAny', 'property_relationship_to_any', [ + 'templateToSegmentF', + ]), ], entities: [ + factory.entity('P1', 'relationshipPartnerTemplate', {}, { sharedId: 'P1sharedId' }), + factory.entity('P2', 'relationshipPartnerTemplate', {}, { sharedId: 'P2sharedId' }), + factory.entity('P3', 'relationshipPartnerTemplate', {}, { sharedId: 'P3sharedId' }), factory.entity( 'A1', 'templateToSegmentA', @@ -101,6 +116,27 @@ const fixtures: DBFixture = { factory.entity('A20', 'templateToSegmentE', { property_multiselect: [], }), + factory.entity('A21', 'templateToSegmentF', { + property_relationship: [{ value: 'P1sharedId', label: 'P1' }], + property_empty_relationship: [], + property_relationship_to_any: [{ value: 'P1sharedId', label: 'P1' }], + }), + factory.entity('A22', 'templateToSegmentF', { + property_relationship: [ + { value: 'P1sharedId', label: 'P1' }, + { value: 'P3sharedId', label: 'P3' }, + ], + property_empty_relationship: [], + property_relationship_to_any: [ + { value: 'P1', label: 'P1' }, + { value: 'A1', label: 'A1' }, + ], + }), + factory.entity('A23', 'templateToSegmentF', { + property_relationship: [], + property_empty_relationship: [], + property_relationship_to_any: [], + }), ], files: [ factory.file('F1', 'A1', 'document', fixturesPdfNameA, 'other', '', [ @@ -154,6 +190,9 @@ const fixtures: DBFixture = { factory.file('F18', 'A18', 'document', fixturesPdfNameH, 'eng'), factory.file('F19', 'A19', 'document', fixturesPdfNameI, 'eng'), factory.file('F20', 'A20', 'document', ficturesPdfNameJ, 'eng'), + factory.file('F21', 'A21', 'document', fixturesPdfNameK, 'eng'), + factory.file('F22', 'A22', 'document', fixturesPdfNameL, 'eng'), + factory.file('F23', 'A23', 'document', fixturesPdfNameM, 'eng'), ], segmentations: [ { @@ -290,6 +329,77 @@ const fixtures: DBFixture = { paragraphs: [], }, }, + { + _id: factory.id('S11'), + filename: fixturesPdfNameK, + xmlname: 'documentK.xml', + fileID: factory.id('F21'), + status: 'ready', + segmentation: { + page_height: 13, + page_width: 13, + paragraphs: [ + { + left: 1, + top: 1, + width: 1, + height: 1, + page_number: 1, + text: 'P1', + }, + ], + }, + }, + { + _id: factory.id('S12'), + filename: fixturesPdfNameL, + xmlname: 'documentL.xml', + fileID: factory.id('F22'), + status: 'ready', + segmentation: { + page_height: 13, + page_width: 13, + paragraphs: [ + { + left: 1, + top: 1, + width: 1, + height: 1, + page_number: 1, + text: 'P1', + }, + { + left: 1, + top: 1, + width: 1, + height: 1, + page_number: 1, + text: 'P2', + }, + ], + }, + }, + { + _id: factory.id('S13'), + filename: fixturesPdfNameM, + xmlname: 'documentM.xml', + fileID: factory.id('F23'), + status: 'ready', + segmentation: { + page_height: 13, + page_width: 13, + paragraphs: [ + { + left: 1, + top: 1, + width: 1, + height: 1, + page_number: 1, + text: 'P3', + }, + ], + }, + }, ], ixsuggestions: [ { @@ -505,6 +615,45 @@ const fixtures: DBFixture = { page: 1, date: 100, }, + { + _id: factory.id('SUG21'), + fileId: factory.id('F21'), + entityId: 'A21', + entityTemplate: factory.idString('templateToSegmentF'), + language: 'en', + propertyName: 'property_relationship', + extractorId: factory.id('extractorWithRelationship'), + suggestedValue: ['P1'], + status: 'ready', + page: 1, + date: 100, + }, + { + _id: factory.id('SUG22'), + fileId: factory.id('F22'), + entityId: 'A22', + entityTemplate: factory.idString('templateToSegmentF'), + language: 'en', + propertyName: 'property_relationship', + extractorId: factory.id('extractorWithRelationship'), + suggestedValue: ['P1', 'P2'], + status: 'ready', + page: 1, + date: 100, + }, + { + _id: factory.id('SUG23'), + fileId: factory.id('F23'), + entityId: 'A23', + entityTemplate: factory.idString('templateToSegmentF'), + language: 'en', + propertyName: 'property_relationship', + extractorId: factory.id('extractorWithRelationship'), + suggestedValue: [], + status: 'ready', + page: 1, + date: 100, + }, ], ixmodels: [ { @@ -555,8 +704,32 @@ const fixtures: DBFixture = { status: 'ready', findingSuggestions: false, }, + { + extractorId: factory.id('extractorWithRelationship'), + creationDate: 200, + status: 'ready', + findingSuggestions: true, + }, + { + extractorId: factory.id('extractorWithEmptyRelationship'), + creationDate: 200, + status: 'ready', + findingSuggestions: true, + }, + { + extractorId: factory.id('extractorWithRelationshipToAny'), + creationDate: 200, + status: 'ready', + findingSuggestions: true, + }, + ], + relationtypes: [ + factory.relationType('related'), + factory.relationType('emptyRelated'), + factory.relationType('relatedToAny'), ], templates: [ + factory.template('relationshipPartnerTemplate'), factory.template('templateToSegmentA', [ factory.property('property1', 'text'), factory.property('property2', 'date'), @@ -577,6 +750,20 @@ const fixtures: DBFixture = { content: factory.id('thesauri1').toString(), }), ]), + factory.template('templateToSegmentF', [ + factory.property('property_relationship', 'relationship', { + content: factory.idString('relationshipPartnerTemplate'), + relationType: factory.idString('related'), + }), + factory.property('property_empty_relationship', 'relationship', { + content: factory.idString('relationshipPartnerTemplate'), + relationType: factory.idString('emptyRelated'), + }), + factory.property('property_relationship_to_any', 'relationship', { + content: '', + relationType: factory.idString('relatedToAny'), + }), + ]), ], dictionaries: [factory.thesauri('thesauri1', ['A', 'B', 'C'])], }; diff --git a/app/api/services/informationextraction/specs/suggestionFormatting.spec.ts b/app/api/services/informationextraction/specs/suggestionFormatting.spec.ts index c9c828c4f5..39f20fd93f 100644 --- a/app/api/services/informationextraction/specs/suggestionFormatting.spec.ts +++ b/app/api/services/informationextraction/specs/suggestionFormatting.spec.ts @@ -24,6 +24,7 @@ const properties: Record = { date: fixtureFactory.property('date_property', 'date'), select: fixtureFactory.property('select_property', 'select'), multiselect: fixtureFactory.property('multiselect_property', 'multiselect'), + relationship: fixtureFactory.property('relationship_property', 'relationship'), }; const entities: Record = { @@ -46,6 +47,12 @@ const entities: Record = { { value: 'B_id', label: 'B' }, ], }), + relationship: fixtureFactory.entity('entity_id', 'entity_template', { + relationship_property: [ + { value: 'related_1_id', label: 'related_1_title' }, + { value: 'related_2_id', label: 'related_2_title' }, + ], + }), }; const currentSuggestionBase = { @@ -102,6 +109,11 @@ const currentSuggestions: Record = { propertyName: 'multiselect_property', suggestedValue: ['A_id', 'B_id'], }, + relationship: { + ...currentSuggestionBase, + propertyName: 'relationship_property', + suggestedValue: ['related_1_id', 'related_2_id'], + }, }; const rawSuggestionBase = { @@ -178,6 +190,13 @@ const validRawSuggestions = { { id: 'D_id', label: 'D' }, ], }, + relationship: { + ...rawSuggestionBase, + values: [ + { id: 'related_1_id', label: 'related_1_title' }, + { id: 'related_3_id', label: 'related_3_title' }, + ], + }, }; describe('formatSuggestion', () => { @@ -325,6 +344,28 @@ describe('formatSuggestion', () => { entity: entities.multiselect, expectedErrorMessage: '/values/0/id: must be string', }, + { + case: 'invalid relationship values type', + property: properties.relationship, + rawSuggestion: { + ...validRawSuggestions.relationship, + values: 1, + }, + currentSuggestion: currentSuggestions.relationship, + entity: entities.relationship, + expectedErrorMessage: '/values: must be array', + }, + { + case: 'invalid relationship values subtype', + property: properties.relationship, + rawSuggestion: { + ...validRawSuggestions.relationship, + values: [{ id: 1, label: 'value_label' }], + }, + currentSuggestion: currentSuggestions.relationship, + entity: entities.relationship, + expectedErrorMessage: '/values/0/id: must be string', + }, ])( 'should throw error if $case', async ({ property, rawSuggestion, currentSuggestion, entity, expectedErrorMessage }) => { @@ -487,6 +528,19 @@ describe('formatSuggestion', () => { segment: 'new context', }, }, + { + case: 'valid relationship suggestions', + property: properties.relationship, + rawSuggestion: validRawSuggestions.relationship, + currentSuggestion: currentSuggestions.relationship, + entity: entities.relationship, + expectedResult: { + ...currentSuggestions.relationship, + date: expect.any(Number), + suggestedValue: ['related_1_id', 'related_3_id'], + segment: 'new context', + }, + }, ])( 'should return formatted suggestion for $case', async ({ property, rawSuggestion, currentSuggestion, entity, expectedResult }) => { diff --git a/app/api/services/informationextraction/specs/uploads/segmentation/documentK.xml b/app/api/services/informationextraction/specs/uploads/segmentation/documentK.xml new file mode 100644 index 0000000000..ff40832804 --- /dev/null +++ b/app/api/services/informationextraction/specs/uploads/segmentation/documentK.xml @@ -0,0 +1,3 @@ + + K + \ No newline at end of file diff --git a/app/api/services/informationextraction/specs/uploads/segmentation/documentL.xml b/app/api/services/informationextraction/specs/uploads/segmentation/documentL.xml new file mode 100644 index 0000000000..3d8093bb49 --- /dev/null +++ b/app/api/services/informationextraction/specs/uploads/segmentation/documentL.xml @@ -0,0 +1,3 @@ + + L + \ No newline at end of file diff --git a/app/api/services/informationextraction/specs/uploads/segmentation/documentM.xml b/app/api/services/informationextraction/specs/uploads/segmentation/documentM.xml new file mode 100644 index 0000000000..77fe91e68a --- /dev/null +++ b/app/api/services/informationextraction/specs/uploads/segmentation/documentM.xml @@ -0,0 +1,3 @@ + + M + \ No newline at end of file diff --git a/app/api/services/informationextraction/suggestionFormatting.ts b/app/api/services/informationextraction/suggestionFormatting.ts index d69118d449..73839f9f2d 100644 --- a/app/api/services/informationextraction/suggestionFormatting.ts +++ b/app/api/services/informationextraction/suggestionFormatting.ts @@ -71,6 +71,7 @@ const VALIDATORS = { return true; }, multiselect: valuesSelectionValidator, + relationship: valuesSelectionValidator, }; const simpleSuggestion = ( @@ -86,6 +87,16 @@ const simpleSuggestion = ( }), }); +function multiValueIdsSuggestion(rawSuggestion: ValuesSelectionSuggestion) { + const suggestedValue = rawSuggestion.values.map(value => value.id); + + const suggestion: Partial = { + suggestedValue, + segment: rawSuggestion.segment_text, + }; + return suggestion; +} + const textFormatter = ( rawSuggestion: RawSuggestion, _currentSuggestion: IXSuggestionType, @@ -174,12 +185,20 @@ const FORMATTERS: Record< throw new Error('Multiselect suggestion is not valid.'); } - const suggestedValue = rawSuggestion.values.map(value => value.id); + const suggestion: Partial = multiValueIdsSuggestion(rawSuggestion); - const suggestion: Partial = { - suggestedValue, - segment: rawSuggestion.segment_text, - }; + return suggestion; + }, + relationship: ( + rawSuggestion: RawSuggestion, + _currentSuggestion: IXSuggestionType, + _entity: EntitySchema + ) => { + if (!VALIDATORS.relationship(rawSuggestion)) { + throw new Error('Relationship suggestion is not valid.'); + } + + const suggestion: Partial = multiValueIdsSuggestion(rawSuggestion); return suggestion; }, diff --git a/app/api/suggestions/specs/fixtures.ts b/app/api/suggestions/specs/fixtures.ts index 32e81655cd..42b54dfbc2 100644 --- a/app/api/suggestions/specs/fixtures.ts +++ b/app/api/suggestions/specs/fixtures.ts @@ -1407,6 +1407,320 @@ const selectAcceptanceFixtureBase: DBFixture = { ], }; +const relationshipAcceptanceFixtureBase: DBFixture = { + settings: _.cloneDeep(ixSettings), + ixextractors: [ + factory.ixExtractor('relationship_extractor', 'relationship_to_source', ['rel_template']), + factory.ixExtractor( + 'relationship_with_inheritance_extractor', + 'relationship_with_inheritance', + ['rel_template'] + ), + factory.ixExtractor('relationship_to_any_extractor', 'relationship_to_any', ['rel_template']), + ], + ixsuggestions: [], + ixmodels: [ + { + _id: testingDB.id(), + status: 'ready', + creationDate: 1, + extractorId: factory.id('relationship_extractor'), + }, + { + _id: testingDB.id(), + status: 'ready', + creationDate: 1, + extractorId: factory.id('relationship_with_inheritance_extractor'), + }, + { + _id: testingDB.id(), + status: 'ready', + creationDate: 1, + extractorId: factory.id('relationship_to_any_extractor'), + }, + ], + relationtypes: [ + factory.relationType('related'), + factory.relationType('related_with_inheritance'), + factory.relationType('related_to_any'), + ], + templates: [ + factory.template('source_template', [factory.property('text_to_inherit', 'text')]), + factory.template('source_template_2', []), + factory.template('rel_template', [ + factory.property('relationship_to_source', 'relationship', { + content: factory.idString('source_template'), + relationType: factory.idString('related'), + }), + factory.property('relationship_with_inheritance', 'relationship', { + content: factory.idString('source_template'), + relationType: factory.idString('related_with_inheritance'), + inherit: { + property: factory.idString('text_to_inherit'), + type: 'text', + }, + }), + factory.property('relationship_to_any', 'relationship', { + content: '', + relationType: factory.idString('related_to_any'), + }), + ]), + ], + entities: [ + // ---------- sources + { + _id: testingDB.id(), + sharedId: 'S1_sId', + title: 'S1', + language: 'en', + metadata: { text_to_inherit: [{ value: 'inherited text' }] }, + template: factory.id('source_template'), + }, + { + _id: testingDB.id(), + sharedId: 'S1_sId', + title: 'S1_es', + language: 'es', + metadata: { text_to_inherit: [{ value: 'inherited text Spanish' }] }, + template: factory.id('source_template'), + }, + { + _id: testingDB.id(), + sharedId: 'S2_sId', + title: 'S2', + language: 'en', + metadata: { text_to_inherit: [{ value: 'inherited text 2' }] }, + template: factory.id('source_template'), + }, + { + _id: testingDB.id(), + sharedId: 'S2_sId', + title: 'S2_es', + language: 'es', + metadata: { text_to_inherit: [{ value: 'inherited text 2 Spanish' }] }, + template: factory.id('source_template'), + }, + { + _id: testingDB.id(), + sharedId: 'S3_sId', + title: 'S3', + language: 'en', + metadata: { text_to_inherit: [{ value: 'inherited text 3' }] }, + template: factory.id('source_template'), + }, + { + _id: testingDB.id(), + sharedId: 'S3_sId', + title: 'S3_es', + language: 'es', + metadata: { text_to_inherit: [{ value: 'inherited text 3 Spanish' }] }, + template: factory.id('source_template'), + }, + // ---------- other sources + { + _id: testingDB.id(), + sharedId: 'other_source', + title: 'Other Source', + language: 'en', + metadata: {}, + template: factory.id('source_template_2'), + }, + { + _id: testingDB.id(), + sharedId: 'other_source', + title: 'Other Source Spanish', + language: 'es', + metadata: {}, + template: factory.id('source_template_2'), + }, + { + _id: testingDB.id(), + sharedId: 'other_source_2', + title: 'Other Source 2', + language: 'en', + metadata: {}, + template: factory.id('source_template_2'), + }, + { + _id: testingDB.id(), + sharedId: 'other_source_2', + title: 'Other Source 2 Spanish', + language: 'es', + metadata: {}, + template: factory.id('source_template_2'), + }, + // ---------- with relationship + { + _id: testingDB.id(), + sharedId: 'entityWithRelationships_sId', + title: 'entityWithRelationships', + language: 'en', + metadata: { + relationship_to_source: [ + { value: 'S1_sId', label: 'S1' }, + { value: 'S2_sId', label: 'S2' }, + ], + relationship_with_inheritance: [ + { + value: 'S1_sId', + label: 'S1', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text', + }, + ], + }, + { + value: 'S2_sId', + label: 'S2', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text 2', + }, + ], + }, + ], + relationship_to_any: [ + { + value: 'S1_sId', + label: 'S1', + }, + { + value: 'other_source', + label: 'Other Source', + }, + ], + }, + template: factory.id('rel_template'), + }, + { + _id: testingDB.id(), + sharedId: 'entityWithRelationships_sId', + title: 'entityWithRelationshipsEs', + language: 'es', + metadata: { + relationship_to_source: [ + { value: 'S1_sId', label: 'S1_es' }, + { value: 'S2_sId', label: 'S2_es' }, + ], + relationship_with_inheritance: [ + { + value: 'S1_sId', + label: 'S1_es', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text Spanish', + }, + ], + }, + { + value: 'S2_sId', + label: 'S2_es', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text 2 Spanish', + }, + ], + }, + ], + relationship_to_any: [ + { + value: 'S1_sId', + label: 'S1_es', + }, + { + value: 'other_source', + label: 'Other Source Spanish', + }, + ], + }, + template: factory.id('rel_template'), + }, + ], + connections: [ + { + _id: testingDB.id(), + entity: 'entityWithRelationships_sId', + hub: factory.id('hub_S1'), + }, + { + _id: testingDB.id(), + entity: 'S1_sId', + hub: factory.id('hub_S1'), + template: factory.id('related'), + }, + { + _id: testingDB.id(), + entity: 'entityWithRelationships_sId', + hub: factory.id('hub_S2'), + }, + { + _id: testingDB.id(), + entity: 'S2_sId', + hub: factory.id('hub_S2'), + template: factory.id('related'), + }, + { + _id: testingDB.id(), + entity: 'entityWithRelationships_sId', + hub: factory.id('hub_S1_inherited'), + }, + { + _id: testingDB.id(), + entity: 'S1_sId', + hub: factory.id('hub_S1_inherited'), + template: factory.id('related_with_inheritance'), + }, + { + _id: testingDB.id(), + entity: 'entityWithRelationships_sId', + hub: factory.id('hub_S2_inherited'), + }, + { + _id: testingDB.id(), + entity: 'S2_sId', + hub: factory.id('hub_S2_inherited'), + template: factory.id('related_with_inheritance'), + }, + { + _id: testingDB.id(), + entity: 'entityWithRelationships_sId', + hub: factory.id('hub_S1_any'), + }, + { + _id: testingDB.id(), + entity: 'S1_sId', + hub: factory.id('hub_S1_any'), + template: factory.id('related_to_any'), + }, + { + _id: testingDB.id(), + entity: 'entityWithRelationships_sId', + hub: factory.id('hub_other_source_any'), + }, + { + _id: testingDB.id(), + entity: 'other_source', + hub: factory.id('hub_other_source_any'), + template: factory.id('related_to_any'), + }, + ], + files: [ + factory.file( + 'fileForEntityWithRelationships', + 'entityWithRelationships_sId', + 'document', + 'documentWithRelationships.pdf', + 'eng', + 'documentWithRelationships.pdf' + ), + ], +}; + export { factory, file2Id, @@ -1423,4 +1737,5 @@ export { suggestionId, shared2AgeSuggestionId, selectAcceptanceFixtureBase, + relationshipAcceptanceFixtureBase, }; diff --git a/app/api/suggestions/specs/routes.spec.ts b/app/api/suggestions/specs/routes.spec.ts index 40e0e0d18c..10bc43cf2a 100644 --- a/app/api/suggestions/specs/routes.spec.ts +++ b/app/api/suggestions/specs/routes.spec.ts @@ -393,10 +393,7 @@ describe('suggestions routes', () => { title: 'The Penguin', }, ]); - expect(search.indexEntities).toHaveBeenCalledWith( - { _id: { $in: [shared6enId] } }, - '+fullText' - ); + expect(search.indexEntities).toHaveBeenCalledWith({ sharedId: 'shared6' }, '+fullText'); }); it('should reject with unauthorized when user has not admin role', async () => { @@ -434,10 +431,10 @@ describe('suggestions routes', () => { const [entity] = await entities.get({ sharedId: 'entityWithSelects2' }); expect(entity.metadata.property_multiselect).toEqual([ { value: 'A', label: 'A' }, - { value: '1B', label: '1B' }, + { value: '1B', label: '1B', parent: { value: '1', label: '1' } }, ]); expect(search.indexEntities).toHaveBeenCalledWith( - { _id: { $in: [factory.id('entityWithSelects2')] } }, + { sharedId: 'entityWithSelects2' }, '+fullText' ); }); diff --git a/app/api/suggestions/specs/suggestions.spec.ts b/app/api/suggestions/specs/suggestions.spec.ts index 2fa5c756f6..fcf5135583 100644 --- a/app/api/suggestions/specs/suggestions.spec.ts +++ b/app/api/suggestions/specs/suggestions.spec.ts @@ -18,7 +18,9 @@ import { suggestionId, shared2AgeSuggestionId, selectAcceptanceFixtureBase, + relationshipAcceptanceFixtureBase, } from './fixtures'; +import { ObjectId } from 'mongodb'; const getSuggestions = async (filter: IXSuggestionsFilter, size = 5) => Suggestions.get(filter, { page: { size, number: 1 } }); @@ -346,18 +348,20 @@ const matchState = (match: boolean = true): IXSuggestionStateType => ({ error: false, }); -const selectSuggestionBase = (propertyName: string, extractorName: string) => ({ - fileId: factory.id('fileForentityWithSelects'), - entityId: 'entityWithSelects', - entityTemplate: factory.id('templateWithSelects').toString(), - propertyName, - extractorId: factory.id(extractorName), - date: 5, - status: 'ready' as 'ready', - error: '', -}); +type SuggestionBase = Pick< + IXSuggestionType, + | 'fileId' + | 'entityId' + | 'entityTemplate' + | 'propertyName' + | 'extractorId' + | 'date' + | 'status' + | 'error' +>; const prepareAndAcceptSuggestion = async ( + suggestionBase: SuggestionBase, suggestedValue: string | string[], language: string, propertyName: string, @@ -368,7 +372,7 @@ const prepareAndAcceptSuggestion = async ( } = {} ) => { const suggestion = { - ...selectSuggestionBase(propertyName, extractorName), + ...suggestionBase, suggestedValue, language, }; @@ -387,6 +391,69 @@ const prepareAndAcceptSuggestion = async ( return { acceptedSuggestion, metadataValues, allFiles }; }; +const selectSuggestionBase = (propertyName: string, extractorName: string): SuggestionBase => ({ + fileId: factory.id('fileForentityWithSelects'), + entityId: 'entityWithSelects', + entityTemplate: factory.id('templateWithSelects').toString(), + propertyName, + extractorId: factory.id(extractorName), + date: 5, + status: 'ready' as 'ready', + error: '', +}); + +const prepareAndAcceptSelectSuggestion = async ( + suggestedValue: string | string[], + language: string, + propertyName: string, + extractorName: string, + acceptanceParameters: { + addedValues?: string[]; + removedValues?: string[]; + } = {} +) => + prepareAndAcceptSuggestion( + selectSuggestionBase(propertyName, extractorName), + suggestedValue, + language, + propertyName, + extractorName, + acceptanceParameters + ); + +const relationshipSuggestionBase = ( + propertyName: string, + extractorName: string +): SuggestionBase => ({ + fileId: factory.id('fileForEntityWithRelationships'), + entityId: 'entityWithRelationships_sId', + entityTemplate: factory.id('rel_template').toString(), + propertyName, + extractorId: factory.id(extractorName), + date: 5, + status: 'ready' as 'ready', + error: '', +}); + +const prepareAndAcceptRelationshipSuggestion = async ( + suggestedValue: string | string[], + language: string, + propertyName: string, + extractorName: string, + acceptanceParameters: { + addedValues?: string[]; + removedValues?: string[]; + } = {} +) => + prepareAndAcceptSuggestion( + relationshipSuggestionBase(propertyName, extractorName), + suggestedValue, + language, + propertyName, + extractorName, + acceptanceParameters + ); + describe('suggestions', () => { afterAll(async () => { await db.disconnect(); @@ -828,14 +895,14 @@ describe('suggestions', () => { .find({ sharedId: 'shared1' }) .toArray(); const ages1 = entities1?.map(entity => entity.metadata.age[0].value); - expect(ages1).toEqual(['17', '17']); + expect(ages1).toEqual([17, 17]); const entities2 = await db.mongodb ?.collection('entities') .find({ sharedId: 'shared2' }) .toArray(); const ages2 = entities2?.map(entity => entity.metadata.age[0].value); - expect(ages2).toEqual(['20', '20', '20']); + expect(ages2).toEqual([20, 20, 20]); }); }); @@ -846,18 +913,14 @@ describe('suggestions', () => { it('should validate that the id exists in the dictionary', async () => { const action = async () => { - await prepareAndAcceptSuggestion('Z', 'en', 'property_select', 'select_extractor'); + await prepareAndAcceptSelectSuggestion('Z', 'en', 'property_select', 'select_extractor'); }; await expect(action()).rejects.toThrow('Id is invalid: Z (Nested Thesaurus).'); }); it('should update entities of all languages, with the properly translated labels', async () => { - const { acceptedSuggestion, metadataValues, allFiles } = await prepareAndAcceptSuggestion( - 'A', - 'en', - 'property_select', - 'select_extractor' - ); + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptSelectSuggestion('A', 'en', 'property_select', 'select_extractor'); expect(acceptedSuggestion.state).toEqual(matchState()); expect(metadataValues).toMatchObject([ [{ value: 'A', label: 'A' }], @@ -874,7 +937,7 @@ describe('suggestions', () => { it('should validate that the ids exist in the dictionary', async () => { const action = async () => { - await prepareAndAcceptSuggestion( + await prepareAndAcceptSelectSuggestion( ['Z', '1A', 'Y', 'A'], 'en', 'property_multiselect', @@ -884,29 +947,41 @@ describe('suggestions', () => { await expect(action()).rejects.toThrow('Ids are invalid: Z, Y (Nested Thesaurus).'); }); - it('should validate that partial acceptance is allowed only for multiselects', async () => { + it('should validate that partial acceptance is allowed only for multiselects/relationships', async () => { const addAction = async () => { - await prepareAndAcceptSuggestion('1A', 'en', 'property_select', 'select_extractor', { - addedValues: ['1A'], - }); + await prepareAndAcceptSelectSuggestion( + '1A', + 'en', + 'property_select', + 'select_extractor', + { + addedValues: ['1A'], + } + ); }; await expect(addAction()).rejects.toThrow( - 'Partial acceptance is only allowed for multiselects.' + 'Partial acceptance is only allowed for multiselects or relationships.' ); const removeAction = async () => { - await prepareAndAcceptSuggestion('1A', 'en', 'property_select', 'select_extractor', { - removedValues: ['1B'], - }); + await prepareAndAcceptSelectSuggestion( + '1A', + 'en', + 'property_select', + 'select_extractor', + { + removedValues: ['1B'], + } + ); }; await expect(removeAction()).rejects.toThrow( - 'Partial acceptance is only allowed for multiselects.' + 'Partial acceptance is only allowed for multiselects or relationships.' ); }); it("should validate that the accepted id's through partial acceptance do exist on the suggestion", async () => { const action = async () => { - await prepareAndAcceptSuggestion( + await prepareAndAcceptSelectSuggestion( ['1A', '1B'], 'en', 'property_multiselect', @@ -923,7 +998,7 @@ describe('suggestions', () => { it("should validate that the id's to remove through partial acceptance do not exist on the suggestion", async () => { const action = async () => { - await prepareAndAcceptSuggestion( + await prepareAndAcceptSelectSuggestion( ['1A', '1B'], 'en', 'property_multiselect', @@ -939,12 +1014,13 @@ describe('suggestions', () => { }); it('should allow full acceptance, and update entites of all languages, with the properly translated labels', async () => { - const { acceptedSuggestion, metadataValues, allFiles } = await prepareAndAcceptSuggestion( - ['1A', '1B'], - 'en', - 'property_multiselect', - 'multiselect_extractor' - ); + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptSelectSuggestion( + ['1A', '1B'], + 'en', + 'property_multiselect', + 'multiselect_extractor' + ); expect(acceptedSuggestion.state).toEqual(matchState()); expect(metadataValues).toMatchObject([ [ @@ -960,15 +1036,16 @@ describe('suggestions', () => { }); it('should allow partial acceptance, and update entites of all languages, with the properly translated labels', async () => { - const { acceptedSuggestion, metadataValues, allFiles } = await prepareAndAcceptSuggestion( - ['B', '1B'], - 'en', - 'property_multiselect', - 'multiselect_extractor', - { - addedValues: ['B'], - } - ); + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptSelectSuggestion( + ['B', '1B'], + 'en', + 'property_multiselect', + 'multiselect_extractor', + { + addedValues: ['B'], + } + ); expect(acceptedSuggestion.state).toEqual(matchState(false)); expect(metadataValues).toMatchObject([ [ @@ -986,15 +1063,16 @@ describe('suggestions', () => { }); it('should do nothing on partial acceptance if the id is already in the entity metadata', async () => { - const { acceptedSuggestion, metadataValues, allFiles } = await prepareAndAcceptSuggestion( - ['1A', '1B'], - 'en', - 'property_multiselect', - 'multiselect_extractor', - { - addedValues: ['1A'], - } - ); + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptSelectSuggestion( + ['1A', '1B'], + 'en', + 'property_multiselect', + 'multiselect_extractor', + { + addedValues: ['1A'], + } + ); expect(acceptedSuggestion.state).toEqual(matchState(false)); expect(metadataValues).toMatchObject([ [ @@ -1010,15 +1088,16 @@ describe('suggestions', () => { }); it('should allow removal through partial acceptance, and update entities of all languages', async () => { - const { acceptedSuggestion, metadataValues, allFiles } = await prepareAndAcceptSuggestion( - ['1A', '1B'], - 'en', - 'property_multiselect', - 'multiselect_extractor', - { - removedValues: ['A'], - } - ); + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptSelectSuggestion( + ['1A', '1B'], + 'en', + 'property_multiselect', + 'multiselect_extractor', + { + removedValues: ['A'], + } + ); expect(acceptedSuggestion.state).toEqual(matchState(false)); expect(metadataValues).toMatchObject([ [{ value: '1A', label: '1A' }], @@ -1028,15 +1107,16 @@ describe('suggestions', () => { }); it('should do nothing on removal through partial acceptance if the id is not in the entity metadata', async () => { - const { acceptedSuggestion, metadataValues, allFiles } = await prepareAndAcceptSuggestion( - ['1A', 'A'], - 'en', - 'property_multiselect', - 'multiselect_extractor', - { - removedValues: ['B'], - } - ); + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptSelectSuggestion( + ['1A', 'A'], + 'en', + 'property_multiselect', + 'multiselect_extractor', + { + removedValues: ['B'], + } + ); expect(acceptedSuggestion.state).toEqual(matchState()); expect(metadataValues).toMatchObject([ [ @@ -1051,6 +1131,328 @@ describe('suggestions', () => { expect(allFiles).toEqual(selectAcceptanceFixtureBase.files); }); }); + + describe('relationship', () => { + beforeEach(async () => { + await db.setupFixturesAndContext(relationshipAcceptanceFixtureBase); + }); + + it('should validate that the entities in the suggestion exist', async () => { + const action = async () => { + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'X_sId', 'S2_sId', 'Y_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor' + ); + }; + await expect(action()).rejects.toThrow( + 'The following sharedIds do not exist in the database: X_sId, Y_sId.' + ); + }); + + it("should validate that the accepted id's through partial acceptance do exist on the suggestion", async () => { + const action = async () => { + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S2_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor', + { + addedValues: ['S1_sId', 'X_sId', 'Y_sId'], + } + ); + }; + await expect(action()).rejects.toThrow( + 'Some of the accepted values do not exist in the suggestion: X_sId, Y_sId. Cannot accept values that are not suggested.' + ); + }); + + it("should validate that the id's to remove through partial acceptance do not exist on the suggestion", async () => { + const action = async () => { + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S2_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor', + { + removedValues: ['S1_sId', 'S0_sId'], + } + ); + }; + await expect(action()).rejects.toThrow( + 'Some of the removed values exist in the suggestion: S1_sId. Cannot remove values that are suggested.' + ); + }); + + it('should allow full acceptance, and update entites of all languages, with the properly translated labels', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S3_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor' + ); + expect(acceptedSuggestion.state).toEqual(matchState(true)); + expect(metadataValues).toMatchObject([ + [ + { value: 'S1_sId', label: 'S1' }, + { value: 'S3_sId', label: 'S3' }, + ], + [ + { value: 'S1_sId', label: 'S1_es' }, + { value: 'S3_sId', label: 'S3_es' }, + ], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + }); + + it('should allow partial acceptance, and update entites of all languages, with the properly translated labels', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S3_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor', + { + addedValues: ['S3_sId'], + } + ); + expect(acceptedSuggestion.state).toEqual(matchState(false)); + expect(metadataValues).toMatchObject([ + [ + { value: 'S1_sId', label: 'S1' }, + { value: 'S2_sId', label: 'S2' }, + { value: 'S3_sId', label: 'S3' }, + ], + [ + { value: 'S1_sId', label: 'S1_es' }, + { value: 'S2_sId', label: 'S2_es' }, + { value: 'S3_sId', label: 'S3_es' }, + ], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + }); + + it('should do nothing on partial acceptance if the id is already in the entity metadata', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S3_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor', + { + addedValues: ['S1_sId'], + } + ); + expect(acceptedSuggestion.state).toEqual(matchState(false)); + expect(metadataValues).toMatchObject([ + [ + { value: 'S1_sId', label: 'S1' }, + { value: 'S2_sId', label: 'S2' }, + ], + [ + { value: 'S1_sId', label: 'S1_es' }, + { value: 'S2_sId', label: 'S2_es' }, + ], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + }); + + it('should allow removal through partial acceptance, and update entities of all languages', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S3_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor', + { + removedValues: ['S2_sId'], + } + ); + expect(acceptedSuggestion.state).toEqual(matchState(false)); + expect(metadataValues).toMatchObject([ + [{ value: 'S1_sId', label: 'S1' }], + [{ value: 'S1_sId', label: 'S1_es' }], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + }); + + it('should do nothing on removal through partial acceptance if the id is not in the entity metadata', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S2_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor', + { + removedValues: ['S3_sId'], + } + ); + expect(acceptedSuggestion.state).toEqual(matchState(true)); + expect(metadataValues).toMatchObject([ + [ + { value: 'S1_sId', label: 'S1' }, + { value: 'S2_sId', label: 'S2' }, + ], + [ + { value: 'S1_sId', label: 'S1_es' }, + { value: 'S2_sId', label: 'S2_es' }, + ], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + }); + + it('should update inherited values per language', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S3_sId'], + 'en', + 'relationship_with_inheritance', + 'relationship_with_inheritance_extractor' + ); + expect(acceptedSuggestion.state).toEqual(matchState(true)); + expect(metadataValues).toMatchObject([ + [ + { + value: 'S1_sId', + label: 'S1', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text', + }, + ], + }, + { + value: 'S3_sId', + label: 'S3', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text 3', + }, + ], + }, + ], + [ + { + value: 'S1_sId', + label: 'S1_es', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text Spanish', + }, + ], + }, + { + value: 'S3_sId', + label: 'S3_es', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text 3 Spanish', + }, + ], + }, + ], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + }); + + it('should check if the suggested entities are of the correct template', async () => { + const action = async () => { + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'other_source'], + 'en', + 'relationship_to_source', + 'relationship_extractor' + ); + }; + await expect(action()).rejects.toThrow( + 'The following sharedIds do not match the content template in the relationship property: other_source.' + ); + }); + + it('should handle relationship properties with any template as content', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S2_sId', 'other_source_2'], + 'en', + 'relationship_to_any', + 'relationship_to_any_extractor' + ); + expect(acceptedSuggestion.state).toEqual(matchState(true)); + expect(metadataValues).toMatchObject([ + [ + { + value: 'S2_sId', + label: 'S2', + }, + { + value: 'other_source_2', + label: 'Other Source 2', + }, + ], + [ + { + value: 'S2_sId', + label: 'S2_es', + }, + { + value: 'other_source_2', + label: 'Other Source 2 Spanish', + }, + ], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + }); + + it('should remove or create connections as necessary', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S3_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor' + ); + expect(acceptedSuggestion.state).toEqual(matchState(true)); + expect(metadataValues).toMatchObject([ + [ + { value: 'S1_sId', label: 'S1' }, + { value: 'S3_sId', label: 'S3' }, + ], + [ + { value: 'S1_sId', label: 'S1_es' }, + { value: 'S3_sId', label: 'S3_es' }, + ], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + + const removedConnection = await db.mongodb + ?.collection('connections') + .findOne({ entity: 'S2_sId', template: factory.id('related') }); + expect(removedConnection).toBeNull(); + + const newConnection = await db.mongodb + ?.collection('connections') + .findOne({ entity: 'S3_sId' }); + expect(newConnection).toMatchObject({ + entity: 'S3_sId', + hub: expect.any(ObjectId), + template: factory.id('related'), + }); + const newHub = newConnection?.hub; + const pairedConnection = await db.mongodb + ?.collection('connections') + .findOne({ entity: 'entityWithRelationships_sId', hub: newHub }); + expect(pairedConnection).toMatchObject({ + entity: 'entityWithRelationships_sId', + hub: newHub, + }); + }); + }); }); describe('save()', () => { diff --git a/app/api/suggestions/suggestions.ts b/app/api/suggestions/suggestions.ts index 225181bcc0..0e5f1ebd21 100644 --- a/app/api/suggestions/suggestions.ts +++ b/app/api/suggestions/suggestions.ts @@ -23,7 +23,7 @@ import { import { objectIndex } from 'shared/data_utils/objectIndex'; import { getSegmentedFilesIds, - propertyTypeIsSelectOrMultiSelect, + propertyTypeIsWithoutExtractedMetadata, } from 'api/services/informationextraction/getFiles'; import { Extractors } from 'api/services/informationextraction/ixextractors'; import { registerEventListeners } from './eventListeners'; @@ -48,7 +48,7 @@ const updateExtractedMetadata = async ( suggestions: IXSuggestionType[], property: PropertySchema ) => { - if (propertyTypeIsSelectOrMultiSelect(property.type)) return; + if (propertyTypeIsWithoutExtractedMetadata(property.type)) return; const fetchedFiles = await files.get({ _id: { $in: suggestions.map(s => s.fileId) } }); const suggestionsByFileId = objectIndex( @@ -205,15 +205,22 @@ const propertyTypesWithAllLanguages = new Set(['numeric', 'date', 'select', 'mul const needsAllLanguages = (propertyType: PropertySchema['type']) => propertyTypesWithAllLanguages.has(propertyType); +const validTypesForPartialAcceptance = new Set(['multiselect', 'relationship']); + +const typeIsValidForPartialAcceptance = (propertyType: string) => + validTypesForPartialAcceptance.has(propertyType); + const validatePartialAcceptanceTypeConstraint = ( acceptedSuggestions: AcceptedSuggestion[], property: PropertySchema ) => { const addedValuesExist = acceptedSuggestions.some(s => s.addedValues); const removedValuesExist = acceptedSuggestions.some(s => s.removedValues); - const multiSelectOnly = addedValuesExist || removedValuesExist; - if (property.type !== 'multiselect' && multiSelectOnly) { - throw new SuggestionAcceptanceError('Partial acceptance is only allowed for multiselects.'); + const partialAcceptanceTriggered = addedValuesExist || removedValuesExist; + if (!typeIsValidForPartialAcceptance(property.type) && partialAcceptanceTriggered) { + throw new SuggestionAcceptanceError( + 'Partial acceptance is only allowed for multiselects or relationships.' + ); } }; diff --git a/app/api/suggestions/updateEntities.ts b/app/api/suggestions/updateEntities.ts index d66e97090c..2639819f95 100644 --- a/app/api/suggestions/updateEntities.ts +++ b/app/api/suggestions/updateEntities.ts @@ -1,9 +1,11 @@ import entities from 'api/entities'; -import translations from 'api/i18n/translations'; +import { checkTypeIsAllowed } from 'api/services/informationextraction/ixextractors'; import thesauri from 'api/thesauri'; import { flatThesaurusValues } from 'api/thesauri/thesauri'; +import { ObjectId } from 'mongodb'; import { arrayBidirectionalDiff } from 'shared/data_utils/arrayBidirectionalDiff'; import { IndexTypes, objectIndex } from 'shared/data_utils/objectIndex'; +import { syncedPromiseLoop } from 'shared/data_utils/promiseUtils'; import { setIntersection } from 'shared/data_utils/setUtils'; import { ObjectIdSchema, PropertySchema } from 'shared/types/commonTypes'; import { EntitySchema } from 'shared/types/entityType'; @@ -19,6 +21,10 @@ interface AcceptedSuggestion { removedValues?: string[]; } +type EntityInfo = Record; + +const fetchNoResources = async () => ({}); + const fetchThesaurus = async (thesaurusId: PropertySchema['content']) => { const dict = await thesauri.getById(thesaurusId); const thesaurusName = dict!.name; @@ -31,39 +37,61 @@ const fetchThesaurus = async (thesaurusId: PropertySchema['content']) => { return { name: thesaurusName, id: thesaurusId, indexedlabels }; }; -const fetchTranslations = async (property: PropertySchema) => { - const trs = await translations.get({ context: property.content }); - const indexed = objectIndex( - trs, - t => t.locale || '', - t => t.contexts?.[0].values +const fetchEntityInfo = async ( + _property: PropertySchema, + acceptedSuggestions: AcceptedSuggestion[], + suggestions: IXSuggestionType[] +): Promise<{ entityInfo: EntityInfo }> => { + const suggestionSharedIds = suggestions.map(s => s.suggestedValue).flat(); + const addedSharedIds = acceptedSuggestions.map(s => s.addedValues || []).flat(); + const expectedSharedIds = Array.from(new Set(suggestionSharedIds.concat(addedSharedIds))); + const entitiesInDb = (await entities.get({ sharedId: { $in: expectedSharedIds } }, [ + 'sharedId', + 'template', + ])) as { sharedId: string; template: ObjectId }[]; + const indexedBySharedId = objectIndex( + entitiesInDb, + e => e.sharedId, + e => e ); - return indexed; + return { entityInfo: indexedBySharedId }; }; const fetchSelectResources = async (property: PropertySchema) => { const thesaurus = await fetchThesaurus(property.content); - const labelTranslations = await fetchTranslations(property); - return { thesaurus, translations: labelTranslations }; + return { thesaurus }; }; const resourceFetchers = { - _default: async () => ({}), + title: fetchNoResources, + text: fetchNoResources, + numeric: fetchNoResources, + date: fetchNoResources, select: fetchSelectResources, multiselect: fetchSelectResources, + relationship: fetchEntityInfo, }; -const fetchResources = async (property: PropertySchema) => { - // @ts-ignore - const fetcher = resourceFetchers[property.type] || resourceFetchers._default; - return fetcher(property); +const fetchResources = async ( + property: PropertySchema, + acceptedSuggestions: AcceptedSuggestion[], + suggestions: IXSuggestionType[] +) => { + const type = checkTypeIsAllowed(property.type); + const fetcher = resourceFetchers[type]; + return fetcher(property, acceptedSuggestions, suggestions); }; +const getAcceptedSuggestion = ( + entity: EntitySchema, + acceptedSuggestionsBySharedId: Record +): AcceptedSuggestion => acceptedSuggestionsBySharedId[entity.sharedId || '']; + const getSuggestion = ( entity: EntitySchema, suggestionsById: Record, acceptedSuggestionsBySharedId: Record -) => suggestionsById[acceptedSuggestionsBySharedId[entity.sharedId || '']._id.toString()]; +) => suggestionsById[getAcceptedSuggestion(entity, acceptedSuggestionsBySharedId)._id.toString()]; const getRawValue = ( entity: EntitySchema, @@ -88,17 +116,6 @@ const checkValuesInThesaurus = ( } }; -const mapLabels = ( - values: string[], - entity: EntitySchema, - thesaurus: { indexedlabels: Record }, - translation: Record> -) => { - const labels = values.map(v => thesaurus.indexedlabels[v]); - const translatedLabels = labels.map(l => translation[entity.language || '']?.[l]); - return values.map((value, index) => ({ value, label: translatedLabels[index] })); -}; - function readAddedValues(acceptedSuggestion: AcceptedSuggestion, suggestionValues: string[]) { const addedValues = acceptedSuggestion.addedValues || []; const addedButNotSuggested = arrayBidirectionalDiff( @@ -146,7 +163,7 @@ function mixFinalValues( return finalValues; } -function arrangeValues( +function arrangeAddedOrRemovedValues( acceptedSuggestion: AcceptedSuggestion, suggestionValues: string[], entity: EntitySchema, @@ -163,36 +180,66 @@ function arrangeValues( return finalValues; } +function checkSharedIds(values: string[], entityInfo: EntityInfo) { + const missingSharedIds = values.filter(v => !(v in entityInfo)); + if (missingSharedIds.length > 0) { + throw new SuggestionAcceptanceError( + `The following sharedIds do not exist in the database: ${missingSharedIds.join(', ')}.` + ); + } +} + +function checkTemplates(property: PropertySchema, values: string[], entityInfo: EntityInfo) { + const { content } = property; + if (!content) return; + const templateId = new ObjectId(content); + const wrongTemplatedSharedIds = values.filter( + v => entityInfo[v].template.toString() !== templateId.toString() + ); + if (wrongTemplatedSharedIds.length > 0) { + throw new SuggestionAcceptanceError( + `The following sharedIds do not match the content template in the relationship property: ${wrongTemplatedSharedIds.join(', ')}.` + ); + } +} + +const getRawValueAsArray = ( + _property: PropertySchema, + entity: EntitySchema, + suggestionsById: Record, + acceptedSuggestionsBySharedId: Record +) => [ + { + value: getRawValue(entity, suggestionsById, acceptedSuggestionsBySharedId), + }, +]; + const valueGetters = { - _default: ( - entity: EntitySchema, - suggestionsById: Record, - acceptedSuggestionsBySharedId: Record - ) => [ - { - value: getRawValue(entity, suggestionsById, acceptedSuggestionsBySharedId), - }, - ], + text: getRawValueAsArray, + date: getRawValueAsArray, + numeric: getRawValueAsArray, select: ( + _property: PropertySchema, entity: EntitySchema, suggestionsById: Record, acceptedSuggestionsBySharedId: Record, resources: any ) => { - const { thesaurus, translations: translation } = resources; + const { thesaurus } = resources; const value = getRawValue(entity, suggestionsById, acceptedSuggestionsBySharedId) as string; checkValuesInThesaurus([value], thesaurus.name, thesaurus.indexedlabels); - return mapLabels([value], entity, thesaurus, translation); + return [{ value }]; }, multiselect: ( + _property: PropertySchema, entity: EntitySchema, suggestionsById: Record, acceptedSuggestionsBySharedId: Record, resources: any ) => { - const { thesaurus, translations: translation } = resources; - const acceptedSuggestion = acceptedSuggestionsBySharedId[entity.sharedId || '']; + const { thesaurus } = resources; + const acceptedSuggestion = getAcceptedSuggestion(entity, acceptedSuggestionsBySharedId); const suggestion = getSuggestion(entity, suggestionsById, acceptedSuggestionsBySharedId); const suggestionValues = getRawValue( entity, @@ -201,14 +248,42 @@ const valueGetters = { ) as string[]; checkValuesInThesaurus(suggestionValues, thesaurus.name, thesaurus.indexedlabels); - const finalValues: string[] = arrangeValues( + const finalValues: string[] = arrangeAddedOrRemovedValues( acceptedSuggestion, suggestionValues, entity, suggestion ); - return mapLabels(finalValues, entity, thesaurus, translation); + return finalValues.map(value => ({ value })); + }, + relationship: ( + property: PropertySchema, + entity: EntitySchema, + suggestionsById: Record, + acceptedSuggestionsBySharedId: Record, + resources: any + ) => { + const { entityInfo } = resources; + + const acceptedSuggestion = getAcceptedSuggestion(entity, acceptedSuggestionsBySharedId); + const suggestion = getSuggestion(entity, suggestionsById, acceptedSuggestionsBySharedId); + const suggestionValues = getRawValue( + entity, + suggestionsById, + acceptedSuggestionsBySharedId + ) as string[]; + checkSharedIds(suggestionValues, entityInfo); + checkTemplates(property, suggestionValues, entityInfo); + + const finalValues: string[] = arrangeAddedOrRemovedValues( + acceptedSuggestion, + suggestionValues, + entity, + suggestion + ); + + return finalValues.map(value => ({ value })); }, }; @@ -219,9 +294,18 @@ const getValue = ( acceptedSuggestionsBySharedId: Record, resources: any ) => { - // @ts-ignore - const getter = valueGetters[property.type] || valueGetters._default; - return getter(entity, suggestionsById, acceptedSuggestionsBySharedId, resources); + const type = checkTypeIsAllowed(property.type); + if (type === 'title') { + throw new SuggestionAcceptanceError('Title should not be handled here.'); + } + const getter = valueGetters[type]; + return getter(property, entity, suggestionsById, acceptedSuggestionsBySharedId, resources); +}; + +const saveEntities = async (entitiesToUpdate: EntitySchema[]) => { + await syncedPromiseLoop(entitiesToUpdate, async (entity: EntitySchema) => { + await entities.save(entity, { user: {}, language: entity.language }); + }); }; const updateEntitiesWithSuggestion = async ( @@ -249,7 +333,7 @@ const updateEntitiesWithSuggestion = async ( s => s ); - const resources = await fetchResources(property); + const resources = await fetchResources(property, acceptedSuggestions, suggestions); const entitiesToUpdate = propertyName !== 'title' @@ -272,7 +356,7 @@ const updateEntitiesWithSuggestion = async ( title: getRawValue(entity, suggestionsById, acceptedSuggestionsBySharedId), })); - await entities.saveMultiple(entitiesToUpdate); + await saveEntities(entitiesToUpdate); }; export { updateEntitiesWithSuggestion, SuggestionAcceptanceError }; diff --git a/app/api/suggestions/updateState.ts b/app/api/suggestions/updateState.ts index 78bb8a1e25..a352033ed0 100644 --- a/app/api/suggestions/updateState.ts +++ b/app/api/suggestions/updateState.ts @@ -2,7 +2,7 @@ import settings from 'api/settings'; import templates from 'api/templates'; import { objectIndex } from 'shared/data_utils/objectIndex'; import { CurrentValue, getSuggestionState, SuggestionValues } from 'shared/getIXSuggestionState'; -import { propertyIsMultiselect } from 'shared/propertyTypes'; +import { propertyIsMultiselect, propertyIsRelationship } from 'shared/propertyTypes'; import { LanguagesListSchema, PropertyTypeSchema } from 'shared/types/commonTypes'; import { IXSuggestionsModel } from './IXSuggestionsModel'; import { @@ -95,7 +95,9 @@ const postProcessCurrentValue = ( suggestion: SuggestionsAggregationResult, propertyType: PropertyTypeSchema ): PostProcessedAggregationResult => { - if (propertyIsMultiselect(propertyType)) return suggestion; + if (propertyIsMultiselect(propertyType) || propertyIsRelationship(propertyType)) { + return suggestion; + } return { ...suggestion, currentValue: suggestion.currentValue.length > 0 ? suggestion.currentValue[0] : '', diff --git a/app/react/Routes.tsx b/app/react/Routes.tsx index 2b175ba633..7a586d3b51 100644 --- a/app/react/Routes.tsx +++ b/app/react/Routes.tsx @@ -113,16 +113,18 @@ const getRoutesLayout = ( )} /> )} /> - )} - loader={IXdashboardLoader(headers)} - /> - )} - /> + + )} + loader={IXdashboardLoader(headers)} + /> + )} + /> + }> {layout} - + {layout} } /> diff --git a/app/react/V2/Components/Forms/MultiselectList.tsx b/app/react/V2/Components/Forms/MultiselectList.tsx index f1d172f781..a981a2aff0 100644 --- a/app/react/V2/Components/Forms/MultiselectList.tsx +++ b/app/react/V2/Components/Forms/MultiselectList.tsx @@ -47,6 +47,7 @@ const MultiselectList = ({ singleSelect = false, allowSelelectAll = false, }: MultiselectListProps) => { + console.log(value); const [selectedItems, setSelectedItems] = useState(value || []); const [showAll, setShowAll] = useState(true); const [searchTerm, setSearchTerm] = useState(''); diff --git a/app/react/V2/Components/Layouts/SettingsContent.tsx b/app/react/V2/Components/Layouts/SettingsContent.tsx index 960ba744f1..8407bc1ff1 100644 --- a/app/react/V2/Components/Layouts/SettingsContent.tsx +++ b/app/react/V2/Components/Layouts/SettingsContent.tsx @@ -1,9 +1,8 @@ /* eslint-disable react/no-multi-comp */ -import { Link } from 'react-router-dom'; import React, { PropsWithChildren } from 'react'; import { Breadcrumb } from 'flowbite-react'; import { ChevronLeftIcon } from '@heroicons/react/20/solid'; -import { Translate } from 'app/I18N'; +import { I18NLink, Translate } from 'app/I18N'; interface SettingsContentProps extends PropsWithChildren { className?: string; @@ -31,12 +30,12 @@ const SettingsContent = ({ children, className }: SettingsContentProps) => ( const SettingsHeader = ({ contextId, title, children, path, className }: SettingsHeaderProps) => (
- + Navigate back - + {Array.from(path?.entries() || []).map(([key, value]) => ( diff --git a/app/react/V2/Components/Layouts/specs/SettingsContent.cy.tsx b/app/react/V2/Components/Layouts/specs/SettingsContent.cy.tsx index e8b5a9c7d2..4bb80c381c 100644 --- a/app/react/V2/Components/Layouts/specs/SettingsContent.cy.tsx +++ b/app/react/V2/Components/Layouts/specs/SettingsContent.cy.tsx @@ -47,7 +47,7 @@ describe('ConfirmationModal', () => { cy.get('[data-testid="settings-content-header"]') .invoke('text') .should('contain', 'Root PathMiddle PathLeafCurrent page'); - cy.get('a[href="/settings"]').should('not.be.visible'); + cy.get('a[href="/en/settings"]').should('not.be.visible'); cy.contains('a', 'Root Path').invoke('attr', 'href').should('include', '#top'); cy.contains('a', 'Middle Path').invoke('attr', 'href').should('include', '#bottom'); cy.contains('a', 'Leaf').invoke('attr', 'href').should('include', '#footer'); @@ -58,6 +58,6 @@ describe('ConfirmationModal', () => { it('should have an arrow to return to settings menu for mobile', () => { cy.viewport(450, 650); render(); - cy.get('a[href="/settings"]').should('be.visible'); + cy.get('a[href="/en/settings"]').should('be.visible'); }); }); diff --git a/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx b/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx index eadacd19f8..1cc8284355 100644 --- a/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx +++ b/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx @@ -92,7 +92,7 @@ const IXSuggestions = () => { }, [templates, extractor]); useMemo(() => { - if (property?.type === 'multiselect') { + if (property?.type === 'multiselect' || property?.type === 'relationship') { const flatenedSuggestions = suggestions.map(suggestion => generateChildrenRows(suggestion as MultiValueSuggestion) ); @@ -268,7 +268,7 @@ const IXSuggestions = () => {
diff --git a/app/react/V2/Routes/Settings/IX/components/ExtractorModal.tsx b/app/react/V2/Routes/Settings/IX/components/ExtractorModal.tsx index e2c0bb24d9..2549f6cd2f 100644 --- a/app/react/V2/Routes/Settings/IX/components/ExtractorModal.tsx +++ b/app/react/V2/Routes/Settings/IX/components/ExtractorModal.tsx @@ -11,7 +11,10 @@ import { InputField } from 'app/V2/Components/Forms/InputField'; import { RadioSelect } from 'app/V2/Components/Forms'; import { propertyIcons } from './Icons'; -const SUPPORTED_PROPERTIES = ['text', 'numeric', 'date', 'select', 'multiselect']; +const SUPPORTED_PROPERTIES = ['text', 'numeric', 'date', 'select', 'multiselect', 'relationship']; +type SupportedProperty = Omit & { + type: 'text' | 'numeric' | 'date' | 'select' | 'multiselect' | 'relationship'; +}; interface ExtractorModalProps { setShowModal: React.Dispatch>; @@ -21,30 +24,11 @@ interface ExtractorModalProps { extractor?: IXExtractorInfo; } -const getPropertyLabel = (property: ClientPropertySchema, templateId: string) => { - let icon: React.ReactNode; - let propertyTypeTranslationKey = 'property text'; +const getPropertyLabel = (property: SupportedProperty, templateId: string) => { + const { type } = property; - switch (property.type) { - case 'numeric': - icon = propertyIcons.numeric; - propertyTypeTranslationKey = 'property numeric'; - break; - case 'date': - icon = propertyIcons.date; - propertyTypeTranslationKey = 'property date'; - break; - case 'select': - icon = propertyIcons.select; - propertyTypeTranslationKey = 'property select'; - break; - case 'multiselect': - icon = propertyIcons.multiselect; - propertyTypeTranslationKey = 'property multiselect'; - break; - default: - icon = propertyIcons.text; - } + const icon = propertyIcons[type]; + const propertyTypeTranslationKey = `property ${type}`; return (
@@ -77,7 +61,7 @@ const formatOptions = (values: string[], templates: ClientTemplateSchema[]) => { SUPPORTED_PROPERTIES.includes(prop.type) ) .map(prop => ({ - label: getPropertyLabel(prop, template._id), + label: getPropertyLabel(prop as SupportedProperty, template._id), value: `${template._id?.toString()}-${prop.name}`, searchLabel: prop.label, })), @@ -114,7 +98,7 @@ const getPropertyForValue = (value: string, templates: ClientTemplateSchema[]) = const matchedProperty = matchedTemplate?.properties.find( property => property.name === propertyName - ); + ) as SupportedProperty; if (matchedProperty) { return getPropertyLabel(matchedProperty, matchedTemplate!._id.toString()); diff --git a/app/react/V2/Routes/Settings/IX/components/Icons.tsx b/app/react/V2/Routes/Settings/IX/components/Icons.tsx index c02e8dc7d3..daab3aa486 100644 --- a/app/react/V2/Routes/Settings/IX/components/Icons.tsx +++ b/app/react/V2/Routes/Settings/IX/components/Icons.tsx @@ -5,6 +5,7 @@ import { NumericPropertyIcon, SelectPropertyIcon, TextPropertyIcon, + RelationshipPropertyIcon, } from 'V2/Components/CustomIcons'; const propertyIcons = { @@ -14,6 +15,7 @@ const propertyIcons = { markdown: , select: , multiselect: , + relationship: , }; export { propertyIcons }; diff --git a/app/react/V2/Routes/Settings/IX/components/PDFSidepanel.tsx b/app/react/V2/Routes/Settings/IX/components/PDFSidepanel.tsx index f78c129e79..40637ddc42 100644 --- a/app/react/V2/Routes/Settings/IX/components/PDFSidepanel.tsx +++ b/app/react/V2/Routes/Settings/IX/components/PDFSidepanel.tsx @@ -62,7 +62,7 @@ const getFormValue = ( value = dateString; } - if (type === 'select' || type === 'multiselect') { + if (type === 'select' || type === 'multiselect' || type === 'relationship') { value = entityMetadata?.map((metadata: MetadataObjectSchema) => metadata.value); } } @@ -367,7 +367,7 @@ const PDFSidepanel = ({ items?: Option[]; } - const renderSelect = (type: 'select' | 'multiselect') => { + const renderSelect = (type: 'select' | 'multiselect' | 'relationship') => { const options: Option[] = []; thesaurus?.values.forEach((value: any) => { options.push({ @@ -400,6 +400,7 @@ const PDFSidepanel = ({ return renderInputText(property?.type); case 'select': case 'multiselect': + case 'relationship': return renderSelect(property?.type); default: return ''; @@ -446,7 +447,7 @@ const PDFSidepanel = ({ {' '}
diff --git a/app/react/V2/Routes/Settings/IX/components/SuggestedValue.tsx b/app/react/V2/Routes/Settings/IX/components/SuggestedValue.tsx index 8c51842657..b9c66d377e 100644 --- a/app/react/V2/Routes/Settings/IX/components/SuggestedValue.tsx +++ b/app/react/V2/Routes/Settings/IX/components/SuggestedValue.tsx @@ -52,7 +52,7 @@ const SuggestedValue = ({ return secondsToDate(value, locale); } - if (type === 'select' || type === 'multiselect') { + if (type === 'select' || type === 'multiselect' || type === 'relationship') { const label = thesaurus?.values.find(v => v.id === value)?.label; return {label}; } @@ -67,7 +67,7 @@ const SuggestedValue = ({ if (type === 'date') { return secondsToDate((suggestion.suggestedValue as string | number) || '', locale); } - if (type === 'select' || type === 'multiselect') { + if (type === 'select' || type === 'multiselect' || type === 'relationship') { const label = thesaurus?.values.find(v => v.id === suggestion.suggestedValue)?.label; return {label}; } diff --git a/app/react/V2/Routes/Settings/IX/components/helpers.ts b/app/react/V2/Routes/Settings/IX/components/helpers.ts index 43bf59487b..7b4e10ad95 100644 --- a/app/react/V2/Routes/Settings/IX/components/helpers.ts +++ b/app/react/V2/Routes/Settings/IX/components/helpers.ts @@ -105,7 +105,7 @@ const updateSuggestionsByEntity = ( if (updatedEntity.metadata[propertyToUpdate]?.length) { const newValue = ( - property?.type === 'multiselect' + property?.type === 'multiselect' || property?.type === 'relationship' ? updatedEntity.metadata[propertyToUpdate]?.map(v => v.value) : updatedEntity.metadata[propertyToUpdate]![0].value ) as SuggestionValue; @@ -119,7 +119,7 @@ const updateSuggestionsByEntity = ( suggestionToUpdate.state.match = suggestionToUpdate.suggestedValue === ''; } - if (property?.type === 'multiselect') { + if (property?.type === 'multiselect' || property?.type === 'relationship') { suggestionToUpdate = generateChildrenRows(suggestionToUpdate as MultiValueSuggestion); } diff --git a/app/shared/getIXSuggestionState.ts b/app/shared/getIXSuggestionState.ts index b743e8ec91..44b807eaaa 100644 --- a/app/shared/getIXSuggestionState.ts +++ b/app/shared/getIXSuggestionState.ts @@ -4,10 +4,14 @@ import { IXSuggestionStateType } from './types/suggestionType'; import { setsEqual } from './data_utils/setUtils'; import { propertyIsMultiselect, + propertyIsRelationship, propertyIsSelect, propertyIsSelectOrMultiSelect, } from './propertyTypes'; +const propertyIsMultiValued = (propertyType: PropertySchema['type']) => + propertyIsMultiselect(propertyType) || propertyIsRelationship(propertyType); + type CurrentValue = string | number | null; interface SuggestionValues { @@ -27,6 +31,7 @@ const sameValueSet = (first: any, second: any) => setsEqual(first || [], second const EQUALITIES: Record boolean> = { date: isSameDate, multiselect: sameValueSet, + relationship: sameValueSet, }; const equalsForType = (type: PropertySchema['type']) => (first: any, second: any) => @@ -67,7 +72,7 @@ class IXSuggestionState implements IXSuggestionStateType { if ( labeledValue || (propertyIsSelect(propertyType) && currentValue) || - (propertyIsMultiselect(propertyType) && + (propertyIsMultiValued(propertyType) && Array.isArray(currentValue) && currentValue.length > 0) ) { @@ -76,7 +81,7 @@ class IXSuggestionState implements IXSuggestionStateType { } setWithValue({ currentValue }: SuggestionValues, propertyType: PropertySchema['type']) { - if (propertyIsMultiselect(propertyType) && Array.isArray(currentValue)) { + if (propertyIsMultiValued(propertyType) && Array.isArray(currentValue)) { this.withValue = currentValue?.length > 0; } else if (currentValue) { this.withValue = true; @@ -103,7 +108,11 @@ class IXSuggestionState implements IXSuggestionStateType { } setHasContext({ segment }: SuggestionValues, propertyType: PropertySchema['type']) { - if (segment || propertyIsSelectOrMultiSelect(propertyType)) { + if ( + segment || + propertyIsSelectOrMultiSelect(propertyType) || + propertyIsRelationship(propertyType) + ) { this.hasContext = true; } } diff --git a/app/shared/propertyTypes.ts b/app/shared/propertyTypes.ts index 344abd80db..3e4889269b 100644 --- a/app/shared/propertyTypes.ts +++ b/app/shared/propertyTypes.ts @@ -49,10 +49,13 @@ const propertyIsMultiselect = (propertyType: PropertySchema['type']) => const propertyIsSelectOrMultiSelect = (propertyType: PropertySchema['type']) => propertyIsSelect(propertyType) || propertyIsMultiselect(propertyType); +const propertyIsRelationship = (propertyType: string) => propertyType === 'relationship'; + export { propertyTypes, getCompatibleTypes, propertyIsSelect, propertyIsMultiselect, propertyIsSelectOrMultiSelect, + propertyIsRelationship, }; diff --git a/app/shared/tsUtils.ts b/app/shared/tsUtils.ts index 0b4417980e..819dc93848 100644 --- a/app/shared/tsUtils.ts +++ b/app/shared/tsUtils.ts @@ -3,6 +3,8 @@ import { isObject, isString } from 'lodash'; import ValidationError from 'ajv/dist/runtime/validation_error'; import { ClientBlobFile } from 'app/istore'; +export type Subset = T; + export const isBlobFile = (file: unknown): file is ClientBlobFile => isObject(file) && isString((file as ClientBlobFile).data);