diff --git a/pom.xml b/pom.xml index fd485e53bd..cf6b79deff 100644 --- a/pom.xml +++ b/pom.xml @@ -178,6 +178,11 @@ jackson-dataformat-yaml ${jackson.version} + + com.fasterxml.jackson.dataformat + jackson-dataformat-csv + ${jackson.version} + com.fasterxml.jackson.datatype jackson-datatype-jdk8 diff --git a/waltz-common/src/main/java/org/finos/waltz/common/MapUtilities.java b/waltz-common/src/main/java/org/finos/waltz/common/MapUtilities.java index 6c1d5df0be..4d1398236d 100644 --- a/waltz-common/src/main/java/org/finos/waltz/common/MapUtilities.java +++ b/waltz-common/src/main/java/org/finos/waltz/common/MapUtilities.java @@ -217,6 +217,11 @@ public static Map indexBy(Collection xs, public static Map countBy(Function keyFn, Collection xs) { + return countBy(xs, keyFn); + } + + public static Map countBy(Collection xs, + Function keyFn) { if (xs == null) { return emptyMap(); } diff --git a/waltz-common/src/main/java/org/finos/waltz/common/SetUtilities.java b/waltz-common/src/main/java/org/finos/waltz/common/SetUtilities.java index 584e55ecca..917f0220e0 100644 --- a/waltz-common/src/main/java/org/finos/waltz/common/SetUtilities.java +++ b/waltz-common/src/main/java/org/finos/waltz/common/SetUtilities.java @@ -182,4 +182,8 @@ public static Set add(Set orig, T... ts) { public static Set remove(Set orig, T... ts) { return minus(orig, asSet(ts)); } + + public static Set compact(T... ts) { + return remove(asSet(ts), (T) null); + } } diff --git a/waltz-common/src/main/java/org/finos/waltz/common/StreamUtilities.java b/waltz-common/src/main/java/org/finos/waltz/common/StreamUtilities.java index 09951e2c3f..f918f64ca7 100644 --- a/waltz-common/src/main/java/org/finos/waltz/common/StreamUtilities.java +++ b/waltz-common/src/main/java/org/finos/waltz/common/StreamUtilities.java @@ -73,6 +73,12 @@ public static Stream concat(Stream... streams) { } + public static Stream lines(String multiLineStr) { + String[] lines = multiLineStr.split("(\n|\r|\r\n)"); + return Stream.of(lines); + } + + public static class Siphon implements Predicate { private final Predicate pred; private final List results = new ArrayList<>(); diff --git a/waltz-common/src/main/java/org/finos/waltz/common/StringUtilities.java b/waltz-common/src/main/java/org/finos/waltz/common/StringUtilities.java index 72be7336aa..0b937cb4de 100644 --- a/waltz-common/src/main/java/org/finos/waltz/common/StringUtilities.java +++ b/waltz-common/src/main/java/org/finos/waltz/common/StringUtilities.java @@ -327,5 +327,37 @@ public static Optional firstNonNull(String... xs) { .findFirst(); } + + public static boolean safeEq(String expected, String actual) { + if(actual == null && expected == null) { + return true; + } else if(actual == null || expected == null) { + return false; + } else { + return actual.equals(expected); + } + } + + + public static boolean safeEqIgnoreCase(String expected, String actual) { + if(actual == null && expected == null) { + return true; + } else if(actual == null || expected == null) { + return false; + } else { + return actual.equalsIgnoreCase(expected); + } + } + + + public static String mkExternalId(String str) { + return sanitizeCharacters(mkSafe(str)) + .toUpperCase() + .replaceAll("&", " AND ") + .replaceAll("[\\s\\-(){}/\\\\,.*;]", "_") + .replaceAll("_+", "_"); + } + + } diff --git a/waltz-common/src/test/java/org/finos/waltz/common/StreamUtilities_linesTest.java b/waltz-common/src/test/java/org/finos/waltz/common/StreamUtilities_linesTest.java new file mode 100644 index 0000000000..12a760c67b --- /dev/null +++ b/waltz-common/src/test/java/org/finos/waltz/common/StreamUtilities_linesTest.java @@ -0,0 +1,49 @@ +package org.finos.waltz.common; + +import org.junit.jupiter.api.Test; + +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class StreamUtilities_linesTest { + + @Test + public void simpleCase(){ + Stream emptyLine = StreamUtilities + .lines(""); + + assertEquals("", emptyLine.collect(Collectors.joining(""))); + + } + + + @Test + public void multiLines(){ + Stream output = StreamUtilities + .lines("a\nb\nc"); + + assertEquals("a#b#c", output.collect(Collectors.joining("#"))); + + } + + + @Test + public void onlyOneLine(){ + Stream output = StreamUtilities + .lines("ddd"); + + assertEquals("", output.collect(Collectors.joining("#", "<", ">"))); + + } + + + @Test + public void emptyLines(){ + Stream output = StreamUtilities + .lines("hello\n\nworld"); + + assertEquals("", output.collect(Collectors.joining("#", "<", ">"))); + } +} diff --git a/waltz-data/src/main/java/org/finos/waltz/data/measurable/MeasurableDao.java b/waltz-data/src/main/java/org/finos/waltz/data/measurable/MeasurableDao.java index bcd9f82b24..565f2d5a02 100644 --- a/waltz-data/src/main/java/org/finos/waltz/data/measurable/MeasurableDao.java +++ b/waltz-data/src/main/java/org/finos/waltz/data/measurable/MeasurableDao.java @@ -298,6 +298,16 @@ public List findByCategoryId(Long categoryId) { } + public List findByCategoryId(Long categoryId, Set statuses) { + return dsl + .select(MEASURABLE.fields()) + .from(MEASURABLE) + .where(MEASURABLE.MEASURABLE_CATEGORY_ID.eq(categoryId)) + .and(MEASURABLE.ENTITY_LIFECYCLE_STATUS.in(statuses)) + .fetch(TO_DOMAIN_MAPPER); + } + + public List findByParentId(Long parentId) { return dsl .select(MEASURABLE.fields()) diff --git a/waltz-integration-test/src/test/java/org/finos/waltz/integration_test/inmem/service/BulkTaxonomyChangeServiceTest.java b/waltz-integration-test/src/test/java/org/finos/waltz/integration_test/inmem/service/BulkTaxonomyChangeServiceTest.java new file mode 100644 index 0000000000..cacc1d56dc --- /dev/null +++ b/waltz-integration-test/src/test/java/org/finos/waltz/integration_test/inmem/service/BulkTaxonomyChangeServiceTest.java @@ -0,0 +1,354 @@ +package org.finos.waltz.integration_test.inmem.service; + +import org.finos.waltz.common.CollectionUtilities; +import org.finos.waltz.common.ListUtilities; +import org.finos.waltz.common.SetUtilities; +import org.finos.waltz.integration_test.inmem.BaseInMemoryIntegrationTest; +import org.finos.waltz.model.EntityKind; +import org.finos.waltz.model.EntityLifecycleStatus; +import org.finos.waltz.model.EntityReference; +import org.finos.waltz.model.bulk_upload.BulkUpdateMode; +import org.finos.waltz.model.bulk_upload.ChangeOperation; +import org.finos.waltz.model.bulk_upload.taxonomy.BulkTaxonomyValidatedItem; +import org.finos.waltz.model.bulk_upload.taxonomy.BulkTaxonomyValidationResult; +import org.finos.waltz.model.bulk_upload.taxonomy.ChangedFieldType; +import org.finos.waltz.model.measurable.Measurable; +import org.finos.waltz.model.user.SystemRole; +import org.finos.waltz.service.measurable.MeasurableService; +import org.finos.waltz.service.taxonomy_management.BulkTaxonomyChangeService; +import org.finos.waltz.service.taxonomy_management.BulkTaxonomyItemParser; +import org.finos.waltz.test_common.helpers.MeasurableHelper; +import org.finos.waltz.test_common.helpers.UserHelper; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static java.lang.String.format; +import static org.finos.waltz.common.CollectionUtilities.all; +import static org.finos.waltz.common.CollectionUtilities.find; +import static org.finos.waltz.common.CollectionUtilities.isEmpty; +import static org.finos.waltz.common.ListUtilities.asList; +import static org.finos.waltz.common.SetUtilities.asSet; +import static org.finos.waltz.model.EntityReference.mkRef; +import static org.finos.waltz.test_common.helpers.NameHelper.mkName; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class BulkTaxonomyChangeServiceTest extends BaseInMemoryIntegrationTest { + + @Autowired + private UserHelper userHelper; + + @Autowired + private MeasurableHelper measurableHelper; + + @Autowired + private MeasurableService measurableService; + + @Autowired + private BulkTaxonomyChangeService taxonomyChangeService; + + + @Test + public void previewBrandNewTaxonomy() { + EntityReference category = setupCategory(); + + BulkTaxonomyValidationResult result = taxonomyChangeService.previewBulk( + category, + mkSimpleTsv(), + BulkTaxonomyItemParser.InputFormat.CSV, + BulkUpdateMode.ADD_ONLY); + + assertNotNull(result, "Expected a result"); + assertNoErrors(result); + + assertTrue( + all(result.validatedItems(), d -> d.changeOperation() == ChangeOperation.ADD), + "Should be all adds as we are using a new taxonomy"); + + assertNoRemovals(result); + assertExternalIdsMatch(result, asList("a1", "a1.1", "a1.2")); + } + + + @Test + public void previewUpdates() { + EntityReference category = setupCategory(); + measurableHelper.createMeasurable("a1", "A1", category.id()); + + BulkTaxonomyValidationResult result = taxonomyChangeService.previewBulk( + category, + mkSimpleTsv(), + BulkTaxonomyItemParser.InputFormat.CSV, + BulkUpdateMode.ADD_ONLY); + + assertNotNull(result, "Expected a result"); + assertNoErrors(result); + + assertNoRemovals(result); + assertExternalIdsMatch(result, asList("a1", "a1.1", "a1.2")); + + assertOperation(result, "a1", ChangeOperation.UPDATE); + assertOperation(result, "a1.1", ChangeOperation.ADD); + assertOperation(result, "a1.2", ChangeOperation.ADD); + + assertChangedFields( + result, + "a1", + asSet(ChangedFieldType.DESCRIPTION, ChangedFieldType.CONCRETE)); + } + + + @Test + public void previewRemovalsIfModeIsReplace() { + EntityReference category = setupCategory(); + measurableHelper.createMeasurable("z1", "Z1", category.id()); + + BulkTaxonomyValidationResult result = taxonomyChangeService.previewBulk( + category, + mkSimpleTsv(), + BulkTaxonomyItemParser.InputFormat.CSV, + BulkUpdateMode.REPLACE); + + assertNotNull(result, "Expected a result"); + assertNoErrors(result); + + assertHasRemovals(result, asSet("z1")); + assertExternalIdsMatch(result, asList("a1", "a1.1", "a1.2")); + + assertOperation(result, "a1", ChangeOperation.ADD); + assertOperation(result, "a1.1", ChangeOperation.ADD); + assertOperation(result, "a1.2", ChangeOperation.ADD); + } + + + + @Test + public void previewRestorationsIfMeasurableWasPreviouslyRemoved() { + EntityReference category = setupCategory(); + long a1_1_id = measurableHelper.createMeasurable("a1.1", "", category.id()); + // set a1_1_id as removed + measurableHelper.updateMeasurableLifecycleStatus(a1_1_id, EntityLifecycleStatus.REMOVED); + + BulkTaxonomyValidationResult result = taxonomyChangeService.previewBulk( + category, + mkSimpleTsv(), + BulkTaxonomyItemParser.InputFormat.CSV, + BulkUpdateMode.REPLACE); + + assertNotNull(result, "Expected a result"); + assertNoErrors(result); + + assertExternalIdsMatch(result, asList("a1", "a1.1", "a1.2")); + + assertOperation(result, "a1", ChangeOperation.ADD); + assertOperation(result, "a1.1", ChangeOperation.RESTORE); + assertOperation(result, "a1.2", ChangeOperation.ADD); + } + + + @Test + public void previewMultipleRemovalsIfModeIsReplace() { + EntityReference category = setupCategory(); + measurableHelper.createMeasurable("z1", "Z1", category.id()); + measurableHelper.createMeasurable("z2", "Z2", category.id()); + + BulkTaxonomyValidationResult result = taxonomyChangeService.previewBulk( + category, + mkSimpleTsv(), + BulkTaxonomyItemParser.InputFormat.CSV, + BulkUpdateMode.REPLACE); + + assertNotNull(result, "Expected a result"); + assertNoErrors(result); + + assertHasRemovals(result, asSet("z1", "z2")); + assertExternalIdsMatch(result, asList("a1", "a1.1", "a1.2")); + + assertOperation(result, "a1", ChangeOperation.ADD); + assertOperation(result, "a1.1", ChangeOperation.ADD); + assertOperation(result, "a1.2", ChangeOperation.ADD); + } + + + + + @Test + public void applyRestoresMeasurablesThatWerePreviouslyRemoved() { + String user = setupUser(); + EntityReference category = setupCategory(); + long a1_1_id = measurableHelper.createMeasurable("a1.1", "", category.id()); + measurableHelper.updateMeasurableLifecycleStatus(a1_1_id, EntityLifecycleStatus.REMOVED); + + BulkTaxonomyValidationResult result = taxonomyChangeService.previewBulk( + category, + mkSimpleTsv(), + BulkTaxonomyItemParser.InputFormat.CSV, + BulkUpdateMode.REPLACE); + + taxonomyChangeService.applyBulk(category, result, user); + + boolean isRestored = measurableService + .findByExternalId("a1.1") + .stream() + .filter(m -> m.categoryId() == category.id()) + .anyMatch(m -> m.entityLifecycleStatus() == EntityLifecycleStatus.ACTIVE); + + assertTrue(isRestored, "previously removed 'a1.1' node has been restored"); + } + + + @Test + public void applyBrandNewTaxonomy() { + EntityReference category = setupCategory(); + String user = setupUser(); + + BulkTaxonomyValidationResult result = taxonomyChangeService.previewBulk( + category, + mkSimpleTsv(), + BulkTaxonomyItemParser.InputFormat.CSV, + BulkUpdateMode.ADD_ONLY); + + taxonomyChangeService.applyBulk(category, result, user); + + List storedMeasurables = measurableService.findByCategoryId(category.id()); + + assertEquals( + asSet("a1", "a1.1", "a1.2"), + SetUtilities.map(storedMeasurables, m -> m.externalId().orElse(null))); + } + + + @Test + public void applyUpdates() { + EntityReference category = setupCategory(); + String user = setupUser(); + measurableHelper.createMeasurable("a1", "A1", category.id()); + + BulkTaxonomyValidationResult result = taxonomyChangeService.previewBulk( + category, + mkSimpleTsv(), + BulkTaxonomyItemParser.InputFormat.CSV, + BulkUpdateMode.ADD_ONLY); + + taxonomyChangeService.applyBulk(category, result, user); + + List storedMeasurables = measurableService.findByCategoryId(category.id()); + + assertEquals( + asSet("a1", "a1.1", "a1.2"), + SetUtilities.map(storedMeasurables, m -> m.externalId().orElse(null))); + + CollectionUtilities + .find(storedMeasurables, m -> m.externalId().equals(Optional.of("a1"))) + .ifPresent(m -> { + assertEquals("Root node", m.description(), "Description should be updated"); + assertFalse(m.concrete(), "Concrete flag should be unset"); + }); + } + + + @Test + public void applyRemovalsIfModeIsReplace() { + EntityReference category = setupCategory(); + String user = setupUser(); + measurableHelper.createMeasurable("z1", "Z1", category.id()); + + BulkTaxonomyValidationResult result = taxonomyChangeService.previewBulk( + category, + mkSimpleTsv(), + BulkTaxonomyItemParser.InputFormat.CSV, + BulkUpdateMode.REPLACE); + + taxonomyChangeService.applyBulk(category, result, user); + + List storedMeasurables = measurableService.findByCategoryId(category.id()); + assertEquals( + asSet("a1", "a1.1", "a1.2"), + SetUtilities.map(storedMeasurables, m -> m.externalId().orElse(null)), + "z1 should be missing"); + } + + // --- HELPERS ----- + + private EntityReference setupCategory() { + String categoryName = mkName("TaxonomyChangeServiceTest", "category"); + long categoryId = measurableHelper.createMeasurableCategory(categoryName); + return mkRef(EntityKind.MEASURABLE_CATEGORY, categoryId, categoryName); + } + + + private String setupUser() { + String username = mkName("TaxonomyChangeServiceTest", "user"); + userHelper.createUserWithSystemRoles(username, asSet(SystemRole.TAXONOMY_EDITOR)); + + return username; + } + + + private void assertOperation(BulkTaxonomyValidationResult result, + String externalId, + ChangeOperation expectedOp) { + Optional item = maybeFindItem(result, externalId); + + assertTrue(item.isPresent(), "Expected to find item: " + externalId); + assertEquals(item.get().changeOperation(), expectedOp, format("Expected item: %s to have op: %s", externalId, expectedOp)); + } + + + private void assertChangedFields(BulkTaxonomyValidationResult result, + String externalId, + Set expectedFields) { + Optional item = maybeFindItem(result, externalId); + + assertTrue(item.isPresent(), "Expected to find item: " + externalId); + assertEquals(expectedFields, item.get().changedFields(), "Expected fields to have been changed"); + } + + + private Optional maybeFindItem(BulkTaxonomyValidationResult result, + String externalId) { + Optional item = find( + result.validatedItems(), + d -> d.parsedItem().externalId().equals(externalId)); + return item; + } + + + private void assertHasRemovals(BulkTaxonomyValidationResult result, Set expectedRemovals) { + assertEquals(expectedRemovals.size(), result.plannedRemovals().size(), "removals expected"); + } + + + private void assertNoRemovals(BulkTaxonomyValidationResult result) { + assertTrue(result.plannedRemovals().isEmpty(), "No removals expected"); + } + + private void assertExternalIdsMatch(BulkTaxonomyValidationResult result, + List expectedExternalIds) { + assertEquals( + expectedExternalIds, + ListUtilities.map(result.validatedItems(), d -> d.parsedItem().externalId()), + "Expected external ids do not match"); + } + + + private void assertNoErrors(BulkTaxonomyValidationResult result) { + assertTrue( + all(result.validatedItems(), d -> isEmpty(d.errors())), + "Should have no errors"); + } + + private String mkSimpleTsv() { + return "externalId, parentExternalId, name, description, concrete\n" + + "a1,, A1, Root node, false\n" + + "a1.1, a1, A1_1, First child, true\n" + + "a1.2, a1, A1_2, Second child, true\n"; + } + +} diff --git a/waltz-jobs/pom.xml b/waltz-jobs/pom.xml index f80806c4c1..1fe30e62a4 100644 --- a/waltz-jobs/pom.xml +++ b/waltz-jobs/pom.xml @@ -75,6 +75,11 @@ org.apache.commons commons-jexl3 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-csv + diff --git a/waltz-jobs/src/main/java/org/finos/waltz/jobs/XlsUtilities.java b/waltz-jobs/src/main/java/org/finos/waltz/jobs/XlsUtilities.java index 9fa5d8ebba..2bf7b4ab10 100644 --- a/waltz-jobs/src/main/java/org/finos/waltz/jobs/XlsUtilities.java +++ b/waltz-jobs/src/main/java/org/finos/waltz/jobs/XlsUtilities.java @@ -18,6 +18,7 @@ package org.finos.waltz.jobs; +import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; @@ -33,7 +34,8 @@ public static T mapStrCell(Row row, int i, Function mapper) { public static String strVal(Row row, int offset) { - return row.getCell(offset).getStringCellValue(); + Cell cell = row.getCell(offset); + return cell == null ? null : cell.getStringCellValue(); } diff --git a/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/PhysicalFlowHarness.java b/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/PhysicalFlowHarness.java index 5fff5e9f87..241df7285a 100644 --- a/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/PhysicalFlowHarness.java +++ b/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/PhysicalFlowHarness.java @@ -18,16 +18,207 @@ package org.finos.waltz.jobs.harness; +import org.finos.waltz.common.MapUtilities; +import org.finos.waltz.common.SetUtilities; +import org.finos.waltz.data.InlineSelectFieldFactory; +import org.finos.waltz.model.EntityKind; +import org.finos.waltz.model.EntityLifecycleStatus; +import org.finos.waltz.model.EntityReference; +import org.finos.waltz.model.Nullable; +import org.finos.waltz.schema.Tables; +import org.finos.waltz.schema.tables.DataType; +import org.finos.waltz.schema.tables.LogicalFlow; +import org.finos.waltz.schema.tables.LogicalFlowDecorator; +import org.finos.waltz.schema.tables.PhysicalFlow; +import org.finos.waltz.schema.tables.PhysicalSpecDataType; +import org.finos.waltz.schema.tables.PhysicalSpecification; import org.finos.waltz.service.DIConfiguration; -import org.finos.waltz.service.physical_flow.PhysicalFlowService; +import org.immutables.value.Value; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.Record10; +import org.jooq.SelectConditionStep; +import org.jooq.lambda.tuple.Tuple2; +import org.jooq.lambda.tuple.Tuple4; import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.supercsv.io.CsvListWriter; +import org.supercsv.prefs.CsvPreference; + +import java.io.FileWriter; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.finos.waltz.common.StringUtilities.mkSafe; +import static org.jooq.lambda.fi.util.function.CheckedConsumer.unchecked; +import static org.jooq.lambda.tuple.Tuple.tuple; public class PhysicalFlowHarness { - public static void main(String[] args) { + @Value.Immutable + interface LogicalFlowKey { + long lfId(); + EntityReference src(); + EntityReference trg(); + } + + @Value.Immutable + interface PhysicalFlowKey { + long lfId(); + long pfId(); + @Nullable + String flowName(); + + @Nullable + String transport(); + @Nullable + String flowExtId(); + @Nullable + String specName(); + @Nullable + String specExtId(); + + @Nullable + String specDescription(); + + @Nullable + String flowDescription(); + } + + + public static void main(String[] args) throws IOException { AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(DIConfiguration.class); - PhysicalFlowService service = ctx.getBean(PhysicalFlowService.class); - System.out.println(service); + DSLContext dsl = ctx.getBean(DSLContext.class); + + PhysicalFlow pf = Tables.PHYSICAL_FLOW; + PhysicalSpecification ps = Tables.PHYSICAL_SPECIFICATION; + PhysicalSpecDataType psdt = Tables.PHYSICAL_SPEC_DATA_TYPE; + LogicalFlow lf = Tables.LOGICAL_FLOW; + DataType dt = Tables.DATA_TYPE; + LogicalFlowDecorator lfd = Tables.LOGICAL_FLOW_DECORATOR; + + Condition lfIsActive = lf.IS_REMOVED.isFalse().and(lf.ENTITY_LIFECYCLE_STATUS.ne(EntityLifecycleStatus.REMOVED.name())); + Condition pfIsActive = pf.IS_REMOVED.isFalse().and(pf.ENTITY_LIFECYCLE_STATUS.ne(EntityLifecycleStatus.REMOVED.name())); + + Collection kinds = SetUtilities.asSet(EntityKind.APPLICATION, EntityKind.END_USER_APPLICATION, EntityKind.ACTOR); + Field srcName = InlineSelectFieldFactory.mkNameField(lf.SOURCE_ENTITY_ID, lf.SOURCE_ENTITY_KIND, kinds); + Field trgName = InlineSelectFieldFactory.mkNameField(lf.TARGET_ENTITY_ID, lf.TARGET_ENTITY_KIND, kinds); + Field srcExtId = InlineSelectFieldFactory.mkExternalIdField(lf.SOURCE_ENTITY_ID, lf.SOURCE_ENTITY_KIND, kinds); + Field trgExtId = InlineSelectFieldFactory.mkExternalIdField(lf.TARGET_ENTITY_ID, lf.TARGET_ENTITY_KIND, kinds); + + SelectConditionStep> lfQry = dsl + .select(lf.ID, lf.SOURCE_ENTITY_KIND, lf.SOURCE_ENTITY_ID, srcName, srcExtId, + lf.TARGET_ENTITY_KIND, lf.TARGET_ENTITY_ID, trgName, trgExtId, + dt.NAME) + .from(lf) + .innerJoin(lfd).on(lfd.LOGICAL_FLOW_ID.eq(lf.ID)) + .innerJoin(dt).on(dt.ID.eq(lfd.DECORATOR_ENTITY_ID).and(lfd.DECORATOR_ENTITY_KIND.eq(EntityKind.DATA_TYPE.name()))) + .where(lfIsActive); + + SelectConditionStep> pfQry = dsl + .select(lf.ID, + pf.ID, pf.NAME, ps.NAME, pf.EXTERNAL_ID, pf.TRANSPORT, + ps.EXTERNAL_ID, + dt.NAME, + ps.DESCRIPTION, pf.DESCRIPTION) + .from(lf) + .innerJoin(pf).on(pf.LOGICAL_FLOW_ID.eq(lf.ID)) + .innerJoin(ps).on(ps.ID.eq(pf.SPECIFICATION_ID)) + .innerJoin(psdt).on(psdt.SPECIFICATION_ID.eq(ps.ID)) + .innerJoin(dt).on(dt.ID.eq(psdt.DATA_TYPE_ID)) + .where(lfIsActive.and(pfIsActive)); + + Map> lfsToDTs = MapUtilities + .groupAndThen( + lfQry.fetch(), + r -> ImmutableLogicalFlowKey.builder() + .lfId(r.get(lf.ID)) + .src(EntityReference.mkRef(EntityKind.valueOf(r.get(lf.SOURCE_ENTITY_KIND)), r.get(lf.SOURCE_ENTITY_ID), r.get(srcName), null, r.get(srcExtId))) + .trg(EntityReference.mkRef(EntityKind.valueOf(r.get(lf.TARGET_ENTITY_KIND)), r.get(lf.TARGET_ENTITY_ID), r.get(trgName), null, r.get(trgExtId))) + .build(), + xs -> xs.stream().map(r -> r.get(dt.NAME)).distinct().sorted().collect(Collectors.toList())); + + Map> pfsToDTs = MapUtilities + .groupAndThen( + pfQry.fetch(), + r -> ImmutablePhysicalFlowKey + .builder() + .lfId(r.get(lf.ID)) + .pfId(r.get(pf.ID)) + .flowName(r.get(pf.NAME)) + .flowExtId(r.get(pf.EXTERNAL_ID)) + .transport(r.get(pf.TRANSPORT)) + .specName(r.get(ps.NAME)) + .specExtId(r.get(ps.EXTERNAL_ID)) + .flowDescription(r.get(pf.DESCRIPTION)) + .specDescription(r.get(ps.DESCRIPTION)) + .build(), + xs -> xs.stream().map(r -> r.get(dt.NAME)).distinct().sorted().collect(Collectors.toList())); + + Map>>> lfToPfs = MapUtilities + .groupBy( + pfsToDTs.entrySet(), + kv -> kv.getKey().lfId(), + kv -> tuple(kv.getKey(), kv.getValue())); + + List, PhysicalFlowKey, List>> big = lfsToDTs + .entrySet() + .stream() + .map(kv -> tuple(kv.getKey(), kv.getValue())) + .flatMap(t -> { + Collection>> pfs = lfToPfs.get(t.v1.lfId()); + return pfs == null + ? Stream.of(t.concat( + tuple((PhysicalFlowKey) null, (List) null))) + : pfs.stream().map(pft -> t.concat( + tuple(pft.v1, pft.v2))); + }) + .collect(Collectors.toList()); + + toCSV(big); + } + + + private static void toCSV(List, PhysicalFlowKey, List>> big) throws IOException { + CsvListWriter writer = new CsvListWriter(new FileWriter("flows_with_comments.tsv"), CsvPreference.TAB_PREFERENCE); + writer.writeHeader( + "Logical Flow Id", + "Source Name", "Source Ext Id", "Source Kind", + "Target Name", "Target Ext Id", "Target Kind", + "Logical DataTypes", + "Physical Flow Id", "Physical Flow Name", "Physical Flow Ext Id", "Physical Flow Transport", + "Specification Name", "Specification Ext Id", + "Specification DataTypes", + "Physical Flow Comment", + "Specification Comment"); + + big.forEach(unchecked(d -> writer.write( + d.v1.lfId(), + d.v1.src().name().orElse(""), d.v1.src().externalId().orElse(""), d.v1.src().kind(), + d.v1.trg().name().orElse(""), d.v1.trg().externalId().orElse(""), d.v1.trg().kind(), + toStr(d.v2), + d.v3 == null ? null : toStr(d.v3.pfId()), + d.v3 == null ? null : mkSafe(d.v3.flowName()), + d.v3 == null ? null : mkSafe(d.v3.flowExtId()), + d.v3 == null ? null : mkSafe(d.v3.transport()), + d.v3 == null ? null : mkSafe(d.v3.specName()), + d.v3 == null ? null : mkSafe(d.v3.specExtId()), + toStr(d.v4), + d.v3 == null ? null : mkSafe(d.v3.flowDescription()), + d.v3 == null ? null : mkSafe(d.v3.specDescription())))); + } + + + private static String toStr(Long d) { + return d == null ? "" : d.toString(); + } + + private static String toStr(List d) { + return d == null ? "" : d.stream().collect(Collectors.joining("; ")); } } diff --git a/waltz-jobs/src/main/java/org/finos/waltz/jobs/tools/DataTypeUtilities.java b/waltz-jobs/src/main/java/org/finos/waltz/jobs/tools/DataTypeUtilities.java new file mode 100644 index 0000000000..ff72fdd702 --- /dev/null +++ b/waltz-jobs/src/main/java/org/finos/waltz/jobs/tools/DataTypeUtilities.java @@ -0,0 +1,391 @@ +package org.finos.waltz.jobs.tools; + +import org.finos.waltz.common.SetUtilities; +import org.finos.waltz.model.EntityKind; +import org.finos.waltz.model.EntityLifecycleStatus; +import org.finos.waltz.model.EntityReference; +import org.finos.waltz.schema.Tables; +import org.finos.waltz.schema.tables.DataType; +import org.finos.waltz.schema.tables.DataTypeUsage; +import org.finos.waltz.schema.tables.FlowClassificationRule; +import org.finos.waltz.schema.tables.LogicalFlow; +import org.finos.waltz.schema.tables.LogicalFlowDecorator; +import org.finos.waltz.schema.tables.PhysicalFlow; +import org.finos.waltz.schema.tables.PhysicalSpecDataType; +import org.finos.waltz.schema.tables.PhysicalSpecification; +import org.finos.waltz.schema.tables.records.LogicalFlowRecord; +import org.finos.waltz.service.DIConfiguration; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.impl.DSL; +import org.jooq.lambda.tuple.Tuple; +import org.jooq.lambda.tuple.Tuple2; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; +import static org.finos.waltz.model.EntityReference.mkRef; + +public class DataTypeUtilities { + + private static final Logger LOG = LoggerFactory.getLogger(DataTypeUtilities.class); + private static final DataType dataType = Tables.DATA_TYPE; + private static final LogicalFlow logicalFlow = Tables.LOGICAL_FLOW; + private static final LogicalFlowDecorator logicalFlowDecorator = Tables.LOGICAL_FLOW_DECORATOR; + private static final FlowClassificationRule flowClassificationRule = Tables.FLOW_CLASSIFICATION_RULE; + private static final PhysicalFlow physicalFlow = Tables.PHYSICAL_FLOW; + private static final PhysicalSpecification physicalSpec = Tables.PHYSICAL_SPECIFICATION; + private static final PhysicalSpecDataType physicalSpecDataType = Tables.PHYSICAL_SPEC_DATA_TYPE; + private static final DataTypeUsage dataTypeUsage = Tables.DATA_TYPE_USAGE; + + public static long changeParent(DSLContext dsl, + String childCode, + String newParentCode) { + LOG.debug("Change parent from: {} to {}", childCode, newParentCode); + + Long childId = dsl + .select(dataType.ID) + .from(dataType) + .where(dataType.CODE.eq(childCode)) + .fetchOne(dataType.ID); + + Long newParentId = dsl + .select(dataType.ID) + .from(dataType) + .where(dataType.CODE.eq(newParentCode)) + .fetchOne(dataType.ID); + + return dsl.update(dataType) + .set(dataType.PARENT_ID, newParentId) + .where(dataType.ID.eq(childId)) + .execute(); + } + + public static Map, LogicalFlowRecord> getAllLogicalFlowsBySourceTargetMap(DSLContext dsl) { + return dsl + .selectFrom(logicalFlow) + .fetch() + .stream() + .collect(toMap( + t -> Tuple.tuple( + mkRef(EntityKind.valueOf(t.getSourceEntityKind()), t.getSourceEntityId()), + mkRef(EntityKind.valueOf(t.getTargetEntityKind()), t.getTargetEntityId())), + t -> t)); + } + + public static Map> getLogicalFlowToPhysicalSpecId(DSLContext dsl) { + return dsl + .select(physicalFlow.LOGICAL_FLOW_ID, physicalFlow.SPECIFICATION_ID) + .from(physicalFlow) + .where(physicalFlow.IS_REMOVED.eq(false) + .and(physicalFlow.ENTITY_LIFECYCLE_STATUS.ne(EntityLifecycleStatus.REMOVED.name()))) + .fetch() + .stream() + .collect(Collectors.groupingBy( + t -> t.get(physicalFlow.LOGICAL_FLOW_ID), + Collectors.mapping( + t -> t.get(physicalFlow.SPECIFICATION_ID), + toSet()))); + } + + private static long removeDataType(DSLContext dsl, + String fromCode) { + return dsl + .delete(dataType) + .where(dataType.CODE.eq(fromCode)) + .execute(); + } + + + public static void migrate(DSLContext dsl, + String fromCode, + String toCode, + boolean deleteOldDataType){ + + Long fromId = dsl + .select(dataType.ID) + .from(dataType) + .where(dataType.CODE.eq(fromCode)) + .fetchOne(dataType.ID); + + Long toId = dsl + .select(dataType.ID) + .from(dataType) + .where(dataType.CODE.eq(toCode)) + .fetchOne(dataType.ID); + + LOG.debug("Migrate from: {}/{} to {}/{}", fromCode, fromId, toCode, toId); + + + // 1) update data_type_usage set code = toCode where code = fromCode + // 2) update logical_flow_decorator set decorator_entity_id = (toId) where decorator_entity_id = (fromId) and decorator_entity_kind = 'DATA_TYPE' + // 3) update physical_spec_data_type set data_type_id = (toId) where data_type_id = (fromId) + // 4) update flow_classification_rules set data_type_id = toId where data_type_id = fromId + // x) delete from dataType where code = 'fromCode' + + migrateDataTypeUsage(dsl, fromId, toId); + migrateFlowClassificationRules(dsl, fromId, toId); + migrateLogicalFlowDecorator(dsl, fromId, toId); + migratePhysicalSpecDataType(dsl, fromId, toId); + + if (deleteOldDataType) { + removeDataType(dsl, fromCode); + } + + } + + private static void migratePhysicalSpecDataType(DSLContext dsl, + Long fromId, + Long toId) { + PhysicalSpecDataType physicSpec = physicalSpecDataType.as("physicSpec"); + Condition notAlreadyExists = DSL + .notExists(DSL + .select(physicSpec.SPECIFICATION_ID) + .from(physicSpec) + .where(physicSpec.SPECIFICATION_ID.eq(physicalSpecDataType.SPECIFICATION_ID) + .and(physicSpec.DATA_TYPE_ID.eq(toId))));; + + int updateCount = dsl + .update(physicalSpecDataType) + .set(physicalSpecDataType.DATA_TYPE_ID, toId) + .where(physicalSpecDataType.DATA_TYPE_ID.eq(fromId) + .and(notAlreadyExists)) + .execute(); + + int rmCount = dsl + .delete(physicalSpecDataType) + .where(physicalSpecDataType.DATA_TYPE_ID.eq(fromId)) + .execute(); + + LOG.info("Migrate Phys Spec Data Type Usage: {} -> {}, updated: {}, removed: {}", fromId, toId, updateCount, rmCount); + + } + + private static void migrateLogicalFlowDecorator(DSLContext dsl, + Long fromId, + Long toId) { + LogicalFlowDecorator decorator = logicalFlowDecorator.as("decorator"); + + Condition notAlreadyExists = DSL.notExists(DSL + .select(decorator.LOGICAL_FLOW_ID) + .from(decorator) + .where(decorator.LOGICAL_FLOW_ID.eq(logicalFlowDecorator.LOGICAL_FLOW_ID) + .and(decorator.DECORATOR_ENTITY_ID.eq(toId)) + .and(decorator.DECORATOR_ENTITY_KIND.eq(EntityKind.DATA_TYPE.name())))); + + int updateCount = dsl + .update(logicalFlowDecorator) + .set(logicalFlowDecorator.DECORATOR_ENTITY_ID, toId) + .where(logicalFlowDecorator.DECORATOR_ENTITY_ID.eq(fromId) + .and(logicalFlowDecorator.DECORATOR_ENTITY_KIND.eq(EntityKind.DATA_TYPE.name())) + .and(notAlreadyExists)) + .execute(); + + int rmCount = dsl + .delete(logicalFlowDecorator) + .where(logicalFlowDecorator.DECORATOR_ENTITY_ID.eq(fromId) + .and(logicalFlowDecorator.DECORATOR_ENTITY_KIND.eq(EntityKind.DATA_TYPE.name()))) + .execute(); + + LOG.info("Migrate Logical Flow Decorator: {} -> {}, updated: {}, removed: {}", fromId, toId, updateCount, rmCount); + } + + + private static void migrateFlowClassificationRules(DSLContext dsl, + Long fromId, + Long toId) { + + FlowClassificationRule authSrc = flowClassificationRule.as("authSrc"); + + Condition notAlreadyExists = DSL.notExists(DSL + .select(authSrc.ID) + .from(authSrc) + .where(authSrc.PARENT_KIND.eq(flowClassificationRule.PARENT_KIND) + .and(authSrc.PARENT_ID.eq(flowClassificationRule.PARENT_ID)) + .and(authSrc.DATA_TYPE_ID.eq(toId)) + .and(authSrc.SUBJECT_ENTITY_ID.eq(flowClassificationRule.SUBJECT_ENTITY_ID) + .and(authSrc.SUBJECT_ENTITY_KIND.eq(flowClassificationRule.SUBJECT_ENTITY_KIND))))); + + int updateCount = dsl + .update(flowClassificationRule) + .set(flowClassificationRule.DATA_TYPE_ID, toId) + .where(flowClassificationRule.DATA_TYPE_ID.eq(fromId) + .and(notAlreadyExists)) + .execute(); + + int rmCount = dsl + .delete(flowClassificationRule) + .where(flowClassificationRule.DATA_TYPE_ID.eq(fromId)) + .execute(); + + LOG.info("Migrate Flow Classification Rules: {} -> {}, updated: {}, removed: {}", fromId, toId, updateCount, rmCount); + } + + + private static void migrateDataTypeUsage(DSLContext dsl, + Long fromId, + Long toId) { + DataTypeUsage dtu = dataTypeUsage.as("dtu"); + + Condition condition = DSL.notExists(DSL + .select(dtu.ENTITY_ID) + .from(dtu) + .where(dataTypeUsage.ENTITY_ID.eq(dtu.ENTITY_ID) + .and(dataTypeUsage.ENTITY_KIND.eq(dtu.ENTITY_KIND)) + .and(dtu.DATA_TYPE_ID.eq(toId)) + .and(dataTypeUsage.USAGE_KIND.eq(dtu.USAGE_KIND)))); + + int updateCount = dsl + .update(dataTypeUsage) + .set(dataTypeUsage.DATA_TYPE_ID, toId) + .where(dataTypeUsage.DATA_TYPE_ID.eq(fromId) + .and(condition)) + .execute(); + + int rmCount = dsl + .delete(dataTypeUsage) + .where(dataTypeUsage.DATA_TYPE_ID.eq(fromId)) + .execute(); + + LOG.info("Migrate DataType Usage: {} -> {}, updated: {}, removed: {}", fromId, toId, updateCount, rmCount); + } + + public static List findLogicalFlowIdsForDataType(DSLContext dsl, Long datatype, Set logicalFlowIds) { + return dsl + .select(logicalFlowDecorator.LOGICAL_FLOW_ID) + .from(logicalFlowDecorator) + .where(logicalFlowDecorator.LOGICAL_FLOW_ID.in(logicalFlowIds) + .and(logicalFlowDecorator.DECORATOR_ENTITY_ID.eq(datatype)) + .and(logicalFlowDecorator.DECORATOR_ENTITY_KIND.eq(EntityKind.DATA_TYPE.name()))) + .fetch(logicalFlowDecorator.LOGICAL_FLOW_ID); + } + + + public static long deleteLogicalFlowDecorators(DSLContext dsl, + Long datatype, + Collection logicalFlowIds) { + return dsl.delete(logicalFlowDecorator) + .where(logicalFlowDecorator.LOGICAL_FLOW_ID.in(logicalFlowIds) + .and(logicalFlowDecorator.DECORATOR_ENTITY_ID.eq(datatype)) + .and(logicalFlowDecorator.DECORATOR_ENTITY_KIND.eq(EntityKind.DATA_TYPE.name()))) + .execute(); + } + + + public static long updatePhysicalFlowDecorators(DSLContext dsl, + Long oldDataType, + Long newDataType, + Set specIds) { + Set existingSpecIds = dsl + .selectDistinct(physicalSpecDataType.SPECIFICATION_ID) + .from(physicalSpecDataType) + .where(physicalSpecDataType.DATA_TYPE_ID.eq(newDataType) + .and(physicalSpecDataType.SPECIFICATION_ID.in(specIds))) + .fetchSet(physicalSpecDataType.SPECIFICATION_ID); + + Set remainingSpecIds = SetUtilities.minus(specIds, existingSpecIds); + + long execute = dsl.update(physicalSpecDataType) + .set(physicalSpecDataType.DATA_TYPE_ID, newDataType) + .where(physicalSpecDataType.SPECIFICATION_ID.in(remainingSpecIds) + .and(physicalSpecDataType.DATA_TYPE_ID.eq(oldDataType))) + .execute(); + + long deleteCount = dsl.delete(physicalSpecDataType) + .where(physicalSpecDataType.SPECIFICATION_ID.in(specIds) + .and(physicalSpecDataType.DATA_TYPE_ID.eq(oldDataType))) + .execute(); + + System.out.println("============= ()() Physical spec decorator removed are " + deleteCount); + + return execute; + } + + public static long deletePhysicalFlowDecorators(DSLContext dsl, + Long datatype, + Set specIds) { + return dsl.delete(physicalSpecDataType) + .where(physicalSpecDataType.SPECIFICATION_ID.in(specIds) + .and(physicalSpecDataType.DATA_TYPE_ID.eq(datatype))) + .execute(); + } + + public static long changeTitleAndDescription(DSLContext dsl, + String title, + String description, + String code) { + return dsl.update(dataType) + .set(dataType.NAME, title) + .set(dataType.DESCRIPTION, description) + .where(dataType.CODE.eq(code)).execute(); + } + + + public static long markDataTypeAsDeprecated(DSLContext dsl, + Set codeListToBeDeprecated) { + String deprecatedIdentifier = "DEPRECATED - "; + dsl + .update(dataType) + .set(dataType.NAME, DSL.concat(deprecatedIdentifier, dataType.NAME)) + .set(dataType.DESCRIPTION, DSL.concat(deprecatedIdentifier, dataType.DESCRIPTION)) + .where(dataType.CODE.in(codeListToBeDeprecated) + .and(dataType.NAME.startsWith(deprecatedIdentifier).not())) + .execute(); + + return dsl + .update(dataType) + .set(dataType.DEPRECATED, true) + .set(dataType.CONCRETE, false) + .where(dataType.CODE.in(codeListToBeDeprecated)) + .execute(); + } + + public static long unMarkDataTypeAsDeprecated(DSLContext dsl, + Set dataTypeCodes) { + return dsl + .update(dataType) + .set(dataType.DEPRECATED, false) + .where(dataType.CODE.in(dataTypeCodes)) + .execute(); + } + + public static long markDataTypeAsConcrete(DSLContext dsl, + Set dataTypeCodes) { + return dsl + .update(dataType) + .set(dataType.CONCRETE, true) + .where(dataType.CODE.in(dataTypeCodes)) + .execute(); + } + + + public static void main(String[] args) throws IOException { + + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(DIConfiguration.class); + DSLContext dsl = ctx.getBean(DSLContext.class); + + String party = "DataTypeCode1"; + String allocation = "DataTypeCode2"; + String instrumentIdentifier = "DataTypeCode3"; + String prospect = "DataTypeCode4"; + String dealEvent = "DataTypeCode5"; + String instrumentStatic = "DataTypeCode6"; + + dsl.transaction(context -> { + DSLContext tx = context.dsl(); + migrate(tx, instrumentIdentifier, instrumentStatic, false); + migrate(tx, prospect, party, false); + migrate(tx, allocation, dealEvent, false); + //throw new RuntimeException("BoooM!"); + }); + } +} diff --git a/waltz-jobs/src/main/java/org/finos/waltz/jobs/tools/ShortestPath.java b/waltz-jobs/src/main/java/org/finos/waltz/jobs/tools/ShortestPath.java index b87aa214f2..73b7a2fadb 100644 --- a/waltz-jobs/src/main/java/org/finos/waltz/jobs/tools/ShortestPath.java +++ b/waltz-jobs/src/main/java/org/finos/waltz/jobs/tools/ShortestPath.java @@ -45,13 +45,11 @@ public class ShortestPath { private static ExternalIdValue[] sourceAssetCodes = new ExternalIdValue[] { - ExternalIdValue.of("assetCode1"), - ExternalIdValue.of("assetCode2"), - ExternalIdValue.of("assetCode3") + ExternalIdValue.of("startId") }; - private static final ExternalIdValue targetAssetCode = ExternalIdValue.of("26877-2"); + private static final ExternalIdValue targetAssetCode = ExternalIdValue.of("endId"); public static void main(String[] args) { diff --git a/waltz-jobs/src/main/java/org/finos/waltz/jobs/tools/importers/ImporterExperiment.java b/waltz-jobs/src/main/java/org/finos/waltz/jobs/tools/importers/ImporterExperiment.java new file mode 100644 index 0000000000..e66f990ad1 --- /dev/null +++ b/waltz-jobs/src/main/java/org/finos/waltz/jobs/tools/importers/ImporterExperiment.java @@ -0,0 +1,77 @@ +package org.finos.waltz.jobs.tools.importers; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import org.finos.waltz.model.Nullable; +import org.immutables.value.Value; + +import java.io.IOException; +import java.io.InputStream; + +import static org.finos.waltz.common.IOUtilities.readAsString; + +public class ImporterExperiment { + + @Value.Immutable + @JsonDeserialize(as = ImmutableMyEntity.class) + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) + interface MyEntity { + String name(); + @JsonAlias({"external_id", "ext_id"}) + String externalId(); + @Nullable String description(); + } + + public static void main(String[] args) { + + try { + go(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static void go() throws IOException { + System.out.println("-- JSON ----------------------"); + { + ObjectMapper mapper = new ObjectMapper(); + InputStream jsonStream = ImporterExperiment.class.getClassLoader().getResourceAsStream("importer-experiment.json"); + MyEntity[] myEntities = mapper.readValue(readAsString(jsonStream), MyEntity[].class); + for (MyEntity myEntity : myEntities) { + System.out.println(myEntity); + } + } + + System.out.println("-- CSV ----------------------"); + { + CsvSchema bootstrapSchema = CsvSchema.emptySchema().withHeader(); + + ObjectMapper mapper = new CsvMapper(); + InputStream csvStream = ImporterExperiment.class.getClassLoader().getResourceAsStream("importer-experiment.csv"); + MappingIterator myEntities = mapper.readerFor(MyEntity.class).with(bootstrapSchema).readValues(readAsString(csvStream)); + for (MyEntity myEntity : myEntities.readAll()) { + System.out.println(myEntity); + } + } + + System.out.println("-- TSV ----------------------"); + { + CsvSchema bootstrapSchema = CsvSchema.emptySchema().withHeader().withColumnSeparator('\t'); + + ObjectMapper mapper = new CsvMapper(); + InputStream csvStream = ImporterExperiment.class.getClassLoader().getResourceAsStream("importer-experiment.tsv"); + MappingIterator myEntities = mapper.readerFor(MyEntity.class).with(bootstrapSchema).readValues(readAsString(csvStream)); + for (MyEntity myEntity : myEntities.readAll()) { + System.out.println(myEntity); + } + } + } + +} diff --git a/waltz-jobs/src/main/resources/importer-experiment.csv b/waltz-jobs/src/main/resources/importer-experiment.csv new file mode 100644 index 0000000000..81575e2a6c --- /dev/null +++ b/waltz-jobs/src/main/resources/importer-experiment.csv @@ -0,0 +1,4 @@ +foo, externalId, description, wibble, Name +hello, h1, Hello World, wobble, Hello +goodbye, g1, Goodbye Cruel World, wobble, World +hi, h2,, wobble, Hi Jess \ No newline at end of file diff --git a/waltz-jobs/src/main/resources/importer-experiment.json b/waltz-jobs/src/main/resources/importer-experiment.json new file mode 100644 index 0000000000..ad7b325f1e --- /dev/null +++ b/waltz-jobs/src/main/resources/importer-experiment.json @@ -0,0 +1,21 @@ +[ + { + "name": "hello", + "externalId": "h1", + "description": "Hello World" + }, + { + "name": "goodbye", + "externalId": "g1", + "description": "Goodbye Cruel World" + }, + { + "name": "mum", + "external_Id": "m1", + "description": "Hi Mum" + }, + { + "ext_Id": "b1", + "description": "Hi Bob" + } +] \ No newline at end of file diff --git a/waltz-jobs/src/main/resources/importer-experiment.tsv b/waltz-jobs/src/main/resources/importer-experiment.tsv new file mode 100644 index 0000000000..61aa38c66c --- /dev/null +++ b/waltz-jobs/src/main/resources/importer-experiment.tsv @@ -0,0 +1,4 @@ +foo externalId description wibble Name +hello h1 Hello World wobble Hello +goodbye g1 Goodbye Cruel World wobble World +hi h2 wobble Hi Jess \ No newline at end of file diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/BulkUpdateMode.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/BulkUpdateMode.java index b69ae98ed2..d764928ac4 100644 --- a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/BulkUpdateMode.java +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/BulkUpdateMode.java @@ -1,6 +1,6 @@ package org.finos.waltz.model.bulk_upload; public enum BulkUpdateMode { - ADD_ONLY, - REPLACE + ADD_ONLY, // will not remove, only add and/or update + REPLACE // will remove things } diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/ChangeOperation.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/ChangeOperation.java new file mode 100644 index 0000000000..17f9e6e19c --- /dev/null +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/ChangeOperation.java @@ -0,0 +1,9 @@ +package org.finos.waltz.model.bulk_upload; + +public enum ChangeOperation { + ADD, + REMOVE, + RESTORE, + UPDATE, + NONE +} diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/BulkTaxonomyApplyResult.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/BulkTaxonomyApplyResult.java new file mode 100644 index 0000000000..904c307847 --- /dev/null +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/BulkTaxonomyApplyResult.java @@ -0,0 +1,18 @@ +package org.finos.waltz.model.bulk_upload.taxonomy; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.immutables.value.Value; + +@Value.Immutable +@JsonSerialize(as = ImmutableBulkTaxonomyApplyResult.class) +public interface BulkTaxonomyApplyResult { + int recordsAdded(); + + int recordsUpdated(); + + int recordsRemoved(); + + int recordsRestored(); + + boolean hierarchyRebuilt(); +} diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/BulkTaxonomyItem.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/BulkTaxonomyItem.java new file mode 100644 index 0000000000..9f503c691a --- /dev/null +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/BulkTaxonomyItem.java @@ -0,0 +1,33 @@ +package org.finos.waltz.model.bulk_upload.taxonomy; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.finos.waltz.model.Nullable; +import org.immutables.value.Value; + +@Value.Immutable +@JsonDeserialize(as = ImmutableBulkTaxonomyItem.class) +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) +public interface BulkTaxonomyItem { + + String name(); + + @JsonAlias({"id", "external_id", "ext_id", "extId", "external id", "ext id"}) + String externalId(); + + @JsonAlias({"parent_id", "parent_external_id", "parent_ext_id", "parentExtId", "parent external id", "parent id", "parent ext id", "external parent id", "external_parent_id", "externalParentId"}) + @Nullable + String parentExternalId(); + + @JsonAlias({"desc"}) + @Nullable + String description(); + + @Value.Default + default boolean concrete() { + return true; + } +} diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/BulkTaxonomyParseResult.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/BulkTaxonomyParseResult.java new file mode 100644 index 0000000000..169d8e192d --- /dev/null +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/BulkTaxonomyParseResult.java @@ -0,0 +1,41 @@ +package org.finos.waltz.model.bulk_upload.taxonomy; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.finos.waltz.model.Nullable; +import org.immutables.value.Value; + +import java.util.List; + +@Value.Immutable +@JsonSerialize(as = ImmutableBulkTaxonomyParseResult.class) +public interface BulkTaxonomyParseResult { + + @Value.Immutable + interface BulkTaxonomyParseError { + String message(); + + @Nullable + Integer line(); + + @Nullable + Integer column(); + } + + List parsedItems(); + + @Nullable String input(); + + @Nullable + BulkTaxonomyParseError error(); + + + static BulkTaxonomyParseResult mkResult(List items, + String input) { + return ImmutableBulkTaxonomyParseResult + .builder() + .parsedItems(items) + .input(input) + .build(); + } + +} diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/BulkTaxonomyValidatedItem.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/BulkTaxonomyValidatedItem.java new file mode 100644 index 0000000000..117f162c01 --- /dev/null +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/BulkTaxonomyValidatedItem.java @@ -0,0 +1,22 @@ +package org.finos.waltz.model.bulk_upload.taxonomy; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.finos.waltz.model.EntityReference; +import org.finos.waltz.model.Nullable; +import org.finos.waltz.model.bulk_upload.ChangeOperation; +import org.immutables.value.Value; + +import java.util.Set; + +@Value.Immutable +@JsonSerialize(as=ImmutableBulkTaxonomyValidatedItem.class) +public interface BulkTaxonomyValidatedItem { + + BulkTaxonomyItem parsedItem(); + ChangeOperation changeOperation(); + Set changedFields(); + Set errors(); + + @Nullable + EntityReference existingItemReference(); +} \ No newline at end of file diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/BulkTaxonomyValidationResult.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/BulkTaxonomyValidationResult.java new file mode 100644 index 0000000000..275863917e --- /dev/null +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/BulkTaxonomyValidationResult.java @@ -0,0 +1,22 @@ +package org.finos.waltz.model.bulk_upload.taxonomy; + + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.finos.waltz.model.Nullable; +import org.finos.waltz.model.measurable.Measurable; +import org.immutables.value.Value; + +import java.util.List; +import java.util.Set; + +@Value.Immutable +@JsonSerialize(as=ImmutableBulkTaxonomyValidationResult.class) +public interface BulkTaxonomyValidationResult { + + List validatedItems(); + + Set plannedRemovals(); + + @Nullable + BulkTaxonomyParseResult.BulkTaxonomyParseError error(); +} diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/ChangedFieldType.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/ChangedFieldType.java new file mode 100644 index 0000000000..7083c90918 --- /dev/null +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/ChangedFieldType.java @@ -0,0 +1,8 @@ +package org.finos.waltz.model.bulk_upload.taxonomy; + +public enum ChangedFieldType { + NAME, + DESCRIPTION, + PARENT_EXTERNAL_ID, + CONCRETE +} diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/ValidationError.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/ValidationError.java new file mode 100644 index 0000000000..1603703270 --- /dev/null +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/taxonomy/ValidationError.java @@ -0,0 +1,7 @@ +package org.finos.waltz.model.bulk_upload.taxonomy; + +public enum ValidationError { + PARENT_NOT_FOUND, + DUPLICATE_EXT_ID, + CYCLE_DETECTED +} diff --git a/waltz-model/src/main/java/org/finos/waltz/model/taxonomy_management/TaxonomyChangeType.java b/waltz-model/src/main/java/org/finos/waltz/model/taxonomy_management/TaxonomyChangeType.java index ef04900729..7fe38ab8f8 100644 --- a/waltz-model/src/main/java/org/finos/waltz/model/taxonomy_management/TaxonomyChangeType.java +++ b/waltz-model/src/main/java/org/finos/waltz/model/taxonomy_management/TaxonomyChangeType.java @@ -30,6 +30,9 @@ public enum TaxonomyChangeType { UPDATE_NAME, UPDATE_DESCRIPTION, UPDATE_CONCRETENESS, - UPDATE_EXTERNAL_ID + UPDATE_EXTERNAL_ID, + BULK_ADD, + BULK_RESTORE, + BULK_UPDATE } diff --git a/waltz-ng/client/common/services/enums/taxonomy-change-type.js b/waltz-ng/client/common/services/enums/taxonomy-change-type.js index 7778cf795d..4270b908da 100644 --- a/waltz-ng/client/common/services/enums/taxonomy-change-type.js +++ b/waltz-ng/client/common/services/enums/taxonomy-change-type.js @@ -13,6 +13,27 @@ export const taxonomyChangeType = { description: null, position: 10 }, + BULK_ADD: { + key: "BULK_ADD", + name: "Bulk Add", + icon: null, + description: null, + position: 10 + }, + BULK_RESTORE: { + key: "BULK_RESTORE", + name: "Bulk Restore", + icon: null, + description: null, + position: 10 + }, + BULK_UPDATE: { + key: "BULK_UPDATE", + name: "Bulk Update", + icon: null, + description: null, + position: 10 + }, DEPRECATE: { key: "DEPRECATE", name: "Deprecate", diff --git a/waltz-ng/client/measurable-category/components/bulk-taxonomy-editor/BulkTaxonomyEditor.svelte b/waltz-ng/client/measurable-category/components/bulk-taxonomy-editor/BulkTaxonomyEditor.svelte new file mode 100644 index 0000000000..84e8fb4c28 --- /dev/null +++ b/waltz-ng/client/measurable-category/components/bulk-taxonomy-editor/BulkTaxonomyEditor.svelte @@ -0,0 +1,331 @@ + + +
+ The bulk taxonomy editor can be used to upload multiple changes to this taxonomy. +
+ +{#if mode === Modes.EDIT} +
+ Help +
+ The bulk change format should look like, column order is not important but the headers are: +
+
External Id
+
This uniquely identifies the item within the category. It should not be changed after it is set.
+ +
ParentExternalId
+
This optionally defines the external id of the parent. If this is changed, the item will be moved accordingly.
+ +
Name
+
Short descriptive name of the item
+ +
Description
+
Short description
+ +
Concrete
+
This determines if the node can be used in mappings against applications. If set to false, it implies the node is abstract and cannot be used in mappings.
+
+ For example: +
+ 
+            externalId	 parentExternalId	 name	 description	 concrete
+            a1		 A1	 Root node	 false
+            a1.1	 a1	 A1_1	 First child	 true
+            a1.2	 a1	 A1_2	 Second child	 true
+
+ Note, removal of items should be done via the Interactive Taxonomy Editor panel. + + +
+ +
+ +
+ + + + + +
+ +
+{/if} + +{#if mode === Modes.LOADING} + +{/if} + +{#if mode === Modes.PREVIEW} + {#if !_.isNil(previewData.error)} +
+ + Could not parse the data, see error message below. + +
{previewData.error.message}
+
+ {/if} + {#if _.isNil(previewData.error)} +
+ + + + + + + + + + + + + + {#each previewData.validatedItems as item} + + + + + + + + + + {/each} + +
ActionExternal IdParent External IdNameDescriptionConcreteErrors
+ {mkOpLabel(item)} + + {item.parsedItem.externalId} + + {item.parsedItem.parentExternalId} + + {item.parsedItem.name} + + {truncateMiddle(item.parsedItem.description)} + + {item.parsedItem.concrete} + + {item.errors} +
+
+ {/if} + + + +{/if} + +{#if mode === Modes.APPLY} + + + + + + + + + + + + + + + + + + + + + + + +
Added Records 0}> + {applyData.recordsAdded} +
Updated Records 0}> + {applyData.recordsUpdated} +
Removed Records 0}> + {applyData.recordsRemoved} +
Restored Records 0}> + {applyData.recordsRestored} +
Hierarchy Rebuilt + {applyData.hierarchyRebuilt} +
+ + {#if applyData.hierarchyRebuilt} +

+ Please note: This change has altered the hierarchy, you will need to reload this page. +

+ {/if} + + + +{/if} + + \ No newline at end of file diff --git a/waltz-ng/client/measurable-category/pages/edit/measurable-category-edit.html b/waltz-ng/client/measurable-category/pages/edit/measurable-category-edit.html index a3669edbaa..7b297125cb 100644 --- a/waltz-ng/client/measurable-category/pages/edit/measurable-category-edit.html +++ b/waltz-ng/client/measurable-category/pages/edit/measurable-category-edit.html @@ -51,88 +51,139 @@
-
- - - -
- - -
- - - - -
-
- - - -
-
- Recently selected nodes are shown here: -
- -
+
+ + + -
- - Clear - -
- - -
-
+ + - -
- - -
- - -
+ + -
- - Close - +
+
+ + +
+
+
+ The bulk rating editor can be used to upload multiple measurable ratings against this taxonomy. + The file format should look like:
- - - -
- - - -
- - + 
+                                A,B,C
+                                1,2,3
+                                5,6,7
+
+
+
+
+ + + +
+ + +
+ + + + +
+
+ + + +
+
+ Recently selected nodes are shown here: +
+ +
+ +
+ + Clear + +
+ + +
-
- - Close - + + +
+ + +
+ + +
+ +
+ + Close + +
+ + + +
+ + + +
+ + +
+
+ + Close + +
+ +
- - +
+
diff --git a/waltz-ng/client/measurable-category/pages/edit/measurable-category-edit.js b/waltz-ng/client/measurable-category/pages/edit/measurable-category-edit.js index 8029f95736..e8e2d1c4c0 100644 --- a/waltz-ng/client/measurable-category/pages/edit/measurable-category-edit.js +++ b/waltz-ng/client/measurable-category/pages/edit/measurable-category-edit.js @@ -21,6 +21,7 @@ import {CORE_API} from "../../../common/services/core-api-utils"; import template from "./measurable-category-edit.html"; import {toEntityRef} from "../../../common/entity-utils"; import toasts from "../../../svelte-stores/toast-store"; +import BulkTaxonomyEditor from "../../components/bulk-taxonomy-editor/BulkTaxonomyEditor.svelte"; const modes = { @@ -29,6 +30,11 @@ const modes = { CHANGE_VIEW: "CHANGE_VIEW" }; +const tabs = { + INTERACTIVE_TAXONOMY: "INTERACTIVE_TAXONOMY", + BULK_TAXONOMY: "BULK_TAXONOMY", + BULK_RATING: "BULK_RATING" +}; const initialState = { changeDomain: null, @@ -38,7 +44,9 @@ const initialState = { selectedChange: null, recentlySelected: [], pendingChanges: [], - mode: modes.SUMMARY + mode: modes.SUMMARY, + activeTab: tabs.INTERACTIVE_TAXONOMY, + BulkTaxonomyEditor }; diff --git a/waltz-ng/client/svelte-stores/taxonomy-management-store.js b/waltz-ng/client/svelte-stores/taxonomy-management-store.js new file mode 100644 index 0000000000..19a4e887b4 --- /dev/null +++ b/waltz-ng/client/svelte-stores/taxonomy-management-store.js @@ -0,0 +1,41 @@ +/* + * Waltz - Enterprise Architecture + * Copyright (C) 2016, 2017, 2018, 2019 Waltz open source project + * See README.md for more information + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific + * + */ + +import {remote} from "./remote"; + +export function mkTaxonomyManagementStore() { + + const bulkPreview = (taxonomyRef, rawText) => remote + .execute( + "POST", + `api/taxonomy-management/bulk/preview/${taxonomyRef.kind}/${taxonomyRef.id}`, + rawText); + + const bulkApply = (taxonomyRef, rawText) => remote + .execute( + "POST", + `api/taxonomy-management/bulk/apply/${taxonomyRef.kind}/${taxonomyRef.id}`, + rawText); + + return { + bulkPreview, + bulkApply + }; +} + +export const taxonomyManagementStore = mkTaxonomyManagementStore(); \ No newline at end of file diff --git a/waltz-service/pom.xml b/waltz-service/pom.xml index 3a4d79c72a..034501bafb 100644 --- a/waltz-service/pom.xml +++ b/waltz-service/pom.xml @@ -92,6 +92,11 @@ commons-jexl3 + + com.fasterxml.jackson.dataformat + jackson-dataformat-csv + + org.junit.jupiter diff --git a/waltz-service/src/main/java/org/finos/waltz/service/measurable/MeasurableService.java b/waltz-service/src/main/java/org/finos/waltz/service/measurable/MeasurableService.java index 3aeaa75cc2..693dac43bc 100644 --- a/waltz-service/src/main/java/org/finos/waltz/service/measurable/MeasurableService.java +++ b/waltz-service/src/main/java/org/finos/waltz/service/measurable/MeasurableService.java @@ -103,6 +103,13 @@ public List findByCategoryId(Long categoryId) { return measurableDao.findByCategoryId(categoryId); } + /** + * returns all measurables for a given category which match the supplied list of statuses + */ + public List findByCategoryId(Long categoryId, Set statuses) { + return measurableDao.findByCategoryId(categoryId, statuses); + } + public List findByParentId(Long parentId) { return measurableDao.findByParentId(parentId); } diff --git a/waltz-service/src/main/java/org/finos/waltz/service/taxonomy_management/BulkTaxonomyChangeService.java b/waltz-service/src/main/java/org/finos/waltz/service/taxonomy_management/BulkTaxonomyChangeService.java new file mode 100644 index 0000000000..36820ca62d --- /dev/null +++ b/waltz-service/src/main/java/org/finos/waltz/service/taxonomy_management/BulkTaxonomyChangeService.java @@ -0,0 +1,527 @@ +/* + * Waltz - Enterprise Architecture + * Copyright (C) 2016, 2017, 2018, 2019 Waltz open source project + * See README.md for more information + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific + * + */ + +package org.finos.waltz.service.taxonomy_management; + +import org.finos.waltz.common.DateTimeUtilities; +import org.finos.waltz.common.StringUtilities; +import org.finos.waltz.common.hierarchy.FlatNode; +import org.finos.waltz.model.EntityKind; +import org.finos.waltz.model.EntityLifecycleStatus; +import org.finos.waltz.model.EntityReference; +import org.finos.waltz.model.bulk_upload.BulkUpdateMode; +import org.finos.waltz.model.bulk_upload.ChangeOperation; +import org.finos.waltz.model.bulk_upload.taxonomy.BulkTaxonomyApplyResult; +import org.finos.waltz.model.bulk_upload.taxonomy.BulkTaxonomyItem; +import org.finos.waltz.model.bulk_upload.taxonomy.BulkTaxonomyParseResult; +import org.finos.waltz.model.bulk_upload.taxonomy.BulkTaxonomyValidatedItem; +import org.finos.waltz.model.bulk_upload.taxonomy.BulkTaxonomyValidationResult; +import org.finos.waltz.model.bulk_upload.taxonomy.ChangedFieldType; +import org.finos.waltz.model.bulk_upload.taxonomy.ImmutableBulkTaxonomyApplyResult; +import org.finos.waltz.model.bulk_upload.taxonomy.ImmutableBulkTaxonomyValidatedItem; +import org.finos.waltz.model.bulk_upload.taxonomy.ImmutableBulkTaxonomyValidationResult; +import org.finos.waltz.model.bulk_upload.taxonomy.ValidationError; +import org.finos.waltz.model.measurable.Measurable; +import org.finos.waltz.model.measurable_category.MeasurableCategory; +import org.finos.waltz.model.taxonomy_management.TaxonomyChangeLifecycleStatus; +import org.finos.waltz.model.taxonomy_management.TaxonomyChangeType; +import org.finos.waltz.schema.tables.records.MeasurableRecord; +import org.finos.waltz.schema.tables.records.TaxonomyChangeRecord; +import org.finos.waltz.service.entity_hierarchy.EntityHierarchyService; +import org.finos.waltz.service.measurable.MeasurableService; +import org.finos.waltz.service.measurable_category.MeasurableCategoryService; +import org.finos.waltz.service.taxonomy_management.BulkTaxonomyItemParser.InputFormat; +import org.finos.waltz.service.user.UserRoleService; +import org.jooq.DSLContext; +import org.jooq.UpdateConditionStep; +import org.jooq.UpdateSetStep; +import org.jooq.impl.DSL; +import org.jooq.lambda.tuple.Tuple2; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.sql.Timestamp; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.lang.String.format; +import static java.util.Collections.emptySet; +import static java.util.Optional.ofNullable; +import static java.util.stream.Collectors.toList; +import static org.finos.waltz.common.Checks.checkNotNull; +import static org.finos.waltz.common.MapUtilities.countBy; +import static org.finos.waltz.common.MapUtilities.indexBy; +import static org.finos.waltz.common.SetUtilities.asSet; +import static org.finos.waltz.common.SetUtilities.compact; +import static org.finos.waltz.common.SetUtilities.filter; +import static org.finos.waltz.common.SetUtilities.union; +import static org.finos.waltz.common.StringUtilities.limit; +import static org.finos.waltz.common.StringUtilities.mkSafe; +import static org.finos.waltz.common.StringUtilities.safeEq; +import static org.finos.waltz.common.hierarchy.HierarchyUtilities.hasCycle; +import static org.finos.waltz.common.hierarchy.HierarchyUtilities.toForest; +import static org.finos.waltz.data.JooqUtilities.summarizeResults; +import static org.finos.waltz.schema.Tables.MEASURABLE; +import static org.finos.waltz.service.taxonomy_management.TaxonomyManagementUtilities.verifyUserHasPermissions; +import static org.jooq.lambda.tuple.Tuple.tuple; + +@Service +public class BulkTaxonomyChangeService { + + private static final Logger LOG = LoggerFactory.getLogger(BulkTaxonomyChangeService.class); + + private final MeasurableCategoryService measurableCategoryService; + private final MeasurableService measurableService; + private final UserRoleService userRoleService; + private final EntityHierarchyService entityHierarchyService; + private final DSLContext dsl; + + + @Autowired + public BulkTaxonomyChangeService(MeasurableCategoryService measurableCategoryService, + MeasurableService measurableService, + UserRoleService userRoleService, + EntityHierarchyService entityHierarchyService, + DSLContext dsl) { + this.measurableCategoryService = measurableCategoryService; + this.measurableService = measurableService; + this.userRoleService = userRoleService; + this.entityHierarchyService = entityHierarchyService; + this.dsl = dsl; + } + + + public BulkTaxonomyValidationResult previewBulk(EntityReference taxonomyRef, + String inputStr, + InputFormat format, + BulkUpdateMode mode) { + if (taxonomyRef.kind() != EntityKind.MEASURABLE_CATEGORY) { + throw new UnsupportedOperationException("Only measurable category bulk updates supported"); + } + + LOG.debug( + "Bulk preview - category:{}, format:{}, inputStr:{}", + taxonomyRef, + format, + limit(inputStr, 40)); + + MeasurableCategory category = measurableCategoryService.getById(taxonomyRef.id()); + checkNotNull(category, "Unknown category: %d", taxonomyRef); + + List existingMeasurables = measurableService.findByCategoryId( + taxonomyRef.id(), + asSet(EntityLifecycleStatus.values())); + + Map existingByExtId = indexBy(existingMeasurables, m -> m.externalId().orElse(null)); + + LOG.debug( + "category: {} = {}({})", + taxonomyRef, + category.name(), + category.externalId()); + + BulkTaxonomyParseResult result = new BulkTaxonomyItemParser().parse(inputStr, format); + + if (result.error() != null) { + return ImmutableBulkTaxonomyValidationResult + .builder() + .error(result.error()) + .build(); + } + + Map givenByExtId = indexBy(result.parsedItems(), BulkTaxonomyItem::externalId); + Set allExtIds = union(existingByExtId.keySet(), givenByExtId.keySet()); + + /* + Validation checks: + + - unique external ids + - all parent external id's exist either in file or in existing taxonomy + - check for cycles by converting to forest + - do a diff to determine + - new + - removed, if mode == replace + - updated, match on external id + */ + + Map countByExtId = countBy(result.parsedItems(), BulkTaxonomyItem::externalId); + + Boolean hasCycle = determineIfTreeHasCycle(existingMeasurables, result); + + List validatedItems = result + .parsedItems() + .stream() + .map(d -> tuple( + d, + existingByExtId.get(d.externalId()))) // => (parsedItem, existingItem?) + .map(t -> { + Tuple2> op = determineOperation(t.v1, t.v2); + boolean isUnique = countByExtId.get(t.v1.externalId()) == 1; + boolean parentExists = StringUtilities.isEmpty(t.v1.parentExternalId()) + || allExtIds.contains(t.v1.parentExternalId()); + boolean isCyclical = hasCycle; + return ImmutableBulkTaxonomyValidatedItem + .builder() + .parsedItem(t.v1) + .changedFields(op.v2) + .changeOperation(op.v1) + .existingItemReference(ofNullable(t.v2) + .map(Measurable::entityReference) + .orElse(null)) + .errors(compact( + isUnique ? null : ValidationError.DUPLICATE_EXT_ID, + parentExists ? null : ValidationError.PARENT_NOT_FOUND, + isCyclical ? ValidationError.CYCLE_DETECTED : null)) + .build(); + }) + .collect(Collectors.toList()); + + Set toRemove = mode == BulkUpdateMode.ADD_ONLY + ? emptySet() + : filter(existingMeasurables, m -> ! givenByExtId.containsKey(m.externalId().get())); + + return ImmutableBulkTaxonomyValidationResult + .builder() + .plannedRemovals(toRemove) + .validatedItems(validatedItems) + .build(); + } + + + public BulkTaxonomyApplyResult applyBulk(EntityReference taxonomyRef, + BulkTaxonomyValidationResult bulkRequest, + String userId) { + if (taxonomyRef.kind() != EntityKind.MEASURABLE_CATEGORY) { + throw new UnsupportedOperationException("Only measurable category bulk updates supported"); + } + verifyUserHasPermissions(measurableCategoryService, userRoleService, userId, taxonomyRef); + Timestamp now = DateTimeUtilities.nowUtcTimestamp(); + + Set toAdd = bulkRequest + .validatedItems() + .stream() + .filter(d -> d.changeOperation() == ChangeOperation.ADD) + .map(BulkTaxonomyValidatedItem::parsedItem) + .map(bulkTaxonomyItem -> { + MeasurableRecord r = new MeasurableRecord(); + r.setMeasurableCategoryId(taxonomyRef.id()); + r.setConcrete(bulkTaxonomyItem.concrete()); + r.setName(bulkTaxonomyItem.name()); + r.setExternalId(bulkTaxonomyItem.externalId()); + r.setExternalParentId(bulkTaxonomyItem.parentExternalId()); + r.setDescription(bulkTaxonomyItem.description()); + r.setLastUpdatedAt(now); + r.setLastUpdatedBy(userId); + r.setProvenance("waltz"); + return r; + }) + .collect(Collectors.toSet()); + + Set> toRestore = bulkRequest + .validatedItems() + .stream() + .filter(d -> d.changeOperation() == ChangeOperation.RESTORE) + .map(d -> DSL + .update(org.finos.waltz.schema.tables.Measurable.MEASURABLE) + .set(org.finos.waltz.schema.tables.Measurable.MEASURABLE.ENTITY_LIFECYCLE_STATUS, EntityLifecycleStatus.ACTIVE.name()) + .where(org.finos.waltz.schema.tables.Measurable.MEASURABLE.MEASURABLE_CATEGORY_ID.eq(taxonomyRef.id())) + .and(org.finos.waltz.schema.tables.Measurable.MEASURABLE.ID.eq(d.existingItemReference().id()))) + .collect(Collectors.toSet()); + + Set> toUpdate = bulkRequest + .validatedItems() + .stream() + .filter(d -> d.changeOperation() == ChangeOperation.UPDATE || d.changeOperation() == ChangeOperation.RESTORE) + .map(d -> { + BulkTaxonomyItem item = d.parsedItem(); + UpdateSetStep upd = DSL.update(org.finos.waltz.schema.tables.Measurable.MEASURABLE); + if (d.changedFields().contains(ChangedFieldType.NAME)) { + upd = upd.set(org.finos.waltz.schema.tables.Measurable.MEASURABLE.NAME, item.name()); + } + if (d.changedFields().contains(ChangedFieldType.PARENT_EXTERNAL_ID)) { + upd = StringUtilities.isEmpty(item.parentExternalId()) + ? upd.setNull(org.finos.waltz.schema.tables.Measurable.MEASURABLE.EXTERNAL_PARENT_ID) + : upd.set(org.finos.waltz.schema.tables.Measurable.MEASURABLE.EXTERNAL_PARENT_ID, item.parentExternalId()); + } + if (d.changedFields().contains(ChangedFieldType.DESCRIPTION)) { + upd = upd.set(org.finos.waltz.schema.tables.Measurable.MEASURABLE.DESCRIPTION, item.description()); + } + if (d.changedFields().contains(ChangedFieldType.CONCRETE)) { + upd = upd.set(org.finos.waltz.schema.tables.Measurable.MEASURABLE.CONCRETE, item.concrete()); + } + return upd + .set(org.finos.waltz.schema.tables.Measurable.MEASURABLE.LAST_UPDATED_AT, now) + .set(org.finos.waltz.schema.tables.Measurable.MEASURABLE.LAST_UPDATED_BY, userId) + .where(org.finos.waltz.schema.tables.Measurable.MEASURABLE.MEASURABLE_CATEGORY_ID.eq(taxonomyRef.id())) + .and(org.finos.waltz.schema.tables.Measurable.MEASURABLE.ID.eq(d.existingItemReference().id())); + }) + .collect(Collectors.toSet()); + + + boolean requiresRebuild = requiresHierarchyRebuild(bulkRequest.validatedItems()); + + BulkTaxonomyApplyResult changeResult = dsl + .transactionResult(ctx -> { + DSLContext tx = ctx.dsl(); + int insertCount = summarizeResults(tx.batchInsert(toAdd).execute()); + int restoreCount = summarizeResults(tx.batch(toRestore).execute()); + int updateCount = summarizeResults(tx.batch(toUpdate).execute()); + + Set changeRecords = mkTaxonomyChangeRecords(tx, taxonomyRef, bulkRequest, userId); + + int insertedChangeRecordCount = summarizeResults(tx.batchInsert(changeRecords).execute()); + + LOG.debug("Added {} new change record entries", insertedChangeRecordCount); + + if (requiresRebuild) { + int updatedParents = updateParentIdsFromExternalIds(tx, taxonomyRef.id()); + LOG.debug("Updated parents: {}", updatedParents); + } + + return ImmutableBulkTaxonomyApplyResult + .builder() + .recordsAdded(insertCount) + .recordsUpdated(updateCount) + .recordsRestored(restoreCount) + .recordsRemoved(0)//TODO: add removeCount + .hierarchyRebuilt(requiresRebuild) + .build(); + }); + + LOG.debug("Result of apply changes: {}", changeResult); + if (requiresRebuild) { + int entriesCreated = entityHierarchyService.buildForMeasurableByCategory(taxonomyRef.id()); + LOG.debug("Recreated hierarchy with {} new entries", entriesCreated); + } + + return changeResult; + } + + + private Set mkTaxonomyChangeRecords(DSLContext tx, + EntityReference taxonomyRef, + BulkTaxonomyValidationResult bulkRequest, + String user) { + + Map> extIdToIdMap = tx + .select(MEASURABLE.EXTERNAL_ID, MEASURABLE.ID, MEASURABLE.NAME) + .from(MEASURABLE) + .where(MEASURABLE.MEASURABLE_CATEGORY_ID.eq(taxonomyRef.id())) + .fetchMap( + MEASURABLE.EXTERNAL_ID, + r -> tuple( + r.get(MEASURABLE.ID), + r.get(MEASURABLE.NAME))); + + return bulkRequest + .validatedItems() + .stream() + .flatMap(d -> toTaxonomyChangeRecords( + d, + taxonomyRef, + extIdToIdMap.get(d.parsedItem().externalId()), + extIdToIdMap.get(d.parsedItem().parentExternalId()), + user)) + .collect(Collectors.toSet()); + } + + + private Stream toTaxonomyChangeRecords(BulkTaxonomyValidatedItem validatedItem, + EntityReference taxonomyReference, + Tuple2 item, + Tuple2 parent, // may be null + String user) { + + if (validatedItem.changeOperation() == ChangeOperation.ADD) { + + TaxonomyChangeRecord r = mkBaseTaxonomyChangeRecord(taxonomyReference, item.v1, user); + r.setChangeType(TaxonomyChangeType.BULK_ADD.name()); + return Stream.of(r); + + } else if (validatedItem.changeOperation() == ChangeOperation.RESTORE || validatedItem.changeOperation() == ChangeOperation.UPDATE) { + + Stream fieldStream = validatedItem + .changedFields() + .stream() + .map(f -> { + TaxonomyChangeRecord r = mkBaseTaxonomyChangeRecord(taxonomyReference, item.v1, user); + switch (f) { + case NAME: + r.setChangeType(TaxonomyChangeType.UPDATE_NAME.name()); + r.setParams(format( + "{ \"name\": \"%s\", \"originalValue\": \"\" }", + validatedItem.parsedItem().name())); + break; + case DESCRIPTION: + r.setChangeType(TaxonomyChangeType.UPDATE_DESCRIPTION.name()); + r.setParams(format( + "{ \"description\": \"%s\", \"originalValue\": \"\" }", + validatedItem.parsedItem().description())); + break; + case CONCRETE: + r.setChangeType(TaxonomyChangeType.UPDATE_CONCRETENESS.name()); + r.setParams(format( + "{ \"concrete\": \"%s\" }", + validatedItem.parsedItem().concrete())); + break; + case PARENT_EXTERNAL_ID: + r.setChangeType(TaxonomyChangeType.MOVE.name()); + r.setParams(format( + "{ \"destinationId\": \"%d\", \"destinationName\": \"%s\", \"originalValue\": \"\" }", + parent == null ? 0 : parent.v1, + parent == null ? "root" : parent.v2)); + break; + default: + r.setChangeType(TaxonomyChangeType.BULK_UPDATE.name()); + } + return r; + }); + + return Stream.concat( + validatedItem.changeOperation() == ChangeOperation.RESTORE + ? Stream.of(mkRestoreRecord(taxonomyReference, item.v1, user)) + : Stream.empty(), + fieldStream); + + } else { + return Stream.empty(); + } + } + + + private TaxonomyChangeRecord mkRestoreRecord(EntityReference taxonomyReference, + Long itemId, + String user) { + TaxonomyChangeRecord r = mkBaseTaxonomyChangeRecord(taxonomyReference, itemId, user); + r.setChangeType(TaxonomyChangeType.BULK_RESTORE.name()); + return r; + } + + + private TaxonomyChangeRecord mkBaseTaxonomyChangeRecord(EntityReference taxonomyReference, + Long itemId, + String user) { + TaxonomyChangeRecord r = new TaxonomyChangeRecord(); + r.setDomainKind(taxonomyReference.kind().name()); + r.setDomainId(taxonomyReference.id()); + r.setPrimaryReferenceKind(EntityKind.MEASURABLE.name()); + r.setPrimaryReferenceId(itemId); + r.setStatus(TaxonomyChangeLifecycleStatus.EXECUTED.name()); + r.setParams("{}"); + r.setCreatedBy(user); + r.setLastUpdatedBy(user); + return r; + } + + + // --- HELPERS ---- + + /** + * This is achieved by doing a self join on the measurable table to find any matching parent and updating + * the child parent_id with the matching parent's id + */ + private int updateParentIdsFromExternalIds(DSLContext tx, + long categoryId) { + org.finos.waltz.schema.tables.Measurable parent = MEASURABLE.as("p"); + + return tx + .update(MEASURABLE) + .set(MEASURABLE.PARENT_ID, DSL + .select(parent.ID) + .from(parent) + .where(parent.EXTERNAL_ID.eq(MEASURABLE.EXTERNAL_PARENT_ID) + .and(parent.MEASURABLE_CATEGORY_ID.eq(categoryId)))) + .where(MEASURABLE.MEASURABLE_CATEGORY_ID.eq(categoryId)) + .execute(); + } + + + public static boolean requiresHierarchyRebuild(Collection items) { + Set opsThatNeedRebuild = asSet( + ChangeOperation.ADD, + ChangeOperation.REMOVE, + ChangeOperation.RESTORE); + + Predicate requiresRebuild = d -> { + boolean deffoNeedsRebuild = opsThatNeedRebuild.contains(d.changeOperation()); + boolean opIsUpdate = d.changeOperation() == ChangeOperation.UPDATE; + boolean parentIdChanged = d.changedFields().contains(ChangedFieldType.PARENT_EXTERNAL_ID); + + return deffoNeedsRebuild || (opIsUpdate && parentIdChanged); + }; + + return items + .stream() + .anyMatch(requiresRebuild); + } + + + private boolean determineIfTreeHasCycle(List existingMeasurables, + BulkTaxonomyParseResult result) { + return Stream + .concat( + existingMeasurables.stream().map(d -> new FlatNode( + d.externalId().orElse(null), + d.externalParentId(), + null)), + result.parsedItems().stream().map(d -> new FlatNode( + d.externalId(), + ofNullable(d.parentExternalId()), + null))) + .collect(Collectors.collectingAndThen(toList(), xs -> hasCycle(toForest(xs)))); + } + + + private Tuple2> determineOperation(BulkTaxonomyItem requiredItem, + Measurable existingItem) { + if (existingItem == null) { + return tuple(ChangeOperation.ADD, emptySet()); + } + + boolean isRestore = existingItem.entityLifecycleStatus() != EntityLifecycleStatus.ACTIVE; + boolean nameMatches = safeEq(requiredItem.name(), existingItem.name()); + boolean descMatches = safeEq(requiredItem.description(), existingItem.description()); + boolean parentExtIdMatches = safeEq(mkSafe(requiredItem.parentExternalId()), mkSafe(existingItem.externalParentId().orElse(null))); + boolean concreteMatches = requiredItem.concrete() == existingItem.concrete(); + + Set changedFields = new HashSet<>(); + if (!nameMatches) { changedFields.add(ChangedFieldType.NAME); } + if (!descMatches) { changedFields.add(ChangedFieldType.DESCRIPTION); } + if (!parentExtIdMatches) { changedFields.add(ChangedFieldType.PARENT_EXTERNAL_ID); } + if (!concreteMatches) { changedFields.add(ChangedFieldType.CONCRETE); } + + ChangeOperation op = isRestore + ? ChangeOperation.RESTORE + : changedFields.isEmpty() + ? ChangeOperation.NONE + : ChangeOperation.UPDATE; + + return tuple(op, changedFields); + + } + + +} diff --git a/waltz-service/src/main/java/org/finos/waltz/service/taxonomy_management/BulkTaxonomyItemParser.java b/waltz-service/src/main/java/org/finos/waltz/service/taxonomy_management/BulkTaxonomyItemParser.java new file mode 100644 index 0000000000..8f68ae9bab --- /dev/null +++ b/waltz-service/src/main/java/org/finos/waltz/service/taxonomy_management/BulkTaxonomyItemParser.java @@ -0,0 +1,134 @@ +package org.finos.waltz.service.taxonomy_management; + +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvParser; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import org.finos.waltz.common.StreamUtilities; +import org.finos.waltz.common.StringUtilities; +import org.finos.waltz.model.bulk_upload.taxonomy.BulkTaxonomyItem; +import org.finos.waltz.model.bulk_upload.taxonomy.BulkTaxonomyParseResult; +import org.finos.waltz.model.bulk_upload.taxonomy.ImmutableBulkTaxonomyParseError; +import org.finos.waltz.model.bulk_upload.taxonomy.ImmutableBulkTaxonomyParseResult; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.finos.waltz.common.StringUtilities.isEmpty; + +public class BulkTaxonomyItemParser { + + public enum InputFormat { + CSV, + TSV, + JSON + } + + + public BulkTaxonomyParseResult parse(String input, InputFormat format) { + if (isEmpty(input)) { + return handleEmptyInput(input); + } + + try { + switch (format) { + case CSV: + return parseCSV(clean(input)); + case TSV: + return parseTSV(clean(input)); + case JSON: + return parseJSON(input); + default: + throw new IllegalArgumentException(format("Unknown format: %s", format)); + } + } catch (IOException e) { + return ImmutableBulkTaxonomyParseResult + .builder() + .input(input) + .error(ImmutableBulkTaxonomyParseError + .builder() + .message(e.getMessage()) + .build()) + .build(); + } + } + + + private String clean(String input) { + return StreamUtilities + .lines(input) + .filter(StringUtilities::isDefined) + .filter(line -> ! line.startsWith("#")) + .collect(Collectors.joining("\n")); + } + + + private BulkTaxonomyParseResult parseJSON(String input) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + MappingIterator items = mapper + .readerFor(BulkTaxonomyItem.class) + .readValues(input); + + return BulkTaxonomyParseResult.mkResult( + items.readAll(), + input); + } + + + private BulkTaxonomyParseResult parseCSV(String input) throws IOException { + List items = attemptToParseDelimited(input, configureCSVSchema()); + return BulkTaxonomyParseResult.mkResult(items, input); + } + + + private BulkTaxonomyParseResult parseTSV(String input) throws IOException { + List items = attemptToParseDelimited(input, configureTSVSchema()); + return BulkTaxonomyParseResult.mkResult(items, input); + } + + + private List attemptToParseDelimited(String input, + CsvSchema bootstrapSchema) throws IOException { + CsvMapper mapper = new CsvMapper(); + mapper.enable(CsvParser.Feature.TRIM_SPACES); + MappingIterator items = mapper + .readerFor(BulkTaxonomyItem.class) + .with(bootstrapSchema) + .readValues(input); + + return items.readAll(); + } + + + private CsvSchema configureCSVSchema() { + return CsvSchema + .emptySchema() + .withHeader(); + } + + + private CsvSchema configureTSVSchema() { + return CsvSchema + .emptySchema() + .withHeader() + .withColumnSeparator('\t'); + } + + + private BulkTaxonomyParseResult handleEmptyInput(String input) { + return ImmutableBulkTaxonomyParseResult + .builder() + .input(input) + .error(ImmutableBulkTaxonomyParseError + .builder() + .message("Cannot parse an empty string") + .column(0) + .line(0) + .build()) + .build(); + } + +} diff --git a/waltz-service/src/main/java/org/finos/waltz/service/taxonomy_management/TaxonomyChangeService.java b/waltz-service/src/main/java/org/finos/waltz/service/taxonomy_management/TaxonomyChangeService.java index 49c982dcc5..c0e6a5e174 100644 --- a/waltz-service/src/main/java/org/finos/waltz/service/taxonomy_management/TaxonomyChangeService.java +++ b/waltz-service/src/main/java/org/finos/waltz/service/taxonomy_management/TaxonomyChangeService.java @@ -23,11 +23,13 @@ import org.finos.waltz.data.taxonomy_management.TaxonomyChangeDao; import org.finos.waltz.model.EntityKind; import org.finos.waltz.model.EntityReference; -import org.finos.waltz.model.exceptions.NotAuthorizedException; import org.finos.waltz.model.measurable.Measurable; -import org.finos.waltz.model.measurable_category.MeasurableCategory; -import org.finos.waltz.model.taxonomy_management.*; -import org.finos.waltz.model.user.SystemRole; +import org.finos.waltz.model.taxonomy_management.ImmutableTaxonomyChangeCommand; +import org.finos.waltz.model.taxonomy_management.ImmutableTaxonomyChangePreview; +import org.finos.waltz.model.taxonomy_management.TaxonomyChangeCommand; +import org.finos.waltz.model.taxonomy_management.TaxonomyChangeLifecycleStatus; +import org.finos.waltz.model.taxonomy_management.TaxonomyChangePreview; +import org.finos.waltz.model.taxonomy_management.TaxonomyChangeType; import org.finos.waltz.service.client_cache_key.ClientCacheKeyService; import org.finos.waltz.service.entity_hierarchy.EntityHierarchyService; import org.finos.waltz.service.measurable.MeasurableService; @@ -43,7 +45,10 @@ import java.util.Map; import static java.util.stream.Collectors.toMap; -import static org.finos.waltz.common.Checks.*; +import static org.finos.waltz.common.Checks.checkFalse; +import static org.finos.waltz.common.Checks.checkNotNull; +import static org.finos.waltz.common.Checks.checkTrue; +import static org.finos.waltz.service.taxonomy_management.TaxonomyManagementUtilities.verifyUserHasPermissions; import static org.jooq.lambda.tuple.Tuple.tuple; @Service @@ -105,8 +110,9 @@ public TaxonomyChangePreview previewById(long id) { } - public TaxonomyChangeCommand submitDraftChange(TaxonomyChangeCommand draftCommand, String userId) { - verifyUserHasPermissions(userId, draftCommand.changeDomain()); + public TaxonomyChangeCommand submitDraftChange(TaxonomyChangeCommand draftCommand, + String userId) { + verifyUserHasPermissions(measurableCategoryService, userRoleService, userId, draftCommand.changeDomain()); checkTrue(draftCommand.status() == TaxonomyChangeLifecycleStatus.DRAFT, "Command must be DRAFT"); TaxonomyChangeCommand commandToSave = ImmutableTaxonomyChangeCommand @@ -133,9 +139,10 @@ public Collection findAllChangesByDomain(EntityReference } - public TaxonomyChangeCommand applyById(long id, String userId) { + public TaxonomyChangeCommand applyById(long id, + String userId) { TaxonomyChangeCommand command = taxonomyChangeDao.getDraftCommandById(id); - verifyUserHasPermissions(userId, command.changeDomain()); + verifyUserHasPermissions(measurableCategoryService, userRoleService, userId, command.changeDomain()); checkFalse(isMoveToSameParent(command), "Measurable cannot set it self as its parent."); @@ -163,8 +170,9 @@ && isHierarchyChange(command)) { } - public boolean removeById(long id, String userId) { - verifyUserHasPermissions(userId); + public boolean removeById(long id, + String userId) { + verifyUserHasPermissions(userRoleService, userId); return taxonomyChangeDao.removeById(id, userId); } @@ -176,26 +184,9 @@ private TaxonomyCommandProcessor getCommandProcessor(TaxonomyChangeCommand comma } - private void verifyUserHasPermissions(String userId, EntityReference changeDomain) { - verifyUserHasPermissions(userId); - - if (changeDomain.kind() == EntityKind.MEASURABLE_CATEGORY) { - MeasurableCategory category = measurableCategoryService.getById(changeDomain.id()); - if (! category.editable()) { - throw new NotAuthorizedException("Unauthorised: Category is not editable"); - } - } - } - - private void verifyUserHasPermissions(String userId) { - if (! userRoleService.hasRole(userId, SystemRole.TAXONOMY_EDITOR.name())) { - throw new NotAuthorizedException(); - } - } - private boolean isMoveToSameParent(TaxonomyChangeCommand command) { String destinationId = command.params().get("destinationId"); - if(isMovingToANode(command, destinationId)) { + if (isMovingToANode(command, destinationId)) { long parentId = Long.parseLong(destinationId); final Measurable parent = measurableService.getById(parentId); @@ -209,10 +200,12 @@ private boolean isMoveToSameParent(TaxonomyChangeCommand command) { private boolean isMoveToANodeWhichIsAlreadyAChild(TaxonomyChangeCommand command) { String destinationId = command.params().get("destinationId"); return isMovingToANode(command, destinationId) && - Long.parseLong(destinationId) == command.primaryReference().id(); + Long.parseLong(destinationId) == command.primaryReference().id(); } - private boolean isMovingToANode(TaxonomyChangeCommand command, String destinationId) { + + private boolean isMovingToANode(TaxonomyChangeCommand command, + String destinationId) { return command.changeType().equals(TaxonomyChangeType.MOVE) && StringUtilities.notEmpty(destinationId); } @@ -224,4 +217,5 @@ private boolean isHierarchyChange(TaxonomyChangeCommand command) { || command.changeType() == TaxonomyChangeType.REMOVE || command.changeType() == TaxonomyChangeType.MOVE; } + } diff --git a/waltz-service/src/main/java/org/finos/waltz/service/taxonomy_management/TaxonomyManagementUtilities.java b/waltz-service/src/main/java/org/finos/waltz/service/taxonomy_management/TaxonomyManagementUtilities.java index d1be2ab553..ae4a1b0cb0 100644 --- a/waltz-service/src/main/java/org/finos/waltz/service/taxonomy_management/TaxonomyManagementUtilities.java +++ b/waltz-service/src/main/java/org/finos/waltz/service/taxonomy_management/TaxonomyManagementUtilities.java @@ -19,17 +19,23 @@ package org.finos.waltz.service.taxonomy_management; import org.finos.waltz.common.SetUtilities; +import org.finos.waltz.model.EntityKind; import org.finos.waltz.model.EntityReference; import org.finos.waltz.model.HierarchyQueryScope; import org.finos.waltz.model.IdSelectionOptions; import org.finos.waltz.model.Severity; +import org.finos.waltz.model.exceptions.NotAuthorizedException; import org.finos.waltz.model.measurable.Measurable; +import org.finos.waltz.model.measurable_category.MeasurableCategory; import org.finos.waltz.model.measurable_rating.MeasurableRating; import org.finos.waltz.model.taxonomy_management.ImmutableTaxonomyChangeImpact; import org.finos.waltz.model.taxonomy_management.ImmutableTaxonomyChangePreview; import org.finos.waltz.model.taxonomy_management.TaxonomyChangeCommand; +import org.finos.waltz.model.user.SystemRole; import org.finos.waltz.service.measurable.MeasurableService; +import org.finos.waltz.service.measurable_category.MeasurableCategoryService; import org.finos.waltz.service.measurable_rating.MeasurableRatingService; +import org.finos.waltz.service.user.UserRoleService; import java.util.List; import java.util.Set; @@ -176,4 +182,25 @@ public static String getExternalIdParam(TaxonomyChangeCommand cmd) { public static boolean getConcreteParam(TaxonomyChangeCommand cmd, boolean dflt) { return cmd.paramAsBoolean("concrete", dflt); } + + public static void verifyUserHasPermissions(MeasurableCategoryService measurableCategoryService, + UserRoleService userRoleService, + String userId, + EntityReference changeDomain) { + verifyUserHasPermissions(userRoleService, userId); + + if (changeDomain.kind() == EntityKind.MEASURABLE_CATEGORY) { + MeasurableCategory category = measurableCategoryService.getById(changeDomain.id()); + if (!category.editable()) { + throw new NotAuthorizedException("Unauthorised: Category is not editable"); + } + } + } + + public static void verifyUserHasPermissions(UserRoleService userRoleService, String userId) { + if (!userRoleService.hasRole(userId, SystemRole.TAXONOMY_EDITOR.name())) { + throw new NotAuthorizedException(); + } + } + } diff --git a/waltz-service/src/test/java/org/finos/waltz/service/taxonomy_management/BulkTaxonomyChangeService_RequiresHierarchyRebuildTest.java b/waltz-service/src/test/java/org/finos/waltz/service/taxonomy_management/BulkTaxonomyChangeService_RequiresHierarchyRebuildTest.java new file mode 100644 index 0000000000..099ff05f3d --- /dev/null +++ b/waltz-service/src/test/java/org/finos/waltz/service/taxonomy_management/BulkTaxonomyChangeService_RequiresHierarchyRebuildTest.java @@ -0,0 +1,92 @@ +package org.finos.waltz.service.taxonomy_management; + +import org.finos.waltz.model.bulk_upload.ChangeOperation; +import org.finos.waltz.model.bulk_upload.taxonomy.BulkTaxonomyValidatedItem; +import org.finos.waltz.model.bulk_upload.taxonomy.ChangedFieldType; +import org.finos.waltz.model.bulk_upload.taxonomy.ImmutableBulkTaxonomyItem; +import org.finos.waltz.model.bulk_upload.taxonomy.ImmutableBulkTaxonomyValidatedItem; +import org.junit.jupiter.api.Test; + +import java.util.Set; + +import static java.util.Collections.emptySet; +import static org.finos.waltz.common.SetUtilities.asSet; +import static org.finos.waltz.service.taxonomy_management.BulkTaxonomyChangeService.requiresHierarchyRebuild; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class BulkTaxonomyChangeService_RequiresHierarchyRebuildTest { + + private static final ImmutableBulkTaxonomyItem defaultItem = ImmutableBulkTaxonomyItem + .builder() + .name("foo") + .externalId("baa") + .build(); + + @Test + void doesNotRequireRebuildIfNoItems() { + assertFalse(requiresHierarchyRebuild(emptySet()), "no items, therefore rebuild not required"); + } + + + @Test + void doesNotRequireRebuildIfItemsOnlyUpdateDecorations() { + Set itemsToCheck = asSet( + ImmutableBulkTaxonomyValidatedItem + .builder() + .parsedItem(defaultItem) + .changeOperation(ChangeOperation.UPDATE) + .changedFields(asSet(ChangedFieldType.NAME)) + .build()); + assertFalse(requiresHierarchyRebuild(itemsToCheck), "changes would not require a hierarchy rebuild"); + } + + + @Test + void requireRebuildIfItemsIncludeAdditions() { + Set itemsToCheck = asSet( + ImmutableBulkTaxonomyValidatedItem + .builder() + .parsedItem(defaultItem) + .changeOperation(ChangeOperation.ADD) + .build()); + assertTrue(requiresHierarchyRebuild(itemsToCheck)); + } + + + @Test + void requireRebuildIfItemsChangeTheirParentID() { + Set itemsToCheck = asSet( + ImmutableBulkTaxonomyValidatedItem + .builder() + .parsedItem(defaultItem) + .changeOperation(ChangeOperation.UPDATE) + .changedFields(asSet(ChangedFieldType.PARENT_EXTERNAL_ID)) + .build()); + assertTrue(requiresHierarchyRebuild(itemsToCheck)); + } + + + @Test + void requireRebuildIfItemsIncludeRemovals() { + Set itemsToCheck = asSet( + ImmutableBulkTaxonomyValidatedItem + .builder() + .parsedItem(defaultItem) + .changeOperation(ChangeOperation.REMOVE) + .build()); + assertTrue(requiresHierarchyRebuild(itemsToCheck)); + } + + + @Test + void requireRebuildIfItemsIncludeRestorations() { + Set itemsToCheck = asSet( + ImmutableBulkTaxonomyValidatedItem + .builder() + .parsedItem(defaultItem) + .changeOperation(ChangeOperation.RESTORE) + .build()); + assertTrue(requiresHierarchyRebuild(itemsToCheck)); + } +} \ No newline at end of file diff --git a/waltz-service/src/test/java/org/finos/waltz/service/taxonomy_management/BulkTaxonomyItemParserTest.java b/waltz-service/src/test/java/org/finos/waltz/service/taxonomy_management/BulkTaxonomyItemParserTest.java new file mode 100644 index 0000000000..2e3fb53fa1 --- /dev/null +++ b/waltz-service/src/test/java/org/finos/waltz/service/taxonomy_management/BulkTaxonomyItemParserTest.java @@ -0,0 +1,137 @@ +package org.finos.waltz.service.taxonomy_management; + +import org.finos.waltz.common.SetUtilities; +import org.finos.waltz.model.bulk_upload.taxonomy.BulkTaxonomyItem; +import org.finos.waltz.model.bulk_upload.taxonomy.BulkTaxonomyParseResult; +import org.finos.waltz.model.bulk_upload.taxonomy.BulkTaxonomyParseResult.BulkTaxonomyParseError; +import org.finos.waltz.service.taxonomy_management.BulkTaxonomyItemParser.InputFormat; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Set; + +import static org.finos.waltz.common.IOUtilities.readAsString; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class BulkTaxonomyItemParserTest { + + private final BulkTaxonomyItemParser parser = new BulkTaxonomyItemParser(); + + @Test + void simpleCSV() { + BulkTaxonomyParseResult result = parser.parse(readTestFile("test-taxonomy-items.csv"), InputFormat.CSV); + assertNull(result.error()); + assertEquals(3, result.parsedItems().size()); + assertNames(result.parsedItems(), "A1", "A1_1", "A1_2"); + } + + + @Test + void dodgyCSV() { + BulkTaxonomyParseResult result = parser.parse(readTestFile("test-bad-taxonomy-items.csv"), InputFormat.CSV); + assertNotNull(result.error()); + assertTrue(result.error().message().contains("externalId")); + } + + + @Test + void canHandleCSVsWithAliasedColumns() { + BulkTaxonomyParseResult result = parser.parse(readTestFile("test-taxonomy-items-aliased.csv"), InputFormat.CSV); + assertNull(result.error()); + assertEquals(3, result.parsedItems().size()); + assertNames(result.parsedItems(), "A1", "A1_1", "A1_2"); + } + + + @Test + void canHandleJSONsWithAliasedProperties() { + BulkTaxonomyParseResult result = parser.parse(readTestFile("test-taxonomy-items-aliased.json"), InputFormat.JSON); + assertNull(result.error()); + assertEquals(3, result.parsedItems().size()); + assertNames(result.parsedItems(), "A1", "A1_1", "A1_2"); + } + + + @Test + void simpleTSV() { + BulkTaxonomyParseResult result = parser.parse(readTestFile("test-taxonomy-items.tsv"), InputFormat.TSV); + assertNull(result.error()); + assertEquals(3, result.parsedItems().size()); + assertNames(result.parsedItems(), "A1", "A1_1", "A1_2"); + } + + + @Test + void simpleJSON() { + BulkTaxonomyParseResult result = parser.parse(readTestFile("test-taxonomy-items.json"), InputFormat.JSON); + assertNull(result.error()); + assertEquals(3, result.parsedItems().size()); + assertNames(result.parsedItems(), "A1", "A1_1", "A1_2"); + } + + + @Test + void parseFailsIfGivenEmpty() { + BulkTaxonomyParseResult result = parser.parse("", InputFormat.TSV); + expectEmptyError(result); + } + + + @Test + void parseFailsIfGivenNull() { + BulkTaxonomyParseResult result = parser.parse(null, InputFormat.TSV); + expectEmptyError(result); + } + + + @Test + void parseFailsIfGivenBlanks() { + BulkTaxonomyParseResult result = parser.parse(" \t\n ", InputFormat.TSV); + expectEmptyError(result); + } + + + @Test + void parseIgnoresEmptyLines() { + BulkTaxonomyParseResult result = parser.parse(readTestFile("test-taxonomy-items-with-blanks.tsv"), InputFormat.TSV); + assertEquals(3, result.parsedItems().size()); + assertNames(result.parsedItems(), "A1", "A1_1", "A1_2"); + } + + + @Test + void parseIgnoresCommentedLines() { + BulkTaxonomyParseResult result = parser.parse(readTestFile("test-taxonomy-items-with-commented-lines.csv"), InputFormat.CSV); + assertEquals(3, result.parsedItems().size()); + assertNames(result.parsedItems(), "A1", "A1_1", "A1_2"); + } + + + // -- HELPERS ------------------ + + private void expectEmptyError(BulkTaxonomyParseResult result) { + BulkTaxonomyParseError err = result.error(); + assertNotNull(err); + assertTrue(err.message().contains("empty")); + assertEquals(0, err.column()); + assertEquals(0, err.line()); + } + + + private String readTestFile(String fileName) { + return readAsString(BulkTaxonomyItemParserTest.class.getResourceAsStream(fileName)); + } + + + private void assertNames(List items, + String... names) { + + Set itemNames = SetUtilities.map(items, BulkTaxonomyItem::name); + Set expectedNames = SetUtilities.asSet(names); + assertEquals(expectedNames, itemNames); + } + +} \ No newline at end of file diff --git a/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-bad-taxonomy-items.csv b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-bad-taxonomy-items.csv new file mode 100644 index 0000000000..1128369b29 --- /dev/null +++ b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-bad-taxonomy-items.csv @@ -0,0 +1,4 @@ +wibble, parentExternalId, name, description, concrete +a1,, A1, Root node, false +a1.1, a1, A1_1, First child, true +a1.2, a1, A1_2, Second child, true diff --git a/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items-aliased.csv b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items-aliased.csv new file mode 100644 index 0000000000..555c2f7d74 --- /dev/null +++ b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items-aliased.csv @@ -0,0 +1,4 @@ +ext_Id, parentextid, name, description, concrete +a1,, A1, Root node, false +a1.1, a1, A1_1, First child, true +a1.2, a1, A1_2, Second child, true diff --git a/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items-aliased.json b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items-aliased.json new file mode 100644 index 0000000000..a46c4d81a3 --- /dev/null +++ b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items-aliased.json @@ -0,0 +1,20 @@ +[ + { + "external_Id": "a1", + "name": "A1", + "description": "Root node", + "concrete": true + }, { + "extId": "a1.1", + "parentExternalId": "a1", + "name": "A1_1", + "description": "First child", + "concrete": true + }, { + "Ext_Id": "a1.2", + "parent_External_Id": "a1", + "name": "A1_2", + "description": "Second child", + "concrete": true + } +] \ No newline at end of file diff --git a/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items-with-blanks.tsv b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items-with-blanks.tsv new file mode 100644 index 0000000000..ba91c9d1da --- /dev/null +++ b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items-with-blanks.tsv @@ -0,0 +1,11 @@ + + + + +externalId parentExternalId name description concrete +a1 A1 Root node false + +a1.1 a1 A1_1 First child true +a1.2 a1 A1_2 Second child true + + diff --git a/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items-with-commented-lines.csv b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items-with-commented-lines.csv new file mode 100644 index 0000000000..fb88acbb64 --- /dev/null +++ b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items-with-commented-lines.csv @@ -0,0 +1,14 @@ +# I am a comment + +externalId, parentExternalId, name, description, concrete + +# Ignore me +a1,, A1, Root node, false + +a1.1, a1, A1_1, First child, true + +a1.2, a1, A1_2, Second child, true +# +# and me! + +# \ No newline at end of file diff --git a/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items.csv b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items.csv new file mode 100644 index 0000000000..806888d200 --- /dev/null +++ b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items.csv @@ -0,0 +1,4 @@ +externalId, parentExternalId, name, description, concrete +a1,, A1, Root node, false +a1.1, a1, A1_1, First child, true +a1.2, a1, A1_2, Second child, true diff --git a/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items.json b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items.json new file mode 100644 index 0000000000..199c4e187f --- /dev/null +++ b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items.json @@ -0,0 +1,20 @@ +[ + { + "externalId": "a1", + "name": "A1", + "description": "Root node", + "concrete": true + }, { + "externalId": "a1.1", + "parentExternalId": "a1", + "name": "A1_1", + "description": "First child", + "concrete": true + }, { + "externalId": "a1.2", + "parentExternalId": "a1", + "name": "A1_2", + "description": "Second child", + "concrete": true + } +] \ No newline at end of file diff --git a/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items.tsv b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items.tsv new file mode 100644 index 0000000000..9c8aa79b58 --- /dev/null +++ b/waltz-service/src/test/resources/org/finos/waltz/service/taxonomy_management/test-taxonomy-items.tsv @@ -0,0 +1,4 @@ +externalId parentExternalId name description concrete +a1 A1 Root node false +a1.1 a1 A1_1 First child true +a1.2 a1 A1_2 Second child true diff --git a/waltz-test-common/src/main/java/org/finos/waltz/test_common/helpers/MeasurableHelper.java b/waltz-test-common/src/main/java/org/finos/waltz/test_common/helpers/MeasurableHelper.java index ce024901b6..142b45da0a 100644 --- a/waltz-test-common/src/main/java/org/finos/waltz/test_common/helpers/MeasurableHelper.java +++ b/waltz-test-common/src/main/java/org/finos/waltz/test_common/helpers/MeasurableHelper.java @@ -1,6 +1,7 @@ package org.finos.waltz.test_common.helpers; import org.finos.waltz.common.CollectionUtilities; +import org.finos.waltz.model.EntityLifecycleStatus; import org.finos.waltz.model.EntityReference; import org.finos.waltz.model.measurable_category.MeasurableCategory; import org.finos.waltz.schema.tables.records.MeasurableCategoryRecord; @@ -16,6 +17,7 @@ import static org.finos.waltz.common.DateTimeUtilities.nowUtcTimestamp; import static org.finos.waltz.common.DateTimeUtilities.toSqlDate; +import static org.finos.waltz.common.StringUtilities.mkExternalId; import static org.finos.waltz.schema.Tables.MEASURABLE; import static org.finos.waltz.schema.Tables.MEASURABLE_CATEGORY; import static org.finos.waltz.schema.Tables.MEASURABLE_RATING; @@ -75,6 +77,14 @@ public void updateCategoryNotEditable(long categoryId) { public long createMeasurable(String name, long categoryId) { + return createMeasurable( + mkExternalId(name), + name, + categoryId); + } + + + public long createMeasurable(String externalId, String name, long categoryId) { return dsl .select(MEASURABLE.ID) .from(MEASURABLE) @@ -87,7 +97,7 @@ public long createMeasurable(String name, long categoryId) { record.setName(name); record.setDescription(name); record.setConcrete(true); - record.setExternalId(name); + record.setExternalId(externalId); record.setProvenance(PROVENANCE); record.setLastUpdatedBy(LAST_UPDATE_USER); record.setLastUpdatedAt(nowUtcTimestamp()); @@ -152,4 +162,11 @@ public void updateMeasurableReadOnly(EntityReference ref, long measurableId) { .execute(); } + public int updateMeasurableLifecycleStatus(long measurableId, EntityLifecycleStatus newStatus) { + return dsl + .update(MEASURABLE) + .set(MEASURABLE.ENTITY_LIFECYCLE_STATUS, newStatus.name()) + .where(MEASURABLE.ID.eq(measurableId)) + .execute(); + } } diff --git a/waltz-web/src/main/java/org/finos/waltz/web/endpoints/api/TaxonomyManagementEndpoint.java b/waltz-web/src/main/java/org/finos/waltz/web/endpoints/api/TaxonomyManagementEndpoint.java index 2ee0e7afee..6eb733c026 100644 --- a/waltz-web/src/main/java/org/finos/waltz/web/endpoints/api/TaxonomyManagementEndpoint.java +++ b/waltz-web/src/main/java/org/finos/waltz/web/endpoints/api/TaxonomyManagementEndpoint.java @@ -18,15 +18,23 @@ package org.finos.waltz.web.endpoints.api; -import org.finos.waltz.service.taxonomy_management.TaxonomyChangeService; -import org.finos.waltz.web.endpoints.Endpoint; +import org.finos.waltz.model.EntityReference; +import org.finos.waltz.model.bulk_upload.BulkUpdateMode; +import org.finos.waltz.model.bulk_upload.taxonomy.BulkTaxonomyValidationResult; import org.finos.waltz.model.taxonomy_management.TaxonomyChangeCommand; +import org.finos.waltz.service.taxonomy_management.BulkTaxonomyChangeService; +import org.finos.waltz.service.taxonomy_management.BulkTaxonomyItemParser.InputFormat; +import org.finos.waltz.service.taxonomy_management.TaxonomyChangeService; import org.finos.waltz.web.WebUtilities; +import org.finos.waltz.web.endpoints.Endpoint; import org.finos.waltz.web.endpoints.EndpointUtilities; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import static org.finos.waltz.web.WebUtilities.getEntityReference; +import static org.finos.waltz.web.WebUtilities.getUsername; import static org.finos.waltz.web.WebUtilities.mkPath; +import static org.finos.waltz.web.WebUtilities.readEnum; import static org.finos.waltz.web.endpoints.EndpointUtilities.deleteForDatum; import static org.finos.waltz.web.endpoints.EndpointUtilities.getForList; import static org.finos.waltz.web.endpoints.EndpointUtilities.postForDatum; @@ -38,11 +46,14 @@ public class TaxonomyManagementEndpoint implements Endpoint { private static final String BASE_URL = mkPath("api", "taxonomy-management"); private final TaxonomyChangeService taxonomyChangeService; + private final BulkTaxonomyChangeService bulkTaxonomyChangeService; @Autowired - public TaxonomyManagementEndpoint(TaxonomyChangeService taxonomyChangeService) { + public TaxonomyManagementEndpoint(TaxonomyChangeService taxonomyChangeService, + BulkTaxonomyChangeService bulkTaxonomyChangeService) { this.taxonomyChangeService = taxonomyChangeService; + this.bulkTaxonomyChangeService = bulkTaxonomyChangeService; } @@ -55,8 +66,34 @@ public void register() { registerApplyPendingChange(mkPath(BASE_URL, "pending-changes", "id", ":id", "apply")); registerFindPendingChangesByDomain(mkPath(BASE_URL, "pending-changes", "by-domain", ":kind", ":id")); registerFindAllChangesByDomain(mkPath(BASE_URL, "all", "by-domain", ":kind", ":id")); + registerPreviewBulkTaxonomyChanges(mkPath(BASE_URL, "bulk", "preview", ":kind", ":id")); + registerApplyBulkTaxonomyChanges(mkPath(BASE_URL, "bulk", "apply", ":kind", ":id")); + } + + + private void registerApplyBulkTaxonomyChanges(String path) { + postForDatum(path, (req, resp) -> { + String userId = getUsername(req); + EntityReference taxonomyRef = getEntityReference(req); + InputFormat format = readEnum(req, "format", InputFormat.class, s -> InputFormat.TSV); + BulkUpdateMode mode = readEnum(req, "mode", BulkUpdateMode.class, s -> BulkUpdateMode.ADD_ONLY); + String body = req.body(); + BulkTaxonomyValidationResult validationResult = bulkTaxonomyChangeService.previewBulk(taxonomyRef, body, format, mode); + return bulkTaxonomyChangeService.applyBulk(taxonomyRef, validationResult, userId); + }); } + private void registerPreviewBulkTaxonomyChanges(String path) { + postForDatum(path, (req, resp) -> { + EntityReference taxonomyRef = getEntityReference(req); + InputFormat format = readEnum(req, "format", InputFormat.class, s -> InputFormat.TSV); + BulkUpdateMode mode = readEnum(req, "mode", BulkUpdateMode.class, s -> BulkUpdateMode.ADD_ONLY); + String body = req.body(); + return bulkTaxonomyChangeService.previewBulk(taxonomyRef, body, format, mode); + }); + } + + private void registerFindAllChangesByDomain(String path) { getForList(path, (req, resp) -> { return taxonomyChangeService.findAllChangesByDomain(WebUtilities.getEntityReference(req));