diff --git a/waltz-common/src/main/java/org/finos/waltz/common/CollectionUtilities.java b/waltz-common/src/main/java/org/finos/waltz/common/CollectionUtilities.java index 0d3676e0ee..f5dc07a510 100644 --- a/waltz-common/src/main/java/org/finos/waltz/common/CollectionUtilities.java +++ b/waltz-common/src/main/java/org/finos/waltz/common/CollectionUtilities.java @@ -18,14 +18,17 @@ package org.finos.waltz.common; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; -import static org.finos.waltz.common.Checks.checkNotEmpty; - public class CollectionUtilities { @@ -242,5 +245,4 @@ public static Long sumInts(Collection values) { return acc; } - } diff --git a/waltz-common/src/main/java/org/finos/waltz/common/ListUtilities.java b/waltz-common/src/main/java/org/finos/waltz/common/ListUtilities.java index 409b817e1d..17353ab7da 100644 --- a/waltz-common/src/main/java/org/finos/waltz/common/ListUtilities.java +++ b/waltz-common/src/main/java/org/finos/waltz/common/ListUtilities.java @@ -24,6 +24,7 @@ import java.util.*; import java.util.function.Function; import java.util.function.Predicate; +import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; import static org.finos.waltz.common.Checks.checkNotNull; @@ -205,6 +206,17 @@ public static T getOrDefault(List xs, int idx, T defaultValue) { } + public static List take(Collection xs, + int amount) { + return ListUtilities + .ensureNotNull(xs) + .stream() + .limit(amount) + .collect(Collectors.toList()); + } + + + /** * Returns a list of distinct values from a given list * @param ts a list of T's with possible duplicates diff --git a/waltz-data/src/main/java/org/finos/waltz/data/flow_classification_rule/FlowClassificationRuleDao.java b/waltz-data/src/main/java/org/finos/waltz/data/flow_classification_rule/FlowClassificationRuleDao.java index 5627319211..8a0152bd60 100644 --- a/waltz-data/src/main/java/org/finos/waltz/data/flow_classification_rule/FlowClassificationRuleDao.java +++ b/waltz-data/src/main/java/org/finos/waltz/data/flow_classification_rule/FlowClassificationRuleDao.java @@ -21,6 +21,7 @@ import org.finos.waltz.data.InlineSelectFieldFactory; import org.finos.waltz.model.EntityKind; import org.finos.waltz.model.EntityReference; +import org.finos.waltz.model.FlowDirection; import org.finos.waltz.model.ImmutableEntityReference; import org.finos.waltz.model.Severity; import org.finos.waltz.model.flow_classification_rule.DiscouragedSource; @@ -372,7 +373,12 @@ public List findExpandedFlowClassificationRu } - public List findFlowClassificationRuleVantagePoints() { + public List findFlowClassificationRuleVantagePoints(FlowDirection direction) { + return findFlowClassificationRuleVantagePoints(FLOW_CLASSIFICATION.DIRECTION.eq(direction.name())); + } + + + private List findFlowClassificationRuleVantagePoints(Condition condition) { SelectSeekStep4, Integer, Integer, Long, Long> select = dsl .select(targetOrgUnitId, declaredOrgUnitLevel, @@ -394,6 +400,7 @@ public List findFlowClassificationRuleVantag .and(ehDataType.KIND.eq(EntityKind.DATA_TYPE.name())) .and(ehDataType.ID.eq(ehDataType.ANCESTOR_ID))) .innerJoin(FLOW_CLASSIFICATION).on(FLOW_CLASSIFICATION_RULE.FLOW_CLASSIFICATION_ID.eq(FLOW_CLASSIFICATION.ID)) + .where(condition) .orderBy( ehOrgUnit.LEVEL.desc(), ehDataType.LEVEL.desc(), @@ -533,7 +540,7 @@ private SelectOnConditionStep baseSelect() { } - public int updatePointToPointFlowClassificationRules() { + public int updatePointToPointFlowClassificationRules(FlowDirection direction) { Condition logicalFlowTargetIsAuthSourceParent = FLOW_CLASSIFICATION_RULE.SUBJECT_ENTITY_ID.eq(LOGICAL_FLOW.SOURCE_ENTITY_ID) .and(LOGICAL_FLOW.SOURCE_ENTITY_KIND.eq(FLOW_CLASSIFICATION_RULE.SUBJECT_ENTITY_KIND) @@ -558,7 +565,7 @@ public int updatePointToPointFlowClassificationRules() { .and(level.ID.eq(level.ANCESTOR_ID) .and(level.KIND.eq(EntityKind.DATA_TYPE.name())))) .innerJoin(FLOW_CLASSIFICATION).on(FLOW_CLASSIFICATION_RULE.FLOW_CLASSIFICATION_ID.eq(FLOW_CLASSIFICATION.ID)) - .where(FLOW_CLASSIFICATION.CODE.ne(LOGICAL_FLOW_DECORATOR.RATING)) + .where(FLOW_CLASSIFICATION.CODE.ne(LOGICAL_FLOW_DECORATOR.RATING).and(FLOW_CLASSIFICATION.DIRECTION.eq(direction.name()))) .orderBy(level.LEVEL) .fetch() .stream() diff --git a/waltz-data/src/main/java/org/finos/waltz/data/licence/search/LicenceSearchDao.java b/waltz-data/src/main/java/org/finos/waltz/data/licence/search/LicenceSearchDao.java index 46d445e3a4..e81fdf1de3 100644 --- a/waltz-data/src/main/java/org/finos/waltz/data/licence/search/LicenceSearchDao.java +++ b/waltz-data/src/main/java/org/finos/waltz/data/licence/search/LicenceSearchDao.java @@ -2,19 +2,15 @@ import org.finos.waltz.common.ListUtilities; import org.finos.waltz.data.SearchDao; -import org.finos.waltz.data.legal_entity.LegalEntityDao; import org.finos.waltz.data.licence.LicenceDao; import org.finos.waltz.model.entity_search.EntitySearchOptions; -import org.finos.waltz.model.legal_entity.LegalEntity; import org.finos.waltz.model.licence.Licence; import org.jooq.Condition; import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; +import org.springframework.stereotype.Repository; -import java.util.ArrayList; import java.util.List; -import java.util.TreeSet; import static java.util.Collections.emptyList; import static org.finos.waltz.common.ListUtilities.concat; @@ -23,7 +19,7 @@ import static org.finos.waltz.data.SearchUtilities.mkTerms; import static org.finos.waltz.schema.Tables.LICENCE; -@Service +@Repository public class LicenceSearchDao implements SearchDao { private final DSLContext dsl; diff --git a/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRule2Harness.java b/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRule2Harness.java new file mode 100644 index 0000000000..1bfbc4aca7 --- /dev/null +++ b/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRule2Harness.java @@ -0,0 +1,269 @@ +/* + * Waltz - Enterprise Architecture + * Copyright (C) 2016, 2017, 2018, 2019 Waltz open source project + * See README.md for more information + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific + * + */ + +package org.finos.waltz.jobs.harness; + +import org.finos.waltz.common.FunctionUtilities; +import org.finos.waltz.common.LoggingUtilities; +import org.finos.waltz.common.MapUtilities; +import org.finos.waltz.common.SetUtilities; +import org.finos.waltz.data.flow_classification_rule.FlowClassificationRuleDao; +import org.finos.waltz.model.EntityKind; +import org.finos.waltz.model.EntityLifecycleStatus; +import org.finos.waltz.model.EntityReference; +import org.finos.waltz.model.FlowDirection; +import org.finos.waltz.model.Nullable; +import org.finos.waltz.model.flow_classification_rule.FlowClassificationRuleVantagePoint; +import org.finos.waltz.schema.Tables; +import org.finos.waltz.schema.tables.Application; +import org.finos.waltz.schema.tables.EntityHierarchy; +import org.finos.waltz.schema.tables.LogicalFlow; +import org.finos.waltz.schema.tables.LogicalFlowDecorator; +import org.finos.waltz.service.DIConfiguration; +import org.finos.waltz.service.flow_classification_rule.FlowClassificationRuleService; +import org.immutables.value.Value; +import org.jooq.DSLContext; +import org.jooq.lambda.function.Function4; +import org.jooq.lambda.tuple.Tuple2; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.finos.waltz.data.JooqUtilities.readRef; +import static org.jooq.lambda.tuple.Tuple.tuple; + + +public class FlowClassificationRule2Harness { + + public static final Logger LOG = LoggerFactory.getLogger(FlowClassificationRule2Harness.class); + + + @Value.Immutable + interface FlowDataType { + EntityReference source(); + EntityReference target(); + @Nullable Long sourceOuId(); + @Nullable Long targetOuId(); + long lfId(); + long lfdId(); + long dtId(); + @Nullable Long outboundRuleId(); + @Nullable Long inboundRuleId(); + } + + + private static final LogicalFlow lf = Tables.LOGICAL_FLOW; + private static final LogicalFlowDecorator lfd = Tables.LOGICAL_FLOW_DECORATOR; + private static final Application srcApp = Tables.APPLICATION.as("srcApp"); + private static final Application targetApp = Tables.APPLICATION.as("targetApp"); + private static final EntityHierarchy eh = Tables.ENTITY_HIERARCHY; + + + public static void main(String[] args) { + LoggingUtilities.configureLogging(); + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(DIConfiguration.class); + DSLContext dsl = ctx.getBean(DSLContext.class); + + FlowClassificationRuleService svc = ctx.getBean(FlowClassificationRuleService.class); + FlowClassificationRuleDao dao = ctx.getBean(FlowClassificationRuleDao.class); + + FunctionUtilities.time("doIt", () -> doIt(dsl, dao)); + +// System.exit(-1); + } + + + private static void doIt(DSLContext dsl, + FlowClassificationRuleDao dao) { + LOG.debug("Loading rule vantage points"); + + List inboundRuleVantagePoints = dao.findFlowClassificationRuleVantagePoints(FlowDirection.INBOUND); + List outboundRuleVantagePoints = dao.findFlowClassificationRuleVantagePoints(FlowDirection.OUTBOUND); + + +// take(allRuleVantagePoints, 10).forEach(System.out::println); + + Set population = fetchFlowDataTypePopulation(dsl); + + LOG.debug( + "Loaded: {} inbound and {} outbound vantage point rules, and a population of: {} flows with datatypes", + inboundRuleVantagePoints.size(), + outboundRuleVantagePoints.size(), + population.size()); + + // take(population, 10).forEach(System.out::println); + + LOG.debug("Loading hierarchies"); + List> ouHierarchy = fetchHierarchy(dsl, EntityKind.ORG_UNIT); + List> dtHierarchy = fetchHierarchy(dsl, EntityKind.DATA_TYPE); +// +// System.out.println(findChildren(ouHierarchy, 10186L)); +// System.out.println(findChildren(dtHierarchy, 33100L)); + + LOG.debug("Applying rules to population"); + Map> lfdIdToOutboundRuleIdMap = applyVantagePoints(FlowDirection.OUTBOUND, outboundRuleVantagePoints, population, ouHierarchy, dtHierarchy); + Map> lfdIdToInboundRuleIdMap = applyVantagePoints(FlowDirection.INBOUND, inboundRuleVantagePoints, population, ouHierarchy, dtHierarchy); + + System.out.println("Curr"); + MapUtilities.countBy(FlowDataType::outboundRuleId, SetUtilities.filter(population, p -> p.outboundRuleId() != null)).entrySet().stream().sorted(Map.Entry.comparingByKey()).limit(20).forEach(System.out::println); + System.out.println("Future"); + MapUtilities.countBy(Map.Entry::getValue, lfdIdToOutboundRuleIdMap.entrySet()).entrySet().stream().sorted(Map.Entry.comparingByKey()).limit(20).forEach(System.out::println); + + } + + + private static Map> applyVantagePoints(FlowDirection direction, + List ruleVantagePoints, + Set population, + List> ouHierarchy, + List> dtHierarchy) { + + Function4, Set, FlowDataType, MatchOutcome> matcher = determineMatcherFn(direction); + + Map> lfdIdToRuleAndOutcomeMap = new HashMap<>(); + ruleVantagePoints + .stream() + .filter(rvp -> rvp.vantagePoint().kind() == EntityKind.ORG_UNIT) + .forEach(rvp -> { + Set childOUs = findChildren(ouHierarchy, rvp.vantagePoint().id()); + Set childDTs = findChildren(dtHierarchy, rvp.dataType().id()); + population.forEach(p -> { + Tuple2 currentRuleAndOutcome = lfdIdToRuleAndOutcomeMap.get(p.lfdId()); + if (currentRuleAndOutcome != null && currentRuleAndOutcome.v2 == MatchOutcome.POSITIVE_MATCH) { + return; // skip, already got a good match + } + MatchOutcome outcome = matcher.apply(rvp, childOUs, childDTs, p); + if (outcome == MatchOutcome.NOT_APPLICABLE) { + // skip + } else if (currentRuleAndOutcome == null) { + lfdIdToRuleAndOutcomeMap.put(p.lfdId(), tuple(rvp.ruleId(), outcome)); + } else if (currentRuleAndOutcome.v2 == MatchOutcome.NEGATIVE_MATCH && outcome == MatchOutcome.POSITIVE_MATCH) { + // override result as we have a positive match + lfdIdToRuleAndOutcomeMap.put(p.lfdId(), tuple(rvp.ruleId(), MatchOutcome.POSITIVE_MATCH)); + } else { + // skip, leave the map alone as a more specific negative rule id already exists + } + }); + }); + + LOG.debug( + "finished processing {} {} rules, {} decorators have outcomes", + ruleVantagePoints.size(), + direction, + lfdIdToRuleAndOutcomeMap.size()); + + return lfdIdToRuleAndOutcomeMap; + } + + enum MatchOutcome { + NOT_APPLICABLE, + NEGATIVE_MATCH, + POSITIVE_MATCH + + } + + private static Function4, Set, FlowDataType, MatchOutcome> determineMatcherFn(FlowDirection direction) { + Function4, Set, FlowDataType, MatchOutcome> inboundMatcher = + (rvp, childOUs, childDTs, p) -> { + boolean subjectMatches = p.target().equals(rvp.subjectReference()); + boolean dtAndOuMatches = childDTs.contains(p.dtId()) && rvp.vantagePoint().kind() == EntityKind.ORG_UNIT && p.sourceOuId() != null && childOUs.contains(p.sourceOuId()); + return determineOutcome(subjectMatches, dtAndOuMatches); + }; + + Function4, Set, FlowDataType, MatchOutcome> outboundMatcher = + (rvp, childOUs, childDTs, p) -> { + boolean subjectMatches = p.source().equals(rvp.subjectReference()); + boolean dtAndOuMatches = childDTs.contains(p.dtId()) && rvp.vantagePoint().kind() == EntityKind.ORG_UNIT && p.targetOuId() != null && childOUs.contains(p.targetOuId()); + return determineOutcome(subjectMatches, dtAndOuMatches); + }; + + return direction == FlowDirection.INBOUND + ? inboundMatcher + : outboundMatcher; + } + + + private static MatchOutcome determineOutcome(boolean subjectMatches, + boolean dtAndOuMatches) { + if (subjectMatches && dtAndOuMatches) { + return MatchOutcome.POSITIVE_MATCH; + } else if (dtAndOuMatches) { + return MatchOutcome.NEGATIVE_MATCH; + } else { + return MatchOutcome.NOT_APPLICABLE; + } + } + + + + private static Set fetchFlowDataTypePopulation(DSLContext dsl) { + LOG.debug("Loading population"); + return dsl + .select(lf.ID, + lfd.ID, lfd.DECORATOR_ENTITY_ID, lfd.INBOUND_FLOW_CLASSIFICATION_RULE_ID, lfd.FLOW_CLASSIFICATION_RULE_ID, + lf.SOURCE_ENTITY_ID, lf.SOURCE_ENTITY_KIND, lf.TARGET_ENTITY_ID, lf.TARGET_ENTITY_KIND, + srcApp.ORGANISATIONAL_UNIT_ID, + targetApp.ORGANISATIONAL_UNIT_ID) + .from(lf) + .innerJoin(lfd).on(lfd.LOGICAL_FLOW_ID.eq(lf.ID).and(lfd.DECORATOR_ENTITY_KIND.eq(EntityKind.DATA_TYPE.name()))) + .leftJoin(srcApp).on(srcApp.ID.eq(lf.SOURCE_ENTITY_ID).and(lf.SOURCE_ENTITY_KIND.eq(EntityKind.APPLICATION.name()))) + .leftJoin(targetApp).on(targetApp.ID.eq(lf.TARGET_ENTITY_ID).and(lf.TARGET_ENTITY_KIND.eq(EntityKind.APPLICATION.name()))) + .where(lf.IS_REMOVED.isFalse() + .and(lf.ENTITY_LIFECYCLE_STATUS.eq(EntityLifecycleStatus.ACTIVE.name()))) + .fetchSet(r -> ImmutableFlowDataType + .builder() + .lfdId(r.get(lfd.ID)) + .dtId(r.get(lfd.DECORATOR_ENTITY_ID)) + .lfId(r.get(lf.ID)) + .source(readRef(r, lf.SOURCE_ENTITY_KIND, lf.SOURCE_ENTITY_ID)) + .target(readRef(r, lf.TARGET_ENTITY_KIND, lf.TARGET_ENTITY_ID)) + .inboundRuleId(r.get(lfd.INBOUND_FLOW_CLASSIFICATION_RULE_ID)) + .outboundRuleId(r.get(lfd.FLOW_CLASSIFICATION_RULE_ID)) + .sourceOuId(r.get(srcApp.ORGANISATIONAL_UNIT_ID)) + .targetOuId(r.get(targetApp.ORGANISATIONAL_UNIT_ID)) + .build()); + } + + + private static List> fetchHierarchy(DSLContext dsl, + EntityKind kind) { + return dsl + .select(eh.ID, eh.ANCESTOR_ID) + .from(eh) + .where(eh.KIND.eq(kind.name())) + .fetch(r -> tuple(r.get(eh.ID), r.get(eh.ANCESTOR_ID))); + } + + + private static Set findChildren(List> hierarchy, + long parentId) { + return hierarchy + .stream() + .filter(t -> t.v2 == parentId) + .map(t -> t.v1) + .collect(Collectors.toSet()); + } + + +} diff --git a/waltz-schema/src/main/resources/liquibase/db.changelog-1.59.xml b/waltz-schema/src/main/resources/liquibase/db.changelog-1.59.xml index dbeef643a1..52111e08ac 100644 --- a/waltz-schema/src/main/resources/liquibase/db.changelog-1.59.xml +++ b/waltz-schema/src/main/resources/liquibase/db.changelog-1.59.xml @@ -100,4 +100,27 @@ remarks="Severity for styling the display of the message shown against flows which have ratings determined by this rule"/> + + 7032: adding inbound_flow_classification_rule_id column to logical flow decorator + + + + + + + + + + + + + + \ No newline at end of file diff --git a/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationCalculator.java b/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationCalculator.java index 297cadbcc7..db8e4fc2d4 100644 --- a/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationCalculator.java +++ b/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationCalculator.java @@ -95,7 +95,7 @@ private int[] update(DataType dataType, EntityReference vantageRef) { vantageRef); IdSelectionOptions selectorOptions = mkOpts(vantageRef); - Select> selector = appIdSelectorFactory.apply(selectorOptions); + Select> appSelector = appIdSelectorFactory.apply(selectorOptions); Set dataTypeDescendents = entityHierarchyDao .findDesendents(dataType.entityReference()) .stream() @@ -103,7 +103,7 @@ private int[] update(DataType dataType, EntityReference vantageRef) { .collect(Collectors.toSet()); Collection impactedDecorators = logicalFlowDecoratorDao - .findByEntityIdSelector(selector, Optional.of(EntityKind.APPLICATION)) + .findByEntityIdSelector(appSelector, Optional.of(EntityKind.APPLICATION)) .stream() .filter(decorator -> dataTypeDescendents.contains(decorator.decoratorEntity().id())) .collect(toList()); diff --git a/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleService.java b/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleService.java index bf593cffa1..c8b7f94fca 100644 --- a/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleService.java +++ b/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleService.java @@ -155,7 +155,7 @@ public long insert(FlowClassificationRuleCreateCommand command, String username) } logInsert(classificationRuleId, command, username); - flowClassificationRuleDao.updatePointToPointFlowClassificationRules(); + flowClassificationRuleDao.updatePointToPointFlowClassificationRules(FlowDirection.OUTBOUND); return classificationRuleId; } @@ -208,7 +208,7 @@ public int fastRecalculateAllFlowRatings() { //finds all the vantage points to apply using parent as selector List flowClassificationRuleVantagePoints = flowClassificationRuleDao - .findFlowClassificationRuleVantagePoints(); + .findFlowClassificationRuleVantagePoints(FlowDirection.OUTBOUND); int updatedRuleDecorators = flowClassificationRuleVantagePoints .stream() @@ -216,7 +216,7 @@ public int fastRecalculateAllFlowRatings() { .sum(); //overrides rating for point to point flows (must run after the above) - int updatedPointToPointDecorators = flowClassificationRuleDao.updatePointToPointFlowClassificationRules(); + int updatedPointToPointDecorators = flowClassificationRuleDao.updatePointToPointFlowClassificationRules(FlowDirection.OUTBOUND); LOG.info( "Updated decorators for: {} for general rules and {} point-to-point flows",