From 74ec6057c42778bb5d037455379969fa143b8011 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 15 Nov 2022 17:42:57 +0200 Subject: [PATCH 001/187] Initial implementation of the optimized TcpLikeProperties (and HyperCubeSet) holding all connections including src_peers, dst_peers and protocols Signed-off-by: Tanya --- nca/CoreDS/DimensionsManager.py | 6 +- nca/CoreDS/MethodSet.py | 27 +++ nca/CoreDS/Peer.py | 17 +- nca/CoreDS/ProtocolNameResolver.py | 4 + nca/CoreDS/ProtocolSet.py | 112 ++++++++++ nca/CoreDS/TcpLikeProperties.py | 194 ++++++++++++++++-- nca/FWRules/ClusterInfo.py | 5 + nca/FWRules/ConnectivityGraph.py | 177 ++++++++++++++-- nca/FWRules/MinimizeFWRules.py | 4 + nca/NetworkConfig/NetworkConfig.py | 27 +++ nca/NetworkConfig/NetworkConfigQuery.py | 22 +- nca/NetworkConfig/NetworkLayer.py | 83 ++++++++ nca/Parsers/GenericIngressLikeYamlParser.py | 71 ++----- nca/Parsers/IngressPolicyYamlParser.py | 14 +- nca/Parsers/IstioPolicyYamlParser.py | 3 +- .../IstioTrafficResourcesYamlParser.py | 25 ++- nca/Parsers/K8sPolicyYamlParser.py | 84 +++++--- nca/Resources/IstioTrafficResources.py | 15 +- nca/Resources/K8sNetworkPolicy.py | 10 + nca/Resources/NetworkPolicy.py | 24 +++ 20 files changed, 775 insertions(+), 149 deletions(-) create mode 100644 nca/CoreDS/ProtocolSet.py diff --git a/nca/CoreDS/DimensionsManager.py b/nca/CoreDS/DimensionsManager.py index 92fc67015..4be932ed3 100644 --- a/nca/CoreDS/DimensionsManager.py +++ b/nca/CoreDS/DimensionsManager.py @@ -5,6 +5,7 @@ from enum import Enum from .CanonicalIntervalSet import CanonicalIntervalSet from .MethodSet import MethodSet +from .ProtocolSet import ProtocolSet from .MinDFA import MinDFA @@ -31,12 +32,15 @@ def __init__(self): dfa_all_words_default = self._get_dfa_from_alphabet_str(self.default_dfa_alphabet_str) ports_interval = CanonicalIntervalSet.get_interval_set(1, 65535) all_methods_interval = MethodSet(True) + all_protocols_interval = ProtocolSet(True) all_peers_interval = CanonicalIntervalSet.get_interval_set(0, 10000) # assuming max possible peer number self.dim_dict = dict() self.dim_dict["src_ports"] = (DimensionsManager.DimensionType.IntervalSet, ports_interval) self.dim_dict["dst_ports"] = (DimensionsManager.DimensionType.IntervalSet, ports_interval) self.dim_dict["methods"] = (DimensionsManager.DimensionType.IntervalSet, all_methods_interval) - self.dim_dict["peers"] = (DimensionsManager.DimensionType.IntervalSet, all_peers_interval) + self.dim_dict["protocols"] = (DimensionsManager.DimensionType.IntervalSet, all_protocols_interval) + self.dim_dict["src_peers"] = (DimensionsManager.DimensionType.IntervalSet, all_peers_interval) + self.dim_dict["dst_peers"] = (DimensionsManager.DimensionType.IntervalSet, all_peers_interval) self.dim_dict["paths"] = (DimensionsManager.DimensionType.DFA, dfa_all_words_default) self.dim_dict["hosts"] = (DimensionsManager.DimensionType.DFA, dfa_all_words_default) diff --git a/nca/CoreDS/MethodSet.py b/nca/CoreDS/MethodSet.py index a75f8b08c..7c653662d 100644 --- a/nca/CoreDS/MethodSet.py +++ b/nca/CoreDS/MethodSet.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache2.0 # +import re from .CanonicalIntervalSet import CanonicalIntervalSet @@ -20,6 +21,32 @@ def __init__(self, all_methods=False): if all_methods: # the whole range self.add_interval(self._whole_range_interval()) + def add_method(self, method): + """ + Adds a given method to the MethodSet if the method is one of the eligible methods (in all_methods_list); + otherwise raises ValueError exception + :param str method: the method to add + """ + index = self.all_methods_list.index(method) + self.add_interval(self.Interval(index, index)) + + def add_methods_from_regex(self, methods_regex): + """ + Adds all methods in methods_regex to the MethodSet + :param str methods_regex: + """ + for index, method in enumerate(self.all_methods_list): + if re.match(methods_regex, method): + self.add_interval(self.Interval(index, index)) + + def set_methods(self, methods): + """ + Sets all methods from the given parameter + :param CanonicalIntervalSet methods: the methods to set + """ + for interval in methods: + self.add_interval(interval) + @staticmethod def _whole_range_interval(): """ diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index cab56b55e..73b60e1a2 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -576,25 +576,28 @@ def get_peer_interval_of(self, peer_set): res.add_interval(CanonicalIntervalSet.Interval(index, index)) return res - def get_peer_list_by_indices(self, peer_inteval_set): + def get_peer_set_by_indices(self, peer_inteval_set): """ Return peer list from interval set of indices :param peer_inteval_set: the interval set of indices into the sorted peer list :return: the list of peers referenced by the indices in the interval set """ - assert len(self.sorted_peer_list) == len(self) - res = [] + my_len = len(self) + if len(self.sorted_peer_list) != my_len: + self.update_sorted_peer_list() + peer_list = [] for interval in peer_inteval_set: - for ind in range(interval.start, interval.end + 1): - res.append(self.sorted_peer_list[ind]) - return res + for ind in range(min(interval.start, my_len), min(interval.end + 1, my_len)): + peer_list.append(self.sorted_peer_list[ind]) + return PeerSet(set(peer_list)) def get_all_peers_interval(self): """ Returns the interval of all peers :return: CanonicalIntervalSet of all peers """ - assert len(self.sorted_peer_list) == len(self) + if len(self.sorted_peer_list) != len(self): + self.update_sorted_peer_list() return CanonicalIntervalSet.get_interval_set(0, len(self.sorted_peer_list) - 1) def is_whole_range(self, peer_interval_set): diff --git a/nca/CoreDS/ProtocolNameResolver.py b/nca/CoreDS/ProtocolNameResolver.py index 9d1563d74..81d6c2e39 100644 --- a/nca/CoreDS/ProtocolNameResolver.py +++ b/nca/CoreDS/ProtocolNameResolver.py @@ -56,6 +56,10 @@ class ProtocolNameResolver: 135: 'MobilityHeader', 136: 'UDPLite', 137: 'MPLSinIP', 138: 'manet', 139: 'HIP', 140: 'Shim6', 141: 'WESP', 142: 'ROHC', 143: 'Ethernet'} + @staticmethod + def get_all_protocols_list(): + return list(ProtocolNameResolver._protocol_name_to_number_dict.keys()) + @staticmethod def get_protocol_name(protocol_number: int) -> str: """ diff --git a/nca/CoreDS/ProtocolSet.py b/nca/CoreDS/ProtocolSet.py new file mode 100644 index 000000000..251890540 --- /dev/null +++ b/nca/CoreDS/ProtocolSet.py @@ -0,0 +1,112 @@ +# +# Copyright 2020- IBM Inc. All rights reserved +# SPDX-License-Identifier: Apache2.0 +# + +from .CanonicalIntervalSet import CanonicalIntervalSet +from .ProtocolNameResolver import ProtocolNameResolver + + +class ProtocolSet(CanonicalIntervalSet): + """ + A class for holding a set of HTTP methods + """ + + # According to https://en.wikipedia.org/wiki/List_of_IP_protocol_numbers + all_protocols_list = ProtocolNameResolver.get_all_protocols_list() + + def __init__(self, all_protocols=False): + """ + :param bool all_protocols: whether to create the object holding all protocols + """ + super().__init__() + if all_protocols: # the whole range + self.add_interval(self._whole_range_interval()) + + def add_protocol(self, protocol): + """ + Adds a given protocol to the ProtocolSet if the method is one of the eligible protocols (in all_protocols_list); + otherwise raises ValueError exception + :param str protocol: the protocol to add + """ + index = self.all_protocols_list.index(protocol) + self.add_interval(self.Interval(index, index)) + + def set_protocols(self, protocols): + """ + Sets all protocols from the given parameter + :param CanonicalIntervalSet protocols: the protocols to set + """ + for interval in protocols: + self.add_interval(interval) + + @staticmethod + def _whole_range_interval(): + """ + :return: the interval representing the whole range (all protocols) + """ + return CanonicalIntervalSet.Interval(0, len(ProtocolSet.all_protocols_list) - 1) + + @staticmethod + def _whole_range_interval_set(): + """ + :return: the interval set representing the whole range (all protocols) + """ + interval = ProtocolSet._whole_range_interval() + return CanonicalIntervalSet.get_interval_set(interval.start, interval.end) + + def is_whole_range(self): + """ + :return: True if the ProtocolSet contains all protocols, False otherwise + """ + return self == self._whole_range_interval_set() + + @staticmethod + def get_protocol_names_from_interval_set(interval_set): + """ + Returns names of protocols represented by a given interval set + :param CanonicalIntervalSet interval_set: the interval set + :return: the list of protocol names + """ + res = [] + for interval in interval_set: + assert interval.start >= 0 and interval.end < len(ProtocolSet.all_protocols_list) + for index in range(interval.start, interval.end + 1): + res.append(ProtocolSet.all_protocols_list[index]) + return res + + @staticmethod + def _get_compl_protocol_names_from_interval_set(interval_set): + """ + Returns names of protocols not included in a given interval set + :param CanonicalIntervalSet interval_set: the interval set + :return: the list of complement protocol names + """ + res = ProtocolSet.all_protocols_list.copy() + for protocol in ProtocolSet.get_protocol_names_from_interval_set(interval_set): + res.remove(protocol) + return res + + def __str__(self): + """ + :return: Compact string representation of the ProtocolSet + """ + if self.is_whole_range(): + return '*' + if not self: + return 'Empty' + protocol_names = self.get_protocol_names_from_interval_set(self) + compl_protocol_names = self._get_compl_protocol_names_from_interval_set(self) + if len(protocol_names) <= len(compl_protocol_names): + values_list = ', '.join(protocol for protocol in protocol_names) + else: + values_list = 'all but ' + ', '.join(protocol for protocol in compl_protocol_names) + + return values_list + + def copy(self): + # new_copy = copy.copy(self) # the copy.copy() keeps the same reference to the interlval_set attribute + new_copy = ProtocolSet() + for interval in self.interval_set: + new_copy.interval_set.append(interval.copy()) + return new_copy diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index b2d29cccd..8e161c168 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -8,7 +8,8 @@ from .DimensionsManager import DimensionsManager from .PortSet import PortSet from .MethodSet import MethodSet -from .Peer import PeerSet +from .ProtocolSet import ProtocolSet +from .Peer import PeerSet, IpBlock class TcpLikeProperties(CanonicalHyperCubeSet): @@ -36,23 +37,25 @@ class TcpLikeProperties(CanonicalHyperCubeSet): (2) calico: +ve and -ve named ports, no src named ports, and no use of operators between these objects. """ - dimensions_list = ["src_ports", "dst_ports", "methods", "paths", "hosts", "peers"] + dimensions_list = ["src_peers", "dst_peers", "protocols", "src_ports", "dst_ports", "methods", "paths", "hosts", ] # TODO: change constructor defaults? either all arguments in "allow all" by default, or "empty" by default - def __init__(self, source_ports=PortSet(), dest_ports=PortSet(), methods=MethodSet(True), paths=None, hosts=None, - peers=None, base_peer_set=None): + def __init__(self, source_ports=PortSet(), dest_ports=PortSet(), protocols=ProtocolSet(True), + methods=MethodSet(True), paths=None, hosts=None, base_peer_set=None, src_peers=None, dst_peers=None): """ This will create all cubes made of the input arguments ranges/regex values. :param PortSet source_ports: The set of source ports (as a set of intervals/ranges) :param PortSet dest_ports: The set of target ports (as a set of intervals/ranges) + :param ProtocolSet protocols: the set of eligible protocols :param MethodSet methods: the set of http request methods :param MinDFA paths: The dfa of http request paths :param MinDFA hosts: The dfa of http request hosts - :param CanonicalIntervalSet peers: the set of (target) peers :param PeerSet base_peer_set: the base peer set which is referenced by the indices in 'peers' + :param CanonicalIntervalSet src_peers: the set of source peers + :param CanonicalIntervalSet dst_peers: the set of target peers """ super().__init__(TcpLikeProperties.dimensions_list) - assert not peers or base_peer_set + assert (not src_peers and not dst_peers) or base_peer_set self.named_ports = {} # a mapping from dst named port (String) to src ports interval set self.excluded_named_ports = {} # a mapping from dst named port (String) to src ports interval set @@ -61,13 +64,23 @@ def __init__(self, source_ports=PortSet(), dest_ports=PortSet(), methods=MethodS # create the cube from input arguments cube = [] active_dims = [] + # The order of dimensions below should be the same as in self.dimensions_list + if src_peers is not None: + cube.append(src_peers) + active_dims.append("src_peers") + if dst_peers is not None: + cube.append(dst_peers) + active_dims.append("dst_peers") + if protocols and not protocols.is_whole_range(): + cube.append(protocols) + active_dims.append("protocols") if not source_ports.is_all(): cube.append(source_ports.port_set) active_dims.append("src_ports") if not dest_ports.is_all(): cube.append(dest_ports.port_set) active_dims.append("dst_ports") - if not methods.is_whole_range(): + if methods and not methods.is_whole_range(): cube.append(methods) active_dims.append("methods") if paths is not None: @@ -76,9 +89,6 @@ def __init__(self, source_ports=PortSet(), dest_ports=PortSet(), methods=MethodS if hosts is not None: cube.append(hosts) active_dims.append("hosts") - if peers is not None: - cube.append(peers) - active_dims.append("peers") if not active_dims: self.set_all() @@ -135,10 +145,10 @@ def get_cube_dict(self, cube, is_txt=False): dim_domain = DimensionsManager().get_dimension_domain_by_name(dim) if dim_domain == dim_values: continue # skip dimensions with all values allowed in a cube - if dim == 'methods': + if dim == 'protocols' or dim == 'methods': values_list = str(dim_values) - elif dim == "peers": - values_list = self.base_peer_set.get_peer_list_by_indices(dim_values) + elif dim == "src_peers" or dim == "dst_peers": + values_list = self.base_peer_set.get_peer_set_by_indices(dim_values) elif dim_type == DimensionsManager.DimensionType.IntervalSet: values_list = dim_values.get_interval_set_list_numbers_and_ranges() if is_txt: @@ -149,6 +159,36 @@ def get_cube_dict(self, cube, is_txt=False): cube_dict[dim] = values_list return cube_dict + def get_cube_dict_with_orig_values(self, cube): + """ + represent the properties cube as dict object, where the values are the original values + with which the cube was built (i.e., MethodSet, PeerSet, etc.) + :param list cube: the values of the input cube + :return: the cube properties in dict representation + :rtype: dict + """ + cube_dict = {} + for i, dim in enumerate(self.active_dimensions): + dim_values = cube[i] + dim_domain = DimensionsManager().get_dimension_domain_by_name(dim) + if dim_domain == dim_values: + continue # skip dimensions with all values allowed in a cube + if dim == 'src_ports' or dim == 'dst_ports': + values = PortSet() + values.port_set = dim_values.copy() + elif dim == 'protocols': + values = ProtocolSet() + values.set_protocols(dim_values) + elif dim == 'methods': + values = MethodSet() + values.set_methods(dim_values) + elif dim == "src_peers" or dim == "dst_peers": + values = self.base_peer_set.get_peer_set_by_indices(dim_values) + else: + values = dim_values + cube_dict[dim] = values + return cube_dict + def get_properties_obj(self): """ get an object for a yaml representation of the protocol's properties @@ -166,7 +206,8 @@ def get_properties_obj(self): def __eq__(self, other): if isinstance(other, TcpLikeProperties): - assert self.base_peer_set == other.base_peer_set + assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ + not other.base_peer_set or self.base_peer_set == other.base_peer_set res = super().__eq__(other) and self.named_ports == other.named_ports and \ self.excluded_named_ports == other.excluded_named_ports return res @@ -188,16 +229,22 @@ def __sub__(self, other): return res def __iand__(self, other): - assert not isinstance(other, TcpLikeProperties) or self.base_peer_set == other.base_peer_set + assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ + not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.has_named_ports() assert not isinstance(other, TcpLikeProperties) or not other.has_named_ports() + if isinstance(other, TcpLikeProperties): + self.base_peer_set |= other.base_peer_set super().__iand__(other) return self def __ior__(self, other): - assert not isinstance(other, TcpLikeProperties) or self.base_peer_set == other.base_peer_set + assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ + not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.excluded_named_ports assert not isinstance(other, TcpLikeProperties) or not other.excluded_named_ports + if isinstance(other, TcpLikeProperties): + self.base_peer_set |= other.base_peer_set super().__ior__(other) if isinstance(other, TcpLikeProperties): res_named_ports = dict({}) @@ -212,9 +259,12 @@ def __ior__(self, other): return self def __isub__(self, other): - assert not isinstance(other, TcpLikeProperties) or self.base_peer_set == other.base_peer_set + assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ + not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.has_named_ports() assert not isinstance(other, TcpLikeProperties) or not other.has_named_ports() +# if isinstance(other, TcpLikeProperties): +# self.base_peer_set |= other.base_peer_set super().__isub__(other) return self @@ -224,7 +274,8 @@ def contained_in(self, other): :return: Whether all (source port, target port) pairs in self also appear in other :rtype: bool """ - assert not isinstance(other, TcpLikeProperties) or self.base_peer_set == other.base_peer_set + assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ + not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.has_named_ports() assert not other.has_named_ports() return super().contained_in(other) @@ -307,6 +358,111 @@ def _get_first_item_str(self): item = self.get_first_item(self.active_dimensions) res_list = [] for i, dim_name in enumerate(self.active_dimensions): - dim_item = item[i] if dim_name != 'methods' else MethodSet.all_methods_list[item[i]] + if dim_name == 'protocols': + dim_item = ProtocolSet.all_protocols_list[item[i]] + elif dim_name == 'methods': + dim_item = MethodSet.all_methods_list[item[i]] + else: + dim_item = item[i] res_list.append(f'{dim_name}={dim_item}') return '[' + ','.join(s for s in res_list) + ']' + + def project_on_one_dimension(self, dim_name): + """ + Build the projection of self to the given dimension + :param str dim_name: the given dimension + :return: the projection on the given dimension, having that dimension type (either IntervalSet or DFA) + """ + if dim_name not in self.active_dimensions: + return None + res = None + for cube in self: + cube_dict = self.get_cube_dict_with_orig_values(cube) + values = cube_dict.get(dim_name) + if values: + res = (res | values) if res else values + return res + + @staticmethod + def make_tcp_like_properties(peer_container, dest_ports, protocols=None, src_peers=None, dst_peers=None, + paths_dfa=None, hosts_dfa=None, methods=None): + """ + get TcpLikeProperties with TCP allowed connections, corresponding to input properties cube. + TcpLikeProperties should not contain named ports: substitute them with corresponding port numbers, per peer + :param PeerContainer peer_container: The set of endpoints and their namespaces + :param PortSet dest_ports: ports set for dest_ports dimension (possibly containing named ports) + :param ProtocolSet protocols: CanonicalIntervalSet obj for protocols dimension + :param PeerSet src_peers: the set of source peers + :param PeerSet dst_peers: the set of target peers + :param MinDFA paths_dfa: MinDFA obj for paths dimension + :param MinDFA hosts_dfa: MinDFA obj for hosts dimension + :param MethodSet methods: CanonicalIntervalSet obj for methods dimension + :return: TcpLikeProperties with TCP allowed connections, corresponding to input properties cube + """ + base_peer_set = peer_container.peer_set.copy() + base_peer_set.add(IpBlock.get_all_ips_block()) + if src_peers: + src_peers_interval = base_peer_set.get_peer_interval_of(src_peers) + else: + src_peers_interval = None + if dst_peers: + dst_peers_interval = base_peer_set.get_peer_interval_of(dst_peers) + else: + dst_peers_interval = None + if not dest_ports.named_ports: + return TcpLikeProperties(source_ports=PortSet(True), dest_ports=dest_ports, + protocols=protocols, methods=methods, + paths=paths_dfa, hosts=hosts_dfa, src_peers=src_peers_interval, + dst_peers=dst_peers_interval, base_peer_set=base_peer_set) + assert dst_peers + assert not dest_ports.port_set + assert len(dest_ports.named_ports) == 1 + port = list(dest_ports.named_ports)[0] + tcp_properties = None + tcp_protocol = ProtocolSet() + tcp_protocol.add_protocol('TCP') + for peer in dst_peers: + named_ports = peer.get_named_ports() + real_port = named_ports.get(port) + if not real_port: + print(f'Warning: Missing named port {port} in the pod {peer}. Ignoring the pod') + continue + if real_port[1] != 'TCP': + print(f'Warning: Illegal protocol {real_port[1]} in the named port {port} of the target pod {peer}.' + f'Ignoring the pod') + continue + peer_in_set = PeerSet() + peer_in_set.add(peer) + ports = PortSet() + ports.add_port(real_port[0]) + props = TcpLikeProperties(source_ports=PortSet(True), dest_ports=ports, + protocols=protocols if protocols else tcp_protocol, methods=methods, + paths=paths_dfa, hosts=hosts_dfa, src_peers=src_peers_interval, + dst_peers=base_peer_set.get_peer_interval_of(peer_in_set), + base_peer_set=base_peer_set) + if tcp_properties: + tcp_properties |= props + else: + tcp_properties = props + + return tcp_properties + + @staticmethod + def make_tcp_like_properties_from_dict(peer_container, cube_dict): + """ + Create TcpLikeProperties from the given cube + :param PeerContainer peer_container: the set of all peers + :param dict cube_dict: the given cube represented as a dictionary + :return: TcpLikeProperties + """ + cube_dict_copy = cube_dict.copy() + dest_ports = cube_dict_copy.pop("dst_ports", PortSet(True)) + protocols = cube_dict_copy.pop("protocols", None) + src_peers = cube_dict_copy.pop("src_peers", None) + dst_peers = cube_dict_copy.pop("dst_peers", None) + paths_dfa = cube_dict_copy.pop("paths", None) + hosts_dfa = cube_dict_copy.pop("hosts", None) + methods = cube_dict_copy.pop("methods", None) + assert not cube_dict_copy + return TcpLikeProperties.make_tcp_like_properties(peer_container, dest_ports, protocols, + src_peers, dst_peers, paths_dfa, hosts_dfa, methods) diff --git a/nca/FWRules/ClusterInfo.py b/nca/FWRules/ClusterInfo.py index dcc02f46a..4405a9596 100644 --- a/nca/FWRules/ClusterInfo.py +++ b/nca/FWRules/ClusterInfo.py @@ -52,6 +52,11 @@ def __init__(self, all_peers, allowed_labels): # labels values for a combination of multiple labels, in a fw-rule self.add_update_pods_labels_map_with_required_conjunction_labels() + def __eq__(self, other): + return self.all_peers == other.all_peers and self.allowed_labels == other.allowed_labels and \ + self.ns_dict == other.ns_dict and self.pods_labels_map == other.pods_labels_map and \ + self.all_label_values_per_ns == other.all_label_values_per_ns + def add_update_pods_labels_map_with_invalid_val(self, all_pods): """ Updating the pods_labels_map with (key,"NO_LABEL_VALUE") for the set of pods without this label diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index 37445a040..645b60d3a 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -4,12 +4,40 @@ # from collections import defaultdict -from nca.CoreDS.Peer import Peer, IpBlock, ClusterEP, Pod +from nca.CoreDS.Peer import Peer, IpBlock, PeerSet, ClusterEP, Pod +from nca.CoreDS.PortSet import PortSet +from nca.CoreDS.ConnectionSet import ConnectionSet +from nca.CoreDS.ProtocolSet import ProtocolSet +from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from .MinimizeFWRules import MinimizeCsFwRules, MinimizeFWRules from .ClusterInfo import ClusterInfo -class ConnectivityGraph: +class ConnectivityGraphPrototype: + def __init__(self, output_config): + """ + Create a ConnectivityGraph object + :param output_config: OutputConfiguration object + """ + # connections_to_peers holds the connectivity graph + self.output_config = output_config + + def _get_peer_name(self, peer): + """ + Get the name of a peer object for connectivity graph + flag indicating if it is ip-block + :param Peer peer: the peer object + :return: tuple(str, bool) + str: the peer name + bool: flag to indicate if peer is ip-block (True) or not (False) + """ + if isinstance(peer, IpBlock): + return peer.get_ip_range_or_cidr_str(), True + if self.output_config.outputEndpoints == 'deployments' and isinstance(peer, Pod): + return peer.workload_name, False + return str(peer), False + + +class ConnectivityGraph(ConnectivityGraphPrototype): """ Represents a connectivity digraph, that is a set of labeled edges, where the nodes are peers and the labels on the edges are the allowed connections between two peers. @@ -23,8 +51,8 @@ def __init__(self, all_peers, allowed_labels, output_config): :param output_config: OutputConfiguration object """ # connections_to_peers holds the connectivity graph + super().__init__(output_config) self.connections_to_peers = defaultdict(list) - self.output_config = output_config if self.output_config.fwRulesOverrideAllowedLabels: allowed_labels = set(label for label in self.output_config.fwRulesOverrideAllowedLabels.split(',')) self.cluster_info = ClusterInfo(all_peers, allowed_labels) @@ -48,19 +76,41 @@ def add_edges(self, connections): """ self.connections_to_peers.update(connections) - def _get_peer_name(self, peer): + def add_edges_from_cube_dict(self, peer_container, cube_dict): """ - Get the name of a peer object for connectivity graph + flag indicating if it is ip-block - :param Peer peer: the peer object - :return: tuple(str, bool) - str: the peer name - bool: flag to indicate if peer is ip-block (True) or not (False) + Add edges to the graph according to the give cube + :param peer_container: the peer_container containing all possible peers + :param dict cube_dict: the given cube in dictionary format """ - if isinstance(peer, IpBlock): - return peer.get_ip_range_or_cidr_str(), True - if self.output_config.outputEndpoints == 'deployments' and isinstance(peer, Pod): - return peer.workload_name, False - return str(peer), False + new_cube_dict = cube_dict.copy() + src_peers = new_cube_dict.get('src_peers') + if src_peers: + new_cube_dict.pop('src_peers') + else: + src_peers = PeerSet() + dst_peers = new_cube_dict.get('dst_peers') + if dst_peers: + new_cube_dict.pop('dst_peers') + else: + dst_peers = PeerSet() + protocols = new_cube_dict.get('protocols') + if protocols: + new_cube_dict.pop('protocols') + + if not protocols and not new_cube_dict: + conns = ConnectionSet(True) + else: + conns = ConnectionSet() + protocol_names = ProtocolSet.get_protocol_names_from_interval_set(protocols) if protocols else ['TCP'] + for protocol in protocol_names: + if new_cube_dict: + conns.add_connections(protocol, TcpLikeProperties.make_tcp_like_properties_from_dict(peer_container, + new_cube_dict)) + else: + conns.add_connections(protocol, TcpLikeProperties(PortSet(True), PortSet(True))) + for src_peer in src_peers: + for dst_peer in dst_peers: + self.connections_to_peers[conns].append((src_peer, dst_peer)) def get_connectivity_dot_format_str(self): """ @@ -70,12 +120,13 @@ def get_connectivity_dot_format_str(self): output_result += 'digraph ' + '{\n' if self.output_config.queryName and self.output_config.configName: output_result += f'\tHEADER [shape="box" label=< {self.output_config.queryName}/' \ - f'{self.output_config.configName} > fontsize=30 color=webmaroon fontcolor=webmaroon];\n' + f'{self.output_config.configName} > fontsize=30 color=webmaroon fontcolor=webmaroon];\n' peer_lines = set() for peer in self.cluster_info.all_peers: peer_name, is_ip_block = self._get_peer_name(peer) peer_color = "red2" if is_ip_block else "blue" - peer_lines.add(f'\t\"{peer_name}\" [label=\"{peer_name}\" color=\"{peer_color}\" fontcolor=\"{peer_color}\"]\n') + peer_lines.add(f'\t\"{peer_name}\" [label=\"{peer_name}\" color=\"{peer_color}\" ' + f'fontcolor=\"{peer_color}\"]\n') edge_lines = set() for connections, peer_pairs in self.connections_to_peers.items(): @@ -223,3 +274,97 @@ def conn_graph_has_fw_rules(self): if not conn: # conn is "no connections": return False return True + + +class ConnectivityGraphOptimized(ConnectivityGraphPrototype): + """ + Represents an optimized connectivity digraph, that is a set of labeled edges, where the nodes are sets of peers + and the labels on the edges are the allowed connections between the two sets of peers. + """ + + def __init__(self, output_config): + """ + Create a ConnectivityGraph object + :param output_config: OutputConfiguration object + """ + super().__init__(output_config) + self.edges = [] # the list of tuples(src_peers, dst_peers, connections) + self.peer_sets = set() # the set of all src/dst PeerSets in the graph + + def get_peer_set_names(self, peer_set): + """ + Convert a given peer_set to a string format for the output + :param peer_set: the given peer_set + :return: the string describing the given peer_set + """ + res_names = '' + res_is_only_ip_block = True + res_is_only_pods = True + for peer in peer_set: + peer_name, is_ip_block = self._get_peer_name(peer) + res_names += ', ' + peer_name if res_names else peer_name + res_is_only_ip_block &= is_ip_block + res_is_only_pods &= not is_ip_block + return res_names, res_is_only_ip_block, res_is_only_pods + + def add_edge(self, cube_dict): + """ + Adding a labeled edge to the graph + :param dict cube_dict: The map from all every dimension to its values + :return: None + """ + new_cube_dict = cube_dict.copy() + src_peers = new_cube_dict['src_peers'] or PeerSet() + dst_peers = new_cube_dict['dst_peers'] or PeerSet() + self.peer_sets.add(src_peers) + self.peer_sets.add(dst_peers) + new_cube_dict.pop('src_peers') + new_cube_dict.pop('dst_peers') + self.edges.append((src_peers, dst_peers, new_cube_dict)) + + def get_connectivity_dot_format_str(self): + """ + :return: a string with content of dot format for connectivity graph + """ + output_result = f'// The Connectivity Graph of {self.output_config.configName}\n' + output_result += 'digraph ' + '{\n' + if self.output_config.queryName and self.output_config.configName: + output_result += f'\tHEADER [shape="box" label=< {self.output_config.queryName}/' \ + f'{self.output_config.configName} > fontsize=30 color=webmaroon fontcolor=webmaroon];\n' + peer_set_lines = set() + for peer_set in self.peer_sets: + peer_set_name, is_only_ip_block, is_only_pods = self.get_peer_set_names(peer_set) + peer_color = "red2" if is_only_ip_block else "blue" if is_only_pods else "black" + peer_set_lines.add(f'\t\"{peer_set_name}\" [label=\"{peer_set_name}\" color=\"{peer_color}\" ' + f'fontcolor=\"{peer_color}\"]\n') + + edge_lines = set() + for src_peer_set, dst_peer_set, cube_dict in self.edges: + if src_peer_set != dst_peer_set and cube_dict: + src_peers_names, _, _ = self.get_peer_set_names(src_peer_set) + dst_peers_names, _, _ = self.get_peer_set_names(dst_peer_set) + line = '\t' + line += f'\"{src_peers_names}\"' + line += ' -> ' + line += f'\"{dst_peers_names}\"' + conn_str = str(cube_dict).replace("protocols:", "") + line += f' [label=\"{conn_str}\" color=\"gold2\" fontcolor=\"darkgreen\"]\n' + edge_lines.add(line) + output_result += ''.join(line for line in sorted(list(peer_set_lines))) + \ + ''.join(line for line in sorted(list(edge_lines))) + '}\n\n' + return output_result + + def get_connectivity_txt_format_str(self): + """ + :return: a string with content of txt format for connectivity graph + """ + output_result = '' + for src_peer_set, dst_peer_set, cube_dict in self.edges: + src_peers_names, _, _ = self.get_peer_set_names(src_peer_set) + dst_peers_names, _, _ = self.get_peer_set_names(dst_peer_set) + output_result += "src_pods: [" + src_peers_names + "] " + output_result += "dst_pods: [" + dst_peers_names + "] " + conn_str = str(cube_dict).replace("protocols:", "") + output_result += "conn: " + conn_str if cube_dict else "All connections" + output_result += '\n' + return output_result diff --git a/nca/FWRules/MinimizeFWRules.py b/nca/FWRules/MinimizeFWRules.py index dfb5214cd..94aa755e5 100644 --- a/nca/FWRules/MinimizeFWRules.py +++ b/nca/FWRules/MinimizeFWRules.py @@ -655,6 +655,10 @@ def __init__(self, fw_rules_map, cluster_info, output_config, results_map): self.output_config = output_config self.results_map = results_map + def __eq__(self, other): + return self.fw_rules_map == other.fw_rules_map and self.cluster_info == other.cluster_info and \ + self.output_config == other.output_config and self.results_map == other.results_map + def get_fw_rules_in_required_format(self, add_txt_header=True, add_csv_header=True): """ :param add_txt_header: bool flag to indicate if header of fw-rules query should be added in txt format diff --git a/nca/NetworkConfig/NetworkConfig.py b/nca/NetworkConfig/NetworkConfig.py index 06a8f33a8..67eca9c87 100644 --- a/nca/NetworkConfig/NetworkConfig.py +++ b/nca/NetworkConfig/NetworkConfig.py @@ -246,6 +246,33 @@ def allowed_connections(self, from_peer, to_peer, layer_name=None): return allowed_conns_res, captured_flag_res, allowed_captured_conns_res, denied_conns_res + def allowed_connections_optimized(self, layer_name=None): + """ + Computes the set of allowed connections between any relevant peers. + :param NetworkLayerName layer_name: The name of the layer to use, if requested to use a specific layer only + :return: allowed_conns: all allowed connections for relevant peers. + :rtype: TcpLikeProperties + """ + if layer_name is not None: + if layer_name not in self.policies_container.layers: + return self.policies_container.layers.empty_layer_allowed_connections_optimized(self.peer_container, + layer_name) + return self.policies_container.layers[layer_name].allowed_connections_optimized(self.peer_container) + + # TODO handle connectivity of hostEndpoints (for calico layer) + + conns_res = None + for layer, layer_obj in self.policies_container.layers.items(): + conns_per_layer = layer_obj.allowed_connections_optimized(self.peer_container) + + # all allowed connections: intersection of all allowed connections from all layers + if conns_res and conns_per_layer: + conns_res &= conns_per_layer + elif not conns_res: + conns_res = conns_per_layer + + return conns_res + def append_policy_to_config(self, policy): """ appends a policy to an existing config diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index f2e49c935..3693b4a9e 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -13,7 +13,7 @@ from nca.CoreDS.Peer import PeerSet, IpBlock, Pod, Peer from nca.Resources.CalicoNetworkPolicy import CalicoNetworkPolicy from nca.Resources.IngressPolicy import IngressPolicy -from nca.FWRules.ConnectivityGraph import ConnectivityGraph +from nca.FWRules.ConnectivityGraph import ConnectivityGraph, ConnectivityGraphOptimized from .NetworkConfig import NetworkConfig from .NetworkLayer import NetworkLayerName @@ -667,6 +667,7 @@ def exec(self): peers.add(peer2) res = QueryAnswer(True) + fw_rules = None if self.output_config.outputFormat == 'dot': conn_graph = ConnectivityGraph(peers, self.config.get_allowed_labels(), self.output_config) conn_graph.add_edges(connections) @@ -676,6 +677,25 @@ def exec(self): conn_graph.add_edges(connections) fw_rules = conn_graph.get_minimized_firewall_rules() res.output_explanation = fw_rules.get_fw_rules_in_required_format() + + all_conns_opt = self.config.allowed_connections_optimized() + if all_conns_opt: + conn_graph2 = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) + conn_graph_opt = ConnectivityGraphOptimized(self.output_config) + for cube in all_conns_opt: + conn_graph2.add_edges_from_cube_dict(self.config.peer_container, + all_conns_opt.get_cube_dict_with_orig_values(cube)) + conn_graph_opt.add_edge(all_conns_opt.get_cube_dict(cube)) + fw_rules2 = conn_graph2.get_minimized_firewall_rules() + res_opt = QueryAnswer(True) + if self.output_config.outputFormat == 'dot': + res_opt.output_explanation = conn_graph_opt.get_connectivity_dot_format_str() + else: + assert fw_rules == fw_rules2 + res_opt.output_explanation = conn_graph_opt.get_connectivity_txt_format_str() + res.output_explanation += "---------------- OPTIMIZED RESULT: -------------\n" +\ + fw_rules2.get_fw_rules_in_required_format() + \ + "\n------------------------------------------------\n\n" # TEMP for debug return res def compute_query_output(self, query_answer): diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index d6894675d..1ec06d8d1 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -7,6 +7,8 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock, HostEP +from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.PortSet import PortSet from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy from nca.Resources.NetworkPolicy import PolicyConnections, NetworkPolicy @@ -93,6 +95,17 @@ def empty_layer_allowed_connections(layer_name, from_peer, to_peer): empty_layer_obj = layer_name.create_network_layer([]) return empty_layer_obj.allowed_connections(from_peer, to_peer) + @staticmethod + def empty_layer_allowed_connections_optimized(peer_container, layer_name): + """ + Get allowed connections between for all relevant peers for an empty layer (no policies). + :param PeerContainer peer_container: holds all the peers + :param NetworkLayerName layer_name: The empty layer name + :rtype: TcpLikeProperties + """ + empty_layer_obj = layer_name.create_network_layer([]) + return empty_layer_obj.allowed_connections_optimized(peer_container) + class NetworkLayer: """ @@ -145,12 +158,33 @@ def allowed_connections(self, from_peer, to_peer): return allowed_conns, captured_flag, allowed_captured_conns, denied_conns + def allowed_connections_optimized(self, peer_container): + """ + Compute per network layer the allowed connections between any relevant peers, + considering all layer's policies (and defaults) + :param PeerContainer peer_container: the peer container holding the peers + :return: all allowed connections + :rtype: TcpLikeProperties + """ + ingress_conns = self._allowed_xgress_conns_optimized(True, peer_container) + egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) + if ingress_conns and egress_conns: + return ingress_conns & egress_conns + else: + return ingress_conns if ingress_conns else egress_conns + def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): """ Implemented by derived classes to get allowed and denied ingress/egress connections between from_peer and to_pee """ return NotImplemented + def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): + """ + Implemented by derived classes to get ingress/egress connections between any relevant peers + """ + return NotImplemented + def collect_policies_conns(self, from_peer, to_peer, is_ingress, captured_func=lambda policy: True): """ @@ -183,6 +217,23 @@ def collect_policies_conns(self, from_peer, to_peer, is_ingress, pass_conns |= policy_conns.pass_conns return allowed_conns, denied_conns, pass_conns, captured_res + def collect_policies_conns_optimized(self, is_ingress): + """ + Collect all connections (between all relevant peers), considering all layer's policies that capture the + relevant peers. + :param bool is_ingress: indicates whether to return ingress connections or egress connections + :return: conns + :rtype: TcpLikeProperties + """ + conns = None + for policy in self.policies_list: + policy_conns = policy.allowed_connections_optimized(is_ingress) + if policy_conns and conns: + conns |= policy_conns + elif not conns: + conns = policy_conns + return conns + class K8sCalicoNetworkLayer(NetworkLayer): @@ -204,6 +255,32 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): return PolicyConnections(captured_res, allowed_conns, denied_conns, all_allowed_conns=allowed_conns | allowed_non_captured_conns) + def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): + conn = self.collect_policies_conns_optimized(is_ingress) + if not conn: + return conn + # add non-captured connections + if is_ingress: + captured_dst_peers = conn.project_on_one_dimension('dst_peers') + if captured_dst_peers: + non_captured_dst_peers = conn.base_peer_set - captured_dst_peers + non_captured_conns = \ + TcpLikeProperties.make_tcp_like_properties(peer_container, dest_ports=PortSet(True), + src_peers=conn.base_peer_set.copy(), + dst_peers=non_captured_dst_peers) + conn |= non_captured_conns + else: + captured_src_peers = conn.project_on_one_dimension('src_peers') + if captured_src_peers: + non_captured_src_peers = conn.base_peer_set - captured_src_peers + non_captured_conns = \ + TcpLikeProperties.make_tcp_like_properties(peer_container, dest_ports=PortSet(True), + src_peers=non_captured_src_peers, + dst_peers=conn.base_peer_set.copy()) + conn |= non_captured_conns + + return conn + class IstioNetworkLayer(NetworkLayer): @@ -224,6 +301,9 @@ def captured_cond_func(policy): return PolicyConnections(captured_res, allowed_conns, denied_conns, all_allowed_conns=allowed_conns | allowed_non_captured_conns) + def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): + return self.collect_policies_conns_optimized(is_ingress) + class IngressNetworkLayer(NetworkLayer): @@ -237,3 +317,6 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): all_allowed_conns = allowed_conns return PolicyConnections(captured=captured_res, allowed_conns=allowed_conns, denied_conns=ConnectionSet(), all_allowed_conns=all_allowed_conns) + + def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): + return None diff --git a/nca/Parsers/GenericIngressLikeYamlParser.py b/nca/Parsers/GenericIngressLikeYamlParser.py index 7746e0438..b2700d9cb 100644 --- a/nca/Parsers/GenericIngressLikeYamlParser.py +++ b/nca/Parsers/GenericIngressLikeYamlParser.py @@ -6,9 +6,8 @@ from nca.CoreDS.MinDFA import MinDFA from nca.CoreDS.DimensionsManager import DimensionsManager -from nca.CoreDS.Peer import PeerSet, IpBlock +from nca.CoreDS.Peer import PeerSet from nca.CoreDS.PortSet import PortSet -from nca.CoreDS.MethodSet import MethodSet from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from nca.Resources.IngressPolicy import IngressPolicyRule from .GenericYamlParser import GenericYamlParser @@ -53,54 +52,6 @@ def parse_regex_host_value(self, regex_value, rule): regex_value = regex_value.replace("*", allowed_chars + '*') return MinDFA.dfa_from_regex(regex_value) - def _make_tcp_like_properties(self, dest_ports, peers, paths_dfa=None, hosts_dfa=None, methods_dfa=None): - """ - get TcpLikeProperties with TCP allowed connections, corresponding to input properties cube. - TcpLikeProperties should not contain named ports: substitute them with corresponding port numbers, per peer - :param PortSet dest_ports: ports set for dest_ports dimension (possibly containing named ports) - :param PeerSet peers: the set of (target) peers - :param MinDFA paths_dfa: MinDFA obj for paths dimension - :param MinDFA hosts_dfa: MinDFA obj for hosts dimension - :return: TcpLikeProperties with TCP allowed connections, corresponding to input properties cube - """ - assert peers - base_peer_set = self.peer_container.peer_set.copy() - base_peer_set.add(IpBlock.get_all_ips_block()) - if not dest_ports.named_ports: - peers_interval = base_peer_set.get_peer_interval_of(peers) - return TcpLikeProperties(source_ports=PortSet(True), dest_ports=dest_ports, - methods=methods_dfa if methods_dfa else MethodSet(True), - paths=paths_dfa, hosts=hosts_dfa, peers=peers_interval, - base_peer_set=base_peer_set) - assert not dest_ports.port_set - assert len(dest_ports.named_ports) == 1 - port = list(dest_ports.named_ports)[0] - tcp_properties = None - for peer in peers: - named_ports = peer.get_named_ports() - real_port = named_ports.get(port) - if not real_port: - self.warning(f'Missing named port {port} in the pod {peer}. Ignoring the pod') - continue - if real_port[1] != 'TCP': - self.warning(f'Illegal protocol {real_port[1]} in the named port {port} ingress target pod {peer}.' - f'Ignoring the pod') - continue - peer_in_set = PeerSet() - peer_in_set.add(peer) - ports = PortSet() - ports.add_port(real_port[0]) - props = TcpLikeProperties(source_ports=PortSet(True), dest_ports=ports, methods=MethodSet(True), - paths=paths_dfa, hosts=hosts_dfa, - peers=base_peer_set.get_peer_interval_of(peer_in_set), - base_peer_set=base_peer_set) - if tcp_properties: - tcp_properties |= props - else: - tcp_properties = props - - return tcp_properties - def _make_allow_rules(self, allowed_conns): """ Make deny rules from the given connections @@ -122,7 +73,8 @@ def _make_rules_from_conns(self, tcp_conns): ports = None paths = None hosts = None - peer_set = None + src_peer_set = None + dst_peer_set = None for i, dim in enumerate(tcp_conns.active_dimensions): if dim == "dst_ports": ports = cube[i] @@ -130,21 +82,24 @@ def _make_rules_from_conns(self, tcp_conns): paths = cube[i] elif dim == "hosts": hosts = cube[i] - elif dim == "peers": - peer_set = PeerSet(set(tcp_conns.base_peer_set.get_peer_list_by_indices(cube[i]))) + elif dim == "src_peers": + src_peer_set = tcp_conns.base_peer_set.get_peer_set_by_indices(cube[i]) + elif dim == "dst_peers": + dst_peer_set = tcp_conns.base_peer_set.get_peer_set_by_indices(cube[i]) else: assert False - if not peer_set: - peer_set = self.peer_container.peer_set.copy() + assert not src_peer_set + if not dst_peer_set: + dst_peer_set = self.peer_container.peer_set.copy() port_set = PortSet() port_set.port_set = ports port_set.named_ports = tcp_conns.named_ports port_set.excluded_named_ports = tcp_conns.excluded_named_ports new_conns = self._get_connection_set_from_properties(port_set, paths_dfa=paths, hosts_dfa=hosts) - if peers_to_conns.get(peer_set): - peers_to_conns[peer_set] |= new_conns # optimize conns for the same peers + if peers_to_conns.get(dst_peer_set): + peers_to_conns[dst_peer_set] |= new_conns # optimize conns for the same peers else: - peers_to_conns[peer_set] = new_conns + peers_to_conns[dst_peer_set] = new_conns for peer_set, conns in peers_to_conns.items(): res.append(IngressPolicyRule(peer_set, conns)) return res diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index 3c781f833..a3f7b179d 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -8,6 +8,7 @@ from nca.CoreDS.DimensionsManager import DimensionsManager from nca.CoreDS.Peer import PeerSet from nca.CoreDS.PortSet import PortSet +from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from nca.Resources.IngressPolicy import IngressPolicy from nca.Resources.NetworkPolicy import NetworkPolicy from .GenericIngressLikeYamlParser import GenericIngressLikeYamlParser @@ -171,10 +172,14 @@ def _make_default_connections(self, hosts_dfa, paths_dfa=None): default_conns = None if self.default_backend_peers: if paths_dfa: - default_conns = self._make_tcp_like_properties(self.default_backend_ports, self.default_backend_peers, - paths_dfa, hosts_dfa) + default_conns = \ + TcpLikeProperties.make_tcp_like_properties(self.peer_container, self.default_backend_ports, + dst_peers=self.default_backend_peers, + paths_dfa=paths_dfa, hosts_dfa=hosts_dfa) else: - default_conns = self._make_tcp_like_properties(self.default_backend_ports, self.default_backend_peers, + default_conns = \ + TcpLikeProperties.make_tcp_like_properties(self.peer_container, self.default_backend_ports, + dst_peers=self.default_backend_peers, hosts_dfa=hosts_dfa) return default_conns @@ -201,7 +206,8 @@ def parse_rule(self, rule): parsed_paths_with_dfa = self.segregate_longest_paths_and_make_dfa(parsed_paths) for (_, paths_dfa, _, peers, ports) in parsed_paths_with_dfa: # every path is converted to allowed connections - conns = self._make_tcp_like_properties(ports, peers, paths_dfa, hosts_dfa) + conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, ports, dst_peers=peers, + paths_dfa=paths_dfa, hosts_dfa=hosts_dfa) if not allowed_conns: allowed_conns = conns else: diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index 5e8faa407..efb535b3f 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -335,8 +335,7 @@ def _parse_method(self, method_str, operation): res = MethodSet() matching_methods = self._parse_istio_regex_from_enumerated_domain(method_str, 'methods') for method in matching_methods: - index = MethodSet.all_methods_list.index(method) - res.add_interval(MethodSet.Interval(index, index)) + res.add_method(method) if not res: if method_str: diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index 4c5c9c08e..7e7dd4d4a 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -7,6 +7,8 @@ from nca.CoreDS.DimensionsManager import DimensionsManager from nca.CoreDS.MinDFA import MinDFA from nca.CoreDS.Peer import PeerSet +from nca.CoreDS.MethodSet import MethodSet +from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from nca.Resources.IstioTrafficResources import Gateway, VirtualService from nca.Resources.IngressPolicy import IngressPolicy from nca.Resources.NetworkPolicy import NetworkPolicy @@ -201,7 +203,8 @@ def parse_istio_regex_string(self, resource, attr_name, vs_name): :param dict resource: the HttpMatchRequest resource :param str attr_name: the name of the StringMatch attribute :param str vs_name: the name of the VirtualService containing this HttpMatchRequest - :return MinDFA: the StringMatch attribute converted to the MinDFA format + :return MinDFA or MethodSet: the StringMatch attribute converted to the MinDFA format + or to the MethodSet format (in case of the 'method' attribute) """ res = resource.get(attr_name) if not res: @@ -223,7 +226,12 @@ def parse_istio_regex_string(self, resource, attr_name, vs_name): else: self.warning(f'illegal attribute {items[0]} in the VirtualService {vs_name}. Ignoring.') return None - return MinDFA.dfa_from_regex(regex) + if attr_name == 'method': + methods = MethodSet() + methods.add_methods_from_regex(regex) + return methods + else: + return MinDFA.dfa_from_regex(regex) def parse_http_match_request(self, route, parsed_route, vs): """ @@ -245,9 +253,9 @@ def parse_http_match_request(self, route, parsed_route, vs): uri_dfa = self.parse_istio_regex_string(item, 'uri', vs.full_name()) if uri_dfa: parsed_route.add_uri_dfa(uri_dfa) - method_dfa = self.parse_istio_regex_string(item, 'method', vs.full_name()) - if method_dfa: - parsed_route.add_method_dfa(method_dfa) + methods = self.parse_istio_regex_string(item, 'method', vs.full_name()) + if methods: + parsed_route.add_methods(methods) def parse_http_route_destinations(self, route, parsed_route, vs): """ @@ -320,8 +328,11 @@ def make_allowed_connections(self, vs, host_dfa): allowed_conns = None for http_route in vs.http_routes: for dest in http_route.destinations: - conns = self._make_tcp_like_properties(dest.port, dest.service.target_pods, http_route.uri_dfa, - host_dfa, http_route.method_dfa) + conns = \ + TcpLikeProperties.make_tcp_like_properties(self.peer_container, dest.port, + dst_peers=dest.service.target_pods, + paths_dfa=http_route.uri_dfa, hosts_dfa=host_dfa, + methods=http_route.methods) if not allowed_conns: allowed_conns = conns else: diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index d809a8fb7..7c854616a 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -9,6 +9,7 @@ from nca.CoreDS.PortSet import PortSet from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from nca.CoreDS.ProtocolNameResolver import ProtocolNameResolver +from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.NetworkPolicy import NetworkPolicy from nca.Resources.K8sNetworkPolicy import K8sNetworkPolicy, K8sPolicyRule from .GenericYamlParser import GenericYamlParser @@ -117,7 +118,7 @@ def parse_label_selector_requirement(self, requirement, namespace_selector): """ Parse a LabelSelectorRequirement element :param dict requirement: The element to parse - :param bool namespace_selector: Whether or not this is in the context of namespaceSelector + :param bool namespace_selector: Whether this is in the context of namespaceSelector :return: A PeerSet containing the peers that satisfy the requirement :rtype: Peer.PeerSet """ @@ -151,7 +152,7 @@ def parse_label_selector(self, label_selector, namespace_selector=False): """ Parse a LabelSelector element (can also come from a NamespaceSelector) :param dict label_selector: The element to parse - :param bool namespace_selector: Whether or not this is a namespaceSelector + :param bool namespace_selector: Whether this is a namespaceSelector :return: A PeerSet containing all the pods captured by this selection :rtype: Peer.PeerSet """ @@ -253,8 +254,8 @@ def parse_port(self, port): """ Parse an element of the "ports" phrase of a policy rule :param dict port: The element to parse - :return: A ConnectionSet representing the allowed connections by this element (protocols X port numbers) - :rtype: ConnectionSet + :return: A protocol and a port_set represented by this element + :rtype: tuple (str, PortSet) """ self.check_fields_validity(port, 'NetworkPolicyPort', {'port': 0, 'protocol': [0, str], 'endPort': [0, int]}, {'protocol': ['TCP', 'UDP', 'SCTP']}) @@ -264,7 +265,6 @@ def parse_port(self, port): if not protocol: protocol = 'TCP' - res = ConnectionSet() dest_port_set = PortSet(port_id is None) if port_id is not None and end_port_num is not None: if isinstance(port_id, str): @@ -293,16 +293,17 @@ def parse_port(self, port): elif end_port_num: self.syntax_error('endPort cannot be defined if the port field is not defined ', port) - res.add_connections(protocol, TcpLikeProperties(PortSet(True), dest_port_set)) # K8s doesn't reason about src ports - return res + return protocol, dest_port_set - def parse_ingress_egress_rule(self, rule, peer_array_key): + def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): """ Parse a single ingres/egress rule, producing a K8sPolicyRule :param dict rule: The rule to parse :param str peer_array_key: The key which defined the peer set ('from' for ingress, 'to' for egress) - :return: A K8sPolicyRule with the proper PeerSet and ConnectionSet - :rtype: K8sPolicyRule + :param Peer.PeerSet policy_selected_pods: The set of pods the policy applies to + :return: A tuple (K8sPolicyRule, TcpLikeProperties) with the proper PeerSet and attributes, where + TcpLikeProperties is an optimized rule format with protocols, src_peers and dst_peers in a HyperCubeSet + :rtype: tuple(K8sPolicyRule, TcpLikeProperties) """ self.check_fields_validity(rule, 'ingress/egress rule', {peer_array_key: [0, list], 'ports': [0, list]}) peer_array = rule.get(peer_array_key, []) @@ -313,18 +314,40 @@ def parse_ingress_egress_rule(self, rule, peer_array_key): else: res_pods = self.peer_container.get_all_peers_group(True) + if peer_array_key == 'from': # ingress + src_pods = res_pods + dst_pods = policy_selected_pods + else: # egress + src_pods = policy_selected_pods + dst_pods = res_pods + ports_array = rule.get('ports', []) if ports_array: - res_ports = ConnectionSet() + res_conns = ConnectionSet() + res_opt_props = None # TcpLikeProperties for port in ports_array: - res_ports |= self.parse_port(port) + protocol, dest_port_set = self.parse_port(port) + protocols = ProtocolSet() + protocols.add_protocol(protocol) + res_conns.add_connections(protocol, TcpLikeProperties(PortSet(True), dest_port_set)) # K8s doesn't reason about src ports + dest_num_port_set = PortSet() + dest_num_port_set.port_set = dest_port_set.port_set.copy() + tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, dest_num_port_set, + protocols, src_pods, dst_pods) + if res_opt_props: + res_opt_props |= tcp_props + else: + res_opt_props = tcp_props +# self.handle_named_ports(dst_pods, protocol, dest_port_set.named_ports, res_opt_props) else: - res_ports = ConnectionSet(True) + res_conns = ConnectionSet(True) + res_opt_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), + src_peers=src_pods, dst_peers=dst_pods) if not res_pods: self.warning('Rule selects no pods', rule) - return K8sPolicyRule(res_pods, res_ports) + return K8sPolicyRule(res_pods, res_conns), res_opt_props def verify_named_ports(self, rule, rule_pods, rule_conns): """ @@ -358,24 +381,27 @@ def parse_ingress_rule(self, rule, policy_selected_pods): Also, checking validity of named ports w.r.t. the policy's captured pods :param dict rule: The dict with the rule fields :param Peer.PeerSet policy_selected_pods: The set of pods the policy applies to - :return: A K8sPolicyRule with the proper PeerSet and ConnectionSet - :rtype: K8sPolicyRule + :return: A tuple (K8sPolicyRule, TcpLikeProperties) with the proper PeerSet and attributes, where + TcpLikeProperties is an optimized rule format with protocols, src_peers and dst_peers in a HyperCubeSet + :rtype: tuple(K8sPolicyRule, TcpLikeProperties) """ - res = self.parse_ingress_egress_rule(rule, 'from') - self.verify_named_ports(rule, policy_selected_pods, res.port_set) - return res + res_rule, res_opt_props = self.parse_ingress_egress_rule(rule, 'from', policy_selected_pods) + self.verify_named_ports(rule, policy_selected_pods, res_rule.port_set) + return res_rule, res_opt_props - def parse_egress_rule(self, rule): + def parse_egress_rule(self, rule, policy_selected_pods): """ Parse a single egress rule, producing a K8sPolicyRule. Also, checking validity of named ports w.r.t. the rule's peer set :param dict rule: The dict with the rule fields - :return: A K8sPolicyRule with the proper PeerSet and ConnectionSet - :rtype: K8sPolicyRule + :param Peer.PeerSet policy_selected_pods: The set of pods the policy applies to + :return: A tuple (K8sPolicyRule, TcpLikeProperties) with the proper PeerSet and attributes, where + TcpLikeProperties is an optimized rule format with protocols, src_peers and dst_peers in a HyperCubeSet + :rtype: tuple(K8sPolicyRule, TcpLikeProperties) """ - res = self.parse_ingress_egress_rule(rule, 'to') - self.verify_named_ports(rule, res.peer_set, res.port_set) - return res + res_rule, res_opt_props = self.parse_ingress_egress_rule(rule, 'to', policy_selected_pods) + self.verify_named_ports(rule, res_rule.peer_set, res_rule.port_set) + return res_rule, res_opt_props def parse_policy(self): """ @@ -424,12 +450,16 @@ def parse_policy(self): ingress_rules = policy_spec.get('ingress', []) if ingress_rules: for ingress_rule in ingress_rules: - res_policy.add_ingress_rule(self.parse_ingress_rule(ingress_rule, res_policy.selected_peers)) + rule, optimized_props = self.parse_ingress_rule(ingress_rule, res_policy.selected_peers) + res_policy.add_ingress_rule(rule) + res_policy.add_optimized_ingress_props(optimized_props) egress_rules = policy_spec.get('egress', []) if egress_rules: for egress_rule in egress_rules: - res_policy.add_egress_rule(self.parse_egress_rule(egress_rule)) + rule, optimized_props = self.parse_egress_rule(egress_rule, res_policy.selected_peers) + res_policy.add_egress_rule(rule) + res_policy.add_optimized_egress_props(optimized_props) res_policy.findings = self.warning_msgs res_policy.referenced_labels = self.referenced_labels diff --git a/nca/Resources/IstioTrafficResources.py b/nca/Resources/IstioTrafficResources.py index e2abb2227..69968bd30 100644 --- a/nca/Resources/IstioTrafficResources.py +++ b/nca/Resources/IstioTrafficResources.py @@ -7,6 +7,7 @@ from nca.CoreDS.Peer import PeerSet from nca.CoreDS.PortSet import PortSet from nca.CoreDS.MinDFA import MinDFA +from nca.CoreDS.MethodSet import MethodSet from .K8sService import K8sService @@ -93,7 +94,7 @@ class HTTPRoute: def __init__(self): self.uri_dfa = None # self.scheme_dfa = None # not supported yet - self.method_dfa = None + self.methods = MethodSet() # self.authority_dfa = None # not supported yet self.destinations = [] @@ -107,15 +108,15 @@ def add_uri_dfa(self, uri_dfa): else: self.uri_dfa = uri_dfa - def add_method_dfa(self, method_dfa): + def add_methods(self, methods): """ - Adds a method_dfa to the http route - :param MinDFA method_dfa: the method_dfa to add + Adds methods to the http route + :param MethodSet methods: the methods to add """ - if self.method_dfa: - self.method_dfa |= method_dfa + if self.methods: + self.methods |= methods else: - self.method_dfa = method_dfa + self.methods = methods.copy() def add_destination(self, service, port): """ diff --git a/nca/Resources/K8sNetworkPolicy.py b/nca/Resources/K8sNetworkPolicy.py index 57b5fd95c..b7a4df8bc 100644 --- a/nca/Resources/K8sNetworkPolicy.py +++ b/nca/Resources/K8sNetworkPolicy.py @@ -62,6 +62,16 @@ def allowed_connections(self, from_peer, to_peer, is_ingress): return PolicyConnections(True, allowed_conns) + def allowed_connections_optimized(self, is_ingress): + """ + Return the set of connections this policy allows between any two peers + (either ingress or egress). + :param bool is_ingress: whether we evaluate ingress rules only or egress rules only + :return: A TcpLikeProperties object containing all allowed connections for relevant peers + :rtype: TcpLikeProperties + """ + return self.optimized_ingress_props if is_ingress else self.optimized_egress_props + def clone_without_rule(self, rule_to_exclude, ingress_rule): """ Makes a copy of 'self' without a given policy rule diff --git a/nca/Resources/NetworkPolicy.py b/nca/Resources/NetworkPolicy.py index 958a8a1ab..a8567f825 100644 --- a/nca/Resources/NetworkPolicy.py +++ b/nca/Resources/NetworkPolicy.py @@ -52,6 +52,8 @@ def __init__(self, name, namespace): self.selected_peers = PeerSet() # The peers affected by this policy self.ingress_rules = [] self.egress_rules = [] + self.optimized_ingress_props = None # all properties in hypercube set format + self.optimized_egress_props = None # all properties in hypercube set format self.affects_ingress = False # whether the policy affects the ingress of the selected peers self.affects_egress = False # whether the policy affects the egress of the selected peers self.findings = [] # accumulated findings which are relevant only to this policy (emptiness and redundancy) @@ -119,6 +121,28 @@ def add_egress_rule(self, rule): """ self.egress_rules.append(rule) + def add_optimized_ingress_props(self, props): + """ + Adding properties to the CanonicalHyperCubeSet of optimized ingress properties + :param CanonicalHyperCubeSet props: The properties to add + :return: None + """ + if self.optimized_ingress_props: + self.optimized_ingress_props |= props + else: + self.optimized_ingress_props = props + + def add_optimized_egress_props(self, props): + """ + Adding properties to the CanonicalHyperCubeSet of optimized egress properties + :param CanonicalHyperCubeSet props: The properties to add + :return: None + """ + if self.optimized_egress_props: + self.optimized_egress_props |= props + else: + self.optimized_egress_props = props + @staticmethod def get_policy_type_from_dict(policy): # noqa: C901 """ From c279de64d1fe09c2324d0b1de6b238921af3463d Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 15 Nov 2022 17:44:34 +0200 Subject: [PATCH 002/187] Extended testcase3 to produce connectivity_map Signed-off-by: Tanya --- .../example_policies/testcase3/testcase3-scheme.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/k8s_testcases/example_policies/testcase3/testcase3-scheme.yaml b/tests/k8s_testcases/example_policies/testcase3/testcase3-scheme.yaml index 3463cade1..564f6fbd0 100644 --- a/tests/k8s_testcases/example_policies/testcase3/testcase3-scheme.yaml +++ b/tests/k8s_testcases/example_policies/testcase3/testcase3-scheme.yaml @@ -13,6 +13,15 @@ networkConfigList: expectedWarnings: 0 queries: + - name: connectivity_map + connectivityMap: + - np1 + - np2 + expected: 0 + outputConfiguration: + outputFormat: txt + fwRulesRunInTestMode: false + - name: not_vacuous vacuity: - np1 From 1b1ebbe90a6c315341461cf1c2aeb5806cf418b8 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 20 Nov 2022 21:14:47 +0200 Subject: [PATCH 003/187] Fixed a problem in HyperCubeSet (wrongly changing self in _and_aux. Added optimized_denied_ingress_props and optimized_denied_egress_props (in addition to allowed ones). Improved non_captured_conns computation Signed-off-by: Tanya --- nca/CoreDS/CanonicalHyperCubeSet.py | 11 +-- nca/NetworkConfig/NetworkConfigQuery.py | 8 +-- nca/NetworkConfig/NetworkLayer.py | 95 ++++++++++++++++--------- nca/Parsers/K8sPolicyYamlParser.py | 12 ++++ nca/Resources/K8sNetworkPolicy.py | 8 ++- nca/Resources/NetworkPolicy.py | 24 ++++--- 6 files changed, 106 insertions(+), 52 deletions(-) diff --git a/nca/CoreDS/CanonicalHyperCubeSet.py b/nca/CoreDS/CanonicalHyperCubeSet.py index abf7a9a75..ad697354f 100644 --- a/nca/CoreDS/CanonicalHyperCubeSet.py +++ b/nca/CoreDS/CanonicalHyperCubeSet.py @@ -263,7 +263,9 @@ def __iand__(self, other): self._override_by_other(other.copy()) return self other_copy = self._prepare_common_active_dimensions(other) - self._and_aux(other_copy) + res = self._and_aux(other_copy) + self.layers = res.layers + self.active_dimensions = res.active_dimensions self._reduce_active_dimensions() return self @@ -275,6 +277,7 @@ def _and_aux(self, other): :return: self """ assert self.active_dimensions == other.active_dimensions + res = self.copy() res_layers = dict() for self_layer in self.layers: for other_layer in other.layers: @@ -291,9 +294,9 @@ def _and_aux(self, other): if new_sub_elem: res_layers[common_elem] = new_sub_elem - self.layers = res_layers - self._apply_layer_elements_union() - return self + res.layers = res_layers + res._apply_layer_elements_union() + return res def __or__(self, other): res = self.copy() diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 3693b4a9e..29de8498d 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -692,10 +692,10 @@ def exec(self): res_opt.output_explanation = conn_graph_opt.get_connectivity_dot_format_str() else: assert fw_rules == fw_rules2 - res_opt.output_explanation = conn_graph_opt.get_connectivity_txt_format_str() - res.output_explanation += "---------------- OPTIMIZED RESULT: -------------\n" +\ - fw_rules2.get_fw_rules_in_required_format() + \ - "\n------------------------------------------------\n\n" # TEMP for debug + # res_opt.output_explanation = conn_graph_opt.get_connectivity_txt_format_str() + # res.output_explanation += "---------------- OPTIMIZED RESULT: -------------\n" +\ + # fw_rules2.get_fw_rules_in_required_format() + \ + # "\n------------------------------------------------\n\n" # TEMP for debug return res def compute_query_output(self, query_answer): diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 1ec06d8d1..288aa6a3d 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -166,12 +166,18 @@ def allowed_connections_optimized(self, peer_container): :return: all allowed connections :rtype: TcpLikeProperties """ - ingress_conns = self._allowed_xgress_conns_optimized(True, peer_container) - egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) - if ingress_conns and egress_conns: - return ingress_conns & egress_conns + allowed_ingress_conns, denied_ingres_conns = self._allowed_xgress_conns_optimized(True, peer_container) + allowed_egress_conns, denied_egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) + if allowed_ingress_conns and allowed_egress_conns: + res = allowed_ingress_conns & allowed_egress_conns else: - return ingress_conns if ingress_conns else egress_conns + res = allowed_ingress_conns if allowed_ingress_conns else allowed_egress_conns + if res: + if denied_ingres_conns: + res -= denied_ingres_conns + if denied_egress_conns: + res -= denied_egress_conns + return res def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): """ @@ -222,18 +228,23 @@ def collect_policies_conns_optimized(self, is_ingress): Collect all connections (between all relevant peers), considering all layer's policies that capture the relevant peers. :param bool is_ingress: indicates whether to return ingress connections or egress connections - :return: conns - :rtype: TcpLikeProperties + :return: allowed_conns and denied_conns + :rtype: tuple (TcpLikeProperties, TcpLikeProperties) """ - conns = None + allowed_conns = None + denied_conns = None for policy in self.policies_list: - policy_conns = policy.allowed_connections_optimized(is_ingress) - if policy_conns and conns: - conns |= policy_conns - elif not conns: - conns = policy_conns - return conns - + policy_allowed_conns, policy_denied_conns = policy.allowed_connections_optimized(is_ingress) + if policy_allowed_conns and allowed_conns: + allowed_conns |= policy_allowed_conns + elif not allowed_conns: + allowed_conns = policy_allowed_conns + if policy_denied_conns and denied_conns: + denied_conns |= policy_denied_conns + elif not denied_conns: + denied_conns = policy_denied_conns + + return allowed_conns, denied_conns class K8sCalicoNetworkLayer(NetworkLayer): @@ -256,30 +267,44 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): all_allowed_conns=allowed_conns | allowed_non_captured_conns) def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): - conn = self.collect_policies_conns_optimized(is_ingress) - if not conn: - return conn - # add non-captured connections + allowed_conn, denied_conns = self.collect_policies_conns_optimized(is_ingress) + if not allowed_conn and not denied_conns: + return None, None + # Note: The below computation of non-captured conns cannot be done during the parse stage, + # since before computing non-captured conns we should collect all policies conns + # compute non-captured connections + base_peer_set = peer_container.peer_set.copy() + base_peer_set.add(IpBlock.get_all_ips_block()) if is_ingress: - captured_dst_peers = conn.project_on_one_dimension('dst_peers') + captured_dst_peers1 = allowed_conn.project_on_one_dimension('dst_peers') if allowed_conn else None + captured_dst_peers2 = denied_conns.project_on_one_dimension('dst_peers') if denied_conns else None + if captured_dst_peers1 and captured_dst_peers2: + captured_dst_peers = captured_dst_peers1 | captured_dst_peers2 + else: + captured_dst_peers = captured_dst_peers1 if captured_dst_peers1 else captured_dst_peers2 if captured_dst_peers: - non_captured_dst_peers = conn.base_peer_set - captured_dst_peers - non_captured_conns = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, dest_ports=PortSet(True), - src_peers=conn.base_peer_set.copy(), - dst_peers=non_captured_dst_peers) - conn |= non_captured_conns + non_captured_dst_peers = base_peer_set - captured_dst_peers + if non_captured_dst_peers: + non_captured_conns = \ + TcpLikeProperties.make_tcp_like_properties(peer_container, dest_ports=PortSet(True), + dst_peers=non_captured_dst_peers) + allowed_conn = (allowed_conn | non_captured_conns) if allowed_conn else non_captured_conns else: - captured_src_peers = conn.project_on_one_dimension('src_peers') + captured_src_peers1 = allowed_conn.project_on_one_dimension('src_peers') if allowed_conn else None + captured_src_peers2 = denied_conns.project_on_one_dimension('src_peers') if denied_conns else None + if captured_src_peers1 and captured_src_peers2: + captured_src_peers = captured_src_peers1 | captured_src_peers2 + else: + captured_src_peers = captured_src_peers1 if captured_src_peers1 else captured_src_peers2 if captured_src_peers: - non_captured_src_peers = conn.base_peer_set - captured_src_peers - non_captured_conns = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, dest_ports=PortSet(True), - src_peers=non_captured_src_peers, - dst_peers=conn.base_peer_set.copy()) - conn |= non_captured_conns + non_captured_src_peers = base_peer_set - captured_src_peers + if non_captured_src_peers: + non_captured_conns = \ + TcpLikeProperties.make_tcp_like_properties(peer_container, dest_ports=PortSet(True), + src_peers=non_captured_src_peers) + allowed_conn = (allowed_conn | non_captured_conns) if allowed_conn else non_captured_conns - return conn + return allowed_conn, denied_conns class IstioNetworkLayer(NetworkLayer): @@ -319,4 +344,4 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): all_allowed_conns=all_allowed_conns) def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): - return None + return None, None diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index 7c854616a..f572e87ac 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -446,6 +446,8 @@ def parse_policy(self): pod_selector = policy_spec.get('podSelector') res_policy.selected_peers = self.parse_label_selector(pod_selector) res_policy.selected_peers &= self.peer_container.get_namespace_pods(self.namespace) + base_peer_set = self.peer_container.peer_set.copy() + base_peer_set.add(Peer.IpBlock.get_all_ips_block()) ingress_rules = policy_spec.get('ingress', []) if ingress_rules: @@ -453,6 +455,11 @@ def parse_policy(self): rule, optimized_props = self.parse_ingress_rule(ingress_rule, res_policy.selected_peers) res_policy.add_ingress_rule(rule) res_policy.add_optimized_ingress_props(optimized_props) + else: + # add denied connections + denied_conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, dest_ports=PortSet(True), + dst_peers=res_policy.selected_peers) + res_policy.add_optimized_ingress_props(denied_conns, False) egress_rules = policy_spec.get('egress', []) if egress_rules: @@ -460,6 +467,11 @@ def parse_policy(self): rule, optimized_props = self.parse_egress_rule(egress_rule, res_policy.selected_peers) res_policy.add_egress_rule(rule) res_policy.add_optimized_egress_props(optimized_props) + else: + # add only non-captured connections + denied_conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, dest_ports=PortSet(True), + src_peers=res_policy.selected_peers) + res_policy.add_optimized_egress_props(denied_conns, False) res_policy.findings = self.warning_msgs res_policy.referenced_labels = self.referenced_labels diff --git a/nca/Resources/K8sNetworkPolicy.py b/nca/Resources/K8sNetworkPolicy.py index b7a4df8bc..6bb740231 100644 --- a/nca/Resources/K8sNetworkPolicy.py +++ b/nca/Resources/K8sNetworkPolicy.py @@ -70,7 +70,13 @@ def allowed_connections_optimized(self, is_ingress): :return: A TcpLikeProperties object containing all allowed connections for relevant peers :rtype: TcpLikeProperties """ - return self.optimized_ingress_props if is_ingress else self.optimized_egress_props + if is_ingress: + allowed = self.optimized_ingress_props.copy() if self.optimized_ingress_props else None + denied = self.optimized_denied_ingress_props.copy() if self.optimized_denied_ingress_props else None + else: + allowed = self.optimized_egress_props.copy() if self.optimized_egress_props else None + denied = self.optimized_denied_egress_props.copy() if self.optimized_denied_egress_props else None + return allowed, denied def clone_without_rule(self, rule_to_exclude, ingress_rule): """ diff --git a/nca/Resources/NetworkPolicy.py b/nca/Resources/NetworkPolicy.py index a8567f825..297c7c740 100644 --- a/nca/Resources/NetworkPolicy.py +++ b/nca/Resources/NetworkPolicy.py @@ -53,7 +53,9 @@ def __init__(self, name, namespace): self.ingress_rules = [] self.egress_rules = [] self.optimized_ingress_props = None # all properties in hypercube set format + self.optimized_denied_ingress_props = None # all denied properties in hypercube set format self.optimized_egress_props = None # all properties in hypercube set format + self.optimized_denied_egress_props = None # all denied properties in hypercube set format self.affects_ingress = False # whether the policy affects the ingress of the selected peers self.affects_egress = False # whether the policy affects the egress of the selected peers self.findings = [] # accumulated findings which are relevant only to this policy (emptiness and redundancy) @@ -121,27 +123,33 @@ def add_egress_rule(self, rule): """ self.egress_rules.append(rule) - def add_optimized_ingress_props(self, props): + def add_optimized_ingress_props(self, props, is_allow=True): """ Adding properties to the CanonicalHyperCubeSet of optimized ingress properties :param CanonicalHyperCubeSet props: The properties to add + :param Bool is_allow: whether these are an allow or deny properties :return: None """ - if self.optimized_ingress_props: - self.optimized_ingress_props |= props + if is_allow: + self.optimized_ingress_props = \ + (self.optimized_ingress_props | props) if self.optimized_ingress_props else props else: - self.optimized_ingress_props = props + self.optimized_denied_ingress_props = \ + (self.optimized_denied_ingress_props | props) if self.optimized_denied_ingress_props else props - def add_optimized_egress_props(self, props): + def add_optimized_egress_props(self, props, is_allow=True): """ Adding properties to the CanonicalHyperCubeSet of optimized egress properties :param CanonicalHyperCubeSet props: The properties to add + :param Bool is_allow: whether these are an allow or deny properties :return: None """ - if self.optimized_egress_props: - self.optimized_egress_props |= props + if is_allow: + self.optimized_egress_props = \ + (self.optimized_egress_props | props) if self.optimized_egress_props else props else: - self.optimized_egress_props = props + self.optimized_denied_egress_props = \ + (self.optimized_denied_egress_props | props) if self.optimized_denied_egress_props else props @staticmethod def get_policy_type_from_dict(policy): # noqa: C901 From 09cfad1cc22e4046b82e653bda41df23c2507564 Mon Sep 17 00:00:00 2001 From: Tanya Date: Mon, 21 Nov 2022 06:45:16 +0200 Subject: [PATCH 004/187] Small fix Signed-off-by: Tanya --- nca/Parsers/K8sPolicyYamlParser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index f572e87ac..e6b4fd093 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -455,7 +455,7 @@ def parse_policy(self): rule, optimized_props = self.parse_ingress_rule(ingress_rule, res_policy.selected_peers) res_policy.add_ingress_rule(rule) res_policy.add_optimized_ingress_props(optimized_props) - else: + elif res_policy.affects_ingress: # add denied connections denied_conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, dest_ports=PortSet(True), dst_peers=res_policy.selected_peers) @@ -467,7 +467,7 @@ def parse_policy(self): rule, optimized_props = self.parse_egress_rule(egress_rule, res_policy.selected_peers) res_policy.add_egress_rule(rule) res_policy.add_optimized_egress_props(optimized_props) - else: + elif res_policy.affects_egress: # add only non-captured connections denied_conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, dest_ports=PortSet(True), src_peers=res_policy.selected_peers) From 94fb117a7cda6f1739c61ec0a5c50dec3d23161c Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 22 Nov 2022 07:49:48 +0200 Subject: [PATCH 005/187] Further fix of the hyper cube set Signed-off-by: Tanya --- nca/CoreDS/CanonicalHyperCubeSet.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nca/CoreDS/CanonicalHyperCubeSet.py b/nca/CoreDS/CanonicalHyperCubeSet.py index ad697354f..e01deccef 100644 --- a/nca/CoreDS/CanonicalHyperCubeSet.py +++ b/nca/CoreDS/CanonicalHyperCubeSet.py @@ -325,6 +325,7 @@ def or_aux(self, other): :return: self """ assert self.active_dimensions == other.active_dimensions + res = self.copy() res_layers = dict() remaining_other_layers = dict() # map from layer_0 elems in orig "other", to remaining parts to be added for layer_elem in other.layers: @@ -347,9 +348,9 @@ def or_aux(self, other): for layer_elem, remaining_layer_elem in remaining_other_layers.items(): if remaining_layer_elem: res_layers[remaining_layer_elem] = other.layers[layer_elem].copy() - self.layers = res_layers + res.layers = res_layers self._apply_layer_elements_union() - return self + return res def __sub__(self, other): res = self.copy() @@ -374,6 +375,7 @@ def sub_aux(self, other): :return: self """ assert self.active_dimensions == other.active_dimensions + res = self.copy() res_layers = dict() for self_layer in self.layers: remaining_self_layer = self._copy_layer_elem(self_layer) @@ -392,9 +394,9 @@ def sub_aux(self, other): res_layers[common_elem] = new_sub_elem if remaining_self_layer: res_layers[remaining_self_layer] = self.layers[self_layer] - self.layers = res_layers + res.layers = res_layers self._apply_layer_elements_union() - return self + return res def _prepare_common_active_dimensions(self, other): """ From 520377addae6f35985c17b70625f8c455cb32e55 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 22 Nov 2022 07:49:48 +0200 Subject: [PATCH 006/187] Further fix of the hyper cube set Signed-off-by: Tanya --- nca/CoreDS/CanonicalHyperCubeSet.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/nca/CoreDS/CanonicalHyperCubeSet.py b/nca/CoreDS/CanonicalHyperCubeSet.py index ad697354f..22cad4371 100644 --- a/nca/CoreDS/CanonicalHyperCubeSet.py +++ b/nca/CoreDS/CanonicalHyperCubeSet.py @@ -313,7 +313,9 @@ def __ior__(self, other): self._override_by_other(other.copy()) return self other_copy = self._prepare_common_active_dimensions(other) - self.or_aux(other_copy) + res = self.or_aux(other_copy) + self.layers = res.layers + self.active_dimensions = res.active_dimensions self._reduce_active_dimensions() return self @@ -325,6 +327,7 @@ def or_aux(self, other): :return: self """ assert self.active_dimensions == other.active_dimensions + res = self.copy() res_layers = dict() remaining_other_layers = dict() # map from layer_0 elems in orig "other", to remaining parts to be added for layer_elem in other.layers: @@ -347,9 +350,9 @@ def or_aux(self, other): for layer_elem, remaining_layer_elem in remaining_other_layers.items(): if remaining_layer_elem: res_layers[remaining_layer_elem] = other.layers[layer_elem].copy() - self.layers = res_layers - self._apply_layer_elements_union() - return self + res.layers = res_layers + res._apply_layer_elements_union() + return res def __sub__(self, other): res = self.copy() @@ -362,7 +365,9 @@ def __isub__(self, other): if not other: return self other_copy = self._prepare_common_active_dimensions(other) - self.sub_aux(other_copy) + res = self.sub_aux(other_copy) + self.layers = res.layers + self.active_dimensions = res.active_dimensions self._reduce_active_dimensions() return self @@ -374,6 +379,7 @@ def sub_aux(self, other): :return: self """ assert self.active_dimensions == other.active_dimensions + res = self.copy() res_layers = dict() for self_layer in self.layers: remaining_self_layer = self._copy_layer_elem(self_layer) @@ -392,9 +398,9 @@ def sub_aux(self, other): res_layers[common_elem] = new_sub_elem if remaining_self_layer: res_layers[remaining_self_layer] = self.layers[self_layer] - self.layers = res_layers - self._apply_layer_elements_union() - return self + res.layers = res_layers + res._apply_layer_elements_union() + return res def _prepare_common_active_dimensions(self, other): """ From 0339215e0622abb10d98b6f2ae138fcf431eeb4c Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 22 Nov 2022 13:13:54 +0200 Subject: [PATCH 007/187] Avoiding redundant and heavy copy of layers. Signed-off-by: Tanya --- nca/CoreDS/CanonicalHyperCubeSet.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nca/CoreDS/CanonicalHyperCubeSet.py b/nca/CoreDS/CanonicalHyperCubeSet.py index 22cad4371..e514403f1 100644 --- a/nca/CoreDS/CanonicalHyperCubeSet.py +++ b/nca/CoreDS/CanonicalHyperCubeSet.py @@ -277,7 +277,8 @@ def _and_aux(self, other): :return: self """ assert self.active_dimensions == other.active_dimensions - res = self.copy() + res = CanonicalHyperCubeSet(self.all_dimensions_list) + res.active_dimensions = self.active_dimensions res_layers = dict() for self_layer in self.layers: for other_layer in other.layers: @@ -327,7 +328,8 @@ def or_aux(self, other): :return: self """ assert self.active_dimensions == other.active_dimensions - res = self.copy() + res = CanonicalHyperCubeSet(self.all_dimensions_list) + res.active_dimensions = self.active_dimensions res_layers = dict() remaining_other_layers = dict() # map from layer_0 elems in orig "other", to remaining parts to be added for layer_elem in other.layers: @@ -379,7 +381,8 @@ def sub_aux(self, other): :return: self """ assert self.active_dimensions == other.active_dimensions - res = self.copy() + res = CanonicalHyperCubeSet(self.all_dimensions_list) + res.active_dimensions = self.active_dimensions res_layers = dict() for self_layer in self.layers: remaining_self_layer = self._copy_layer_elem(self_layer) From b9810b34d72e162ea007028ba2cf09bedeecbbb6 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 22 Nov 2022 14:03:23 +0200 Subject: [PATCH 008/187] General changes from the Optimized_HC_set branch. Signed-off-by: Tanya --- nca/CoreDS/MethodSet.py | 27 +++++++++++++++++++ nca/CoreDS/Peer.py | 17 +++++++----- nca/CoreDS/TcpLikeProperties.py | 20 +++++++++----- nca/FWRules/ClusterInfo.py | 5 ++++ nca/FWRules/MinimizeFWRules.py | 4 +++ nca/Parsers/GenericIngressLikeYamlParser.py | 2 +- nca/Parsers/IstioPolicyYamlParser.py | 3 +-- .../IstioTrafficResourcesYamlParser.py | 19 ++++++++----- nca/Resources/IstioTrafficResources.py | 15 ++++++----- 9 files changed, 83 insertions(+), 29 deletions(-) diff --git a/nca/CoreDS/MethodSet.py b/nca/CoreDS/MethodSet.py index a75f8b08c..7c653662d 100644 --- a/nca/CoreDS/MethodSet.py +++ b/nca/CoreDS/MethodSet.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache2.0 # +import re from .CanonicalIntervalSet import CanonicalIntervalSet @@ -20,6 +21,32 @@ def __init__(self, all_methods=False): if all_methods: # the whole range self.add_interval(self._whole_range_interval()) + def add_method(self, method): + """ + Adds a given method to the MethodSet if the method is one of the eligible methods (in all_methods_list); + otherwise raises ValueError exception + :param str method: the method to add + """ + index = self.all_methods_list.index(method) + self.add_interval(self.Interval(index, index)) + + def add_methods_from_regex(self, methods_regex): + """ + Adds all methods in methods_regex to the MethodSet + :param str methods_regex: + """ + for index, method in enumerate(self.all_methods_list): + if re.match(methods_regex, method): + self.add_interval(self.Interval(index, index)) + + def set_methods(self, methods): + """ + Sets all methods from the given parameter + :param CanonicalIntervalSet methods: the methods to set + """ + for interval in methods: + self.add_interval(interval) + @staticmethod def _whole_range_interval(): """ diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index cab56b55e..73b60e1a2 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -576,25 +576,28 @@ def get_peer_interval_of(self, peer_set): res.add_interval(CanonicalIntervalSet.Interval(index, index)) return res - def get_peer_list_by_indices(self, peer_inteval_set): + def get_peer_set_by_indices(self, peer_inteval_set): """ Return peer list from interval set of indices :param peer_inteval_set: the interval set of indices into the sorted peer list :return: the list of peers referenced by the indices in the interval set """ - assert len(self.sorted_peer_list) == len(self) - res = [] + my_len = len(self) + if len(self.sorted_peer_list) != my_len: + self.update_sorted_peer_list() + peer_list = [] for interval in peer_inteval_set: - for ind in range(interval.start, interval.end + 1): - res.append(self.sorted_peer_list[ind]) - return res + for ind in range(min(interval.start, my_len), min(interval.end + 1, my_len)): + peer_list.append(self.sorted_peer_list[ind]) + return PeerSet(set(peer_list)) def get_all_peers_interval(self): """ Returns the interval of all peers :return: CanonicalIntervalSet of all peers """ - assert len(self.sorted_peer_list) == len(self) + if len(self.sorted_peer_list) != len(self): + self.update_sorted_peer_list() return CanonicalIntervalSet.get_interval_set(0, len(self.sorted_peer_list) - 1) def is_whole_range(self, peer_interval_set): diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index b2d29cccd..26a58b051 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -138,7 +138,7 @@ def get_cube_dict(self, cube, is_txt=False): if dim == 'methods': values_list = str(dim_values) elif dim == "peers": - values_list = self.base_peer_set.get_peer_list_by_indices(dim_values) + values_list = self.base_peer_set.get_peer_set_by_indices(dim_values) elif dim_type == DimensionsManager.DimensionType.IntervalSet: values_list = dim_values.get_interval_set_list_numbers_and_ranges() if is_txt: @@ -166,7 +166,7 @@ def get_properties_obj(self): def __eq__(self, other): if isinstance(other, TcpLikeProperties): - assert self.base_peer_set == other.base_peer_set + assert not self.base_peer_set or not other.base_peer_set or self.base_peer_set == other.base_peer_set res = super().__eq__(other) and self.named_ports == other.named_ports and \ self.excluded_named_ports == other.excluded_named_ports return res @@ -188,16 +188,22 @@ def __sub__(self, other): return res def __iand__(self, other): - assert not isinstance(other, TcpLikeProperties) or self.base_peer_set == other.base_peer_set + assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ + not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.has_named_ports() assert not isinstance(other, TcpLikeProperties) or not other.has_named_ports() + if isinstance(other, TcpLikeProperties): + self.base_peer_set |= other.base_peer_set super().__iand__(other) return self def __ior__(self, other): - assert not isinstance(other, TcpLikeProperties) or self.base_peer_set == other.base_peer_set + assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ + not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.excluded_named_ports assert not isinstance(other, TcpLikeProperties) or not other.excluded_named_ports + if isinstance(other, TcpLikeProperties): + self.base_peer_set |= other.base_peer_set super().__ior__(other) if isinstance(other, TcpLikeProperties): res_named_ports = dict({}) @@ -212,7 +218,8 @@ def __ior__(self, other): return self def __isub__(self, other): - assert not isinstance(other, TcpLikeProperties) or self.base_peer_set == other.base_peer_set + assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ + not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.has_named_ports() assert not isinstance(other, TcpLikeProperties) or not other.has_named_ports() super().__isub__(other) @@ -224,7 +231,8 @@ def contained_in(self, other): :return: Whether all (source port, target port) pairs in self also appear in other :rtype: bool """ - assert not isinstance(other, TcpLikeProperties) or self.base_peer_set == other.base_peer_set + assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ + not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.has_named_ports() assert not other.has_named_ports() return super().contained_in(other) diff --git a/nca/FWRules/ClusterInfo.py b/nca/FWRules/ClusterInfo.py index dcc02f46a..4405a9596 100644 --- a/nca/FWRules/ClusterInfo.py +++ b/nca/FWRules/ClusterInfo.py @@ -52,6 +52,11 @@ def __init__(self, all_peers, allowed_labels): # labels values for a combination of multiple labels, in a fw-rule self.add_update_pods_labels_map_with_required_conjunction_labels() + def __eq__(self, other): + return self.all_peers == other.all_peers and self.allowed_labels == other.allowed_labels and \ + self.ns_dict == other.ns_dict and self.pods_labels_map == other.pods_labels_map and \ + self.all_label_values_per_ns == other.all_label_values_per_ns + def add_update_pods_labels_map_with_invalid_val(self, all_pods): """ Updating the pods_labels_map with (key,"NO_LABEL_VALUE") for the set of pods without this label diff --git a/nca/FWRules/MinimizeFWRules.py b/nca/FWRules/MinimizeFWRules.py index dfb5214cd..94aa755e5 100644 --- a/nca/FWRules/MinimizeFWRules.py +++ b/nca/FWRules/MinimizeFWRules.py @@ -655,6 +655,10 @@ def __init__(self, fw_rules_map, cluster_info, output_config, results_map): self.output_config = output_config self.results_map = results_map + def __eq__(self, other): + return self.fw_rules_map == other.fw_rules_map and self.cluster_info == other.cluster_info and \ + self.output_config == other.output_config and self.results_map == other.results_map + def get_fw_rules_in_required_format(self, add_txt_header=True, add_csv_header=True): """ :param add_txt_header: bool flag to indicate if header of fw-rules query should be added in txt format diff --git a/nca/Parsers/GenericIngressLikeYamlParser.py b/nca/Parsers/GenericIngressLikeYamlParser.py index 7746e0438..f50ec1648 100644 --- a/nca/Parsers/GenericIngressLikeYamlParser.py +++ b/nca/Parsers/GenericIngressLikeYamlParser.py @@ -131,7 +131,7 @@ def _make_rules_from_conns(self, tcp_conns): elif dim == "hosts": hosts = cube[i] elif dim == "peers": - peer_set = PeerSet(set(tcp_conns.base_peer_set.get_peer_list_by_indices(cube[i]))) + peer_set = tcp_conns.base_peer_set.get_peer_set_by_indices(cube[i]) else: assert False if not peer_set: diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index 5e8faa407..efb535b3f 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -335,8 +335,7 @@ def _parse_method(self, method_str, operation): res = MethodSet() matching_methods = self._parse_istio_regex_from_enumerated_domain(method_str, 'methods') for method in matching_methods: - index = MethodSet.all_methods_list.index(method) - res.add_interval(MethodSet.Interval(index, index)) + res.add_method(method) if not res: if method_str: diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index 4c5c9c08e..0f78074ea 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -7,6 +7,7 @@ from nca.CoreDS.DimensionsManager import DimensionsManager from nca.CoreDS.MinDFA import MinDFA from nca.CoreDS.Peer import PeerSet +from nca.CoreDS.MethodSet import MethodSet from nca.Resources.IstioTrafficResources import Gateway, VirtualService from nca.Resources.IngressPolicy import IngressPolicy from nca.Resources.NetworkPolicy import NetworkPolicy @@ -201,7 +202,8 @@ def parse_istio_regex_string(self, resource, attr_name, vs_name): :param dict resource: the HttpMatchRequest resource :param str attr_name: the name of the StringMatch attribute :param str vs_name: the name of the VirtualService containing this HttpMatchRequest - :return MinDFA: the StringMatch attribute converted to the MinDFA format + :return MinDFA or MethodSet: the StringMatch attribute converted to the MinDFA format + or to the MethodSet format (in case of the 'method' attribute) """ res = resource.get(attr_name) if not res: @@ -223,7 +225,12 @@ def parse_istio_regex_string(self, resource, attr_name, vs_name): else: self.warning(f'illegal attribute {items[0]} in the VirtualService {vs_name}. Ignoring.') return None - return MinDFA.dfa_from_regex(regex) + if attr_name == 'method': + methods = MethodSet() + methods.add_methods_from_regex(regex) + return methods + else: + return MinDFA.dfa_from_regex(regex) def parse_http_match_request(self, route, parsed_route, vs): """ @@ -245,9 +252,9 @@ def parse_http_match_request(self, route, parsed_route, vs): uri_dfa = self.parse_istio_regex_string(item, 'uri', vs.full_name()) if uri_dfa: parsed_route.add_uri_dfa(uri_dfa) - method_dfa = self.parse_istio_regex_string(item, 'method', vs.full_name()) - if method_dfa: - parsed_route.add_method_dfa(method_dfa) + methods = self.parse_istio_regex_string(item, 'method', vs.full_name()) + if methods: + parsed_route.add_methods(methods) def parse_http_route_destinations(self, route, parsed_route, vs): """ @@ -321,7 +328,7 @@ def make_allowed_connections(self, vs, host_dfa): for http_route in vs.http_routes: for dest in http_route.destinations: conns = self._make_tcp_like_properties(dest.port, dest.service.target_pods, http_route.uri_dfa, - host_dfa, http_route.method_dfa) + host_dfa, http_route.methods) if not allowed_conns: allowed_conns = conns else: diff --git a/nca/Resources/IstioTrafficResources.py b/nca/Resources/IstioTrafficResources.py index e2abb2227..69968bd30 100644 --- a/nca/Resources/IstioTrafficResources.py +++ b/nca/Resources/IstioTrafficResources.py @@ -7,6 +7,7 @@ from nca.CoreDS.Peer import PeerSet from nca.CoreDS.PortSet import PortSet from nca.CoreDS.MinDFA import MinDFA +from nca.CoreDS.MethodSet import MethodSet from .K8sService import K8sService @@ -93,7 +94,7 @@ class HTTPRoute: def __init__(self): self.uri_dfa = None # self.scheme_dfa = None # not supported yet - self.method_dfa = None + self.methods = MethodSet() # self.authority_dfa = None # not supported yet self.destinations = [] @@ -107,15 +108,15 @@ def add_uri_dfa(self, uri_dfa): else: self.uri_dfa = uri_dfa - def add_method_dfa(self, method_dfa): + def add_methods(self, methods): """ - Adds a method_dfa to the http route - :param MinDFA method_dfa: the method_dfa to add + Adds methods to the http route + :param MethodSet methods: the methods to add """ - if self.method_dfa: - self.method_dfa |= method_dfa + if self.methods: + self.methods |= methods else: - self.method_dfa = method_dfa + self.methods = methods.copy() def add_destination(self, service, port): """ From e82ac990ef982e9428a16cde6749695fc02fa5c3 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 22 Nov 2022 15:16:49 +0200 Subject: [PATCH 009/187] General changes from the Optimized_HC_set branch. Signed-off-by: Tanya --- nca/CoreDS/CanonicalHyperCubeSet.py | 46 ++++++++++++++++++----------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/nca/CoreDS/CanonicalHyperCubeSet.py b/nca/CoreDS/CanonicalHyperCubeSet.py index abf7a9a75..b0003d4a8 100644 --- a/nca/CoreDS/CanonicalHyperCubeSet.py +++ b/nca/CoreDS/CanonicalHyperCubeSet.py @@ -263,7 +263,9 @@ def __iand__(self, other): self._override_by_other(other.copy()) return self other_copy = self._prepare_common_active_dimensions(other) - self._and_aux(other_copy) + res = self._and_aux(other_copy) + self.layers = res.layers + self.active_dimensions = res.active_dimensions self._reduce_active_dimensions() return self @@ -275,6 +277,8 @@ def _and_aux(self, other): :return: self """ assert self.active_dimensions == other.active_dimensions + res = CanonicalHyperCubeSet(self.all_dimensions_list) + res.active_dimensions = self.active_dimensions res_layers = dict() for self_layer in self.layers: for other_layer in other.layers: @@ -282,7 +286,7 @@ def _and_aux(self, other): if not common_elem: continue if self._is_last_dimension(): - res_layers[common_elem] = self.layers[self_layer] + res_layers[common_elem] = self.layers[self_layer].copy() continue # TODO: use type hint to avoid warning on access to a protected member? # self_sub_elem: CanonicalHyperCubeSet = self.layers[self_layer] @@ -291,9 +295,9 @@ def _and_aux(self, other): if new_sub_elem: res_layers[common_elem] = new_sub_elem - self.layers = res_layers - self._apply_layer_elements_union() - return self + res.layers = res_layers + res._apply_layer_elements_union() + return res def __or__(self, other): res = self.copy() @@ -310,7 +314,9 @@ def __ior__(self, other): self._override_by_other(other.copy()) return self other_copy = self._prepare_common_active_dimensions(other) - self.or_aux(other_copy) + res = self.or_aux(other_copy) + self.layers = res.layers + self.active_dimensions = res.active_dimensions self._reduce_active_dimensions() return self @@ -322,6 +328,8 @@ def or_aux(self, other): :return: self """ assert self.active_dimensions == other.active_dimensions + res = CanonicalHyperCubeSet(self.all_dimensions_list) + res.active_dimensions = self.active_dimensions res_layers = dict() remaining_other_layers = dict() # map from layer_0 elems in orig "other", to remaining parts to be added for layer_elem in other.layers: @@ -337,16 +345,16 @@ def or_aux(self, other): if self._is_last_dimension(): res_layers[common_elem] = CanonicalHyperCubeSet.empty_interval continue - new_sub_elem = (self.layers[self_layer].copy()).or_aux(other.layers[other_layer]) + new_sub_elem = self.layers[self_layer].or_aux(other.layers[other_layer]) res_layers[common_elem] = new_sub_elem if remaining_self_layer: - res_layers[remaining_self_layer] = self.layers[self_layer] + res_layers[remaining_self_layer] = self.layers[self_layer].copy() for layer_elem, remaining_layer_elem in remaining_other_layers.items(): if remaining_layer_elem: res_layers[remaining_layer_elem] = other.layers[layer_elem].copy() - self.layers = res_layers - self._apply_layer_elements_union() - return self + res.layers = res_layers + res._apply_layer_elements_union() + return res def __sub__(self, other): res = self.copy() @@ -359,7 +367,9 @@ def __isub__(self, other): if not other: return self other_copy = self._prepare_common_active_dimensions(other) - self.sub_aux(other_copy) + res = self.sub_aux(other_copy) + self.layers = res.layers + self.active_dimensions = res.active_dimensions self._reduce_active_dimensions() return self @@ -371,6 +381,8 @@ def sub_aux(self, other): :return: self """ assert self.active_dimensions == other.active_dimensions + res = CanonicalHyperCubeSet(self.all_dimensions_list) + res.active_dimensions = self.active_dimensions res_layers = dict() for self_layer in self.layers: remaining_self_layer = self._copy_layer_elem(self_layer) @@ -383,15 +395,15 @@ def sub_aux(self, other): # do not add common_elem to self.layers here because result is empty continue # sub-elements subtraction - new_sub_elem = (self.layers[self_layer].copy()).sub_aux(other.layers[other_layer]) + new_sub_elem = self.layers[self_layer].sub_aux(other.layers[other_layer]) if bool(new_sub_elem): # add remaining new_sub_elem if not empty, under common res_layers[common_elem] = new_sub_elem if remaining_self_layer: - res_layers[remaining_self_layer] = self.layers[self_layer] - self.layers = res_layers - self._apply_layer_elements_union() - return self + res_layers[remaining_self_layer] = self.layers[self_layer].copy() + res.layers = res_layers + res._apply_layer_elements_union() + return res def _prepare_common_active_dimensions(self, other): """ From 6b3b3644b6edc5673cdbc995ed4ccce05acd418d Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 22 Nov 2022 15:22:58 +0200 Subject: [PATCH 010/187] Avoiding lint warnings Signed-off-by: Tanya Signed-off-by: Tanya --- nca/FWRules/ClusterInfo.py | 6 +++--- nca/FWRules/MinimizeFWRules.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nca/FWRules/ClusterInfo.py b/nca/FWRules/ClusterInfo.py index 4405a9596..3f5c3279f 100644 --- a/nca/FWRules/ClusterInfo.py +++ b/nca/FWRules/ClusterInfo.py @@ -53,9 +53,9 @@ def __init__(self, all_peers, allowed_labels): self.add_update_pods_labels_map_with_required_conjunction_labels() def __eq__(self, other): - return self.all_peers == other.all_peers and self.allowed_labels == other.allowed_labels and \ - self.ns_dict == other.ns_dict and self.pods_labels_map == other.pods_labels_map and \ - self.all_label_values_per_ns == other.all_label_values_per_ns + return self.all_peers == other.all_peers and self.allowed_labels == other.allowed_labels \ + and self.ns_dict == other.ns_dict and self.pods_labels_map == other.pods_labels_map \ + and self.all_label_values_per_ns == other.all_label_values_per_ns def add_update_pods_labels_map_with_invalid_val(self, all_pods): """ diff --git a/nca/FWRules/MinimizeFWRules.py b/nca/FWRules/MinimizeFWRules.py index 94aa755e5..7da7d9a3f 100644 --- a/nca/FWRules/MinimizeFWRules.py +++ b/nca/FWRules/MinimizeFWRules.py @@ -656,8 +656,8 @@ def __init__(self, fw_rules_map, cluster_info, output_config, results_map): self.results_map = results_map def __eq__(self, other): - return self.fw_rules_map == other.fw_rules_map and self.cluster_info == other.cluster_info and \ - self.output_config == other.output_config and self.results_map == other.results_map + return self.fw_rules_map == other.fw_rules_map and self.cluster_info == other.cluster_info \ + and self.output_config == other.output_config and self.results_map == other.results_map def get_fw_rules_in_required_format(self, add_txt_header=True, add_csv_header=True): """ From cfb5ee7cdbfffce962abfd3173c17b6f5b9dee79 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 22 Nov 2022 15:24:35 +0200 Subject: [PATCH 011/187] Avoiding lint warnings Signed-off-by: Tanya Signed-off-by: Tanya --- nca/FWRules/ClusterInfo.py | 4 ++-- nca/FWRules/MinimizeFWRules.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nca/FWRules/ClusterInfo.py b/nca/FWRules/ClusterInfo.py index 3f5c3279f..79e79185a 100644 --- a/nca/FWRules/ClusterInfo.py +++ b/nca/FWRules/ClusterInfo.py @@ -54,8 +54,8 @@ def __init__(self, all_peers, allowed_labels): def __eq__(self, other): return self.all_peers == other.all_peers and self.allowed_labels == other.allowed_labels \ - and self.ns_dict == other.ns_dict and self.pods_labels_map == other.pods_labels_map \ - and self.all_label_values_per_ns == other.all_label_values_per_ns + and self.ns_dict == other.ns_dict and self.pods_labels_map == other.pods_labels_map \ + and self.all_label_values_per_ns == other.all_label_values_per_ns def add_update_pods_labels_map_with_invalid_val(self, all_pods): """ diff --git a/nca/FWRules/MinimizeFWRules.py b/nca/FWRules/MinimizeFWRules.py index 7da7d9a3f..8cd66f5cd 100644 --- a/nca/FWRules/MinimizeFWRules.py +++ b/nca/FWRules/MinimizeFWRules.py @@ -657,7 +657,7 @@ def __init__(self, fw_rules_map, cluster_info, output_config, results_map): def __eq__(self, other): return self.fw_rules_map == other.fw_rules_map and self.cluster_info == other.cluster_info \ - and self.output_config == other.output_config and self.results_map == other.results_map + and self.output_config == other.output_config and self.results_map == other.results_map def get_fw_rules_in_required_format(self, add_txt_header=True, add_csv_header=True): """ From 28e211be6cdc4729e5e588dbc70510d32379481c Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 27 Nov 2022 14:44:45 +0200 Subject: [PATCH 012/187] Added support of IpBlocks in optimized hyper cube set implementation. Signed-off-by: Tanya --- nca/CoreDS/DimensionsManager.py | 3 +- nca/CoreDS/Peer.py | 92 ++++++++++++++++++++++---------- nca/CoreDS/TcpLikeProperties.py | 3 +- nca/FWRules/ConnectivityGraph.py | 10 ++-- 4 files changed, 72 insertions(+), 36 deletions(-) diff --git a/nca/CoreDS/DimensionsManager.py b/nca/CoreDS/DimensionsManager.py index 4be932ed3..4be0bc5f4 100644 --- a/nca/CoreDS/DimensionsManager.py +++ b/nca/CoreDS/DimensionsManager.py @@ -6,6 +6,7 @@ from .CanonicalIntervalSet import CanonicalIntervalSet from .MethodSet import MethodSet from .ProtocolSet import ProtocolSet +from .Peer import PeerSet from .MinDFA import MinDFA @@ -33,7 +34,7 @@ def __init__(self): ports_interval = CanonicalIntervalSet.get_interval_set(1, 65535) all_methods_interval = MethodSet(True) all_protocols_interval = ProtocolSet(True) - all_peers_interval = CanonicalIntervalSet.get_interval_set(0, 10000) # assuming max possible peer number + all_peers_interval = PeerSet.get_all_peers_and_ip_blocks_interval() self.dim_dict = dict() self.dim_dict["src_ports"] = (DimensionsManager.DimensionType.IntervalSet, ports_interval) self.dim_dict["dst_ports"] = (DimensionsManager.DimensionType.IntervalSet, ports_interval) diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index 73b60e1a2..b788fd6ee 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -448,9 +448,21 @@ class PeerSet(set): Note #2: __contains__ is implemented under the assumption that item arg is from disjoint_ip_blocks() """ + ipv4_highest_number = int(ip_network('0.0.0.0/0').broadcast_address) + ipv6_highest_number = int(ip_network('::/0').broadcast_address) + pod_highest_number = 9999 # assuming maximum 10,000 pods + gap_width = 5 # the gap is needed to avoid mixed-type intervals union + min_ipv4_index = 0 + max_ipv4_index = min_ipv4_index + ipv4_highest_number + min_ipv6_index = max_ipv4_index + gap_width + max_ipv6_index = min_ipv6_index + ipv6_highest_number + min_pod_index = max_ipv6_index + gap_width + max_pod_index = min_pod_index + pod_highest_number + def __init__(self, peer_set=None): super().__init__(peer_set or set()) self.sorted_peer_list = [] # for converting PeerSet to CanonicalIntervalSet + self.last_size_when_updated_sorted_peer_list = 0 def __contains__(self, item): if isinstance(item, IpBlock): # a special check here because an IpBlock may be contained in another IpBlock @@ -477,6 +489,7 @@ def copy(self): # res = PeerSet(set(elem.copy() for elem in self)) res = PeerSet(super().copy()) res.sorted_peer_list = self.sorted_peer_list + res.last_size_when_updated_sorted_peer_list = self.last_size_when_updated_sorted_peer_list return res # TODO: what is expected for ipblock name/namespace result on intersection? @@ -525,7 +538,7 @@ def __hash__(self): Note: PeerSet is a mutable type. Use with caution! :return: hash value for this object. """ - self.update_sorted_peer_list() + self.update_sorted_peer_list_if_needed() return hash(','.join(str(peer.full_name()) for peer in self.sorted_peer_list)) def rep(self): @@ -554,12 +567,23 @@ def get_ip_block_canonical_form(self): res |= elem return res - def update_sorted_peer_list(self): + @staticmethod + def get_all_peers_and_ip_blocks_interval(): + res = CanonicalIntervalSet() + res.add_interval(CanonicalIntervalSet.Interval(PeerSet.min_ipv4_index, PeerSet.max_ipv4_index)) + res.add_interval(CanonicalIntervalSet.Interval(PeerSet.min_ipv6_index, PeerSet.max_ipv6_index)) + res.add_interval(CanonicalIntervalSet.Interval(PeerSet.min_pod_index, PeerSet.max_pod_index)) + return res + + def update_sorted_peer_list_if_needed(self): """ create self.sorted_peer_list from non IpBlock pods :return: None """ - self.sorted_peer_list = sorted(list(elem for elem in self), key=by_full_name) + if self.last_size_when_updated_sorted_peer_list != len(self): + self.sorted_peer_list = \ + sorted(list(elem for elem in self if not isinstance(elem, IpBlock)), key=by_full_name) + self.last_size_when_updated_sorted_peer_list = len(self) def get_peer_interval_of(self, peer_set): """ @@ -568,12 +592,24 @@ def get_peer_interval_of(self, peer_set): :return: CanonicalIntervalSet for the peer_set """ res = CanonicalIntervalSet() - if len(self.sorted_peer_list) != len(self): - # should update sorted_peer_list - self.update_sorted_peer_list() + self.update_sorted_peer_list_if_needed() for index, peer in enumerate(self.sorted_peer_list): if peer in peer_set: - res.add_interval(CanonicalIntervalSet.Interval(index, index)) + assert not isinstance(peer, IpBlock) + res.add_interval(CanonicalIntervalSet.Interval(self.min_pod_index + index, + self.min_pod_index + index)) + # Now pick IpBlocks + for ipb in peer_set: + if isinstance(ipb, IpBlock): + for cidr in ipb: + if isinstance(cidr.start.address, ipaddress.IPv4Address): + res.add_interval(CanonicalIntervalSet.Interval(self.min_ipv4_index + int(cidr.start), + self.min_ipv4_index + int(cidr.end))) + elif isinstance(cidr.start.address, ipaddress.IPv6Address): + res.add_interval(CanonicalIntervalSet.Interval(self.min_ipv6_index + int(cidr.start), + self.min_ipv6_index + int(cidr.end))) + else: + assert False return res def get_peer_set_by_indices(self, peer_inteval_set): @@ -582,32 +618,30 @@ def get_peer_set_by_indices(self, peer_inteval_set): :param peer_inteval_set: the interval set of indices into the sorted peer list :return: the list of peers referenced by the indices in the interval set """ - my_len = len(self) - if len(self.sorted_peer_list) != my_len: - self.update_sorted_peer_list() + self.update_sorted_peer_list_if_needed() peer_list = [] for interval in peer_inteval_set: - for ind in range(min(interval.start, my_len), min(interval.end + 1, my_len)): - peer_list.append(self.sorted_peer_list[ind]) + if interval.end <= self.max_ipv4_index: + # this is IPv4Address + start = ipaddress.IPv4Address(interval.start - self.min_ipv4_index) + end = ipaddress.IPv4Address(interval.end - self.min_ipv4_index) + ipb = IpBlock(interval=CanonicalIntervalSet.Interval(IPNetworkAddress(start), IPNetworkAddress(end))) + peer_list.append(ipb) + elif interval.end <= self.max_ipv6_index: + # this is IPv6Address + start = ipaddress.IPv6Address(interval.start - self.min_ipv6_index) + end = ipaddress.IPv6Address(interval.end - self.min_ipv6_index) + ipb = IpBlock(interval=CanonicalIntervalSet.Interval(IPNetworkAddress(start), IPNetworkAddress(end))) + peer_list.append(ipb) + else: + # this is Pod + assert interval.end <= self.max_pod_index + curr_pods_max_ind = len(self)-1 + for ind in range(min(interval.start-self.min_pod_index, curr_pods_max_ind), + min(interval.end-self.min_pod_index, curr_pods_max_ind) + 1): + peer_list.append(self.sorted_peer_list[ind]) return PeerSet(set(peer_list)) - def get_all_peers_interval(self): - """ - Returns the interval of all peers - :return: CanonicalIntervalSet of all peers - """ - if len(self.sorted_peer_list) != len(self): - self.update_sorted_peer_list() - return CanonicalIntervalSet.get_interval_set(0, len(self.sorted_peer_list) - 1) - - def is_whole_range(self, peer_interval_set): - """ - Returns True iff the given peer interval set includes all peers - :param peer_interval_set: the given peer interval set - :return: bool whether the given interval set includes all peers - """ - return peer_interval_set == self.get_all_peers_interval() - def by_full_name(elem): """ diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index 304d25f87..0885f7546 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -9,7 +9,7 @@ from .PortSet import PortSet from .MethodSet import MethodSet from .ProtocolSet import ProtocolSet -from .Peer import PeerSet, IpBlock +from .Peer import PeerSet class TcpLikeProperties(CanonicalHyperCubeSet): @@ -399,7 +399,6 @@ def make_tcp_like_properties(peer_container, dest_ports, protocols=None, src_pee :return: TcpLikeProperties with TCP allowed connections, corresponding to input properties cube """ base_peer_set = peer_container.peer_set.copy() - base_peer_set.add(IpBlock.get_all_ips_block()) if src_peers: src_peers_interval = base_peer_set.get_peer_interval_of(src_peers) else: diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index 645b60d3a..7d659668a 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -314,12 +314,14 @@ def add_edge(self, cube_dict): :return: None """ new_cube_dict = cube_dict.copy() - src_peers = new_cube_dict['src_peers'] or PeerSet() - dst_peers = new_cube_dict['dst_peers'] or PeerSet() + src_peers = new_cube_dict.get('src_peers') or PeerSet() + dst_peers = new_cube_dict.get('dst_peers') or PeerSet() self.peer_sets.add(src_peers) self.peer_sets.add(dst_peers) - new_cube_dict.pop('src_peers') - new_cube_dict.pop('dst_peers') + if src_peers: + new_cube_dict.pop('src_peers') + if dst_peers: + new_cube_dict.pop('dst_peers') self.edges.append((src_peers, dst_peers, new_cube_dict)) def get_connectivity_dot_format_str(self): From d5d97a604ab5441d68440b25b2f0454628730251 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 27 Nov 2022 19:10:30 +0200 Subject: [PATCH 013/187] More accurate update of base_peer_set. Signed-off-by: Tanya --- nca/CoreDS/TcpLikeProperties.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index 0885f7546..1e8488c26 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -232,8 +232,6 @@ def __iand__(self, other): not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.has_named_ports() assert not isinstance(other, TcpLikeProperties) or not other.has_named_ports() - if isinstance(other, TcpLikeProperties): - self.base_peer_set |= other.base_peer_set super().__iand__(other) return self @@ -262,8 +260,6 @@ def __isub__(self, other): not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.has_named_ports() assert not isinstance(other, TcpLikeProperties) or not other.has_named_ports() -# if isinstance(other, TcpLikeProperties): -# self.base_peer_set |= other.base_peer_set super().__isub__(other) return self From 0a2cb45fea071d80df00b0ec3a630bb2067d0ac9 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 29 Nov 2022 10:23:29 +0200 Subject: [PATCH 014/187] Added dedundant fw_rules creation for testing (to be further removed). Signed-off-by: Tanya --- nca/NetworkConfig/NetworkConfigQuery.py | 2 ++ tests/istio_tests_failed_runtime_check.csv | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 tests/istio_tests_failed_runtime_check.csv diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index cedc906fc..f749d00ad 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -690,6 +690,8 @@ def exec(self): res_opt = QueryAnswer(True) if self.output_config.outputFormat == 'dot': res_opt.output_explanation = conn_graph_opt.get_connectivity_dot_format_str() + fw_rules = conn_graph.get_minimized_firewall_rules() # Temp for debugging + assert fw_rules == fw_rules2 # Temp for debugging else: assert fw_rules == fw_rules2 # res_opt.output_explanation = conn_graph_opt.get_connectivity_txt_format_str() diff --git a/tests/istio_tests_failed_runtime_check.csv b/tests/istio_tests_failed_runtime_check.csv deleted file mode 100644 index 0bef3012b..000000000 --- a/tests/istio_tests_failed_runtime_check.csv +++ /dev/null @@ -1,2 +0,0 @@ -test_name,expected_run_time (seconds),actual_run_time (seconds) -C:\Users\TATYANAVEKSLER\Documents\Formal\Network Policies\NCA\GitHub\network-config-analyzer\tests\fw_rules_tests\policies\istio-bookinfo-test-request-attrs-scheme.yaml,0.68,3.64 From 080095913e20c3ad504b12deec93f4825132ea66 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 4 Dec 2022 20:40:20 +0200 Subject: [PATCH 015/187] Initial implementation of Calico optimized connections handling. Fixed protocol handling: 0 is a valid protocol number (HOPOPT). Allowing any protocol in the range [0...255], though ProtocolNameResolver does not contain names of all of the possible 256 protocols. Fixed handling non-captured peers in K8S (cannot be handled as denied). Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 26 ++-- nca/CoreDS/Peer.py | 10 +- nca/CoreDS/ProtocolNameResolver.py | 14 +- nca/CoreDS/ProtocolSet.py | 41 ++++-- nca/CoreDS/TcpLikeProperties.py | 137 +++++++++++++----- nca/FWRules/ConnectivityGraph.py | 10 +- nca/NetworkConfig/NetworkLayer.py | 45 +++--- nca/Parsers/CalicoPolicyYamlParser.py | 55 +++++-- nca/Parsers/IngressPolicyYamlParser.py | 9 +- .../IstioTrafficResourcesYamlParser.py | 3 +- nca/Parsers/K8sPolicyYamlParser.py | 14 +- nca/Resources/CalicoNetworkPolicy.py | 17 +++ nca/Resources/K8sNetworkPolicy.py | 15 +- .../testcase5/testcase5-scheme.yaml | 9 ++ 14 files changed, 292 insertions(+), 113 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 413e1ea51..116d18e6c 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -16,10 +16,10 @@ class ConnectionSet: _icmp_protocols = {1, 58} port_supporting_protocols = {6, 17, 132} _max_protocol_num = 255 - _min_protocol_num = 1 + _min_protocol_num = 0 def __init__(self, allow_all=False): - self.allowed_protocols = {} # a map from protocol number (1-255) to allowed properties (ports, icmp) + self.allowed_protocols = {} # a map from protocol number (0-255) to allowed properties (ports, icmp) self.allow_all = allow_all # Shortcut to represent all connections, and then allowed_protocols is to be ignored def __bool__(self): @@ -377,20 +377,26 @@ def copy(self): @staticmethod def protocol_supports_ports(protocol): """ - :param protocol: Protocol number + :param protocol: Protocol number or name :return: Whether the given protocol has ports :rtype: bool """ - return protocol in ConnectionSet.port_supporting_protocols + prot = protocol + if isinstance(protocol, str): + prot = ProtocolNameResolver.get_protocol_number(protocol) + return prot in ConnectionSet.port_supporting_protocols @staticmethod def protocol_is_icmp(protocol): """ - :param protocol: Protocol number + :param protocol: Protocol number or name :return: Whether the protocol is icmp or icmpv6 :rtype: bool """ - return protocol in ConnectionSet._icmp_protocols + prot = protocol + if isinstance(protocol, str): + prot = ProtocolNameResolver.get_protocol_number(protocol) + return prot in ConnectionSet._icmp_protocols def add_connections(self, protocol, properties=True): """ @@ -402,8 +408,8 @@ def add_connections(self, protocol, properties=True): """ if isinstance(protocol, str): protocol = ProtocolNameResolver.get_protocol_number(protocol) - if protocol < 1 or protocol > 255: - raise Exception('Protocol must be in the range 1-255') + if not ProtocolNameResolver.is_valid_protocol(protocol): + raise Exception('Protocol must be in the range 0-255') if not bool(properties): # if properties are empty, there is nothing to add return if protocol in self.allowed_protocols: @@ -419,8 +425,8 @@ def remove_protocol(self, protocol): """ if isinstance(protocol, str): protocol = ProtocolNameResolver.get_protocol_number(protocol) - if protocol < 1 or protocol > 255: - raise Exception('Protocol must be in the range 1-255') + if not ProtocolNameResolver.is_valid_protocol(protocol): + raise Exception('Protocol must be in the range 0-255') if protocol not in self.allowed_protocols: return del self.allowed_protocols[protocol] diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index b788fd6ee..65c2380ac 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -612,15 +612,15 @@ def get_peer_interval_of(self, peer_set): assert False return res - def get_peer_set_by_indices(self, peer_inteval_set): + def get_peer_set_by_indices(self, peer_interval_set): """ - Return peer list from interval set of indices - :param peer_inteval_set: the interval set of indices into the sorted peer list - :return: the list of peers referenced by the indices in the interval set + Return peer set from interval set of indices + :param peer_interval_set: the interval set of indices into the sorted peer list + :return: the PeerSet of peers referenced by the indices in the interval set """ self.update_sorted_peer_list_if_needed() peer_list = [] - for interval in peer_inteval_set: + for interval in peer_interval_set: if interval.end <= self.max_ipv4_index: # this is IPv4Address start = ipaddress.IPv4Address(interval.start - self.min_ipv4_index) diff --git a/nca/CoreDS/ProtocolNameResolver.py b/nca/CoreDS/ProtocolNameResolver.py index 81d6c2e39..886f963d9 100644 --- a/nca/CoreDS/ProtocolNameResolver.py +++ b/nca/CoreDS/ProtocolNameResolver.py @@ -56,6 +56,10 @@ class ProtocolNameResolver: 135: 'MobilityHeader', 136: 'UDPLite', 137: 'MPLSinIP', 138: 'manet', 139: 'HIP', 140: 'Shim6', 141: 'WESP', 142: 'ROHC', 143: 'Ethernet'} + @staticmethod + def is_valid_protocol(protocol): + return protocol >= 0 and protocol <= 255 + @staticmethod def get_all_protocols_list(): return list(ProtocolNameResolver._protocol_name_to_number_dict.keys()) @@ -68,8 +72,8 @@ def get_protocol_name(protocol_number: int) -> str: its 'name' for lack of a specific one. :rtype: str """ - if protocol_number < 1 or protocol_number > 255: - raise Exception('Protocol number must be in the range 1-255') + if not ProtocolNameResolver.is_valid_protocol(protocol_number): + raise Exception('Protocol number must be in the range 0-255') return ProtocolNameResolver._protocol_number_to_name_dict.get(protocol_number, str(protocol_number)) @@ -84,8 +88,10 @@ def get_protocol_number(protocol_name: str) -> int: return protocol_name protocol_num = ProtocolNameResolver._protocol_name_to_number_dict.get(protocol_name) - if not protocol_num: - raise Exception('Unknown protocol name: ' + protocol_name) + if protocol_num is None: + protocol_num = int(protocol_name) + if not ProtocolNameResolver.is_valid_protocol(protocol_num): + raise Exception('Unknown protocol name: ' + protocol_name) return protocol_num diff --git a/nca/CoreDS/ProtocolSet.py b/nca/CoreDS/ProtocolSet.py index 251890540..c5355eedf 100644 --- a/nca/CoreDS/ProtocolSet.py +++ b/nca/CoreDS/ProtocolSet.py @@ -12,8 +12,9 @@ class ProtocolSet(CanonicalIntervalSet): A class for holding a set of HTTP methods """ - # According to https://en.wikipedia.org/wiki/List_of_IP_protocol_numbers - all_protocols_list = ProtocolNameResolver.get_all_protocols_list() + #all_protocols_list = ProtocolNameResolver.get_all_protocols_list() + min_protocol_num = 0 + max_protocol_num = 255 def __init__(self, all_protocols=False): """ @@ -25,12 +26,25 @@ def __init__(self, all_protocols=False): def add_protocol(self, protocol): """ - Adds a given protocol to the ProtocolSet if the method is one of the eligible protocols (in all_protocols_list); - otherwise raises ValueError exception - :param str protocol: the protocol to add + Adds a given protocol to the ProtocolSet if the protocol is one of the eligible protocols + (i.e., protocol in [min_protocol_num...max_protocol_num]); + otherwise raises exception + :param int protocol: the protocol to add """ - index = self.all_protocols_list.index(protocol) - self.add_interval(self.Interval(index, index)) + if not ProtocolNameResolver.is_valid_protocol(protocol): + raise Exception('Protocol must be in the range 0-255') + self.add_interval(self.Interval(protocol, protocol)) + + def remove_protocol(self, protocol): + """ + Removes a given protocol from the ProtocolSet if the protocol is one of the eligible protocols + (i.e., protocol in [min_protocol_num...max_protocol_num]); + otherwise raises exception + :param int protocol: the protocol to remove + """ + if not ProtocolNameResolver.is_valid_protocol(protocol): + raise Exception('Protocol must be in the range 0-255') + self.add_hole(self.Interval(protocol, protocol)) def set_protocols(self, protocols): """ @@ -45,7 +59,7 @@ def _whole_range_interval(): """ :return: the interval representing the whole range (all protocols) """ - return CanonicalIntervalSet.Interval(0, len(ProtocolSet.all_protocols_list) - 1) + return CanonicalIntervalSet.Interval(ProtocolSet.min_protocol_num, ProtocolSet.max_protocol_num) @staticmethod def _whole_range_interval_set(): @@ -70,9 +84,10 @@ def get_protocol_names_from_interval_set(interval_set): """ res = [] for interval in interval_set: - assert interval.start >= 0 and interval.end < len(ProtocolSet.all_protocols_list) + assert interval.start >= ProtocolSet.min_protocol_num and interval.end <= ProtocolSet.max_protocol_num for index in range(interval.start, interval.end + 1): - res.append(ProtocolSet.all_protocols_list[index]) + name = ProtocolNameResolver.get_protocol_name(index) + res.append(name if name else str(index)) return res @staticmethod @@ -82,10 +97,8 @@ def _get_compl_protocol_names_from_interval_set(interval_set): :param CanonicalIntervalSet interval_set: the interval set :return: the list of complement protocol names """ - res = ProtocolSet.all_protocols_list.copy() - for protocol in ProtocolSet.get_protocol_names_from_interval_set(interval_set): - res.remove(protocol) - return res + res_interval_set = ProtocolSet._whole_range_interval_set() - interval_set + return ProtocolSet.get_protocol_names_from_interval_set(res_interval_set) def __str__(self): """ diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index 1e8488c26..2cdbb776a 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -10,6 +10,7 @@ from .MethodSet import MethodSet from .ProtocolSet import ProtocolSet from .Peer import PeerSet +from .ProtocolNameResolver import ProtocolNameResolver class TcpLikeProperties(CanonicalHyperCubeSet): @@ -379,13 +380,14 @@ def project_on_one_dimension(self, dim_name): return res @staticmethod - def make_tcp_like_properties(peer_container, dest_ports, protocols=None, src_peers=None, dst_peers=None, + def make_tcp_like_properties(peer_container, src_ports, dst_ports, protocols=None, src_peers=None, dst_peers=None, paths_dfa=None, hosts_dfa=None, methods=None): """ get TcpLikeProperties with TCP allowed connections, corresponding to input properties cube. TcpLikeProperties should not contain named ports: substitute them with corresponding port numbers, per peer :param PeerContainer peer_container: The set of endpoints and their namespaces - :param PortSet dest_ports: ports set for dest_ports dimension (possibly containing named ports) + :param PortSet src_ports: ports set for src_ports dimension (possibly containing named ports) + :param PortSet dst_ports: ports set for dst_ports dimension (possibly containing named ports) :param ProtocolSet protocols: CanonicalIntervalSet obj for protocols dimension :param PeerSet src_peers: the set of source peers :param PeerSet dst_peers: the set of target peers @@ -403,42 +405,106 @@ def make_tcp_like_properties(peer_container, dest_ports, protocols=None, src_pee dst_peers_interval = base_peer_set.get_peer_interval_of(dst_peers) else: dst_peers_interval = None - if not dest_ports.named_ports: - return TcpLikeProperties(source_ports=PortSet(True), dest_ports=dest_ports, + if not src_ports.named_ports and not dst_ports.named_ports: + return TcpLikeProperties(source_ports=src_ports, dest_ports=dst_ports, protocols=protocols, methods=methods, paths=paths_dfa, hosts=hosts_dfa, src_peers=src_peers_interval, dst_peers=dst_peers_interval, base_peer_set=base_peer_set) - assert dst_peers - assert not dest_ports.port_set - assert len(dest_ports.named_ports) == 1 - port = list(dest_ports.named_ports)[0] tcp_properties = None - tcp_protocol = ProtocolSet() - tcp_protocol.add_protocol('TCP') - for peer in dst_peers: - named_ports = peer.get_named_ports() - real_port = named_ports.get(port) - if not real_port: - print(f'Warning: Missing named port {port} in the pod {peer}. Ignoring the pod') - continue - if real_port[1] != 'TCP': - print(f'Warning: Illegal protocol {real_port[1]} in the named port {port} of the target pod {peer}.' - f'Ignoring the pod') - continue - peer_in_set = PeerSet() - peer_in_set.add(peer) - ports = PortSet() - ports.add_port(real_port[0]) - props = TcpLikeProperties(source_ports=PortSet(True), dest_ports=ports, - protocols=protocols if protocols else tcp_protocol, methods=methods, - paths=paths_dfa, hosts=hosts_dfa, src_peers=src_peers_interval, - dst_peers=base_peer_set.get_peer_interval_of(peer_in_set), - base_peer_set=base_peer_set) - if tcp_properties: - tcp_properties |= props + if src_ports.named_ports and dst_ports.named_ports: + assert src_peers + assert dst_peers + assert not src_ports.port_set + assert not dst_ports.port_set + assert len(src_ports.named_ports) == 1 and len(dst_ports.named_ports) == 1 + src_named_port = list(src_ports.named_ports)[0] + dst_named_port = list(dst_ports.named_ports)[0] + tcp_protocol = ProtocolSet() + tcp_protocol.add_protocol(ProtocolNameResolver.get_protocol_number('TCP')) + for src_peer in src_peers: + src_peer_named_ports = src_peer.get_named_ports() + real_src_port = src_peer_named_ports.get(src_named_port) + if not real_src_port: + print(f'Warning: Missing named port {src_named_port} in the pod {src_peer}. Ignoring the pod') + continue + if real_src_port[1] not in protocols: + print(f'Warning: Illegal protocol {real_src_port[1]} in the named port {src_named_port} ' + f'of the target pod {src_peer}. Ignoring the pod') + continue + src_peer_in_set = PeerSet() + src_peer_in_set.add(src_peer) + real_src_ports = PortSet() + real_src_ports.add_port(real_src_port[0]) + for dst_peer in dst_peers: + dst_peer_named_ports = dst_peer.get_named_ports() + real_dst_port = dst_peer_named_ports.get(dst_named_port) + if not real_dst_port: + print(f'Warning: Missing named port {dst_named_port} in the pod {dst_peer}. Ignoring the pod') + continue + if real_dst_port[1] not in protocols: + print(f'Warning: Illegal protocol {real_dst_port[1]} in the named port {dst_named_port} ' + f'of the target pod {dst_peer}. Ignoring the pod') + continue + dst_peer_in_set = PeerSet() + dst_peer_in_set.add(dst_peer) + real_dst_ports = PortSet() + real_dst_ports.add_port(real_dst_port[0]) + + props = TcpLikeProperties(source_ports=real_src_ports, dest_ports=real_dst_ports, + protocols=protocols if protocols else tcp_protocol, methods=methods, + paths=paths_dfa, hosts=hosts_dfa, + src_peers=base_peer_set.get_peer_interval_of(src_peer_in_set), + dst_peers=base_peer_set.get_peer_interval_of(dst_peer_in_set), + base_peer_set=base_peer_set) + + if tcp_properties: + tcp_properties |= props + else: + tcp_properties = props + else: + # either only src_ports or only dst_ports contain named ports + if src_ports.named_ports: + port_set_with_named_ports = src_ports + peers_for_named_ports = src_peers else: - tcp_properties = props - + port_set_with_named_ports = dst_ports + peers_for_named_ports = dst_peers + assert peers_for_named_ports + assert not port_set_with_named_ports.port_set + assert len(port_set_with_named_ports.named_ports) == 1 + port = list(port_set_with_named_ports.named_ports)[0] + tcp_protocol = ProtocolSet() + tcp_protocol.add_protocol(ProtocolNameResolver.get_protocol_number('TCP')) + for peer in peers_for_named_ports: + named_ports = peer.get_named_ports() + real_port = named_ports.get(port) + if not real_port: + print(f'Warning: Missing named port {port} in the pod {peer}. Ignoring the pod') + continue + if real_port[1] not in protocols: + print(f'Warning: Illegal protocol {real_port[1]} in the named port {port} of the target pod {peer}.' + f'Ignoring the pod') + continue + peer_in_set = PeerSet() + peer_in_set.add(peer) + ports = PortSet() + ports.add_port(real_port[0]) + if src_ports.named_ports: + props = TcpLikeProperties(source_ports=ports, dest_ports=dst_ports, + protocols=protocols if protocols else tcp_protocol, methods=methods, + paths=paths_dfa, hosts=hosts_dfa, + src_peers=base_peer_set.get_peer_interval_of(peer_in_set), + dst_peers=dst_peers_interval, base_peer_set=base_peer_set) + else: + props = TcpLikeProperties(source_ports=src_ports, dest_ports=ports, + protocols=protocols if protocols else tcp_protocol, methods=methods, + paths=paths_dfa, hosts=hosts_dfa, src_peers=src_peers_interval, + dst_peers=base_peer_set.get_peer_interval_of(peer_in_set), + base_peer_set=base_peer_set) + if tcp_properties: + tcp_properties |= props + else: + tcp_properties = props return tcp_properties @staticmethod @@ -450,7 +516,8 @@ def make_tcp_like_properties_from_dict(peer_container, cube_dict): :return: TcpLikeProperties """ cube_dict_copy = cube_dict.copy() - dest_ports = cube_dict_copy.pop("dst_ports", PortSet(True)) + src_ports = cube_dict_copy.pop("src_ports", PortSet(True)) + dst_ports = cube_dict_copy.pop("dst_ports", PortSet(True)) protocols = cube_dict_copy.pop("protocols", None) src_peers = cube_dict_copy.pop("src_peers", None) dst_peers = cube_dict_copy.pop("dst_peers", None) @@ -458,5 +525,5 @@ def make_tcp_like_properties_from_dict(peer_container, cube_dict): hosts_dfa = cube_dict_copy.pop("hosts", None) methods = cube_dict_copy.pop("methods", None) assert not cube_dict_copy - return TcpLikeProperties.make_tcp_like_properties(peer_container, dest_ports, protocols, + return TcpLikeProperties.make_tcp_like_properties(peer_container, src_ports, dst_ports, protocols, src_peers, dst_peers, paths_dfa, hosts_dfa, methods) diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index 7d659668a..40c9ba3cb 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -9,6 +9,7 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.ProtocolSet import ProtocolSet from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ICMPDataSet import ICMPDataSet from .MinimizeFWRules import MinimizeCsFwRules, MinimizeFWRules from .ClusterInfo import ClusterInfo @@ -104,10 +105,17 @@ def add_edges_from_cube_dict(self, peer_container, cube_dict): protocol_names = ProtocolSet.get_protocol_names_from_interval_set(protocols) if protocols else ['TCP'] for protocol in protocol_names: if new_cube_dict: + # TODO - support ICMP + assert ConnectionSet.protocol_supports_ports(protocol) conns.add_connections(protocol, TcpLikeProperties.make_tcp_like_properties_from_dict(peer_container, new_cube_dict)) else: - conns.add_connections(protocol, TcpLikeProperties(PortSet(True), PortSet(True))) + if ConnectionSet.protocol_supports_ports(protocol): + conns.add_connections(protocol, TcpLikeProperties(PortSet(True), PortSet(True))) + elif ConnectionSet.protocol_is_icmp(protocol): + conns.add_connections(protocol, ICMPDataSet(add_all=True)) + else: + conns.add_connections(protocol, True) for src_peer in src_peers: for dst_peer in dst_peers: self.connections_to_peers[conns].append((src_peer, dst_peer)) diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 288aa6a3d..9050bf25f 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -6,7 +6,7 @@ from enum import Enum from nca.CoreDS.ConnectionSet import ConnectionSet -from nca.CoreDS.Peer import IpBlock, HostEP +from nca.CoreDS.Peer import IpBlock, HostEP, PeerSet from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from nca.CoreDS.PortSet import PortSet from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy @@ -228,13 +228,15 @@ def collect_policies_conns_optimized(self, is_ingress): Collect all connections (between all relevant peers), considering all layer's policies that capture the relevant peers. :param bool is_ingress: indicates whether to return ingress connections or egress connections - :return: allowed_conns and denied_conns - :rtype: tuple (TcpLikeProperties, TcpLikeProperties) + :return: allowed_conns, denied_conns and set of peers to be added to captured peers + :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) """ allowed_conns = None denied_conns = None + add_to_captured = None for policy in self.policies_list: - policy_allowed_conns, policy_denied_conns = policy.allowed_connections_optimized(is_ingress) + policy_allowed_conns, policy_denied_conns, policy_add_to_captured = \ + policy.allowed_connections_optimized(is_ingress) if policy_allowed_conns and allowed_conns: allowed_conns |= policy_allowed_conns elif not allowed_conns: @@ -243,8 +245,13 @@ def collect_policies_conns_optimized(self, is_ingress): denied_conns |= policy_denied_conns elif not denied_conns: denied_conns = policy_denied_conns + if policy_add_to_captured and add_to_captured: + add_to_captured |= policy_add_to_captured + elif not add_to_captured: + add_to_captured = policy_add_to_captured + + return allowed_conns, denied_conns, add_to_captured - return allowed_conns, denied_conns class K8sCalicoNetworkLayer(NetworkLayer): @@ -267,7 +274,7 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): all_allowed_conns=allowed_conns | allowed_non_captured_conns) def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): - allowed_conn, denied_conns = self.collect_policies_conns_optimized(is_ingress) + allowed_conn, denied_conns, add_to_captured = self.collect_policies_conns_optimized(is_ingress) if not allowed_conn and not denied_conns: return None, None # Note: The below computation of non-captured conns cannot be done during the parse stage, @@ -276,33 +283,27 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): base_peer_set = peer_container.peer_set.copy() base_peer_set.add(IpBlock.get_all_ips_block()) if is_ingress: - captured_dst_peers1 = allowed_conn.project_on_one_dimension('dst_peers') if allowed_conn else None - captured_dst_peers2 = denied_conns.project_on_one_dimension('dst_peers') if denied_conns else None - if captured_dst_peers1 and captured_dst_peers2: - captured_dst_peers = captured_dst_peers1 | captured_dst_peers2 - else: - captured_dst_peers = captured_dst_peers1 if captured_dst_peers1 else captured_dst_peers2 + captured_dst_peers1 = allowed_conn.project_on_one_dimension('dst_peers') if allowed_conn else PeerSet() + captured_dst_peers2 = denied_conns.project_on_one_dimension('dst_peers') if denied_conns else PeerSet() + captured_dst_peers = captured_dst_peers1 | captured_dst_peers2 | add_to_captured if captured_dst_peers: non_captured_dst_peers = base_peer_set - captured_dst_peers if non_captured_dst_peers: non_captured_conns = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, dest_ports=PortSet(True), + TcpLikeProperties.make_tcp_like_properties(peer_container, PortSet(True), PortSet(True), dst_peers=non_captured_dst_peers) - allowed_conn = (allowed_conn | non_captured_conns) if allowed_conn else non_captured_conns + allowed_conn = (allowed_conn | non_captured_conns) if allowed_conn else non_captured_conns else: - captured_src_peers1 = allowed_conn.project_on_one_dimension('src_peers') if allowed_conn else None - captured_src_peers2 = denied_conns.project_on_one_dimension('src_peers') if denied_conns else None - if captured_src_peers1 and captured_src_peers2: - captured_src_peers = captured_src_peers1 | captured_src_peers2 - else: - captured_src_peers = captured_src_peers1 if captured_src_peers1 else captured_src_peers2 + captured_src_peers1 = allowed_conn.project_on_one_dimension('src_peers') if allowed_conn else PeerSet() + captured_src_peers2 = denied_conns.project_on_one_dimension('src_peers') if denied_conns else PeerSet() + captured_src_peers = captured_src_peers1 | captured_src_peers2 | add_to_captured if captured_src_peers: non_captured_src_peers = base_peer_set - captured_src_peers if non_captured_src_peers: non_captured_conns = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, dest_ports=PortSet(True), + TcpLikeProperties.make_tcp_like_properties(peer_container, PortSet(True), PortSet(True), src_peers=non_captured_src_peers) - allowed_conn = (allowed_conn | non_captured_conns) if allowed_conn else non_captured_conns + allowed_conn = (allowed_conn | non_captured_conns) if allowed_conn else non_captured_conns return allowed_conn, denied_conns diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index a09aec36e..ee9fd6bec 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -9,6 +9,7 @@ from nca.CoreDS.Peer import PeerSet, IpBlock from nca.CoreDS.PortSet import PortSet from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ProtocolSet import ProtocolSet from nca.CoreDS.ICMPDataSet import ICMPDataSet from nca.CoreDS.ConnectionSet import ConnectionSet from nca.Resources.NetworkPolicy import NetworkPolicy @@ -411,11 +412,11 @@ def _parse_protocol(self, protocol, rule): :return: The protocol number :rtype: int """ - if not protocol: + if protocol is None: return None if isinstance(protocol, int): - if protocol < 1 or protocol > 255: - self.syntax_error('protocol must be a string or an integer in the range 1-255', rule) + if not ProtocolNameResolver.is_valid_protocol(protocol): + self.syntax_error('protocol must be a string or an integer in the range 0-255', rule) return protocol if protocol not in ['TCP', 'UDP', 'ICMP', 'ICMPv6', 'SCTP', 'UDPLite']: @@ -429,8 +430,9 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): :param bool is_ingress: Whether this is an ingress rule :param PeerSet policy_selected_eps: The endpoints the policy captured :param bool is_profile: Whether the parsed policy is a Profile object - :return: A CalicoPolicyRule with the proper PeerSets, ConnectionSets and Action - :rtype: CalicoPolicyRule + :return: A tuple (CalicoPolicyRule, TcpLikeProperties) with the proper PeerSets, ConnectionSets and Action, + where TcpLikeProperties is an optimized rule format with protocols, src_peers and dst_peers in a HyperCubeSet + :rtype: tuple(CalicoPolicyRule, TcpLikeProperties) """ allowed_keys = {'action': 1, 'protocol': 0, 'notProtocol': 0, 'icmp': 0, 'notICMP': 0, 'ipVersion': 0, 'source': 0, 'destination': 0, 'http': 2} @@ -465,7 +467,10 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): src_res_pods &= policy_selected_eps connections = ConnectionSet() + tcp_props = None if protocol is not None: + protocols = ProtocolSet() + protocols.add_protocol(protocol) if not_protocol is not None: if protocol == not_protocol: self.warning('Protocol and notProtocol are conflicting, no traffic will be matched', rule) @@ -474,16 +479,34 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): else: if protocol_supports_ports: connections.add_connections(protocol, TcpLikeProperties(src_res_ports, dst_res_ports)) + src_num_port_set = PortSet() + src_num_port_set.port_set = src_res_ports.port_set.copy() + dst_num_port_set = PortSet() + dst_num_port_set.port_set = dst_res_ports.port_set.copy() + tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, src_num_port_set, + dst_num_port_set, protocols, + src_res_pods, dst_res_pods) elif ConnectionSet.protocol_is_icmp(protocol): connections.add_connections(protocol, self._parse_icmp(rule.get('icmp'), rule.get('notICMP'))) + # TODO - update tcp_props else: connections.add_connections(protocol, True) + tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), + PortSet(True), protocols, + src_res_pods, dst_res_pods) elif not_protocol is not None: connections.add_all_connections() connections.remove_protocol(not_protocol) + protocols = ProtocolSet(True) + protocols.remove_protocol(not_protocol) + tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), + PortSet(True), protocols, + src_res_pods, dst_res_pods) else: connections.allow_all = True - + tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), + PortSet(True), ProtocolSet(True), + src_res_pods, dst_res_pods) self._verify_named_ports(rule, dst_res_pods, connections) if not src_res_pods and policy_selected_eps and (is_ingress or not is_profile): @@ -491,7 +514,7 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): if not dst_res_pods and policy_selected_eps and (not is_ingress or not is_profile): self.warning('Rule selects no destination endpoints', rule) - return CalicoPolicyRule(src_res_pods, dst_res_pods, connections, action) + return CalicoPolicyRule(src_res_pods, dst_res_pods, connections, action), tcp_props def _verify_named_ports(self, rule, rule_eps, rule_conns): """ @@ -631,12 +654,26 @@ def parse_policy(self): self.syntax_error('order is not allowed in the spec of a Profile', policy_spec) for ingress_rule in policy_spec.get('ingress', []): - rule = self._parse_xgress_rule(ingress_rule, True, res_policy.selected_peers, is_profile) + rule, optimized_props = self._parse_xgress_rule(ingress_rule, True, res_policy.selected_peers, is_profile) res_policy.add_ingress_rule(rule) + if rule.action != CalicoPolicyRule.ActionType.Pass: + # handle the order of rules + if rule.action == CalicoPolicyRule.ActionType.Allow and res_policy.optimized_denied_ingress_props: + optimized_props -= res_policy.optimized_denied_ingress_props + elif rule.action == CalicoPolicyRule.ActionType.Deny and res_policy.optimized_ingress_props: + optimized_props -= res_policy.optimized_ingress_props + res_policy.add_optimized_ingress_props(optimized_props, rule.action == CalicoPolicyRule.ActionType.Allow) for egress_rule in policy_spec.get('egress', []): - rule = self._parse_xgress_rule(egress_rule, False, res_policy.selected_peers, is_profile) + rule, optimized_props = self._parse_xgress_rule(egress_rule, False, res_policy.selected_peers, is_profile) res_policy.add_egress_rule(rule) + if rule.action != CalicoPolicyRule.ActionType.Pass: + # handle the order of rules + if rule.action == CalicoPolicyRule.ActionType.Allow and res_policy.optimized_denied_egress_props: + optimized_props -= res_policy.optimized_denied_egress_props + elif rule.action == CalicoPolicyRule.ActionType.Deny and res_policy.optimized_egress_props: + optimized_props -= res_policy.optimized_egress_props + res_policy.add_optimized_egress_props(optimized_props, rule.action == CalicoPolicyRule.ActionType.Allow) self._apply_extra_labels(policy_spec, is_profile, res_policy.name) res_policy.findings = self.warning_msgs diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index a3f7b179d..2d61430ac 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -173,12 +173,14 @@ def _make_default_connections(self, hosts_dfa, paths_dfa=None): if self.default_backend_peers: if paths_dfa: default_conns = \ - TcpLikeProperties.make_tcp_like_properties(self.peer_container, self.default_backend_ports, + TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), + self.default_backend_ports, dst_peers=self.default_backend_peers, paths_dfa=paths_dfa, hosts_dfa=hosts_dfa) else: default_conns = \ - TcpLikeProperties.make_tcp_like_properties(self.peer_container, self.default_backend_ports, + TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), + self.default_backend_ports, dst_peers=self.default_backend_peers, hosts_dfa=hosts_dfa) return default_conns @@ -206,7 +208,8 @@ def parse_rule(self, rule): parsed_paths_with_dfa = self.segregate_longest_paths_and_make_dfa(parsed_paths) for (_, paths_dfa, _, peers, ports) in parsed_paths_with_dfa: # every path is converted to allowed connections - conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, ports, dst_peers=peers, + conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), ports, + dst_peers=peers, paths_dfa=paths_dfa, hosts_dfa=hosts_dfa) if not allowed_conns: allowed_conns = conns diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index 7e7dd4d4a..042814dbd 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -7,6 +7,7 @@ from nca.CoreDS.DimensionsManager import DimensionsManager from nca.CoreDS.MinDFA import MinDFA from nca.CoreDS.Peer import PeerSet +from nca.CoreDS.PortSet import PortSet from nca.CoreDS.MethodSet import MethodSet from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from nca.Resources.IstioTrafficResources import Gateway, VirtualService @@ -329,7 +330,7 @@ def make_allowed_connections(self, vs, host_dfa): for http_route in vs.http_routes: for dest in http_route.destinations: conns = \ - TcpLikeProperties.make_tcp_like_properties(self.peer_container, dest.port, + TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), dest.port, dst_peers=dest.service.target_pods, paths_dfa=http_route.uri_dfa, hosts_dfa=host_dfa, methods=http_route.methods) diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index 8f6660fe9..e2ff7535f 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -327,12 +327,15 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): res_opt_props = None # TcpLikeProperties for port in ports_array: protocol, dest_port_set = self.parse_port(port) + if isinstance(protocol, str): + protocol = ProtocolNameResolver.get_protocol_number(protocol) protocols = ProtocolSet() protocols.add_protocol(protocol) res_conns.add_connections(protocol, TcpLikeProperties(PortSet(True), dest_port_set)) # K8s doesn't reason about src ports dest_num_port_set = PortSet() dest_num_port_set.port_set = dest_port_set.port_set.copy() - tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, dest_num_port_set, + tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, + PortSet(True), dest_num_port_set, protocols, src_pods, dst_pods) if res_opt_props: res_opt_props |= tcp_props @@ -341,7 +344,8 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): # self.handle_named_ports(dst_pods, protocol, dest_port_set.named_ports, res_opt_props) else: res_conns = ConnectionSet(True) - res_opt_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), + res_opt_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, + PortSet(True), PortSet(True), src_peers=src_pods, dst_peers=dst_pods) if not res_pods: @@ -455,7 +459,8 @@ def parse_policy(self): res_policy.add_optimized_ingress_props(optimized_props) elif res_policy.affects_ingress: # add denied connections - denied_conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, dest_ports=PortSet(True), + denied_conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, + PortSet(True), PortSet(True), dst_peers=res_policy.selected_peers) res_policy.add_optimized_ingress_props(denied_conns, False) @@ -467,7 +472,8 @@ def parse_policy(self): res_policy.add_optimized_egress_props(optimized_props) elif res_policy.affects_egress: # add only non-captured connections - denied_conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, dest_ports=PortSet(True), + denied_conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, + PortSet(True), PortSet(True), src_peers=res_policy.selected_peers) res_policy.add_optimized_egress_props(denied_conns, False) diff --git a/nca/Resources/CalicoNetworkPolicy.py b/nca/Resources/CalicoNetworkPolicy.py index 082c92cf1..0df9dc8f0 100644 --- a/nca/Resources/CalicoNetworkPolicy.py +++ b/nca/Resources/CalicoNetworkPolicy.py @@ -118,6 +118,23 @@ def allowed_connections(self, from_peer, to_peer, is_ingress): return PolicyConnections(True, allowed_conns, denied_conns, pass_conns) + def allowed_connections_optimized(self, is_ingress): + """ + Evaluate the set of connections this policy allows/denies/passes between any two peers + :param bool is_ingress: whether we evaluate ingress rules only or egress rules only + A TcpLikeProperties object containing all allowed connections for relevant peers, + TcpLikeProperties object containing all denied connections, + and the peer set of captured peers that are not a part of allowed connections. + :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) + """ + if is_ingress: + allowed = self.optimized_ingress_props.copy() if self.optimized_ingress_props else None + denied = self.optimized_denied_ingress_props.copy() if self.optimized_denied_ingress_props else None + else: + allowed = self.optimized_egress_props.copy() if self.optimized_egress_props else None + denied = self.optimized_denied_egress_props.copy() if self.optimized_denied_egress_props else None + return allowed, denied, Peer.PeerSet() + def clone_without_rule(self, rule_to_exclude, ingress_rule): """ Makes a copy of 'self' without a given policy rule diff --git a/nca/Resources/K8sNetworkPolicy.py b/nca/Resources/K8sNetworkPolicy.py index 6bb740231..47a3d48c3 100644 --- a/nca/Resources/K8sNetworkPolicy.py +++ b/nca/Resources/K8sNetworkPolicy.py @@ -67,16 +67,21 @@ def allowed_connections_optimized(self, is_ingress): Return the set of connections this policy allows between any two peers (either ingress or egress). :param bool is_ingress: whether we evaluate ingress rules only or egress rules only - :return: A TcpLikeProperties object containing all allowed connections for relevant peers - :rtype: TcpLikeProperties + :return: A TcpLikeProperties object containing all allowed connections for relevant peers, + None for denied connections (K8s does not have denied), + and the peer set of captured peers that are not a part of allowed connections. + :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) """ + add_to_captured = Peer.PeerSet() if is_ingress: allowed = self.optimized_ingress_props.copy() if self.optimized_ingress_props else None - denied = self.optimized_denied_ingress_props.copy() if self.optimized_denied_ingress_props else None + if self.optimized_denied_ingress_props: + add_to_captured = self.optimized_denied_ingress_props.project_on_one_dimension('dst_peers') else: allowed = self.optimized_egress_props.copy() if self.optimized_egress_props else None - denied = self.optimized_denied_egress_props.copy() if self.optimized_denied_egress_props else None - return allowed, denied + if self.optimized_denied_egress_props: + add_to_captured = self.optimized_denied_egress_props.project_on_one_dimension('src_peers') + return allowed, None, add_to_captured def clone_without_rule(self, rule_to_exclude, ingress_rule): """ diff --git a/tests/calico_testcases/example_policies/testcase5/testcase5-scheme.yaml b/tests/calico_testcases/example_policies/testcase5/testcase5-scheme.yaml index 8e08de60a..79fc1c1b0 100644 --- a/tests/calico_testcases/example_policies/testcase5/testcase5-scheme.yaml +++ b/tests/calico_testcases/example_policies/testcase5/testcase5-scheme.yaml @@ -13,6 +13,15 @@ networkConfigList: expectedWarnings: 0 queries: + - name: connectivity_map + connectivityMap: + - np_allowFirst + - np_denyFirst + expected: 0 + outputConfiguration: + outputFormat: txt + fwRulesRunInTestMode: false + - name: policies_not_vacuous vacuity: - np_denyFirst From 05a496c2e0dbeaa1b77090cd0c4d75c571db9280 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 6 Dec 2022 13:37:05 +0200 Subject: [PATCH 016/187] Fixed the construction of connectivity graph (when some of src_peer or dst_peers dimensions is all values). Added optimization for fw_rules_map - join different entries having the same values (fw_rules). Signed-off-by: Tanya --- nca/FWRules/ConnectivityGraph.py | 4 ++-- nca/FWRules/MinimizeFWRules.py | 27 +++++++++++++++++++++++++ nca/NetworkConfig/NetworkConfigQuery.py | 5 +++-- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index 40c9ba3cb..11b5310c5 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -88,12 +88,12 @@ def add_edges_from_cube_dict(self, peer_container, cube_dict): if src_peers: new_cube_dict.pop('src_peers') else: - src_peers = PeerSet() + src_peers = peer_container.get_all_peers_group(True) dst_peers = new_cube_dict.get('dst_peers') if dst_peers: new_cube_dict.pop('dst_peers') else: - dst_peers = PeerSet() + dst_peers = peer_container.get_all_peers_group(True) protocols = new_cube_dict.get('protocols') if protocols: new_cube_dict.pop('protocols') diff --git a/nca/FWRules/MinimizeFWRules.py b/nca/FWRules/MinimizeFWRules.py index 8cd66f5cd..23fd003c1 100644 --- a/nca/FWRules/MinimizeFWRules.py +++ b/nca/FWRules/MinimizeFWRules.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache2.0 # +from collections import defaultdict import yaml from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock, ClusterEP, Pod, HostEP @@ -659,6 +660,32 @@ def __eq__(self, other): return self.fw_rules_map == other.fw_rules_map and self.cluster_info == other.cluster_info \ and self.output_config == other.output_config and self.results_map == other.results_map + @staticmethod + def same_peers(fw_rules_list1, fw_rules_list2): + # assuming the same lists order + if len(fw_rules_list1) != len(fw_rules_list2): + return False + for index, rule1 in enumerate(fw_rules_list1): + rule2 = fw_rules_list2[index] + if rule1.src != rule2.src or rule1.dst != rule2.dst: + return False + return True + + def unite_fw_rules_with_same_peers(self): + new_fw_rules_map = self.fw_rules_map + self.fw_rules_map = defaultdict(list) + while new_fw_rules_map: + the_conn, the_fw_rules = new_fw_rules_map.popitem() + conns_to_remove = [] + for conn, fw_rules in new_fw_rules_map.items(): + if self.same_peers(fw_rules, the_fw_rules): + the_conn |= conn + conns_to_remove.append(conn) + for r in the_fw_rules: r.conn = the_conn + self.fw_rules_map[the_conn] = the_fw_rules + for conn in conns_to_remove: + new_fw_rules_map.pop(conn) + def get_fw_rules_in_required_format(self, add_txt_header=True, add_csv_header=True): """ :param add_txt_header: bool flag to indicate if header of fw-rules query should be added in txt format diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index f749d00ad..9fa3cad6f 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -687,13 +687,14 @@ def exec(self): all_conns_opt.get_cube_dict_with_orig_values(cube)) conn_graph_opt.add_edge(all_conns_opt.get_cube_dict(cube)) fw_rules2 = conn_graph2.get_minimized_firewall_rules() + fw_rules2.unite_fw_rules_with_same_peers() res_opt = QueryAnswer(True) if self.output_config.outputFormat == 'dot': res_opt.output_explanation = conn_graph_opt.get_connectivity_dot_format_str() fw_rules = conn_graph.get_minimized_firewall_rules() # Temp for debugging - assert fw_rules == fw_rules2 # Temp for debugging + assert fw_rules.fw_rules_map == fw_rules2.fw_rules_map # Temp for debugging else: - assert fw_rules == fw_rules2 + assert fw_rules.fw_rules_map == fw_rules2.fw_rules_map # res_opt.output_explanation = conn_graph_opt.get_connectivity_txt_format_str() # res.output_explanation += "---------------- OPTIMIZED RESULT: -------------\n" +\ # fw_rules2.get_fw_rules_in_required_format() + \ From bb6842c74b1ba5bb79c87fa10fe0c269ec9d8256 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 11 Dec 2022 18:37:57 +0200 Subject: [PATCH 017/187] Multiple fixes: 1. To represent No connections, do not build TcpLikeProperties (TcpLikeProperties with no dimensions represent All connections). 2. Support subsets in query in optimized solution. 3. For comparison of optimized solution to the original one, add connections from peers to themselves. 4. More accurate comparison for 'dot' connectivity queries. 5. Generalized convert_named_ports (to not assume dimensions order) 6. Handling the possibility when projection on one dimension is empty. Signed-off-by: Tanya --- nca/CoreDS/TcpLikeProperties.py | 16 ++++++++-- nca/NetworkConfig/NetworkConfigQuery.py | 37 +++++++++++++++++++++-- nca/NetworkConfig/NetworkLayer.py | 4 +-- nca/Parsers/K8sPolicyYamlParser.py | 40 +++++++++++++------------ nca/Resources/NetworkPolicy.py | 4 +++ 5 files changed, 75 insertions(+), 26 deletions(-) diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index 2cdbb776a..1b93655cc 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -301,20 +301,21 @@ def convert_named_ports(self, named_ports, protocol): my_excluded_named_ports = self.excluded_named_ports self.excluded_named_ports = {} + active_dims = ["src_ports", "dst_ports"] for port in my_named_ports: real_port = named_ports.get(port) if real_port and real_port[1] == protocol: real_port_number = real_port[0] rectangle = [my_named_ports[port], CanonicalIntervalSet.get_interval_set(real_port_number, real_port_number)] - self.add_cube(rectangle) + self.add_cube(rectangle, active_dims) for port in my_excluded_named_ports: real_port = named_ports.get(port) if real_port and real_port[1] == protocol: real_port_number = real_port[0] rectangle = [my_excluded_named_ports[port], CanonicalIntervalSet.get_interval_set(real_port_number, real_port_number)] - self.add_hole(rectangle) + self.add_hole(rectangle, active_dims) def copy(self): res = TcpLikeProperties() @@ -370,7 +371,16 @@ def project_on_one_dimension(self, dim_name): :return: the projection on the given dimension, having that dimension type (either IntervalSet or DFA) """ if dim_name not in self.active_dimensions: - return None + if dim_name == "src_peers" or dim_name == "dst_peers": + return PeerSet() + elif dim_name == "src_ports" or dim_name == "dst_ports": + return PortSet() + elif dim_name == "protocols": + return ProtocolSet() + elif dim_name == "methods": + return MethodSet() + else: + return None res = None for cube in self: cube_dict = self.get_cube_dict_with_orig_values(cube) diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 9fa3cad6f..7e49b83f2 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -11,6 +11,8 @@ from nca.Utils.OutputConfiguration import OutputConfiguration from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import PeerSet, IpBlock, Pod, Peer +from nca.CoreDS.PortSet import PortSet +from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from nca.Resources.CalicoNetworkPolicy import CalicoNetworkPolicy from nca.Resources.IngressPolicy import IngressPolicy from nca.FWRules.ConnectivityGraph import ConnectivityGraph, ConnectivityGraphOptimized @@ -616,6 +618,20 @@ def is_in_subset(self, peer): return False + def compute_subset(self, peers): + """ + Compures all peers that are in the defined subset out of the given peer set + :param PeerSet peers: the given peer set + :return: peers in the defined subset + """ + if not self.output_config.subset: + return peers + res = PeerSet() + for peer in peers: + if self.is_in_subset(peer): + res.add(peer) + return res + @staticmethod def are_labels_all_included(target_labels, pool_labels): for key, val in target_labels.items(): @@ -680,8 +696,21 @@ def exec(self): all_conns_opt = self.config.allowed_connections_optimized() if all_conns_opt: + subset_peers = self.compute_subset(peers_to_compare) + src_peers_in_subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, + PortSet(True), PortSet(True), + src_peers=subset_peers) + dst_peers_in_subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, + PortSet(True), PortSet(True), + dst_peers=subset_peers) + all_conns_opt &= src_peers_in_subset_conns | dst_peers_in_subset_conns conn_graph2 = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) conn_graph_opt = ConnectivityGraphOptimized(self.output_config) + # Add connections from peer to itself + auto_conns = defaultdict(list) + for peer in subset_peers: + auto_conns[ConnectionSet(True)].append((peer, peer)) + conn_graph2.add_edges(auto_conns) for cube in all_conns_opt: conn_graph2.add_edges_from_cube_dict(self.config.peer_container, all_conns_opt.get_cube_dict_with_orig_values(cube)) @@ -691,8 +720,12 @@ def exec(self): res_opt = QueryAnswer(True) if self.output_config.outputFormat == 'dot': res_opt.output_explanation = conn_graph_opt.get_connectivity_dot_format_str() - fw_rules = conn_graph.get_minimized_firewall_rules() # Temp for debugging - assert fw_rules.fw_rules_map == fw_rules2.fw_rules_map # Temp for debugging + # Tanya: temp for debugging + orig_conn_graph = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) + orig_conn_graph.add_edges(connections) + orig_fw_rules = orig_conn_graph.get_minimized_firewall_rules() + assert orig_fw_rules.fw_rules_map == fw_rules2.fw_rules_map + # Tanya: end temp for debugging else: assert fw_rules.fw_rules_map == fw_rules2.fw_rules_map # res_opt.output_explanation = conn_graph_opt.get_connectivity_txt_format_str() diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 9050bf25f..af0ca68a4 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -233,7 +233,7 @@ def collect_policies_conns_optimized(self, is_ingress): """ allowed_conns = None denied_conns = None - add_to_captured = None + add_to_captured = PeerSet() for policy in self.policies_list: policy_allowed_conns, policy_denied_conns, policy_add_to_captured = \ policy.allowed_connections_optimized(is_ingress) @@ -275,7 +275,7 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): allowed_conn, denied_conns, add_to_captured = self.collect_policies_conns_optimized(is_ingress) - if not allowed_conn and not denied_conns: + if not allowed_conn and not denied_conns and not add_to_captured: return None, None # Note: The below computation of non-captured conns cannot be done during the parse stage, # since before computing non-captured conns we should collect all policies conns diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index e2ff7535f..0d9dbee2e 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -320,34 +320,36 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): else: # egress src_pods = policy_selected_pods dst_pods = res_pods - + + res_opt_props = None # TcpLikeProperties ports_array = rule.get('ports', []) if ports_array: res_conns = ConnectionSet() - res_opt_props = None # TcpLikeProperties for port in ports_array: protocol, dest_port_set = self.parse_port(port) if isinstance(protocol, str): protocol = ProtocolNameResolver.get_protocol_number(protocol) - protocols = ProtocolSet() - protocols.add_protocol(protocol) - res_conns.add_connections(protocol, TcpLikeProperties(PortSet(True), dest_port_set)) # K8s doesn't reason about src ports - dest_num_port_set = PortSet() - dest_num_port_set.port_set = dest_port_set.port_set.copy() - tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, - PortSet(True), dest_num_port_set, - protocols, src_pods, dst_pods) - if res_opt_props: - res_opt_props |= tcp_props - else: - res_opt_props = tcp_props -# self.handle_named_ports(dst_pods, protocol, dest_port_set.named_ports, res_opt_props) + res_conns.add_connections(protocol, TcpLikeProperties(PortSet(True), + dest_port_set)) # K8s doesn't reason about src ports + if src_pods and dst_pods: + protocols = ProtocolSet() + protocols.add_protocol(protocol) + + dest_num_port_set = PortSet() + dest_num_port_set.port_set = dest_port_set.port_set.copy() + tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, + PortSet(True), dest_num_port_set, + protocols, src_pods, dst_pods) + if res_opt_props: + res_opt_props |= tcp_props + else: + res_opt_props = tcp_props else: res_conns = ConnectionSet(True) - res_opt_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, - PortSet(True), PortSet(True), - src_peers=src_pods, dst_peers=dst_pods) - + if src_pods and dst_pods: + res_opt_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, + PortSet(True), PortSet(True), + src_peers=src_pods, dst_peers=dst_pods) if not res_pods: self.warning('Rule selects no pods', rule) diff --git a/nca/Resources/NetworkPolicy.py b/nca/Resources/NetworkPolicy.py index 297c7c740..6477f2a57 100644 --- a/nca/Resources/NetworkPolicy.py +++ b/nca/Resources/NetworkPolicy.py @@ -130,6 +130,8 @@ def add_optimized_ingress_props(self, props, is_allow=True): :param Bool is_allow: whether these are an allow or deny properties :return: None """ + if not props: + return if is_allow: self.optimized_ingress_props = \ (self.optimized_ingress_props | props) if self.optimized_ingress_props else props @@ -144,6 +146,8 @@ def add_optimized_egress_props(self, props, is_allow=True): :param Bool is_allow: whether these are an allow or deny properties :return: None """ + if not props: + return if is_allow: self.optimized_egress_props = \ (self.optimized_egress_props | props) if self.optimized_egress_props else props From 0d4bd28e95a1638276e336f34a2388c23b24f091 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 18 Dec 2022 16:39:45 +0200 Subject: [PATCH 018/187] Properly handling 'False' represented by TcpLikeProperties. Properly handling HostEPs in optimized TcpLikeProperties. Signed-off-by: Tanya --- nca/CoreDS/TcpLikeProperties.py | 2 +- nca/NetworkConfig/NetworkConfigQuery.py | 7 +++-- nca/NetworkConfig/NetworkLayer.py | 31 ++++++++++--------- nca/Parsers/CalicoPolicyYamlParser.py | 40 ++++++++++++++----------- 4 files changed, 42 insertions(+), 38 deletions(-) diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index 1b93655cc..a3d92314b 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -38,7 +38,7 @@ class TcpLikeProperties(CanonicalHyperCubeSet): (2) calico: +ve and -ve named ports, no src named ports, and no use of operators between these objects. """ - dimensions_list = ["src_peers", "dst_peers", "protocols", "src_ports", "dst_ports", "methods", "paths", "hosts", ] + dimensions_list = ["src_peers", "dst_peers", "protocols", "src_ports", "dst_ports", "methods", "paths", "hosts"] # TODO: change constructor defaults? either all arguments in "allow all" by default, or "empty" by default def __init__(self, source_ports=PortSet(), dest_ports=PortSet(), protocols=ProtocolSet(True), diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 7e49b83f2..61beb45f1 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -10,7 +10,7 @@ from nca.Utils.OutputConfiguration import OutputConfiguration from nca.CoreDS.ConnectionSet import ConnectionSet -from nca.CoreDS.Peer import PeerSet, IpBlock, Pod, Peer +from nca.CoreDS.Peer import PeerSet, IpBlock, Pod, Peer, HostEP from nca.CoreDS.PortSet import PortSet from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from nca.Resources.CalicoNetworkPolicy import CalicoNetworkPolicy @@ -706,10 +706,11 @@ def exec(self): all_conns_opt &= src_peers_in_subset_conns | dst_peers_in_subset_conns conn_graph2 = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) conn_graph_opt = ConnectivityGraphOptimized(self.output_config) - # Add connections from peer to itself + # Add connections from peer to itself (except for HEPs) auto_conns = defaultdict(list) for peer in subset_peers: - auto_conns[ConnectionSet(True)].append((peer, peer)) + if not isinstance(peer, HostEP): + auto_conns[ConnectionSet(True)].append((peer, peer)) conn_graph2.add_edges(auto_conns) for cube in all_conns_opt: conn_graph2.add_edges_from_cube_dict(self.config.peer_container, diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index af0ca68a4..58ce0062b 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -275,35 +275,34 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): allowed_conn, denied_conns, add_to_captured = self.collect_policies_conns_optimized(is_ingress) - if not allowed_conn and not denied_conns and not add_to_captured: - return None, None # Note: The below computation of non-captured conns cannot be done during the parse stage, # since before computing non-captured conns we should collect all policies conns # compute non-captured connections base_peer_set = peer_container.peer_set.copy() base_peer_set.add(IpBlock.get_all_ips_block()) + base_peer_set_no_hep = PeerSet(set([peer for peer in base_peer_set if not isinstance(peer, HostEP)])) if is_ingress: captured_dst_peers1 = allowed_conn.project_on_one_dimension('dst_peers') if allowed_conn else PeerSet() captured_dst_peers2 = denied_conns.project_on_one_dimension('dst_peers') if denied_conns else PeerSet() captured_dst_peers = captured_dst_peers1 | captured_dst_peers2 | add_to_captured - if captured_dst_peers: - non_captured_dst_peers = base_peer_set - captured_dst_peers - if non_captured_dst_peers: - non_captured_conns = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, PortSet(True), PortSet(True), - dst_peers=non_captured_dst_peers) - allowed_conn = (allowed_conn | non_captured_conns) if allowed_conn else non_captured_conns + non_captured_dst_peers = base_peer_set_no_hep - captured_dst_peers + if non_captured_dst_peers: + non_captured_conns = \ + TcpLikeProperties.make_tcp_like_properties(peer_container, PortSet(True), PortSet(True), + src_peers=base_peer_set, + dst_peers=non_captured_dst_peers) + allowed_conn = (allowed_conn | non_captured_conns) if allowed_conn else non_captured_conns else: captured_src_peers1 = allowed_conn.project_on_one_dimension('src_peers') if allowed_conn else PeerSet() captured_src_peers2 = denied_conns.project_on_one_dimension('src_peers') if denied_conns else PeerSet() captured_src_peers = captured_src_peers1 | captured_src_peers2 | add_to_captured - if captured_src_peers: - non_captured_src_peers = base_peer_set - captured_src_peers - if non_captured_src_peers: - non_captured_conns = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, PortSet(True), PortSet(True), - src_peers=non_captured_src_peers) - allowed_conn = (allowed_conn | non_captured_conns) if allowed_conn else non_captured_conns + non_captured_src_peers = base_peer_set_no_hep - captured_src_peers + if non_captured_src_peers: + non_captured_conns = \ + TcpLikeProperties.make_tcp_like_properties(peer_container, PortSet(True), PortSet(True), + src_peers=non_captured_src_peers, + dst_peers=base_peer_set) + allowed_conn = (allowed_conn | non_captured_conns) if allowed_conn else non_captured_conns return allowed_conn, denied_conns diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index ee9fd6bec..098142cac 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -479,34 +479,38 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): else: if protocol_supports_ports: connections.add_connections(protocol, TcpLikeProperties(src_res_ports, dst_res_ports)) - src_num_port_set = PortSet() - src_num_port_set.port_set = src_res_ports.port_set.copy() - dst_num_port_set = PortSet() - dst_num_port_set.port_set = dst_res_ports.port_set.copy() - tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, src_num_port_set, - dst_num_port_set, protocols, - src_res_pods, dst_res_pods) + if src_res_pods and dst_res_pods: + src_num_port_set = PortSet() + src_num_port_set.port_set = src_res_ports.port_set.copy() + dst_num_port_set = PortSet() + dst_num_port_set.port_set = dst_res_ports.port_set.copy() + tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, src_num_port_set, + dst_num_port_set, protocols, + src_res_pods, dst_res_pods) elif ConnectionSet.protocol_is_icmp(protocol): connections.add_connections(protocol, self._parse_icmp(rule.get('icmp'), rule.get('notICMP'))) # TODO - update tcp_props else: connections.add_connections(protocol, True) - tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), - PortSet(True), protocols, - src_res_pods, dst_res_pods) + if src_res_pods and dst_res_pods: + tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), + PortSet(True), protocols, + src_res_pods, dst_res_pods) elif not_protocol is not None: connections.add_all_connections() connections.remove_protocol(not_protocol) - protocols = ProtocolSet(True) - protocols.remove_protocol(not_protocol) - tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), - PortSet(True), protocols, - src_res_pods, dst_res_pods) + if src_res_pods and dst_res_pods: + protocols = ProtocolSet(True) + protocols.remove_protocol(not_protocol) + tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), + PortSet(True), protocols, + src_res_pods, dst_res_pods) else: connections.allow_all = True - tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), - PortSet(True), ProtocolSet(True), - src_res_pods, dst_res_pods) + if src_res_pods and dst_res_pods: + tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), + PortSet(True), ProtocolSet(True), + src_res_pods, dst_res_pods) self._verify_named_ports(rule, dst_res_pods, connections) if not src_res_pods and policy_selected_eps and (is_ingress or not is_profile): From 47d5bd08d3971396516481568560daba9fb2d519 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 1 Jan 2023 16:26:31 +0200 Subject: [PATCH 019/187] Added support to ICMP data in optimized HC set (handled in TCPLikeProperties) Fixed handling of non-captured pods in optimized solution. Added using True/False HC_set (make_all_properties()/make_empty_properties()) Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 4 +- nca/CoreDS/ICMPDataSet.py | 17 +++ nca/CoreDS/PortSet.py | 3 + nca/CoreDS/ProtocolSet.py | 31 ++-- nca/CoreDS/TcpLikeProperties.py | 144 ++++++++++++++---- nca/FWRules/ConnectivityGraph.py | 38 ++++- nca/NetworkConfig/NetworkConfig.py | 11 +- nca/NetworkConfig/NetworkConfigQuery.py | 16 +- nca/NetworkConfig/NetworkLayer.py | 66 ++++---- nca/Parsers/CalicoPolicyYamlParser.py | 92 +++++++---- nca/Parsers/IngressPolicyYamlParser.py | 14 +- .../IstioTrafficResourcesYamlParser.py | 2 +- nca/Parsers/K8sPolicyYamlParser.py | 29 +--- nca/Resources/CalicoNetworkPolicy.py | 12 +- nca/Resources/K8sNetworkPolicy.py | 14 +- nca/Resources/NetworkPolicy.py | 9 +- .../testcase14-icmp/testcase14-scheme.yaml | 15 ++ .../testcase20-scheme.yaml | 8 + 18 files changed, 361 insertions(+), 164 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 116d18e6c..9ab636611 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -438,7 +438,7 @@ def _add_all_connections_of_protocol(self, protocol): :return: None """ if self.protocol_supports_ports(protocol): - self.allowed_protocols[protocol] = TcpLikeProperties(PortSet(True), PortSet(True)) + self.allowed_protocols[protocol] = TcpLikeProperties.make_all_properties() elif self.protocol_is_icmp(protocol): self.allowed_protocols[protocol] = ICMPDataSet(add_all=True) else: @@ -547,7 +547,7 @@ def print_diff(self, other, self_name, other_name): @staticmethod def get_all_tcp_connections(): tcp_conns = ConnectionSet() - tcp_conns.add_connections('TCP', TcpLikeProperties(PortSet(True), PortSet(True))) + tcp_conns.add_connections('TCP', TcpLikeProperties.make_all_properties()) return tcp_conns @staticmethod diff --git a/nca/CoreDS/ICMPDataSet.py b/nca/CoreDS/ICMPDataSet.py index d4f6b3806..bd67ff47e 100644 --- a/nca/CoreDS/ICMPDataSet.py +++ b/nca/CoreDS/ICMPDataSet.py @@ -56,6 +56,23 @@ def get_properties_obj(self): cubes_list.append(cube_str) return res_obj + def get_cube_dict_with_orig_values(self, cube): + """ + represent the properties cube as dict object, where the values are the original values + with which the cube was built (i.e., icmp_type and icmp_code) + :param list cube: the values of the input cube + :return: the cube properties in dict representation + :rtype: dict + """ + cube_dict = {} + for i, dim in enumerate(self.active_dimensions): + dim_values = cube[i] + dim_domain = DimensionsManager().get_dimension_domain_by_name(dim) + if dim_domain == dim_values: + continue # skip dimensions with all values allowed in a cube + cube_dict[dim] = dim_values + return cube_dict + def copy(self): new_copy = copy.copy(self) return new_copy diff --git a/nca/CoreDS/PortSet.py b/nca/CoreDS/PortSet.py index 2c451916f..938bc1dbb 100644 --- a/nca/CoreDS/PortSet.py +++ b/nca/CoreDS/PortSet.py @@ -82,3 +82,6 @@ def __isub__(self, other): def is_all(self): return self.port_set == PortSet.all_ports_interval + + def is_empty(self): + return not self.port_set and not self.named_ports diff --git a/nca/CoreDS/ProtocolSet.py b/nca/CoreDS/ProtocolSet.py index c5355eedf..cd3f8edeb 100644 --- a/nca/CoreDS/ProtocolSet.py +++ b/nca/CoreDS/ProtocolSet.py @@ -9,10 +9,8 @@ class ProtocolSet(CanonicalIntervalSet): """ - A class for holding a set of HTTP methods + A class for holding a set of protocols """ - - #all_protocols_list = ProtocolNameResolver.get_all_protocols_list() min_protocol_num = 0 max_protocol_num = 255 @@ -24,27 +22,42 @@ def __init__(self, all_protocols=False): if all_protocols: # the whole range self.add_interval(self._whole_range_interval()) + def __contains__(self, protocol): + if isinstance(protocol, str): + protocol_num = ProtocolNameResolver.get_protocol_number(protocol) + else: + protocol_num = protocol + return super().__contains__(protocol_num) + def add_protocol(self, protocol): """ Adds a given protocol to the ProtocolSet if the protocol is one of the eligible protocols (i.e., protocol in [min_protocol_num...max_protocol_num]); otherwise raises exception - :param int protocol: the protocol to add + :param Union[int, str] protocol: the protocol to add """ - if not ProtocolNameResolver.is_valid_protocol(protocol): + if isinstance(protocol, str): + protocol_num = ProtocolNameResolver.get_protocol_number(protocol) + else: + protocol_num = protocol + if not ProtocolNameResolver.is_valid_protocol(protocol_num): raise Exception('Protocol must be in the range 0-255') - self.add_interval(self.Interval(protocol, protocol)) + self.add_interval(self.Interval(protocol_num, protocol_num)) def remove_protocol(self, protocol): """ Removes a given protocol from the ProtocolSet if the protocol is one of the eligible protocols (i.e., protocol in [min_protocol_num...max_protocol_num]); otherwise raises exception - :param int protocol: the protocol to remove + :param Union[int,str] protocol: the protocol to remove """ - if not ProtocolNameResolver.is_valid_protocol(protocol): + if isinstance(protocol, str): + protocol_num = ProtocolNameResolver.get_protocol_number(protocol) + else: + protocol_num = protocol + if not ProtocolNameResolver.is_valid_protocol(protocol_num): raise Exception('Protocol must be in the range 0-255') - self.add_hole(self.Interval(protocol, protocol)) + self.add_hole(self.Interval(protocol_num, protocol_num)) def set_protocols(self, protocols): """ diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index a3d92314b..a8ebca8d0 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -38,11 +38,14 @@ class TcpLikeProperties(CanonicalHyperCubeSet): (2) calico: +ve and -ve named ports, no src named ports, and no use of operators between these objects. """ - dimensions_list = ["src_peers", "dst_peers", "protocols", "src_ports", "dst_ports", "methods", "paths", "hosts"] + dimensions_list = ["src_peers", "dst_peers", "protocols", "src_ports", "dst_ports", "methods", "paths", "hosts", + "icmp_type", "icmp_code"] # TODO: change constructor defaults? either all arguments in "allow all" by default, or "empty" by default - def __init__(self, source_ports=PortSet(), dest_ports=PortSet(), protocols=ProtocolSet(True), - methods=MethodSet(True), paths=None, hosts=None, base_peer_set=None, src_peers=None, dst_peers=None): + def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protocols=ProtocolSet(True), + methods=MethodSet(True), paths=None, hosts=None, icmp_type=None, icmp_code=None, + base_peer_set=None, src_peers=None, dst_peers=None, + create_empty=False): """ This will create all cubes made of the input arguments ranges/regex values. :param PortSet source_ports: The set of source ports (as a set of intervals/ranges) @@ -51,6 +54,8 @@ def __init__(self, source_ports=PortSet(), dest_ports=PortSet(), protocols=Proto :param MethodSet methods: the set of http request methods :param MinDFA paths: The dfa of http request paths :param MinDFA hosts: The dfa of http request hosts + :param CanonicalIntervalSet icmp_type: ICMP-specific parameter (type dimension) + :param CanonicalIntervalSet icmp_code: ICMP-specific parameter (code dimension) :param PeerSet base_peer_set: the base peer set which is referenced by the indices in 'peers' :param CanonicalIntervalSet src_peers: the set of source peers :param CanonicalIntervalSet dst_peers: the set of target peers @@ -60,7 +65,9 @@ def __init__(self, source_ports=PortSet(), dest_ports=PortSet(), protocols=Proto self.named_ports = {} # a mapping from dst named port (String) to src ports interval set self.excluded_named_ports = {} # a mapping from dst named port (String) to src ports interval set - self.base_peer_set = base_peer_set if base_peer_set else PeerSet() + self.base_peer_set = base_peer_set.copy() if base_peer_set else PeerSet() + if create_empty: + return # create the cube from input arguments cube = [] @@ -90,6 +97,12 @@ def __init__(self, source_ports=PortSet(), dest_ports=PortSet(), protocols=Proto if hosts is not None: cube.append(hosts) active_dims.append("hosts") + if icmp_type: + cube.append(icmp_type) + active_dims.append("icmp_type") + if icmp_code: + cube.append(icmp_code) + active_dims.append("icmp_code") if not active_dims: self.set_all() @@ -207,7 +220,7 @@ def get_properties_obj(self): def __eq__(self, other): if isinstance(other, TcpLikeProperties): - assert not self.base_peer_set or not other.base_peer_set or self.base_peer_set == other.base_peer_set + # assert not self.base_peer_set or not other.base_peer_set or self.base_peer_set == other.base_peer_set res = super().__eq__(other) and self.named_ports == other.named_ports and \ self.excluded_named_ports == other.excluded_named_ports return res @@ -229,22 +242,23 @@ def __sub__(self, other): return res def __iand__(self, other): - assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ - not other.base_peer_set or self.base_peer_set == other.base_peer_set + # assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ + # not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.has_named_ports() assert not isinstance(other, TcpLikeProperties) or not other.has_named_ports() super().__iand__(other) + if isinstance(other, TcpLikeProperties): + self.base_peer_set |= other.base_peer_set return self def __ior__(self, other): - assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ - not other.base_peer_set or self.base_peer_set == other.base_peer_set + # assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ + # not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.excluded_named_ports assert not isinstance(other, TcpLikeProperties) or not other.excluded_named_ports - if isinstance(other, TcpLikeProperties): - self.base_peer_set |= other.base_peer_set super().__ior__(other) if isinstance(other, TcpLikeProperties): + self.base_peer_set |= other.base_peer_set res_named_ports = dict({}) for port_name in self.named_ports: res_named_ports[port_name] = self.named_ports[port_name] @@ -257,11 +271,13 @@ def __ior__(self, other): return self def __isub__(self, other): - assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ - not other.base_peer_set or self.base_peer_set == other.base_peer_set + # assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ + # not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.has_named_ports() assert not isinstance(other, TcpLikeProperties) or not other.has_named_ports() super().__isub__(other) + if isinstance(other, TcpLikeProperties): + self.base_peer_set |= other.base_peer_set return self def contained_in(self, other): @@ -270,8 +286,8 @@ def contained_in(self, other): :return: Whether all (source port, target port) pairs in self also appear in other :rtype: bool """ - assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ - not other.base_peer_set or self.base_peer_set == other.base_peer_set + # assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ + # not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.has_named_ports() assert not other.has_named_ports() return super().contained_in(other) @@ -356,7 +372,7 @@ def _get_first_item_str(self): res_list = [] for i, dim_name in enumerate(self.active_dimensions): if dim_name == 'protocols': - dim_item = ProtocolSet.all_protocols_list[item[i]] + dim_item = ProtocolNameResolver.get_protocol_name(item[i]) elif dim_name == 'methods': dim_item = MethodSet.all_methods_list[item[i]] else: @@ -390,8 +406,10 @@ def project_on_one_dimension(self, dim_name): return res @staticmethod - def make_tcp_like_properties(peer_container, src_ports, dst_ports, protocols=None, src_peers=None, dst_peers=None, - paths_dfa=None, hosts_dfa=None, methods=None): + def make_tcp_like_properties(peer_container, src_ports=PortSet(True), dst_ports=PortSet(True), + protocols=ProtocolSet(True), src_peers=None, dst_peers=None, + paths_dfa=None, hosts_dfa=None, methods=MethodSet(True), + icmp_type=None, icmp_code=None, exclude_same_src_dst_peers=True): """ get TcpLikeProperties with TCP allowed connections, corresponding to input properties cube. TcpLikeProperties should not contain named ports: substitute them with corresponding port numbers, per peer @@ -404,22 +422,40 @@ def make_tcp_like_properties(peer_container, src_ports, dst_ports, protocols=Non :param MinDFA paths_dfa: MinDFA obj for paths dimension :param MinDFA hosts_dfa: MinDFA obj for hosts dimension :param MethodSet methods: CanonicalIntervalSet obj for methods dimension + :param CanonicalIntervalSet icmp_type: ICMP-specific parameter (type dimension) + :param CanonicalIntervalSet icmp_code: ICMP-specific parameter (code dimension) :return: TcpLikeProperties with TCP allowed connections, corresponding to input properties cube """ - base_peer_set = peer_container.peer_set.copy() + base_peer_set = peer_container.peer_set if src_peers: + base_peer_set |= src_peers src_peers_interval = base_peer_set.get_peer_interval_of(src_peers) else: src_peers_interval = None if dst_peers: + base_peer_set |= dst_peers dst_peers_interval = base_peer_set.get_peer_interval_of(dst_peers) else: dst_peers_interval = None - if not src_ports.named_ports and not dst_ports.named_ports: - return TcpLikeProperties(source_ports=src_ports, dest_ports=dst_ports, - protocols=protocols, methods=methods, - paths=paths_dfa, hosts=hosts_dfa, src_peers=src_peers_interval, - dst_peers=dst_peers_interval, base_peer_set=base_peer_set) + exclude_props = TcpLikeProperties.make_empty_properties(peer_container) + if exclude_same_src_dst_peers and src_peers and dst_peers: + same_src_dst_peers = src_peers & dst_peers + for peer in same_src_dst_peers: + ps = PeerSet() + ps.add(peer) + peer_interval = base_peer_set.get_peer_interval_of(ps) + exclude_props |= TcpLikeProperties(src_peers=peer_interval, + dst_peers=peer_interval, + base_peer_set=base_peer_set) + if (not src_ports.named_ports or not src_peers) and (not dst_ports.named_ports or not dst_peers): + # Should not resolve named ports + res = TcpLikeProperties(source_ports=src_ports, dest_ports=dst_ports, + protocols=protocols, methods=methods, paths=paths_dfa, hosts=hosts_dfa, + icmp_type=icmp_type, icmp_code=icmp_code, + src_peers=src_peers_interval, dst_peers=dst_peers_interval, + base_peer_set=base_peer_set) + return res - exclude_props if exclude_props else res + # Resolving named ports tcp_properties = None if src_ports.named_ports and dst_ports.named_ports: assert src_peers @@ -463,6 +499,7 @@ def make_tcp_like_properties(peer_container, src_ports, dst_ports, protocols=Non props = TcpLikeProperties(source_ports=real_src_ports, dest_ports=real_dst_ports, protocols=protocols if protocols else tcp_protocol, methods=methods, paths=paths_dfa, hosts=hosts_dfa, + icmp_type=icmp_type, icmp_code=icmp_code, src_peers=base_peer_set.get_peer_interval_of(src_peer_in_set), dst_peers=base_peer_set.get_peer_interval_of(dst_peer_in_set), base_peer_set=base_peer_set) @@ -503,19 +540,22 @@ def make_tcp_like_properties(peer_container, src_ports, dst_ports, protocols=Non props = TcpLikeProperties(source_ports=ports, dest_ports=dst_ports, protocols=protocols if protocols else tcp_protocol, methods=methods, paths=paths_dfa, hosts=hosts_dfa, + icmp_type=icmp_type, icmp_code=icmp_code, src_peers=base_peer_set.get_peer_interval_of(peer_in_set), dst_peers=dst_peers_interval, base_peer_set=base_peer_set) else: props = TcpLikeProperties(source_ports=src_ports, dest_ports=ports, protocols=protocols if protocols else tcp_protocol, methods=methods, - paths=paths_dfa, hosts=hosts_dfa, src_peers=src_peers_interval, + paths=paths_dfa, hosts=hosts_dfa, + icmp_type=icmp_type, icmp_code=icmp_code, + src_peers=src_peers_interval, dst_peers=base_peer_set.get_peer_interval_of(peer_in_set), base_peer_set=base_peer_set) if tcp_properties: tcp_properties |= props else: tcp_properties = props - return tcp_properties + return tcp_properties - exclude_props @staticmethod def make_tcp_like_properties_from_dict(peer_container, cube_dict): @@ -534,6 +574,54 @@ def make_tcp_like_properties_from_dict(peer_container, cube_dict): paths_dfa = cube_dict_copy.pop("paths", None) hosts_dfa = cube_dict_copy.pop("hosts", None) methods = cube_dict_copy.pop("methods", None) + icmp_type = cube_dict_copy.pop("icmp_type", None) + icmp_code = cube_dict_copy.pop("icmp_code", None) assert not cube_dict_copy - return TcpLikeProperties.make_tcp_like_properties(peer_container, src_ports, dst_ports, protocols, - src_peers, dst_peers, paths_dfa, hosts_dfa, methods) + return TcpLikeProperties.make_tcp_like_properties(peer_container, src_ports=src_ports, dst_ports=dst_ports, + protocols=protocols, src_peers=src_peers, dst_peers=dst_peers, + paths_dfa=paths_dfa, hosts_dfa=hosts_dfa, methods=methods, + icmp_type=icmp_type, icmp_code=icmp_code) + + @staticmethod + def make_empty_properties(peer_container=None): + return TcpLikeProperties(base_peer_set=peer_container.peer_set if peer_container else None, create_empty=True) + + @staticmethod + def make_all_properties(peer_container=None): + return TcpLikeProperties(base_peer_set=peer_container.peer_set if peer_container else None) + + ####################################### ICMP-related functions ####################################### + + @staticmethod + def make_icmp_properties(peer_container, protocol="", src_peers=None, dst_peers=None, + icmp_type=None, icmp_code=None): + if protocol: + icmp_protocol_set = ProtocolSet() + icmp_protocol_set.add_protocol(ProtocolNameResolver.get_protocol_number(protocol)) + else: + icmp_protocol_set = ProtocolSet(True) + icmp_type_interval = None + icmp_code_interval = None + if icmp_type: + icmp_type_interval = CanonicalIntervalSet.get_interval_set(icmp_type, icmp_type) + if icmp_code: + icmp_code_interval = CanonicalIntervalSet.get_interval_set(icmp_code, icmp_code) + return TcpLikeProperties.make_tcp_like_properties(peer_container=peer_container, protocols=icmp_protocol_set, + src_peers=src_peers, dst_peers=dst_peers, + icmp_type=icmp_type_interval, icmp_code=icmp_code_interval) + + @staticmethod + def make_all_but_given_icmp_properties(peer_container, protocol="", src_peers=None, dst_peers=None, + icmp_type=None, icmp_code=None): + if protocol: + icmp_protocol_set = ProtocolSet() + icmp_protocol_set.add_protocol(ProtocolNameResolver.get_protocol_number(protocol)) + else: + icmp_protocol_set = ProtocolSet(True) + all_icmp_props = TcpLikeProperties.make_tcp_like_properties(peer_container=peer_container, + protocols=icmp_protocol_set, + src_peers=src_peers, dst_peers=dst_peers) + given_icmp_props = TcpLikeProperties.make_icmp_properties(peer_container, protocol=protocol, + src_peers=src_peers, dst_peers=dst_peers, + icmp_type=icmp_type, icmp_code=icmp_code,) + return all_icmp_props-given_icmp_props diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index 11b5310c5..c98b87bf6 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -105,15 +105,13 @@ def add_edges_from_cube_dict(self, peer_container, cube_dict): protocol_names = ProtocolSet.get_protocol_names_from_interval_set(protocols) if protocols else ['TCP'] for protocol in protocol_names: if new_cube_dict: - # TODO - support ICMP - assert ConnectionSet.protocol_supports_ports(protocol) conns.add_connections(protocol, TcpLikeProperties.make_tcp_like_properties_from_dict(peer_container, new_cube_dict)) else: if ConnectionSet.protocol_supports_ports(protocol): - conns.add_connections(protocol, TcpLikeProperties(PortSet(True), PortSet(True))) + conns.add_connections(protocol, TcpLikeProperties.make_all_properties()) elif ConnectionSet.protocol_is_icmp(protocol): - conns.add_connections(protocol, ICMPDataSet(add_all=True)) + conns.add_connections(protocol, TcpLikeProperties.make_all_properties()) else: conns.add_connections(protocol, True) for src_peer in src_peers: @@ -153,6 +151,38 @@ def get_connectivity_dot_format_str(self): ''.join(line for line in sorted(list(edge_lines))) + '}\n\n' return output_result + def convert_to_tcp_like_properties(self, peer_container): + """ + Used for testing of the optimized solution: converting connectivity graph back to TcpLikeProperties + :param peer_container: The peer container + :return: TcpLikeProperties representing the connectivity graph + """ + res = TcpLikeProperties.make_empty_properties() + for item in self.connections_to_peers.items(): + if item[0].allow_all: + for peer_pair in item[1]: + res |= TcpLikeProperties.make_tcp_like_properties(peer_container, + src_peers=PeerSet({peer_pair[0]}), + dst_peers=PeerSet({peer_pair[1]})) + else: + for prot in item[0].allowed_protocols.items(): + protocols = ProtocolSet() + protocols.add_protocol(prot[0]) + if isinstance(prot[1], bool): + res |= TcpLikeProperties.make_tcp_like_properties(peer_container, protocols=protocols, + src_peers=PeerSet({peer_pair[0]}), + dst_peers=PeerSet({peer_pair[1]})) + continue + for cube in prot[1]: + cube_dict = prot[1].get_cube_dict_with_orig_values(cube) + cube_dict["protocols"] = protocols + for peer_pair in item[1]: + new_cube_dict = cube_dict.copy() + new_cube_dict["src_peers"] = PeerSet({peer_pair[0]}) + new_cube_dict["dst_peers"] = PeerSet({peer_pair[1]}) + res |= TcpLikeProperties.make_tcp_like_properties_from_dict(peer_container, new_cube_dict) + return res + def get_minimized_firewall_rules(self): """ computes and returns minimized firewall rules from original connectivity graph diff --git a/nca/NetworkConfig/NetworkConfig.py b/nca/NetworkConfig/NetworkConfig.py index 67eca9c87..5b3a4b323 100644 --- a/nca/NetworkConfig/NetworkConfig.py +++ b/nca/NetworkConfig/NetworkConfig.py @@ -6,6 +6,7 @@ from dataclasses import dataclass, field from nca.CoreDS import Peer from nca.CoreDS.ConnectionSet import ConnectionSet +from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from nca.Resources.NetworkPolicy import NetworkPolicy from .NetworkLayer import NetworkLayersContainer, NetworkLayerName @@ -259,17 +260,11 @@ def allowed_connections_optimized(self, layer_name=None): layer_name) return self.policies_container.layers[layer_name].allowed_connections_optimized(self.peer_container) - # TODO handle connectivity of hostEndpoints (for calico layer) - - conns_res = None + conns_res = TcpLikeProperties.make_all_properties() # all connections for layer, layer_obj in self.policies_container.layers.items(): conns_per_layer = layer_obj.allowed_connections_optimized(self.peer_container) - # all allowed connections: intersection of all allowed connections from all layers - if conns_res and conns_per_layer: - conns_res &= conns_per_layer - elif not conns_res: - conns_res = conns_per_layer + conns_res &= conns_per_layer return conns_res diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 61beb45f1..9650f0bc5 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -698,10 +698,8 @@ def exec(self): if all_conns_opt: subset_peers = self.compute_subset(peers_to_compare) src_peers_in_subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, - PortSet(True), PortSet(True), src_peers=subset_peers) dst_peers_in_subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, - PortSet(True), PortSet(True), dst_peers=subset_peers) all_conns_opt &= src_peers_in_subset_conns | dst_peers_in_subset_conns conn_graph2 = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) @@ -709,32 +707,34 @@ def exec(self): # Add connections from peer to itself (except for HEPs) auto_conns = defaultdict(list) for peer in subset_peers: - if not isinstance(peer, HostEP): + if not isinstance(peer, HostEP) and not isinstance(peer, IpBlock): auto_conns[ConnectionSet(True)].append((peer, peer)) conn_graph2.add_edges(auto_conns) for cube in all_conns_opt: conn_graph2.add_edges_from_cube_dict(self.config.peer_container, all_conns_opt.get_cube_dict_with_orig_values(cube)) conn_graph_opt.add_edge(all_conns_opt.get_cube_dict(cube)) - fw_rules2 = conn_graph2.get_minimized_firewall_rules() - fw_rules2.unite_fw_rules_with_same_peers() res_opt = QueryAnswer(True) if self.output_config.outputFormat == 'dot': res_opt.output_explanation = conn_graph_opt.get_connectivity_dot_format_str() # Tanya: temp for debugging orig_conn_graph = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) orig_conn_graph.add_edges(connections) - orig_fw_rules = orig_conn_graph.get_minimized_firewall_rules() - assert orig_fw_rules.fw_rules_map == fw_rules2.fw_rules_map + self.compare_conn_graphs(orig_conn_graph, conn_graph2) # Tanya: end temp for debugging else: - assert fw_rules.fw_rules_map == fw_rules2.fw_rules_map + self.compare_conn_graphs(conn_graph, conn_graph2) # res_opt.output_explanation = conn_graph_opt.get_connectivity_txt_format_str() # res.output_explanation += "---------------- OPTIMIZED RESULT: -------------\n" +\ # fw_rules2.get_fw_rules_in_required_format() + \ # "\n------------------------------------------------\n\n" # TEMP for debug return res + def compare_conn_graphs(self, conn_graph1, conn_graph2): + tcp_props1 = conn_graph1.convert_to_tcp_like_properties(self.config.peer_container) + tcp_props2 = conn_graph2.convert_to_tcp_like_properties(self.config.peer_container) + assert tcp_props1 == tcp_props2 + def compute_query_output(self, query_answer): return self.get_query_output(query_answer, only_explanation=query_answer.bool_result) diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 58ce0062b..8c363fa6a 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -168,15 +168,15 @@ def allowed_connections_optimized(self, peer_container): """ allowed_ingress_conns, denied_ingres_conns = self._allowed_xgress_conns_optimized(True, peer_container) allowed_egress_conns, denied_egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) - if allowed_ingress_conns and allowed_egress_conns: - res = allowed_ingress_conns & allowed_egress_conns - else: - res = allowed_ingress_conns if allowed_ingress_conns else allowed_egress_conns - if res: - if denied_ingres_conns: - res -= denied_ingres_conns - if denied_egress_conns: - res -= denied_egress_conns + res = allowed_ingress_conns & allowed_egress_conns + res -= denied_ingres_conns + res -= denied_egress_conns + # exclude IpBlock->IpBlock connections + all_ips_peer_set = PeerSet({IpBlock.get_all_ips_block()}) + excluded_conns = TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=all_ips_peer_set, + dst_peers=all_ips_peer_set, + exclude_same_src_dst_peers=False) + res -= excluded_conns return res def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): @@ -231,26 +231,26 @@ def collect_policies_conns_optimized(self, is_ingress): :return: allowed_conns, denied_conns and set of peers to be added to captured peers :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) """ - allowed_conns = None - denied_conns = None - add_to_captured = PeerSet() + allowed_conns = TcpLikeProperties.make_empty_properties() + denied_conns = TcpLikeProperties.make_empty_properties() + captured = PeerSet() for policy in self.policies_list: - policy_allowed_conns, policy_denied_conns, policy_add_to_captured = \ + policy_allowed_conns, policy_denied_conns, policy_captured = \ policy.allowed_connections_optimized(is_ingress) - if policy_allowed_conns and allowed_conns: - allowed_conns |= policy_allowed_conns - elif not allowed_conns: - allowed_conns = policy_allowed_conns - if policy_denied_conns and denied_conns: - denied_conns |= policy_denied_conns - elif not denied_conns: - denied_conns = policy_denied_conns - if policy_add_to_captured and add_to_captured: - add_to_captured |= policy_add_to_captured - elif not add_to_captured: - add_to_captured = policy_add_to_captured - - return allowed_conns, denied_conns, add_to_captured + + policy_denied_conns -= allowed_conns + #policy_denied_conns -= pass_conns # Preparation for handling of pass + policy_allowed_conns -= denied_conns + #policy_allowed_conns -= pass_conns # Preparation for handling of pass + #policy_pass_conns -= denied_conns + #policy_pass_conns -= allowed_conns + #pass_conns |= policy_pass_conns + + allowed_conns |= policy_allowed_conns + denied_conns |= policy_denied_conns + captured |= policy_captured + + return allowed_conns, denied_conns, captured class K8sCalicoNetworkLayer(NetworkLayer): @@ -288,9 +288,10 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): non_captured_dst_peers = base_peer_set_no_hep - captured_dst_peers if non_captured_dst_peers: non_captured_conns = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, PortSet(True), PortSet(True), + TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=base_peer_set, - dst_peers=non_captured_dst_peers) + dst_peers=non_captured_dst_peers, + exclude_same_src_dst_peers=False) allowed_conn = (allowed_conn | non_captured_conns) if allowed_conn else non_captured_conns else: captured_src_peers1 = allowed_conn.project_on_one_dimension('src_peers') if allowed_conn else PeerSet() @@ -299,9 +300,10 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): non_captured_src_peers = base_peer_set_no_hep - captured_src_peers if non_captured_src_peers: non_captured_conns = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, PortSet(True), PortSet(True), + TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=non_captured_src_peers, - dst_peers=base_peer_set) + dst_peers=base_peer_set, + exclude_same_src_dst_peers=False) allowed_conn = (allowed_conn | non_captured_conns) if allowed_conn else non_captured_conns return allowed_conn, denied_conns @@ -344,4 +346,4 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): all_allowed_conns=all_allowed_conns) def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): - return None, None + return TcpLikeProperties.make_empty_properties(), TcpLikeProperties.make_empty_properties() diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index 098142cac..53e74e32a 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -361,13 +361,18 @@ def _parse_entity_rule(self, entity_rule, protocol_supports_ports): return self._get_rule_peers(entity_rule), self._get_rule_ports(entity_rule, protocol_supports_ports) - def _parse_icmp(self, icmp_data, not_icmp_data): + def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): """ Parse the icmp and notICMP parts of a rule :param dict icmp_data: :param dict not_icmp_data: - :return: an ICMPDataSet object representing the allowed ICMP connections - :rtype: ICMPDataSet + :param: str protocol: the ICMP-like protocol + :param PeerSet src_pods: the source pods + :param PeerSet dst_pods: the destination pods + :return: a tuple (ICMPDataSet, TcpLikeProperties), + where ICMPDataSet is an object representing the allowed ICMP connections, + TcpLikeProperties is an optimized-format ICMP connections, including src and dst pods. + :rtype: tuple (ICMPDataSet, TcpLikeProperties) """ icmp_type = icmp_data.get('type') if icmp_data is not None else None icmp_code = icmp_data.get('code') if icmp_data is not None else None @@ -386,23 +391,55 @@ def _parse_icmp(self, icmp_data, not_icmp_data): if err: self.syntax_error(err, not_icmp_data) - res = ICMPDataSet(icmp_data is None and not_icmp_data is None) + #res = ICMPDataSet(icmp_data is None and not_icmp_data is None) + res = TcpLikeProperties.make_icmp_properties(self.peer_container) + if src_pods and dst_pods: + opt_props = TcpLikeProperties.make_icmp_properties(self.peer_container, protocol=protocol, + src_peers=src_pods, dst_peers=dst_pods) + else: + opt_props = TcpLikeProperties.make_empty_properties(self.peer_container) if icmp_data is not None: - res.add_to_set(icmp_type, icmp_code) + #res.add_to_set(icmp_type, icmp_code) + res = TcpLikeProperties.make_icmp_properties(self.peer_container, icmp_type=icmp_type, icmp_code=icmp_code) + if src_pods and dst_pods: + opt_props = TcpLikeProperties.make_icmp_properties(self.peer_container, protocol=protocol, + src_peers=src_pods, dst_peers=dst_pods, + icmp_type=icmp_type, icmp_code=icmp_code) + else: + opt_props = TcpLikeProperties.make_empty_properties(self.peer_container) if not_icmp_data is not None: if icmp_type == not_icmp_type and icmp_code == not_icmp_code: - res = ICMPDataSet() + #res = ICMPDataSet() + res = TcpLikeProperties.make_empty_properties(self.peer_container) + opt_props = TcpLikeProperties.make_empty_properties(self.peer_container) self.warning('icmp and notICMP are conflicting - no traffic will be matched', not_icmp_data) elif icmp_type == not_icmp_type and icmp_code is None: - tmp = ICMPDataSet() # this is the only case where it makes sense to combine icmp and notICMP - tmp.add_to_set(not_icmp_type, not_icmp_code) + #tmp = ICMPDataSet() # this is the only case where it makes sense to combine icmp and notICMP + #tmp.add_to_set(not_icmp_type, not_icmp_code) + tmp = TcpLikeProperties.make_icmp_properties(self.peer_container, icmp_type=not_icmp_type, + icmp_code=not_icmp_code) res -= tmp + tmp_opt_props = TcpLikeProperties.make_icmp_properties(self.peer_container, protocol=protocol, + src_peers=src_pods, dst_peers=dst_pods, + icmp_type=not_icmp_type, + icmp_code=not_icmp_code) + opt_props -= tmp_opt_props else: self.warning('notICMP has no effect', not_icmp_data) elif not_icmp_data is not None: - res.add_all_but_given_pair(not_icmp_type, not_icmp_code) + #res.add_all_but_given_pair(not_icmp_type, not_icmp_code) + res = TcpLikeProperties.make_all_but_given_icmp_properties(self.peer_container, + icmp_type=not_icmp_type, + icmp_code=not_icmp_code) + if src_pods and dst_pods: + opt_props = TcpLikeProperties.make_all_but_given_icmp_properties(self.peer_container, protocol=protocol, + src_peers=src_pods, dst_peers=dst_pods, + icmp_type=not_icmp_type, + icmp_code=not_icmp_code) + else: + opt_props = TcpLikeProperties.make_empty_properties(self.peer_container) - return res + return res, opt_props def _parse_protocol(self, protocol, rule): """ @@ -467,7 +504,7 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): src_res_pods &= policy_selected_eps connections = ConnectionSet() - tcp_props = None + tcp_props = TcpLikeProperties.make_empty_properties(self.peer_container) if protocol is not None: protocols = ProtocolSet() protocols.add_protocol(protocol) @@ -478,39 +515,42 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): self.warning('notProtocol field has no effect', rule) else: if protocol_supports_ports: - connections.add_connections(protocol, TcpLikeProperties(src_res_ports, dst_res_ports)) + connections.add_connections(protocol, TcpLikeProperties.make_tcp_like_properties( + self.peer_container, src_ports=src_res_ports, dst_ports=dst_res_ports)) if src_res_pods and dst_res_pods: src_num_port_set = PortSet() src_num_port_set.port_set = src_res_ports.port_set.copy() dst_num_port_set = PortSet() dst_num_port_set.port_set = dst_res_ports.port_set.copy() - tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, src_num_port_set, - dst_num_port_set, protocols, - src_res_pods, dst_res_pods) + tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, + src_ports=src_num_port_set, + dst_ports=dst_num_port_set, + protocols=protocols, + src_peers=src_res_pods, + dst_peers=dst_res_pods) elif ConnectionSet.protocol_is_icmp(protocol): - connections.add_connections(protocol, self._parse_icmp(rule.get('icmp'), rule.get('notICMP'))) - # TODO - update tcp_props + icmp_props, tcp_props = self._parse_icmp(rule.get('icmp'), rule.get('notICMP'), + protocol, src_res_pods, dst_res_pods) + connections.add_connections(protocol, icmp_props) else: connections.add_connections(protocol, True) if src_res_pods and dst_res_pods: - tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), - PortSet(True), protocols, - src_res_pods, dst_res_pods) + tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, protocols=protocols, + src_peers=src_res_pods, + dst_peers=dst_res_pods) elif not_protocol is not None: connections.add_all_connections() connections.remove_protocol(not_protocol) if src_res_pods and dst_res_pods: protocols = ProtocolSet(True) protocols.remove_protocol(not_protocol) - tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), - PortSet(True), protocols, - src_res_pods, dst_res_pods) + tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, protocols=protocols, + src_peers=src_res_pods, dst_peers=dst_res_pods) else: connections.allow_all = True if src_res_pods and dst_res_pods: - tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), - PortSet(True), ProtocolSet(True), - src_res_pods, dst_res_pods) + tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, + src_peers=src_res_pods, dst_peers=dst_res_pods) self._verify_named_ports(rule, dst_res_pods, connections) if not src_res_pods and policy_selected_eps and (is_ingress or not is_profile): diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index 2d61430ac..655af8050 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -173,14 +173,14 @@ def _make_default_connections(self, hosts_dfa, paths_dfa=None): if self.default_backend_peers: if paths_dfa: default_conns = \ - TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), - self.default_backend_ports, + TcpLikeProperties.make_tcp_like_properties(self.peer_container, + dst_ports=self.default_backend_ports, dst_peers=self.default_backend_peers, paths_dfa=paths_dfa, hosts_dfa=hosts_dfa) else: default_conns = \ - TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), - self.default_backend_ports, + TcpLikeProperties.make_tcp_like_properties(self.peer_container, + dst_ports=self.default_backend_ports, dst_peers=self.default_backend_peers, hosts_dfa=hosts_dfa) return default_conns @@ -208,9 +208,9 @@ def parse_rule(self, rule): parsed_paths_with_dfa = self.segregate_longest_paths_and_make_dfa(parsed_paths) for (_, paths_dfa, _, peers, ports) in parsed_paths_with_dfa: # every path is converted to allowed connections - conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), ports, - dst_peers=peers, - paths_dfa=paths_dfa, hosts_dfa=hosts_dfa) + conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, dst_ports=ports, + dst_peers=peers, paths_dfa=paths_dfa, + hosts_dfa=hosts_dfa) if not allowed_conns: allowed_conns = conns else: diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index 042814dbd..faa470b8b 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -330,7 +330,7 @@ def make_allowed_connections(self, vs, host_dfa): for http_route in vs.http_routes: for dest in http_route.destinations: conns = \ - TcpLikeProperties.make_tcp_like_properties(self.peer_container, PortSet(True), dest.port, + TcpLikeProperties.make_tcp_like_properties(self.peer_container, dst_ports=dest.port, dst_peers=dest.service.target_pods, paths_dfa=http_route.uri_dfa, hosts_dfa=host_dfa, methods=http_route.methods) diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index 0d9dbee2e..a11cdc8ae 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -321,7 +321,7 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): src_pods = policy_selected_pods dst_pods = res_pods - res_opt_props = None # TcpLikeProperties + res_opt_props = TcpLikeProperties.make_empty_properties(self.peer_container) # TcpLikeProperties ports_array = rule.get('ports', []) if ports_array: res_conns = ConnectionSet() @@ -329,8 +329,8 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): protocol, dest_port_set = self.parse_port(port) if isinstance(protocol, str): protocol = ProtocolNameResolver.get_protocol_number(protocol) - res_conns.add_connections(protocol, TcpLikeProperties(PortSet(True), - dest_port_set)) # K8s doesn't reason about src ports + res_conns.add_connections(protocol, TcpLikeProperties.make_tcp_like_properties( + self.peer_container, dst_ports=dest_port_set)) # K8s doesn't reason about src ports if src_pods and dst_pods: protocols = ProtocolSet() protocols.add_protocol(protocol) @@ -338,17 +338,14 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): dest_num_port_set = PortSet() dest_num_port_set.port_set = dest_port_set.port_set.copy() tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, - PortSet(True), dest_num_port_set, - protocols, src_pods, dst_pods) - if res_opt_props: - res_opt_props |= tcp_props - else: - res_opt_props = tcp_props + dst_ports=dest_num_port_set, + protocols=protocols, + src_peers=src_pods, dst_peers=dst_pods) + res_opt_props |= tcp_props else: res_conns = ConnectionSet(True) if src_pods and dst_pods: res_opt_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, - PortSet(True), PortSet(True), src_peers=src_pods, dst_peers=dst_pods) if not res_pods: self.warning('Rule selects no pods', rule) @@ -459,12 +456,6 @@ def parse_policy(self): rule, optimized_props = self.parse_ingress_rule(ingress_rule, res_policy.selected_peers) res_policy.add_ingress_rule(rule) res_policy.add_optimized_ingress_props(optimized_props) - elif res_policy.affects_ingress: - # add denied connections - denied_conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, - PortSet(True), PortSet(True), - dst_peers=res_policy.selected_peers) - res_policy.add_optimized_ingress_props(denied_conns, False) egress_rules = policy_spec.get('egress', []) if egress_rules: @@ -472,12 +463,6 @@ def parse_policy(self): rule, optimized_props = self.parse_egress_rule(egress_rule, res_policy.selected_peers) res_policy.add_egress_rule(rule) res_policy.add_optimized_egress_props(optimized_props) - elif res_policy.affects_egress: - # add only non-captured connections - denied_conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, - PortSet(True), PortSet(True), - src_peers=res_policy.selected_peers) - res_policy.add_optimized_egress_props(denied_conns, False) res_policy.findings = self.warning_msgs res_policy.referenced_labels = self.referenced_labels diff --git a/nca/Resources/CalicoNetworkPolicy.py b/nca/Resources/CalicoNetworkPolicy.py index 0df9dc8f0..25516ce6b 100644 --- a/nca/Resources/CalicoNetworkPolicy.py +++ b/nca/Resources/CalicoNetworkPolicy.py @@ -128,12 +128,14 @@ def allowed_connections_optimized(self, is_ingress): :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) """ if is_ingress: - allowed = self.optimized_ingress_props.copy() if self.optimized_ingress_props else None - denied = self.optimized_denied_ingress_props.copy() if self.optimized_denied_ingress_props else None + allowed = self.optimized_ingress_props.copy() + denied = self.optimized_denied_ingress_props.copy() + captured = self.selected_peers if self.affects_ingress else Peer.PeerSet() else: - allowed = self.optimized_egress_props.copy() if self.optimized_egress_props else None - denied = self.optimized_denied_egress_props.copy() if self.optimized_denied_egress_props else None - return allowed, denied, Peer.PeerSet() + allowed = self.optimized_egress_props.copy() + denied = self.optimized_denied_egress_props.copy() + captured = self.selected_peers if self.affects_egress else Peer.PeerSet() + return allowed, denied, captured def clone_without_rule(self, rule_to_exclude, ingress_rule): """ diff --git a/nca/Resources/K8sNetworkPolicy.py b/nca/Resources/K8sNetworkPolicy.py index 47a3d48c3..b6cd580db 100644 --- a/nca/Resources/K8sNetworkPolicy.py +++ b/nca/Resources/K8sNetworkPolicy.py @@ -4,6 +4,7 @@ # from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS import Peer +from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from .NetworkPolicy import PolicyConnections, NetworkPolicy @@ -72,16 +73,13 @@ def allowed_connections_optimized(self, is_ingress): and the peer set of captured peers that are not a part of allowed connections. :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) """ - add_to_captured = Peer.PeerSet() if is_ingress: - allowed = self.optimized_ingress_props.copy() if self.optimized_ingress_props else None - if self.optimized_denied_ingress_props: - add_to_captured = self.optimized_denied_ingress_props.project_on_one_dimension('dst_peers') + allowed = self.optimized_ingress_props.copy() + captured = self.selected_peers if self.affects_ingress else Peer.PeerSet() else: - allowed = self.optimized_egress_props.copy() if self.optimized_egress_props else None - if self.optimized_denied_egress_props: - add_to_captured = self.optimized_denied_egress_props.project_on_one_dimension('src_peers') - return allowed, None, add_to_captured + allowed = self.optimized_egress_props.copy() + captured = self.selected_peers if self.affects_egress else Peer.PeerSet() + return allowed, TcpLikeProperties.make_empty_properties(), captured def clone_without_rule(self, rule_to_exclude, ingress_rule): """ diff --git a/nca/Resources/NetworkPolicy.py b/nca/Resources/NetworkPolicy.py index 6477f2a57..e716e760b 100644 --- a/nca/Resources/NetworkPolicy.py +++ b/nca/Resources/NetworkPolicy.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import PeerSet +from nca.CoreDS.TcpLikeProperties import TcpLikeProperties class NetworkPolicy: @@ -52,10 +53,10 @@ def __init__(self, name, namespace): self.selected_peers = PeerSet() # The peers affected by this policy self.ingress_rules = [] self.egress_rules = [] - self.optimized_ingress_props = None # all properties in hypercube set format - self.optimized_denied_ingress_props = None # all denied properties in hypercube set format - self.optimized_egress_props = None # all properties in hypercube set format - self.optimized_denied_egress_props = None # all denied properties in hypercube set format + self.optimized_ingress_props = TcpLikeProperties.make_empty_properties() # allowed properties in hypercube set format + self.optimized_denied_ingress_props = TcpLikeProperties.make_empty_properties() # denied properties in hypercube set format + self.optimized_egress_props = TcpLikeProperties.make_empty_properties() # allowed properties in hypercube set format + self.optimized_denied_egress_props = TcpLikeProperties.make_empty_properties() # denied properties in hypercube set format self.affects_ingress = False # whether the policy affects the ingress of the selected peers self.affects_egress = False # whether the policy affects the egress of the selected peers self.findings = [] # accumulated findings which are relevant only to this policy (emptiness and redundancy) diff --git a/tests/calico_testcases/example_policies/testcase14-icmp/testcase14-scheme.yaml b/tests/calico_testcases/example_policies/testcase14-icmp/testcase14-scheme.yaml index c4adf08b9..e4df8217f 100644 --- a/tests/calico_testcases/example_policies/testcase14-icmp/testcase14-scheme.yaml +++ b/tests/calico_testcases/example_policies/testcase14-icmp/testcase14-scheme.yaml @@ -51,6 +51,21 @@ networkConfigList: expectedWarnings: 0 queries: + - name: connectivity_map + connectivityMap: + - icmp-no-type + - match-icmp-only-between-namespaces + - np_kube-system-non-matching-icmps + - match-icmp-also-within-default + - default-open-but-vacuous + - match-not-icmp + - match-icmp-within-kube-system + - no-match-notICMP + expected: 0 + outputConfiguration: + outputFormat: txt + fwRulesRunInTestMode: false + - name: policies_not_empty emptiness: - match-icmp-only-between-namespaces diff --git a/tests/calico_testcases/example_policies/testcase20-hostendpoint/testcase20-scheme.yaml b/tests/calico_testcases/example_policies/testcase20-hostendpoint/testcase20-scheme.yaml index d350aab68..87ff0948b 100644 --- a/tests/calico_testcases/example_policies/testcase20-hostendpoint/testcase20-scheme.yaml +++ b/tests/calico_testcases/example_policies/testcase20-hostendpoint/testcase20-scheme.yaml @@ -35,6 +35,14 @@ networkConfigList: expectedWarnings: 0 queries: + - name: connectivity_map + connectivityMap: + - Eran_gnps + expected: 0 + outputConfiguration: + outputFormat: txt + fwRulesRunInTestMode: false + - name: sanity_Eran_gnps sanity: - Eran_gnps From 0a8d53f3976f90c024f7fffd10201bb6d3d28650 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 3 Jan 2023 15:39:31 +0200 Subject: [PATCH 020/187] Added command line flag optimized_run, having 3 possibilities: 'false' - only original run 'true' - only optimized run 'debug'- both runs and comparison of their results. Printing parsing time, queries time and total run time. Signed-off-by: Tanya --- nca/NetworkConfig/NetworkConfig.py | 6 +- nca/NetworkConfig/NetworkConfigQuery.py | 107 ++++++++++++------------ nca/NetworkConfig/PoliciesFinder.py | 9 +- nca/NetworkConfig/ResourcesHandler.py | 16 ++-- nca/Parsers/CalicoPolicyYamlParser.py | 40 ++++----- nca/Parsers/K8sPolicyYamlParser.py | 8 +- nca/SchemeRunner.py | 18 ++-- nca/nca_cli.py | 10 ++- 8 files changed, 115 insertions(+), 99 deletions(-) diff --git a/nca/NetworkConfig/NetworkConfig.py b/nca/NetworkConfig/NetworkConfig.py index 5b3a4b323..60c53b298 100644 --- a/nca/NetworkConfig/NetworkConfig.py +++ b/nca/NetworkConfig/NetworkConfig.py @@ -48,7 +48,7 @@ class NetworkConfig: The class also contains the core algorithm of computing allowed connections between two endpoints. """ - def __init__(self, name, peer_container, policies_container): + def __init__(self, name, peer_container, policies_container, optimized_run='false'): """ :param str name: A name for this config :param PeerContainer peer_container: The set of endpoints and their namespaces @@ -56,6 +56,7 @@ def __init__(self, name, peer_container, policies_container): self.name = name self.peer_container = peer_container self.policies_container = policies_container + self.optimized_run = optimized_run self.allowed_labels = None self.referenced_ip_blocks = None @@ -105,7 +106,8 @@ def clone_without_policies(self, name): :rtype: NetworkConfig """ policies_container = PoliciesContainer() - res = NetworkConfig(name, peer_container=self.peer_container, policies_container=policies_container) + res = NetworkConfig(name, peer_container=self.peer_container, policies_container=policies_container, + optimized_run=self.optimized_run) return res def clone_without_policy(self, policy_to_exclude): diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 9650f0bc5..239384404 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -649,52 +649,54 @@ def exec(self): connections = defaultdict(list) peers = PeerSet() peers_to_compare |= ref_ip_blocks - - for peer1 in peers_to_compare: - for peer2 in peers_to_compare: - if self.is_in_subset(peer1): - peers.add(peer1) - elif not self.is_in_subset(peer2): - continue # skipping pairs if none of them are in the given subset - if isinstance(peer1, IpBlock) and isinstance(peer2, IpBlock): - continue # skipping pairs with ip-blocks for both src and dst - if peer1 == peer2: - # cannot restrict pod's connection to itself - connections[ConnectionSet(True)].append((peer1, peer2)) - else: - conns, _, _, _ = self.config.allowed_connections(peer1, peer2) - if conns: - # TODO: consider separate connectivity maps for config that involves istio - - # one that handles non-TCP connections, and one for TCP - # TODO: consider avoid "hiding" egress allowed connections, even though they are - # not covered by authorization policies - if self.config.policies_container.layers.does_contain_single_layer(NetworkLayerName.Istio) and \ - self.output_config.connectivityFilterIstioEdges: - should_filter, modified_conns = self.filter_istio_edge(peer2, conns) - if not should_filter: - connections[modified_conns].append((peer1, peer2)) + res = QueryAnswer(True) + conn_graph = None + if self.config.optimized_run != 'true': + for peer1 in peers_to_compare: + for peer2 in peers_to_compare: + if self.is_in_subset(peer1): + peers.add(peer1) + elif not self.is_in_subset(peer2): + continue # skipping pairs if none of them are in the given subset + if isinstance(peer1, IpBlock) and isinstance(peer2, IpBlock): + continue # skipping pairs with ip-blocks for both src and dst + if peer1 == peer2: + # cannot restrict pod's connection to itself + connections[ConnectionSet(True)].append((peer1, peer2)) + else: + conns, _, _, _ = self.config.allowed_connections(peer1, peer2) + if conns: + # TODO: consider separate connectivity maps for config that involves istio - + # one that handles non-TCP connections, and one for TCP + # TODO: consider avoid "hiding" egress allowed connections, even though they are + # not covered by authorization policies + if self.config.policies_container.layers.does_contain_single_layer(NetworkLayerName.Istio) and \ + self.output_config.connectivityFilterIstioEdges: + should_filter, modified_conns = self.filter_istio_edge(peer2, conns) + if not should_filter: + connections[modified_conns].append((peer1, peer2)) + # collect both peers, even if one of them is not in the subset + peers.add(peer1) + peers.add(peer2) + else: + connections[conns].append((peer1, peer2)) # collect both peers, even if one of them is not in the subset peers.add(peer1) peers.add(peer2) - else: - connections[conns].append((peer1, peer2)) - # collect both peers, even if one of them is not in the subset - peers.add(peer1) - peers.add(peer2) - res = QueryAnswer(True) - fw_rules = None - if self.output_config.outputFormat == 'dot': - conn_graph = ConnectivityGraph(peers, self.config.get_allowed_labels(), self.output_config) - conn_graph.add_edges(connections) - res.output_explanation = conn_graph.get_connectivity_dot_format_str() - else: - conn_graph = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) - conn_graph.add_edges(connections) - fw_rules = conn_graph.get_minimized_firewall_rules() - res.output_explanation = fw_rules.get_fw_rules_in_required_format() - - all_conns_opt = self.config.allowed_connections_optimized() + if self.output_config.outputFormat == 'dot': + conn_graph = ConnectivityGraph(peers, self.config.get_allowed_labels(), self.output_config) + conn_graph.add_edges(connections) + res.output_explanation = conn_graph.get_connectivity_dot_format_str() + else: + conn_graph = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) + conn_graph.add_edges(connections) + fw_rules = conn_graph.get_minimized_firewall_rules() + res.output_explanation = fw_rules.get_fw_rules_in_required_format() + + all_conns_opt = TcpLikeProperties.make_empty_properties() + if self.config.optimized_run != 'false': + all_conns_opt = self.config.allowed_connections_optimized() if all_conns_opt: subset_peers = self.compute_subset(peers_to_compare) src_peers_in_subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, @@ -703,7 +705,7 @@ def exec(self): dst_peers=subset_peers) all_conns_opt &= src_peers_in_subset_conns | dst_peers_in_subset_conns conn_graph2 = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) - conn_graph_opt = ConnectivityGraphOptimized(self.output_config) + #conn_graph_opt = ConnectivityGraphOptimized(self.output_config) # Add connections from peer to itself (except for HEPs) auto_conns = defaultdict(list) for peer in subset_peers: @@ -713,17 +715,18 @@ def exec(self): for cube in all_conns_opt: conn_graph2.add_edges_from_cube_dict(self.config.peer_container, all_conns_opt.get_cube_dict_with_orig_values(cube)) - conn_graph_opt.add_edge(all_conns_opt.get_cube_dict(cube)) - res_opt = QueryAnswer(True) + #conn_graph_opt.add_edge(all_conns_opt.get_cube_dict(cube)) if self.output_config.outputFormat == 'dot': - res_opt.output_explanation = conn_graph_opt.get_connectivity_dot_format_str() - # Tanya: temp for debugging - orig_conn_graph = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) - orig_conn_graph.add_edges(connections) - self.compare_conn_graphs(orig_conn_graph, conn_graph2) - # Tanya: end temp for debugging + res.output_explanation = conn_graph2.get_connectivity_dot_format_str() + if self.config.optimized_run == 'debug': + orig_conn_graph = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) + orig_conn_graph.add_edges(connections) + self.compare_conn_graphs(orig_conn_graph, conn_graph2) else: - self.compare_conn_graphs(conn_graph, conn_graph2) + fw_rules2 = conn_graph2.get_minimized_firewall_rules() + res.output_explanation = fw_rules2.get_fw_rules_in_required_format() + if self.config.optimized_run == 'debug': + self.compare_conn_graphs(conn_graph, conn_graph2) # res_opt.output_explanation = conn_graph_opt.get_connectivity_txt_format_str() # res.output_explanation += "---------------- OPTIMIZED RESULT: -------------\n" +\ # fw_rules2.get_fw_rules_in_required_format() + \ diff --git a/nca/NetworkConfig/PoliciesFinder.py b/nca/NetworkConfig/PoliciesFinder.py index 279f9d3d6..cfd7a3b60 100644 --- a/nca/NetworkConfig/PoliciesFinder.py +++ b/nca/NetworkConfig/PoliciesFinder.py @@ -21,10 +21,11 @@ class PoliciesFinder: This class is responsible for finding the network policies in the relevant input resources The class contains several ways to build the set of policies (from cluster, from file-system, from GitHub). """ - def __init__(self): + def __init__(self, optimized_run='false'): self.policies_container = PoliciesContainer() self._parse_queue = deque() self.peer_container = None + self.optimized_run=optimized_run def set_peer_container(self, peer_container): """ @@ -64,11 +65,11 @@ def parse_policies_in_parse_queue(self): istio_traffic_parser = None for policy, file_name, policy_type in self._parse_queue: if policy_type == NetworkPolicy.PolicyType.CalicoProfile: - parsed_element = CalicoPolicyYamlParser(policy, self.peer_container, file_name) + parsed_element = CalicoPolicyYamlParser(policy, self.peer_container, file_name, self.optimized_run) # only during parsing adding extra labels from profiles (not supporting profiles with rules) parsed_element.parse_policy() elif policy_type == NetworkPolicy.PolicyType.K8sNetworkPolicy: - parsed_element = K8sPolicyYamlParser(policy, self.peer_container, file_name) + parsed_element = K8sPolicyYamlParser(policy, self.peer_container, file_name, self.optimized_run) self._add_policy(parsed_element.parse_policy()) elif policy_type == NetworkPolicy.PolicyType.IstioAuthorizationPolicy: parsed_element = IstioPolicyYamlParser(policy, self.peer_container, file_name) @@ -88,7 +89,7 @@ def parse_policies_in_parse_queue(self): istio_traffic_parser = IstioTrafficResourcesYamlParser(self.peer_container) istio_traffic_parser.parse_virtual_service(policy, file_name) else: - parsed_element = CalicoPolicyYamlParser(policy, self.peer_container, file_name) + parsed_element = CalicoPolicyYamlParser(policy, self.peer_container, file_name, self.optimized_run) self._add_policy(parsed_element.parse_policy()) if istio_traffic_parser: istio_traffic_policies = istio_traffic_parser.create_istio_traffic_policies() diff --git a/nca/NetworkConfig/ResourcesHandler.py b/nca/NetworkConfig/ResourcesHandler.py index 93faf8d30..f529a9a6f 100644 --- a/nca/NetworkConfig/ResourcesHandler.py +++ b/nca/NetworkConfig/ResourcesHandler.py @@ -29,7 +29,7 @@ def __init__(self): self.global_pods_finder = None self.global_ns_finder = None - def set_global_peer_container(self, global_ns_list, global_pod_list, global_resource_list): + def set_global_peer_container(self, global_ns_list, global_pod_list, global_resource_list, optimized_run='false'): """ builds the global peer container based on global input resources, it also saves the global pods and namespaces finder, to use in case specific configs missing one of them. @@ -39,11 +39,12 @@ def set_global_peer_container(self, global_ns_list, global_pod_list, global_reso :param Union[list[str], None] global_resource_list: list of global entries of namespaces/pods to handle in case specific list is None """ - global_resources_parser = ResourcesParser() + global_resources_parser = ResourcesParser(optimized_run) self._set_config_peer_container(global_ns_list, global_pod_list, global_resource_list, 'global', True, global_resources_parser) - def get_network_config(self, np_list, ns_list, pod_list, resource_list, config_name='global', save_flag=False): + def get_network_config(self, np_list, ns_list, pod_list, resource_list, config_name='global', save_flag=False, + optimized_run='false'): """ First tries to build a peer_container using the input resources (NetworkConfigs's resources) If fails, it uses the global peer container. @@ -58,7 +59,7 @@ def get_network_config(self, np_list, ns_list, pod_list, resource_list, config_n will save the peer container as global to use it for base config's peer resources in case are missing :rtype NetworkConfig """ - resources_parser = ResourcesParser() + resources_parser = ResourcesParser(optimized_run) # build peer container peer_container = \ self._set_config_peer_container(ns_list, pod_list, resource_list, config_name, save_flag, resources_parser) @@ -71,7 +72,8 @@ def get_network_config(self, np_list, ns_list, pod_list, resource_list, config_n # build and return the networkConfig return NetworkConfig(name=config_name, peer_container=peer_container, - policies_container=resources_parser.policies_finder.policies_container) + policies_container=resources_parser.policies_finder.policies_container, + optimized_run=optimized_run) def _set_config_peer_container(self, ns_list, pod_list, resource_list, config_name, save_flag, resources_parser): success, res_type = resources_parser.parse_lists_for_topology(ns_list, pod_list, resource_list) @@ -123,8 +125,8 @@ class ResourcesParser: """ This class parses the input resources for topology (pods, namespaces, services) and policies. """ - def __init__(self): - self.policies_finder = PoliciesFinder() + def __init__(self, optimized_run='false'): + self.policies_finder = PoliciesFinder(optimized_run) self.pods_finder = PodsFinder() self.ns_finder = NamespacesFinder() self.services_finder = ServicesFinder() diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index 53e74e32a..f8afd2a72 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -22,7 +22,7 @@ class CalicoPolicyYamlParser(GenericYamlParser): A parser for Calico NetworkPolicy/GlobalNetworkPolicy/Profile objects """ - def __init__(self, policy, peer_container, policy_file_name=''): + def __init__(self, policy, peer_container, policy_file_name='', optimized_run='false'): """ :param dict policy: The policy object as provided by the yaml parser :param PeerContainer peer_container: The policy will be evaluated against this set of peers @@ -34,6 +34,7 @@ def __init__(self, policy, peer_container, policy_file_name=''): self.namespace = None # collecting labels used in calico network policy for fw-rules computation self.referenced_labels = set() + self.optimized_run = optimized_run def _parse_selector_expr(self, expr, origin_map, namespace, is_namespace_selector): """ @@ -393,25 +394,21 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): #res = ICMPDataSet(icmp_data is None and not_icmp_data is None) res = TcpLikeProperties.make_icmp_properties(self.peer_container) - if src_pods and dst_pods: + opt_props = TcpLikeProperties.make_empty_properties(self.peer_container) + if self.optimized_run != 'false' and src_pods and dst_pods: opt_props = TcpLikeProperties.make_icmp_properties(self.peer_container, protocol=protocol, src_peers=src_pods, dst_peers=dst_pods) - else: - opt_props = TcpLikeProperties.make_empty_properties(self.peer_container) if icmp_data is not None: #res.add_to_set(icmp_type, icmp_code) res = TcpLikeProperties.make_icmp_properties(self.peer_container, icmp_type=icmp_type, icmp_code=icmp_code) - if src_pods and dst_pods: + if self.optimized_run != 'false' and src_pods and dst_pods: opt_props = TcpLikeProperties.make_icmp_properties(self.peer_container, protocol=protocol, src_peers=src_pods, dst_peers=dst_pods, icmp_type=icmp_type, icmp_code=icmp_code) - else: - opt_props = TcpLikeProperties.make_empty_properties(self.peer_container) if not_icmp_data is not None: if icmp_type == not_icmp_type and icmp_code == not_icmp_code: #res = ICMPDataSet() res = TcpLikeProperties.make_empty_properties(self.peer_container) - opt_props = TcpLikeProperties.make_empty_properties(self.peer_container) self.warning('icmp and notICMP are conflicting - no traffic will be matched', not_icmp_data) elif icmp_type == not_icmp_type and icmp_code is None: #tmp = ICMPDataSet() # this is the only case where it makes sense to combine icmp and notICMP @@ -419,11 +416,12 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): tmp = TcpLikeProperties.make_icmp_properties(self.peer_container, icmp_type=not_icmp_type, icmp_code=not_icmp_code) res -= tmp - tmp_opt_props = TcpLikeProperties.make_icmp_properties(self.peer_container, protocol=protocol, - src_peers=src_pods, dst_peers=dst_pods, - icmp_type=not_icmp_type, - icmp_code=not_icmp_code) - opt_props -= tmp_opt_props + if self.optimized_run != 'false': + tmp_opt_props = TcpLikeProperties.make_icmp_properties(self.peer_container, protocol=protocol, + src_peers=src_pods, dst_peers=dst_pods, + icmp_type=not_icmp_type, + icmp_code=not_icmp_code) + opt_props -= tmp_opt_props else: self.warning('notICMP has no effect', not_icmp_data) elif not_icmp_data is not None: @@ -431,13 +429,11 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): res = TcpLikeProperties.make_all_but_given_icmp_properties(self.peer_container, icmp_type=not_icmp_type, icmp_code=not_icmp_code) - if src_pods and dst_pods: + if self.optimized_run != 'false' and src_pods and dst_pods: opt_props = TcpLikeProperties.make_all_but_given_icmp_properties(self.peer_container, protocol=protocol, src_peers=src_pods, dst_peers=dst_pods, icmp_type=not_icmp_type, icmp_code=not_icmp_code) - else: - opt_props = TcpLikeProperties.make_empty_properties(self.peer_container) return res, opt_props @@ -517,7 +513,7 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): if protocol_supports_ports: connections.add_connections(protocol, TcpLikeProperties.make_tcp_like_properties( self.peer_container, src_ports=src_res_ports, dst_ports=dst_res_ports)) - if src_res_pods and dst_res_pods: + if self.optimized_run != 'false' and src_res_pods and dst_res_pods: src_num_port_set = PortSet() src_num_port_set.port_set = src_res_ports.port_set.copy() dst_num_port_set = PortSet() @@ -534,21 +530,21 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): connections.add_connections(protocol, icmp_props) else: connections.add_connections(protocol, True) - if src_res_pods and dst_res_pods: + if self.optimized_run != 'false' and src_res_pods and dst_res_pods: tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, protocols=protocols, src_peers=src_res_pods, dst_peers=dst_res_pods) elif not_protocol is not None: connections.add_all_connections() connections.remove_protocol(not_protocol) - if src_res_pods and dst_res_pods: + if self.optimized_run != 'false' and src_res_pods and dst_res_pods: protocols = ProtocolSet(True) protocols.remove_protocol(not_protocol) tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, protocols=protocols, src_peers=src_res_pods, dst_peers=dst_res_pods) else: connections.allow_all = True - if src_res_pods and dst_res_pods: + if self.optimized_run != 'false' and src_res_pods and dst_res_pods: tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, src_peers=src_res_pods, dst_peers=dst_res_pods) self._verify_named_ports(rule, dst_res_pods, connections) @@ -700,7 +696,7 @@ def parse_policy(self): for ingress_rule in policy_spec.get('ingress', []): rule, optimized_props = self._parse_xgress_rule(ingress_rule, True, res_policy.selected_peers, is_profile) res_policy.add_ingress_rule(rule) - if rule.action != CalicoPolicyRule.ActionType.Pass: + if self.optimized_run != 'false' and rule.action != CalicoPolicyRule.ActionType.Pass: # handle the order of rules if rule.action == CalicoPolicyRule.ActionType.Allow and res_policy.optimized_denied_ingress_props: optimized_props -= res_policy.optimized_denied_ingress_props @@ -711,7 +707,7 @@ def parse_policy(self): for egress_rule in policy_spec.get('egress', []): rule, optimized_props = self._parse_xgress_rule(egress_rule, False, res_policy.selected_peers, is_profile) res_policy.add_egress_rule(rule) - if rule.action != CalicoPolicyRule.ActionType.Pass: + if self.optimized_run != 'false' and rule.action != CalicoPolicyRule.ActionType.Pass: # handle the order of rules if rule.action == CalicoPolicyRule.ActionType.Allow and res_policy.optimized_denied_egress_props: optimized_props -= res_policy.optimized_denied_egress_props diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index a11cdc8ae..6ab838037 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -20,7 +20,7 @@ class K8sPolicyYamlParser(GenericYamlParser): A parser for k8s NetworkPolicy objects """ - def __init__(self, policy, peer_container, policy_file_name=''): + def __init__(self, policy, peer_container, policy_file_name='', optimized_run='false'): """ :param dict policy: The policy object as provided by the yaml parser :param PeerContainer peer_container: The policy will be evaluated against this set of peers @@ -31,6 +31,7 @@ def __init__(self, policy, peer_container, policy_file_name=''): self.peer_container = peer_container self.namespace = None self.referenced_labels = set() + self.optimized_run = optimized_run def check_dns_subdomain_name(self, value, key_container): """ @@ -331,10 +332,9 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): protocol = ProtocolNameResolver.get_protocol_number(protocol) res_conns.add_connections(protocol, TcpLikeProperties.make_tcp_like_properties( self.peer_container, dst_ports=dest_port_set)) # K8s doesn't reason about src ports - if src_pods and dst_pods: + if self.optimized_run != 'false' and src_pods and dst_pods: protocols = ProtocolSet() protocols.add_protocol(protocol) - dest_num_port_set = PortSet() dest_num_port_set.port_set = dest_port_set.port_set.copy() tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, @@ -344,7 +344,7 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): res_opt_props |= tcp_props else: res_conns = ConnectionSet(True) - if src_pods and dst_pods: + if self.optimized_run != 'false' and src_pods and dst_pods: res_opt_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, src_peers=src_pods, dst_peers=dst_pods) if not res_pods: diff --git a/nca/SchemeRunner.py b/nca/SchemeRunner.py index bb515b151..fdbcd9e5a 100644 --- a/nca/SchemeRunner.py +++ b/nca/SchemeRunner.py @@ -18,7 +18,7 @@ class SchemeRunner(GenericYamlParser): This class takes a scheme file, build all its network configurations and runs all its queries """ - def __init__(self, scheme_file_name, output_format=None, output_path=None): + def __init__(self, scheme_file_name, output_format=None, output_path=None, optimized_run='false'): GenericYamlParser.__init__(self, scheme_file_name) self.network_configs = {} self.global_res = 0 @@ -27,6 +27,7 @@ def __init__(self, scheme_file_name, output_format=None, output_path=None): self.output_config_from_cli_args['outputFormat'] = output_format if output_path is not None: self.output_config_from_cli_args['outputPath'] = output_path + self.optimized_run=optimized_run with open(scheme_file_name) as scheme_file: yaml = YAML() @@ -73,7 +74,7 @@ def _handle_resources_list(self, resources_list): input_file_list.append(resource_path) return input_file_list - def _add_config(self, config_entry, resources_handler): + def _add_config(self, config_entry, resources_handler, optimized_run): """ Produces a NetworkConfig object for a given entry in the scheme file. Increases self.global_res if the number of warnings/error in the config does not match the expected number. @@ -101,7 +102,7 @@ def _add_config(self, config_entry, resources_handler): expected_error = config_entry.get('expectedError') try: network_config = resources_handler.get_network_config(np_list, ns_list, pod_list, resource_list, - config_name) + config_name, optimized_run=optimized_run) if not network_config: self.warning(f'networkPolicyList {network_config.name} contains no networkPolicies', np_list) @@ -140,13 +141,20 @@ def run_scheme(self): global_ns_list = self._handle_resources_list(self.scheme.get('namespaceList', None)) global_resource_list = self._handle_resources_list(self.scheme.get('resourceList', None)) resources_handler = ResourcesHandler() - resources_handler.set_global_peer_container(global_ns_list, global_pod_list, global_resource_list) + resources_handler.set_global_peer_container(global_ns_list, global_pod_list, global_resource_list, + self.optimized_run) # specified configs (non-global) + start = time.time() for config_entry in self.scheme.get('networkConfigList', []): - self._add_config(config_entry, resources_handler) + self._add_config(config_entry, resources_handler, self.optimized_run) + end_parse = time.time() self.run_queries(self.scheme.get('queries', [])) + end_queries = time.time() + print(f'Parsing time: {(end_parse - start):6.2f} seconds') + print(f'Queries time: {(end_queries - end_parse):6.2f} seconds') + print(f'Total time: {(end_queries - start):6.2f} seconds') return self.global_res def get_query_output_config_obj(self, query): diff --git a/nca/nca_cli.py b/nca/nca_cli.py index b452b7b4f..00e7dbb21 100644 --- a/nca/nca_cli.py +++ b/nca/nca_cli.py @@ -141,7 +141,7 @@ def run_args(args): :rtype: int """ if args.scheme: - return SchemeRunner(args.scheme, args.output_format, args.file_out).run_scheme() + return SchemeRunner(args.scheme, args.output_format, args.file_out, args.optimized_run).run_scheme() ns_list = args.ns_list pod_list = args.pod_list resource_list = args.resource_list @@ -214,7 +214,7 @@ def run_args(args): resources_handler = ResourcesHandler() network_config = resources_handler.get_network_config(_make_recursive(np_list), _make_recursive(ns_list), _make_recursive(pod_list), _make_recursive(resource_list), - save_flag=pair_query_flag) + save_flag=pair_query_flag, optimized_run=args.optimized_run) if pair_query_flag: base_np_list = args.base_np_list base_resource_list = args.base_resource_list @@ -223,7 +223,8 @@ def run_args(args): base_network_config = resources_handler.get_network_config(_make_recursive(base_np_list), _make_recursive(base_ns_list), _make_recursive(base_pod_list), - _make_recursive(base_resource_list)) + _make_recursive(base_resource_list), + optimized_run=args.optimized_run) if base_as_second: network_configs_array = [network_config, base_network_config] else: @@ -310,6 +311,9 @@ def nca_main(argv=None): parser.add_argument('--version', '-v', action='store_true', help='Print version and exit') parser.add_argument('--output_endpoints', choices=['pods', 'deployments'], help='Choose endpoints type in output (pods/deployments)', default='deployments') + parser.add_argument('--optimized_run', '-opt', type=str, + help='Whether to run optimized run (-opt=true), original run (-opt=false) - the default ' + 'or the comparison of the both (debug)', default='false') args = parser.parse_args(argv) From 183eaeb6c5e25130e79744b4843a2e2ce09deeee Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 8 Jan 2023 13:11:01 +0200 Subject: [PATCH 021/187] Small fixes; Workaround for the bug in HC set: using mutual contained_in, instead of == Signed-off-by: Tanya --- nca/CoreDS/TcpLikeProperties.py | 3 ++- nca/FWRules/ConnectivityGraph.py | 3 ++- nca/NetworkConfig/NetworkConfigQuery.py | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index a8ebca8d0..e34fee92a 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -580,7 +580,8 @@ def make_tcp_like_properties_from_dict(peer_container, cube_dict): return TcpLikeProperties.make_tcp_like_properties(peer_container, src_ports=src_ports, dst_ports=dst_ports, protocols=protocols, src_peers=src_peers, dst_peers=dst_peers, paths_dfa=paths_dfa, hosts_dfa=hosts_dfa, methods=methods, - icmp_type=icmp_type, icmp_code=icmp_code) + icmp_type=icmp_type, icmp_code=icmp_code, + exclude_same_src_dst_peers=False) @staticmethod def make_empty_properties(peer_container=None): diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index c98b87bf6..371141285 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -163,7 +163,8 @@ def convert_to_tcp_like_properties(self, peer_container): for peer_pair in item[1]: res |= TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=PeerSet({peer_pair[0]}), - dst_peers=PeerSet({peer_pair[1]})) + dst_peers=PeerSet({peer_pair[1]}), + exclude_same_src_dst_peers=False) else: for prot in item[0].allowed_protocols.items(): protocols = ProtocolSet() diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 239384404..177aa42dc 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -709,7 +709,7 @@ def exec(self): # Add connections from peer to itself (except for HEPs) auto_conns = defaultdict(list) for peer in subset_peers: - if not isinstance(peer, HostEP) and not isinstance(peer, IpBlock): + if not isinstance(peer, IpBlock): auto_conns[ConnectionSet(True)].append((peer, peer)) conn_graph2.add_edges(auto_conns) for cube in all_conns_opt: @@ -736,7 +736,9 @@ def exec(self): def compare_conn_graphs(self, conn_graph1, conn_graph2): tcp_props1 = conn_graph1.convert_to_tcp_like_properties(self.config.peer_container) tcp_props2 = conn_graph2.convert_to_tcp_like_properties(self.config.peer_container) - assert tcp_props1 == tcp_props2 + assert tcp_props1.contained_in(tcp_props2) and tcp_props2.contained_in(tcp_props1) # workaround for == + # The following assert exposes the bug in HC set + assert not tcp_props1.contained_in(tcp_props2) or not tcp_props2.contained_in(tcp_props1) or tcp_props1 == tcp_props2 def compute_query_output(self, query_answer): return self.get_query_output(query_answer, only_explanation=query_answer.bool_result) From 8e34573dd1977f3f8f287b709a82367ed6593c5c Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 8 Jan 2023 13:27:12 +0200 Subject: [PATCH 022/187] Small fixes; Signed-off-by: Tanya --- nca/FWRules/ConnectivityGraph.py | 4 +--- nca/NetworkConfig/NetworkConfigQuery.py | 7 +++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index 371141285..2cbdf2ce0 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -170,9 +170,7 @@ def convert_to_tcp_like_properties(self, peer_container): protocols = ProtocolSet() protocols.add_protocol(prot[0]) if isinstance(prot[1], bool): - res |= TcpLikeProperties.make_tcp_like_properties(peer_container, protocols=protocols, - src_peers=PeerSet({peer_pair[0]}), - dst_peers=PeerSet({peer_pair[1]})) + res |= TcpLikeProperties.make_tcp_like_properties(peer_container, protocols=protocols) continue for cube in prot[1]: cube_dict = prot[1].get_cube_dict_with_orig_values(cube) diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 177aa42dc..018c77785 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -10,8 +10,7 @@ from nca.Utils.OutputConfiguration import OutputConfiguration from nca.CoreDS.ConnectionSet import ConnectionSet -from nca.CoreDS.Peer import PeerSet, IpBlock, Pod, Peer, HostEP -from nca.CoreDS.PortSet import PortSet +from nca.CoreDS.Peer import PeerSet, IpBlock, Pod, Peer from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from nca.Resources.CalicoNetworkPolicy import CalicoNetworkPolicy from nca.Resources.IngressPolicy import IngressPolicy @@ -705,7 +704,7 @@ def exec(self): dst_peers=subset_peers) all_conns_opt &= src_peers_in_subset_conns | dst_peers_in_subset_conns conn_graph2 = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) - #conn_graph_opt = ConnectivityGraphOptimized(self.output_config) + # conn_graph_opt = ConnectivityGraphOptimized(self.output_config) # Add connections from peer to itself (except for HEPs) auto_conns = defaultdict(list) for peer in subset_peers: @@ -715,7 +714,7 @@ def exec(self): for cube in all_conns_opt: conn_graph2.add_edges_from_cube_dict(self.config.peer_container, all_conns_opt.get_cube_dict_with_orig_values(cube)) - #conn_graph_opt.add_edge(all_conns_opt.get_cube_dict(cube)) + # conn_graph_opt.add_edge(all_conns_opt.get_cube_dict(cube)) if self.output_config.outputFormat == 'dot': res.output_explanation = conn_graph2.get_connectivity_dot_format_str() if self.config.optimized_run == 'debug': From 99572287edc135a30362cec2e1bf6e974b4dd763 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 8 Jan 2023 13:50:22 +0200 Subject: [PATCH 023/187] Fixed building tcp_like_properties form connectivity graph Signed-off-by: Tanya Signed-off-by: Tanya --- nca/FWRules/ConnectivityGraph.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index 2cbdf2ce0..9df73157e 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -170,7 +170,11 @@ def convert_to_tcp_like_properties(self, peer_container): protocols = ProtocolSet() protocols.add_protocol(prot[0]) if isinstance(prot[1], bool): - res |= TcpLikeProperties.make_tcp_like_properties(peer_container, protocols=protocols) + for peer_pair in item[1]: + res |= TcpLikeProperties.make_tcp_like_properties(peer_container, protocols=protocols, + src_peers=PeerSet({peer_pair[0]}), + dst_peers=PeerSet({peer_pair[1]}), + exclude_same_src_dst_peers=False) continue for cube in prot[1]: cube_dict = prot[1].get_cube_dict_with_orig_values(cube) From 004742b33337bbd48ae84021bc95892dc19efbd7 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 8 Jan 2023 16:30:50 +0200 Subject: [PATCH 024/187] Optimized the comparison between original and optimized connections (for -opt=debug option) Added more debug prints. Better handling of peer_set copying in TcpLikeProperties. Signed-off-by: Tanya --- nca/CoreDS/TcpLikeProperties.py | 8 +++--- nca/FWRules/ConnectivityGraph.py | 1 + nca/NetworkConfig/NetworkConfigQuery.py | 37 ++++++++++++++++--------- nca/SchemeRunner.py | 2 +- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index e34fee92a..3c8512a02 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -65,7 +65,7 @@ def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protoco self.named_ports = {} # a mapping from dst named port (String) to src ports interval set self.excluded_named_ports = {} # a mapping from dst named port (String) to src ports interval set - self.base_peer_set = base_peer_set.copy() if base_peer_set else PeerSet() + self.base_peer_set = base_peer_set if base_peer_set else PeerSet() if create_empty: return @@ -426,7 +426,7 @@ def make_tcp_like_properties(peer_container, src_ports=PortSet(True), dst_ports= :param CanonicalIntervalSet icmp_code: ICMP-specific parameter (code dimension) :return: TcpLikeProperties with TCP allowed connections, corresponding to input properties cube """ - base_peer_set = peer_container.peer_set + base_peer_set = peer_container.peer_set.copy() if src_peers: base_peer_set |= src_peers src_peers_interval = base_peer_set.get_peer_interval_of(src_peers) @@ -585,11 +585,11 @@ def make_tcp_like_properties_from_dict(peer_container, cube_dict): @staticmethod def make_empty_properties(peer_container=None): - return TcpLikeProperties(base_peer_set=peer_container.peer_set if peer_container else None, create_empty=True) + return TcpLikeProperties(base_peer_set=peer_container.peer_set.copy() if peer_container else None, create_empty=True) @staticmethod def make_all_properties(peer_container=None): - return TcpLikeProperties(base_peer_set=peer_container.peer_set if peer_container else None) + return TcpLikeProperties(base_peer_set=peer_container.peer_set.copy() if peer_container else None) ####################################### ICMP-related functions ####################################### diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index 9df73157e..cfbae0443 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -170,6 +170,7 @@ def convert_to_tcp_like_properties(self, peer_container): protocols = ProtocolSet() protocols.add_protocol(prot[0]) if isinstance(prot[1], bool): +# res |= TcpLikeProperties.make_tcp_like_properties(peer_container, protocols=protocols) # wrong for peer_pair in item[1]: res |= TcpLikeProperties.make_tcp_like_properties(peer_container, protocols=protocols, src_peers=PeerSet({peer_pair[0]}), diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 018c77785..e90753894 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -4,6 +4,7 @@ # import itertools import os +import time from dataclasses import dataclass from collections import defaultdict from enum import Enum @@ -651,7 +652,8 @@ def exec(self): res = QueryAnswer(True) conn_graph = None if self.config.optimized_run != 'true': - for peer1 in peers_to_compare: + peers1_start = time.time() + for peer1_cnt, peer1 in enumerate(peers_to_compare): for peer2 in peers_to_compare: if self.is_in_subset(peer1): peers.add(peer1) @@ -682,7 +684,10 @@ def exec(self): # collect both peers, even if one of them is not in the subset peers.add(peer1) peers.add(peer2) - + #peers2_end = time.time() + #print(f'Original loop: {peer1_cnt+1} / {len(peers_to_compare)} peers, time: {(peers2_end - peers1_start):6.2f} seconds') + peers1_end = time.time() + print(f'Original loop: time: {(peers1_end - peers1_start):6.2f} seconds') if self.output_config.outputFormat == 'dot': conn_graph = ConnectivityGraph(peers, self.config.get_allowed_labels(), self.output_config) conn_graph.add_edges(connections) @@ -694,6 +699,7 @@ def exec(self): res.output_explanation = fw_rules.get_fw_rules_in_required_format() all_conns_opt = TcpLikeProperties.make_empty_properties() + opt_start = time.time() if self.config.optimized_run != 'false': all_conns_opt = self.config.allowed_connections_optimized() if all_conns_opt: @@ -705,12 +711,13 @@ def exec(self): all_conns_opt &= src_peers_in_subset_conns | dst_peers_in_subset_conns conn_graph2 = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) # conn_graph_opt = ConnectivityGraphOptimized(self.output_config) - # Add connections from peer to itself (except for HEPs) - auto_conns = defaultdict(list) + # Add connections from peer to itself (except for IPs) for peer in subset_peers: if not isinstance(peer, IpBlock): - auto_conns[ConnectionSet(True)].append((peer, peer)) - conn_graph2.add_edges(auto_conns) + all_conns_opt |= TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, + src_peers=PeerSet({peer}), + dst_peers=PeerSet({peer}), + exclude_same_src_dst_peers=False) for cube in all_conns_opt: conn_graph2.add_edges_from_cube_dict(self.config.peer_container, all_conns_opt.get_cube_dict_with_orig_values(cube)) @@ -720,24 +727,28 @@ def exec(self): if self.config.optimized_run == 'debug': orig_conn_graph = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) orig_conn_graph.add_edges(connections) - self.compare_conn_graphs(orig_conn_graph, conn_graph2) + opt_end = time.time() + print(f'Opt time: {(opt_end - opt_start):6.2f} seconds') + self.compare_orig_to_opt_conn(orig_conn_graph, all_conns_opt) else: fw_rules2 = conn_graph2.get_minimized_firewall_rules() res.output_explanation = fw_rules2.get_fw_rules_in_required_format() + opt_end = time.time() + print(f'Opt time: {(opt_end - opt_start):6.2f} seconds') if self.config.optimized_run == 'debug': - self.compare_conn_graphs(conn_graph, conn_graph2) + self.compare_orig_to_opt_conn(conn_graph, all_conns_opt) # res_opt.output_explanation = conn_graph_opt.get_connectivity_txt_format_str() # res.output_explanation += "---------------- OPTIMIZED RESULT: -------------\n" +\ # fw_rules2.get_fw_rules_in_required_format() + \ # "\n------------------------------------------------\n\n" # TEMP for debug return res - def compare_conn_graphs(self, conn_graph1, conn_graph2): - tcp_props1 = conn_graph1.convert_to_tcp_like_properties(self.config.peer_container) - tcp_props2 = conn_graph2.convert_to_tcp_like_properties(self.config.peer_container) - assert tcp_props1.contained_in(tcp_props2) and tcp_props2.contained_in(tcp_props1) # workaround for == + def compare_orig_to_opt_conn(self, orig_conn_graph, opt_props): + print("Converting orig_conn_graph to tcp_like_properties...") + orig_tcp_props = orig_conn_graph.convert_to_tcp_like_properties(self.config.peer_container) + assert orig_tcp_props.contained_in(opt_props) and opt_props.contained_in(orig_tcp_props) # workaround for == # The following assert exposes the bug in HC set - assert not tcp_props1.contained_in(tcp_props2) or not tcp_props2.contained_in(tcp_props1) or tcp_props1 == tcp_props2 + assert not orig_tcp_props.contained_in(opt_props) or not opt_props.contained_in(orig_tcp_props) or orig_tcp_props == opt_props def compute_query_output(self, query_answer): return self.get_query_output(query_answer, only_explanation=query_answer.bool_result) diff --git a/nca/SchemeRunner.py b/nca/SchemeRunner.py index fdbcd9e5a..97d880231 100644 --- a/nca/SchemeRunner.py +++ b/nca/SchemeRunner.py @@ -149,7 +149,7 @@ def run_scheme(self): for config_entry in self.scheme.get('networkConfigList', []): self._add_config(config_entry, resources_handler, self.optimized_run) end_parse = time.time() - + print(f'Finished parsing in {(end_parse - start):6.2f} seconds') self.run_queries(self.scheme.get('queries', [])) end_queries = time.time() print(f'Parsing time: {(end_parse - start):6.2f} seconds') From a27899dde2cc061cba62dbae253683c0e9ca2da1 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 15 Jan 2023 15:36:31 +0200 Subject: [PATCH 025/187] Implemented optimized Istio policy handling. Further optimization - calculating ref_ip_blocks only in non-optimized run. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 14 +++++++++- nca/NetworkConfig/NetworkConfig.py | 34 ++++++++++++++++------- nca/NetworkConfig/NetworkConfigQuery.py | 24 ++++++++++------- nca/NetworkConfig/NetworkLayer.py | 25 ++++++++++++++--- nca/Parsers/IstioPolicyYamlParser.py | 36 ++++++++++++++++++------- nca/Resources/CalicoNetworkPolicy.py | 2 +- nca/Resources/IstioNetworkPolicy.py | 21 +++++++++++++++ 7 files changed, 121 insertions(+), 35 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 9ab636611..ec55be9b6 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -3,10 +3,10 @@ # SPDX-License-Identifier: Apache2.0 # from .CanonicalIntervalSet import CanonicalIntervalSet -from .PortSet import PortSet from .TcpLikeProperties import TcpLikeProperties from .ICMPDataSet import ICMPDataSet from .ProtocolNameResolver import ProtocolNameResolver +from .ProtocolSet import ProtocolSet class ConnectionSet: @@ -544,6 +544,18 @@ def print_diff(self, other, self_name, other_name): return 'No diff.' + def convert_to_tcp_like_properties(self, peer_container): + if self.allow_all: + return TcpLikeProperties.make_all_properties(peer_container) + + res = TcpLikeProperties.make_empty_properties(peer_container) + for protocol, properties in self.allowed_protocols.items(): + protocols = ProtocolSet() + protocols.add_protocol(protocol) + this_prop = TcpLikeProperties.make_tcp_like_properties(peer_container, protocols=protocols) + res |= (this_prop & properties) + return res + @staticmethod def get_all_tcp_connections(): tcp_conns = ConnectionSet() diff --git a/nca/NetworkConfig/NetworkConfig.py b/nca/NetworkConfig/NetworkConfig.py index 60c53b298..7e45c53ba 100644 --- a/nca/NetworkConfig/NetworkConfig.py +++ b/nca/NetworkConfig/NetworkConfig.py @@ -7,6 +7,7 @@ from nca.CoreDS import Peer from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.NetworkPolicy import NetworkPolicy from .NetworkLayer import NetworkLayersContainer, NetworkLayerName @@ -249,7 +250,7 @@ def allowed_connections(self, from_peer, to_peer, layer_name=None): return allowed_conns_res, captured_flag_res, allowed_captured_conns_res, denied_conns_res - def allowed_connections_optimized(self, layer_name=None): + def allowed_connections_optimized(self, connectivityFilterIstioEdges, layer_name=None): """ Computes the set of allowed connections between any relevant peers. :param NetworkLayerName layer_name: The name of the layer to use, if requested to use a specific layer only @@ -258,15 +259,28 @@ def allowed_connections_optimized(self, layer_name=None): """ if layer_name is not None: if layer_name not in self.policies_container.layers: - return self.policies_container.layers.empty_layer_allowed_connections_optimized(self.peer_container, - layer_name) - return self.policies_container.layers[layer_name].allowed_connections_optimized(self.peer_container) - - conns_res = TcpLikeProperties.make_all_properties() # all connections - for layer, layer_obj in self.policies_container.layers.items(): - conns_per_layer = layer_obj.allowed_connections_optimized(self.peer_container) - # all allowed connections: intersection of all allowed connections from all layers - conns_res &= conns_per_layer + conns_res = self.policies_container.layers.empty_layer_allowed_connections_optimized(self.peer_container, + layer_name) + else: + conns_res = self.policies_container.layers[layer_name].allowed_connections_optimized(self.peer_container) + else: + conns_res = TcpLikeProperties.make_all_properties() # all connections + for layer, layer_obj in self.policies_container.layers.items(): + conns_per_layer = layer_obj.allowed_connections_optimized(self.peer_container) + # all allowed connections: intersection of all allowed connections from all layers + conns_res &= conns_per_layer + + if self.policies_container.layers.does_contain_single_layer(NetworkLayerName.Istio) \ + and connectivityFilterIstioEdges: + protocols = ProtocolSet() + protocols.add_protocol('TCP') + dst_peers_no_ip = conns_res.project_on_one_dimension('dst_peers') if conns_res else Peer.PeerSet() + dst_peers_no_ip -= Peer.IpBlock.get_all_ips_block_peer_set() + if dst_peers_no_ip: + conns_res &= TcpLikeProperties.make_tcp_like_properties(self.peer_container, dst_peers=dst_peers_no_ip, + protocols=protocols) + else: + conns_res = TcpLikeProperties.make_empty_properties(self.peer_container) return conns_res diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index e90753894..7d5a8dfa3 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -642,16 +642,16 @@ def are_labels_all_included(target_labels, pool_labels): def exec(self): self.output_config.configName = os.path.basename(self.config.name) if self.config.name.startswith('./') else \ self.config.name - peers_to_compare = self.config.peer_container.get_all_peers_group() - - ref_ip_blocks = IpBlock.disjoint_ip_blocks(self.config.get_referenced_ip_blocks(), - IpBlock.get_all_ips_block_peer_set()) connections = defaultdict(list) - peers = PeerSet() - peers_to_compare |= ref_ip_blocks res = QueryAnswer(True) conn_graph = None if self.config.optimized_run != 'true': + peers_to_compare = self.config.peer_container.get_all_peers_group() + + ref_ip_blocks = IpBlock.disjoint_ip_blocks(self.config.get_referenced_ip_blocks(), + IpBlock.get_all_ips_block_peer_set()) + peers_to_compare |= ref_ip_blocks + peers = PeerSet() peers1_start = time.time() for peer1_cnt, peer1 in enumerate(peers_to_compare): for peer2 in peers_to_compare: @@ -701,15 +701,18 @@ def exec(self): all_conns_opt = TcpLikeProperties.make_empty_properties() opt_start = time.time() if self.config.optimized_run != 'false': - all_conns_opt = self.config.allowed_connections_optimized() + all_conns_opt = self.config.allowed_connections_optimized(self.output_config.connectivityFilterIstioEdges) if all_conns_opt: - subset_peers = self.compute_subset(peers_to_compare) + opt_peers_to_compare = self.config.peer_container.get_all_peers_group() + # add all relevant IpBlocks, used in connections + opt_peers_to_compare |= all_conns_opt.project_on_one_dimension('src_peers') \ + | all_conns_opt.project_on_one_dimension('dst_peers') + subset_peers = self.compute_subset(opt_peers_to_compare) src_peers_in_subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, src_peers=subset_peers) dst_peers_in_subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, dst_peers=subset_peers) all_conns_opt &= src_peers_in_subset_conns | dst_peers_in_subset_conns - conn_graph2 = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) # conn_graph_opt = ConnectivityGraphOptimized(self.output_config) # Add connections from peer to itself (except for IPs) for peer in subset_peers: @@ -718,6 +721,7 @@ def exec(self): src_peers=PeerSet({peer}), dst_peers=PeerSet({peer}), exclude_same_src_dst_peers=False) + conn_graph2 = ConnectivityGraph(opt_peers_to_compare, self.config.get_allowed_labels(), self.output_config) for cube in all_conns_opt: conn_graph2.add_edges_from_cube_dict(self.config.peer_container, all_conns_opt.get_cube_dict_with_orig_values(cube)) @@ -725,7 +729,7 @@ def exec(self): if self.output_config.outputFormat == 'dot': res.output_explanation = conn_graph2.get_connectivity_dot_format_str() if self.config.optimized_run == 'debug': - orig_conn_graph = ConnectivityGraph(peers_to_compare, self.config.get_allowed_labels(), self.output_config) + orig_conn_graph = ConnectivityGraph(peers, self.config.get_allowed_labels(), self.output_config) orig_conn_graph.add_edges(connections) opt_end = time.time() print(f'Opt time: {(opt_end - opt_start):6.2f} seconds') diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 8c363fa6a..af9936403 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -282,6 +282,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): base_peer_set.add(IpBlock.get_all_ips_block()) base_peer_set_no_hep = PeerSet(set([peer for peer in base_peer_set if not isinstance(peer, HostEP)])) if is_ingress: + # TODO - probably captured_dst_peers1 and captured_dst_peers2 calculation is redundant (all included in add_to_captured) captured_dst_peers1 = allowed_conn.project_on_one_dimension('dst_peers') if allowed_conn else PeerSet() captured_dst_peers2 = denied_conns.project_on_one_dimension('dst_peers') if denied_conns else PeerSet() captured_dst_peers = captured_dst_peers1 | captured_dst_peers2 | add_to_captured @@ -292,7 +293,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): src_peers=base_peer_set, dst_peers=non_captured_dst_peers, exclude_same_src_dst_peers=False) - allowed_conn = (allowed_conn | non_captured_conns) if allowed_conn else non_captured_conns + allowed_conn |= non_captured_conns else: captured_src_peers1 = allowed_conn.project_on_one_dimension('src_peers') if allowed_conn else PeerSet() captured_src_peers2 = denied_conns.project_on_one_dimension('src_peers') if denied_conns else PeerSet() @@ -304,7 +305,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): src_peers=non_captured_src_peers, dst_peers=base_peer_set, exclude_same_src_dst_peers=False) - allowed_conn = (allowed_conn | non_captured_conns) if allowed_conn else non_captured_conns + allowed_conn |= non_captured_conns return allowed_conn, denied_conns @@ -329,8 +330,24 @@ def captured_cond_func(policy): all_allowed_conns=allowed_conns | allowed_non_captured_conns) def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): - return self.collect_policies_conns_optimized(is_ingress) - + allowed_conn, denied_conns, captured = self.collect_policies_conns_optimized(is_ingress) + base_peer_set = peer_container.peer_set.copy() + base_peer_set.add(IpBlock.get_all_ips_block()) + if is_ingress: + non_captured_peers = base_peer_set - captured + if non_captured_peers: + non_captured_conns = \ + TcpLikeProperties.make_tcp_like_properties(peer_container, + src_peers=base_peer_set, dst_peers=non_captured_peers, + exclude_same_src_dst_peers=False) + allowed_conn |= non_captured_conns + else: + non_captured_conns = \ + TcpLikeProperties.make_tcp_like_properties(peer_container, + src_peers=base_peer_set, dst_peers=base_peer_set, + exclude_same_src_dst_peers=False) + allowed_conn |= non_captured_conns + return allowed_conn, denied_conns class IngressNetworkLayer(NetworkLayer): diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index efb535b3f..823e8c04e 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -10,6 +10,7 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.PortSet import PortSet from nca.CoreDS.MethodSet import MethodSet +from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy, IstioPolicyRule from nca.Resources.IstioTrafficResources import istio_root_namespace from nca.Resources.NetworkPolicy import NetworkPolicy @@ -455,12 +456,14 @@ def parse_source(self, source_dict): # A match occurs when at least one source, one operation and all conditions matches the request # https://istio.io/latest/docs/reference/config/security/authorization-policy/#Rule - def parse_ingress_rule(self, rule): + def parse_ingress_rule(self, rule, selected_peers): """ Parse a single ingress rule, producing a IstioPolicyRule. :param dict rule: The dict with the rule fields - :return: A IstioPolicyRule with the proper PeerSet and ConnectionSet - :rtype: IstioPolicyRule + :param PeerSet selected_peers: The selected peers of the policy + :return: A tuple (IstioPolicyRule, TcpLikeProperties) with the proper PeerSet and ConnectionSet, + where TcpLikeProperties is an optimized rule format in a HyperCubeSet format + :rtype: tuple(IstioPolicyRule, TcpLikeProperties) """ if rule is None: self.syntax_error('Authorization policy rule cannot be null. ') @@ -482,30 +485,41 @@ def parse_ingress_rule(self, rule): to_array = self.get_key_array_and_validate_not_empty(rule, 'to') # currently parsing only ports # TODO: extend operations parsing to include other attributes + tcp_props = TcpLikeProperties.make_empty_properties(self.peer_container) if to_array is not None: connections = ConnectionSet() for operation_dict in to_array: - connections |= self.parse_operation(operation_dict) + conns = self.parse_operation(operation_dict) + connections |= conns + tcp_props |= conns.convert_to_tcp_like_properties(self.peer_container) else: # no 'to' in the rule => all connections allowed connections = ConnectionSet(True) + tcp_props = TcpLikeProperties.make_all_properties(self.peer_container) # condition possible result value: # source-ip (from) , source-namespace (from) [Peerset], destination.port (to) [ConnectionSet] # should update either res_pods or connections according to the condition condition_array = rule.get('when') # this array can be empty (unlike 'to' and 'from') # the combined condition ("AND" of all conditions) should be applied + condition_conns = ConnectionSet(True) if condition_array is not None: for condition in condition_array: condition_res = self.parse_condition(condition) if isinstance(condition_res, PeerSet): res_peers &= condition_res elif isinstance(condition_res, ConnectionSet): - connections &= condition_res - + condition_conns &= condition_res if not res_peers: self.warning('Rule selects no pods', rule) - - return IstioPolicyRule(res_peers, connections) + condition_props = condition_conns.convert_to_tcp_like_properties(self.peer_container) + if not res_peers or not selected_peers: + condition_props = TcpLikeProperties.make_empty_properties(self.peer_container) + else: + condition_props &= TcpLikeProperties.make_tcp_like_properties(self.peer_container, src_peers=res_peers, + dst_peers=selected_peers) + connections &= condition_conns + tcp_props &= condition_props + return IstioPolicyRule(res_peers, connections), tcp_props @staticmethod def parse_policy_action(action): @@ -552,7 +566,11 @@ def parse_policy(self): pod_selector = policy_spec.get('selector') res_policy.selected_peers = self.update_policy_peers(pod_selector, 'matchLabels') for ingress_rule in policy_spec.get('rules', []): - res_policy.add_ingress_rule(self.parse_ingress_rule(ingress_rule)) + rule, optimized_props = self.parse_ingress_rule(ingress_rule, res_policy.selected_peers) + res_policy.add_ingress_rule(rule) + res_policy.add_optimized_ingress_props(optimized_props, + res_policy.action == IstioNetworkPolicy.ActionType.Allow) + res_policy.add_optimized_egress_props(TcpLikeProperties.make_all_properties(self.peer_container)) if not res_policy.ingress_rules and res_policy.action == IstioNetworkPolicy.ActionType.Deny: self.syntax_error("DENY action without rules is meaningless as it will never be triggered") diff --git a/nca/Resources/CalicoNetworkPolicy.py b/nca/Resources/CalicoNetworkPolicy.py index 25516ce6b..b32c21181 100644 --- a/nca/Resources/CalicoNetworkPolicy.py +++ b/nca/Resources/CalicoNetworkPolicy.py @@ -122,7 +122,7 @@ def allowed_connections_optimized(self, is_ingress): """ Evaluate the set of connections this policy allows/denies/passes between any two peers :param bool is_ingress: whether we evaluate ingress rules only or egress rules only - A TcpLikeProperties object containing all allowed connections for relevant peers, + :return: A TcpLikeProperties object containing all allowed connections for relevant peers, TcpLikeProperties object containing all denied connections, and the peer set of captured peers that are not a part of allowed connections. :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) diff --git a/nca/Resources/IstioNetworkPolicy.py b/nca/Resources/IstioNetworkPolicy.py index 714c2cb6f..d5f84c757 100644 --- a/nca/Resources/IstioNetworkPolicy.py +++ b/nca/Resources/IstioNetworkPolicy.py @@ -92,6 +92,27 @@ def allowed_connections(self, from_peer, to_peer, is_ingress): return PolicyConnections(True, allowed_conns, denied_conns) + def allowed_connections_optimized(self, is_ingress): + """ + Evaluate the set of connections this policy allows/denied/passed between any two peers + :param bool is_ingress: whether we evaluate ingress rules only or egress rules only + :return: A TcpLikeProperties object containing all allowed connections for relevant peers, + TcpLikeProperties object containing all denied connections, + and the peer set of captured peers that are not a part of allowed connections. + :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) + """ + if is_ingress: + allowed = self.optimized_ingress_props.copy() + denied = self.optimized_denied_ingress_props.copy() + captured = self.selected_peers if \ + (self.affects_ingress and self.action == IstioNetworkPolicy.ActionType.Allow) else PeerSet() + else: + allowed = self.optimized_egress_props.copy() + denied = self.optimized_denied_egress_props.copy() + captured = self.selected_peers if \ + (self.affects_egress and self.action == IstioNetworkPolicy.ActionType.Allow) else PeerSet() + return allowed, denied, captured + def referenced_ip_blocks(self): """ :return: A set of all ipblocks referenced in one of the policy rules (one Peer object per one ip range) From ebae572262d351898d24bacac4f84e816025d2ee Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 15 Jan 2023 18:52:17 +0200 Subject: [PATCH 026/187] Added Ingress policy support in the optimized solution. Improved comments. Signed-off-by: Tanya --- nca/NetworkConfig/NetworkLayer.py | 20 +++++++++++++++++++- nca/Parsers/IngressPolicyYamlParser.py | 26 ++++++++++++-------------- nca/Resources/CalicoNetworkPolicy.py | 2 +- nca/Resources/IngressPolicy.py | 21 +++++++++++++++++++++ nca/Resources/IstioNetworkPolicy.py | 2 +- nca/Resources/K8sNetworkPolicy.py | 2 +- nca/Resources/NetworkPolicy.py | 12 ++++-------- 7 files changed, 59 insertions(+), 26 deletions(-) diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index af9936403..85a668f77 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -363,4 +363,22 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): all_allowed_conns=all_allowed_conns) def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): - return TcpLikeProperties.make_empty_properties(), TcpLikeProperties.make_empty_properties() + allowed_conn, denied_conns, captured = self.collect_policies_conns_optimized(is_ingress) + base_peer_set = peer_container.peer_set.copy() + base_peer_set.add(IpBlock.get_all_ips_block()) + if is_ingress: + non_captured_conns = \ + TcpLikeProperties.make_tcp_like_properties(peer_container, + src_peers=base_peer_set, dst_peers=base_peer_set, + exclude_same_src_dst_peers=False) + allowed_conn |= non_captured_conns + else: + non_captured_peers = base_peer_set - captured + if non_captured_peers: + non_captured_conns = \ + TcpLikeProperties.make_tcp_like_properties(peer_container, + src_peers=non_captured_peers, dst_peers=base_peer_set, + exclude_same_src_dst_peers=False) + allowed_conn |= non_captured_conns + return allowed_conn, denied_conns + diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index 655af8050..36ce966e2 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -9,6 +9,7 @@ from nca.CoreDS.Peer import PeerSet from nca.CoreDS.PortSet import PortSet from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.IngressPolicy import IngressPolicy from nca.Resources.NetworkPolicy import NetworkPolicy from .GenericIngressLikeYamlParser import GenericIngressLikeYamlParser @@ -169,7 +170,7 @@ def _make_default_connections(self, hosts_dfa, paths_dfa=None): :param MinDFA paths_dfa: the paths for the default connections :return: TcpLikeProperties containing default connections or None (when no default backend exists) """ - default_conns = None + default_conns = TcpLikeProperties.make_empty_properties(self.peer_container) if self.default_backend_peers: if paths_dfa: default_conns = \ @@ -224,10 +225,7 @@ def parse_rule(self, rule): default_conns = self._make_default_connections(hosts_dfa, paths_remainder_dfa) else: # no paths --> everything for this host goes to the default backend or is denied default_conns = self._make_default_connections(hosts_dfa) - if allowed_conns and default_conns: - allowed_conns |= default_conns - elif default_conns: - allowed_conns = default_conns + allowed_conns |= default_conns return allowed_conns, hosts_dfa def parse_policy(self): @@ -256,14 +254,11 @@ def parse_policy(self): self.peer_container.get_pods_with_service_name_containing_given_string('ingress-nginx') if not res_policy.selected_peers: self.warning("No ingress-nginx pods found, the Ingress policy will have no effect") - allowed_conns = None + allowed_conns = TcpLikeProperties.make_empty_properties(self.peer_container) all_hosts_dfa = None for ingress_rule in policy_spec.get('rules', []): conns, hosts_dfa = self.parse_rule(ingress_rule) - if not allowed_conns: - allowed_conns = conns - else: - allowed_conns |= conns + allowed_conns |= conns if hosts_dfa: if not all_hosts_dfa: all_hosts_dfa = hosts_dfa @@ -274,12 +269,15 @@ def parse_policy(self): # every host not captured by the ingress rules goes to the default backend hosts_remainder_dfa = DimensionsManager().get_dimension_domain_by_name('hosts') - all_hosts_dfa default_conns = self._make_default_connections(hosts_remainder_dfa) - if allowed_conns and default_conns: - allowed_conns |= default_conns - elif default_conns: - allowed_conns = default_conns + allowed_conns |= default_conns assert allowed_conns res_policy.add_rules(self._make_allow_rules(allowed_conns)) + protocols = ProtocolSet() + protocols.add_protocol('TCP') + allowed_conns &= TcpLikeProperties.make_tcp_like_properties(self.peer_container, + src_peers=res_policy.selected_peers, + protocols=protocols) + res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs return res_policy diff --git a/nca/Resources/CalicoNetworkPolicy.py b/nca/Resources/CalicoNetworkPolicy.py index b32c21181..c1f1ac293 100644 --- a/nca/Resources/CalicoNetworkPolicy.py +++ b/nca/Resources/CalicoNetworkPolicy.py @@ -124,7 +124,7 @@ def allowed_connections_optimized(self, is_ingress): :param bool is_ingress: whether we evaluate ingress rules only or egress rules only :return: A TcpLikeProperties object containing all allowed connections for relevant peers, TcpLikeProperties object containing all denied connections, - and the peer set of captured peers that are not a part of allowed connections. + and the peer set of captured peers by this policy. :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) """ if is_ingress: diff --git a/nca/Resources/IngressPolicy.py b/nca/Resources/IngressPolicy.py index e372cd765..9cff6c23d 100644 --- a/nca/Resources/IngressPolicy.py +++ b/nca/Resources/IngressPolicy.py @@ -5,6 +5,8 @@ from enum import IntEnum from nca.CoreDS.ConnectionSet import ConnectionSet +from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.Peer import PeerSet from .NetworkPolicy import PolicyConnections, NetworkPolicy @@ -95,6 +97,25 @@ def allowed_connections(self, from_peer, to_peer, is_ingress): return PolicyConnections(True, allowed_conns, denied_conns) + def allowed_connections_optimized(self, is_ingress): + """ + Evaluate the set of connections this ingress resource allows between any two peers + :param bool is_ingress: For compatibility with other policies. + Will return the set of allowed connections only for is_ingress being False. + :return: A TcpLikeProperties object containing all allowed connections for any peers, + TcpLikeProperties object containing all denied connections, + and the peer set of captured peers by this policy. + :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) + """ + + if is_ingress: + allowed = TcpLikeProperties.make_empty_properties() + captured = PeerSet() + else: + allowed = self.optimized_egress_props.copy() + captured = self.selected_peers if self.affects_egress else PeerSet() + return allowed, TcpLikeProperties.make_empty_properties(), captured + def has_empty_rules(self, _config_name=''): """ Checks whether the policy contains empty rules (rules that do not select any peers) diff --git a/nca/Resources/IstioNetworkPolicy.py b/nca/Resources/IstioNetworkPolicy.py index d5f84c757..162dbc835 100644 --- a/nca/Resources/IstioNetworkPolicy.py +++ b/nca/Resources/IstioNetworkPolicy.py @@ -98,7 +98,7 @@ def allowed_connections_optimized(self, is_ingress): :param bool is_ingress: whether we evaluate ingress rules only or egress rules only :return: A TcpLikeProperties object containing all allowed connections for relevant peers, TcpLikeProperties object containing all denied connections, - and the peer set of captured peers that are not a part of allowed connections. + and the peer set of captured peers by this policy. :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) """ if is_ingress: diff --git a/nca/Resources/K8sNetworkPolicy.py b/nca/Resources/K8sNetworkPolicy.py index b6cd580db..742fa1107 100644 --- a/nca/Resources/K8sNetworkPolicy.py +++ b/nca/Resources/K8sNetworkPolicy.py @@ -70,7 +70,7 @@ def allowed_connections_optimized(self, is_ingress): :param bool is_ingress: whether we evaluate ingress rules only or egress rules only :return: A TcpLikeProperties object containing all allowed connections for relevant peers, None for denied connections (K8s does not have denied), - and the peer set of captured peers that are not a part of allowed connections. + and the peer set of captured peers by this policy. :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) """ if is_ingress: diff --git a/nca/Resources/NetworkPolicy.py b/nca/Resources/NetworkPolicy.py index e716e760b..6e3d8cf2d 100644 --- a/nca/Resources/NetworkPolicy.py +++ b/nca/Resources/NetworkPolicy.py @@ -134,11 +134,9 @@ def add_optimized_ingress_props(self, props, is_allow=True): if not props: return if is_allow: - self.optimized_ingress_props = \ - (self.optimized_ingress_props | props) if self.optimized_ingress_props else props + self.optimized_ingress_props |= props else: - self.optimized_denied_ingress_props = \ - (self.optimized_denied_ingress_props | props) if self.optimized_denied_ingress_props else props + self.optimized_denied_ingress_props |= props def add_optimized_egress_props(self, props, is_allow=True): """ @@ -150,11 +148,9 @@ def add_optimized_egress_props(self, props, is_allow=True): if not props: return if is_allow: - self.optimized_egress_props = \ - (self.optimized_egress_props | props) if self.optimized_egress_props else props + self.optimized_egress_props |= props else: - self.optimized_denied_egress_props = \ - (self.optimized_denied_egress_props | props) if self.optimized_denied_egress_props else props + self.optimized_denied_egress_props |= props @staticmethod def get_policy_type_from_dict(policy): # noqa: C901 From 25118de2d6e29664c2c38125de1f0ef64e30b9c9 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 15 Jan 2023 19:19:06 +0200 Subject: [PATCH 027/187] Added Istio Ingress policy support in the optimized solution. Signed-off-by: Tanya --- nca/Parsers/IngressPolicyYamlParser.py | 5 ++--- nca/Parsers/IstioTrafficResourcesYamlParser.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index 36ce966e2..7420224a3 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -275,9 +275,8 @@ def parse_policy(self): res_policy.add_rules(self._make_allow_rules(allowed_conns)) protocols = ProtocolSet() protocols.add_protocol('TCP') - allowed_conns &= TcpLikeProperties.make_tcp_like_properties(self.peer_container, - src_peers=res_policy.selected_peers, - protocols=protocols) + allowed_conns &= TcpLikeProperties.make_tcp_like_properties(self.peer_container, protocols=protocols, + src_peers=res_policy.selected_peers) res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs return res_policy diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index faa470b8b..ae4fb021f 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -7,8 +7,8 @@ from nca.CoreDS.DimensionsManager import DimensionsManager from nca.CoreDS.MinDFA import MinDFA from nca.CoreDS.Peer import PeerSet -from nca.CoreDS.PortSet import PortSet from nca.CoreDS.MethodSet import MethodSet +from nca.CoreDS.ProtocolSet import ProtocolSet from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from nca.Resources.IstioTrafficResources import Gateway, VirtualService from nca.Resources.IngressPolicy import IngressPolicy @@ -326,7 +326,7 @@ def make_allowed_connections(self, vs, host_dfa): :param MinDFA host_dfa: the hosts attribute :return: TcpLikeProperties with TCP allowed connections """ - allowed_conns = None + allowed_conns = TcpLikeProperties.make_empty_properties(self.peer_container) for http_route in vs.http_routes: for dest in http_route.destinations: conns = \ @@ -334,10 +334,7 @@ def make_allowed_connections(self, vs, host_dfa): dst_peers=dest.service.target_pods, paths_dfa=http_route.uri_dfa, hosts_dfa=host_dfa, methods=http_route.methods) - if not allowed_conns: - allowed_conns = conns - else: - allowed_conns |= conns + allowed_conns |= conns return allowed_conns def create_istio_traffic_policies(self): @@ -388,6 +385,11 @@ def create_istio_traffic_policies(self): allowed_conns = self.make_allowed_connections(vs, host_dfa) if allowed_conns: res_policy.add_rules(self._make_allow_rules(allowed_conns)) + protocols = ProtocolSet() + protocols.add_protocol('TCP') + allowed_conns &= TcpLikeProperties.make_tcp_like_properties(self.peer_container, protocols=protocols, + src_peers=res_policy.selected_peers) + res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs vs_policies.append(res_policy) if not vs_policies: From 7780876bb3601b42d399390c0b32652db5bfa396 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 24 Jan 2023 16:04:04 +0200 Subject: [PATCH 028/187] Further optimization: converting HC set directly to fw rules. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 93 +++++++++++++++++++++++++ nca/CoreDS/TcpLikeProperties.py | 26 +++---- nca/FWRules/ConnectivityGraph.py | 7 +- nca/FWRules/FWRule.py | 90 +++++++++++++++++++++++- nca/NetworkConfig/NetworkConfigQuery.py | 46 ++++++------ nca/NetworkConfig/NetworkLayer.py | 24 +++---- 6 files changed, 224 insertions(+), 62 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index ec55be9b6..6d5d09e2d 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -2,11 +2,15 @@ # Copyright 2020- IBM Inc. All rights reserved # SPDX-License-Identifier: Apache2.0 # + +from collections import defaultdict from .CanonicalIntervalSet import CanonicalIntervalSet from .TcpLikeProperties import TcpLikeProperties from .ICMPDataSet import ICMPDataSet from .ProtocolNameResolver import ProtocolNameResolver from .ProtocolSet import ProtocolSet +from .Peer import PeerSet, IpBlock +from nca.FWRules import FWRule class ConnectionSet: @@ -568,3 +572,92 @@ def get_non_tcp_connections(): res.add_all_connections([ProtocolNameResolver.get_protocol_number('TCP')]) return res # return ConnectionSet(True) - ConnectionSet.get_all_TCP_connections() + + # TODO - after moving to the optimized HC set implementation, + # get rid of ConnectionSet and move the code below to TcpLikeProperties.py + @staticmethod + def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container): + fw_rules_map = defaultdict(list) + for cube in tcp_props: + cube_dict = tcp_props.get_cube_dict_with_orig_values(cube) + new_cube_dict = cube_dict.copy() + src_peers = new_cube_dict.get('src_peers') + if src_peers: + new_cube_dict.pop('src_peers') + else: + src_peers = peer_container.get_all_peers_group(True) + dst_peers = new_cube_dict.get('dst_peers') + if dst_peers: + new_cube_dict.pop('dst_peers') + else: + dst_peers = peer_container.get_all_peers_group(True) + protocols = new_cube_dict.get('protocols') + if protocols: + new_cube_dict.pop('protocols') + if not protocols and not new_cube_dict: + conns = ConnectionSet(True) + else: + conns = ConnectionSet() + protocol_names = ProtocolSet.get_protocol_names_from_interval_set(protocols) if protocols else ['TCP'] + for protocol in protocol_names: + if new_cube_dict: + conns.add_connections(protocol, TcpLikeProperties.make_tcp_like_properties_from_dict(peer_container, + new_cube_dict)) + else: + if ConnectionSet.protocol_supports_ports(protocol): + conns.add_connections(protocol, TcpLikeProperties.make_all_properties(peer_container)) + elif ConnectionSet.protocol_is_icmp(protocol): + conns.add_connections(protocol, TcpLikeProperties.make_all_properties(peer_container)) + else: + conns.add_connections(protocol, True) + # create FWRules for src_peers and dst_peers + fw_rules_map[conns] += ConnectionSet.create_fw_rules_list_from_conns(conns, src_peers, dst_peers, + cluster_info) + return fw_rules_map + + @staticmethod + def create_fw_rules_list_from_conns(conns, src_peers, dst_peers, cluster_info): + src_fw_elements = ConnectionSet.split_peer_set_to_fw_rule_elements(src_peers, cluster_info) + dst_fw_elements = ConnectionSet.split_peer_set_to_fw_rule_elements(dst_peers, cluster_info) + fw_rules_list = [] + for src_elem in src_fw_elements: + for dst_elem in dst_fw_elements: + fw_rules_list.append(FWRule.FWRule(src_elem, dst_elem, conns)) + return fw_rules_list + + @staticmethod + def split_peer_set_to_fw_rule_elements(peer_set, cluster_info): + res = [] + peer_set_copy = peer_set.copy() + ns_set = set() + # first, split by namespaces + while peer_set_copy: + peer = list(peer_set_copy)[0] + if isinstance(peer, IpBlock): + res.append(FWRule.IPBlockElement(peer)) + peer_set_copy.remove(peer) + continue + ns_peers = PeerSet(cluster_info.ns_dict[peer.namespace]) + if ns_peers.issubset(peer_set_copy): + ns_set.add(peer.namespace) + else: + # TODO try to split the element below by labels + res.append(FWRule.PeerSetElement(ns_peers & peer_set_copy)) + peer_set_copy -= ns_peers + if ns_set: + res.append(FWRule.FWRuleElement(ns_set)) + + return res + + @staticmethod + def fw_rules_to_tcp_properties(fw_rules, peer_container): + res = TcpLikeProperties.make_empty_properties(peer_container) + for fw_rules_list in fw_rules.fw_rules_map.values(): + for fw_rule in fw_rules_list: + conn_props = fw_rule.conn.convert_to_tcp_like_properties(peer_container) + src_peers = PeerSet(fw_rule.src.get_peer_set(fw_rules.cluster_info)) + dst_peers = PeerSet(fw_rule.dst.get_peer_set(fw_rules.cluster_info)) + rule_props = TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=src_peers, + dst_peers=dst_peers) & conn_props + res |= rule_props + return res diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index 3c8512a02..11d30aff9 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -405,11 +405,18 @@ def project_on_one_dimension(self, dim_name): res = (res | values) if res else values return res + @staticmethod + def cube_dict_to_str(cube_dict): + res = "" + for item in cube_dict.items(): + res += str(item[0]) + " : " + str(item[1]) + return res + @staticmethod def make_tcp_like_properties(peer_container, src_ports=PortSet(True), dst_ports=PortSet(True), protocols=ProtocolSet(True), src_peers=None, dst_peers=None, paths_dfa=None, hosts_dfa=None, methods=MethodSet(True), - icmp_type=None, icmp_code=None, exclude_same_src_dst_peers=True): + icmp_type=None, icmp_code=None): """ get TcpLikeProperties with TCP allowed connections, corresponding to input properties cube. TcpLikeProperties should not contain named ports: substitute them with corresponding port numbers, per peer @@ -437,16 +444,6 @@ def make_tcp_like_properties(peer_container, src_ports=PortSet(True), dst_ports= dst_peers_interval = base_peer_set.get_peer_interval_of(dst_peers) else: dst_peers_interval = None - exclude_props = TcpLikeProperties.make_empty_properties(peer_container) - if exclude_same_src_dst_peers and src_peers and dst_peers: - same_src_dst_peers = src_peers & dst_peers - for peer in same_src_dst_peers: - ps = PeerSet() - ps.add(peer) - peer_interval = base_peer_set.get_peer_interval_of(ps) - exclude_props |= TcpLikeProperties(src_peers=peer_interval, - dst_peers=peer_interval, - base_peer_set=base_peer_set) if (not src_ports.named_ports or not src_peers) and (not dst_ports.named_ports or not dst_peers): # Should not resolve named ports res = TcpLikeProperties(source_ports=src_ports, dest_ports=dst_ports, @@ -454,7 +451,7 @@ def make_tcp_like_properties(peer_container, src_ports=PortSet(True), dst_ports= icmp_type=icmp_type, icmp_code=icmp_code, src_peers=src_peers_interval, dst_peers=dst_peers_interval, base_peer_set=base_peer_set) - return res - exclude_props if exclude_props else res + return res # Resolving named ports tcp_properties = None if src_ports.named_ports and dst_ports.named_ports: @@ -555,7 +552,7 @@ def make_tcp_like_properties(peer_container, src_ports=PortSet(True), dst_ports= tcp_properties |= props else: tcp_properties = props - return tcp_properties - exclude_props + return tcp_properties @staticmethod def make_tcp_like_properties_from_dict(peer_container, cube_dict): @@ -580,8 +577,7 @@ def make_tcp_like_properties_from_dict(peer_container, cube_dict): return TcpLikeProperties.make_tcp_like_properties(peer_container, src_ports=src_ports, dst_ports=dst_ports, protocols=protocols, src_peers=src_peers, dst_peers=dst_peers, paths_dfa=paths_dfa, hosts_dfa=hosts_dfa, methods=methods, - icmp_type=icmp_type, icmp_code=icmp_code, - exclude_same_src_dst_peers=False) + icmp_type=icmp_type, icmp_code=icmp_code) @staticmethod def make_empty_properties(peer_container=None): diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index 4a7301f33..7e6b25c25 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -161,19 +161,16 @@ def convert_to_tcp_like_properties(self, peer_container): for peer_pair in item[1]: res |= TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=PeerSet({peer_pair[0]}), - dst_peers=PeerSet({peer_pair[1]}), - exclude_same_src_dst_peers=False) + dst_peers=PeerSet({peer_pair[1]})) else: for prot in item[0].allowed_protocols.items(): protocols = ProtocolSet() protocols.add_protocol(prot[0]) if isinstance(prot[1], bool): -# res |= TcpLikeProperties.make_tcp_like_properties(peer_container, protocols=protocols) # wrong for peer_pair in item[1]: res |= TcpLikeProperties.make_tcp_like_properties(peer_container, protocols=protocols, src_peers=PeerSet({peer_pair[0]}), - dst_peers=PeerSet({peer_pair[1]}), - exclude_same_src_dst_peers=False) + dst_peers=PeerSet({peer_pair[1]})) continue for cube in prot[1]: cube_dict = prot[1].get_cube_dict_with_orig_values(cube) diff --git a/nca/FWRules/FWRule.py b/nca/FWRules/FWRule.py index 640ca18bd..308d05988 100644 --- a/nca/FWRules/FWRule.py +++ b/nca/FWRules/FWRule.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache2.0 # -from nca.CoreDS.Peer import ClusterEP, IpBlock, Pod +from nca.CoreDS.Peer import ClusterEP, IpBlock, Pod, PeerSet from nca.Resources.K8sNamespace import K8sNamespace from .ClusterInfo import ClusterInfo @@ -203,6 +203,13 @@ def get_pods_set(self, cluster_info): res |= cluster_info.ns_dict[ns] return res + def get_peer_set(self, cluster_info): + """ + :param cluster_info: an object of type ClusterInfo, with relevant cluster topology info + :return: a PeerSet (pods and/or IpBlocks) represented by this element + """ + return PeerSet(self.get_pods_set(cluster_info)) + @staticmethod def create_fw_elements_from_base_element(base_elem): """ @@ -350,6 +357,80 @@ def get_pods_set(self, cluster_info): return res +class PeerSetElement(FWRuleElement): + """ + This is the class for PeerSet element in fw rule + """ + + def __init__(self, element, output_as_deployment=True): + """ + Create an object of type PeerSetElement + :param element: an element of type PeerSet + """ + super().__init__({list(element)[0].namespace}) + self.element = element + self.output_as_deployment = output_as_deployment + + def get_elem_list_obj(self): + """ + :return: list[string] for the field src_pods or dst_pods in representation for yaml/json object + """ + return [str(self._get_pods_names())] + + def get_pod_str(self): + """ + :return: string for the field src_pods or dst_pods in representation for txt rule format + """ + return f'[{self._get_pods_names()}]' + + def _get_pods_names(self): + res = '' + unique_names = set() + for peer in self.element: + if self.output_as_deployment and isinstance(peer, Pod) and peer.owner_name: + if peer.owner_name not in unique_names: + res += (', ' if res else '') + peer.owner_name + unique_names.add(peer.owner_name) + else: + res += (', ' if res else '') + peer.name + return res + + def __str__(self): + """ + :return: string of the represented element + """ + return f'pods: {self.get_pod_str()}' + + def get_elem_str(self, is_src): + """ + :param is_src: bool flag to indicate if element is src (True) or dst (False) + :return: string of the represented element with src or dst description of fields + """ + pods_prefix = ' src_pods: ' if is_src else ' dst_pods: ' + suffix = ' ' if is_src else '' + return pods_prefix + self.get_pod_str() + suffix + + def __hash__(self): + return hash(str(self)) + + def __eq__(self, other): + return isinstance(other, PeerSetElement) and self.element == other.element and super().__eq__(other) + + def get_pods_set(self, cluster_info): + """ + :param cluster_info: an object of type ClusterInfo, with relevant cluster topology info + :return: a set of pods in the cluster represented by this element + """ + return self.element + + def get_peer_set(self, cluster_info): + """ + :param cluster_info: an object of type ClusterInfo, with relevant cluster topology info + :return: a PeerSet (pods and/or IpBlocks) represented by this element + """ + return self.get_pods_set(cluster_info) + + # TODO: should it be a sub-type of FWRuleElement? class IPBlockElement(FWRuleElement): """ @@ -408,6 +489,13 @@ def get_pods_set(self, cluster_info): # an ip block element does not represent any pods return set() + def get_peer_set(self, cluster_info): + """ + :param cluster_info: an object of type ClusterInfo, with relevant cluster topology info + :return: a PeerSet (pods and/or IpBlocks) represented by this element + """ + return PeerSet({self.element}) + class FWRule: """ diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 674eaeca9..54c7c31c6 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -14,6 +14,8 @@ from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from .NetworkConfig import NetworkConfig from nca.FWRules.ConnectivityGraph import ConnectivityGraph +from nca.FWRules.MinimizeFWRules import MinimizeFWRules +from nca.FWRules.ClusterInfo import ClusterInfo from nca.Resources.CalicoNetworkPolicy import CalicoNetworkPolicy from nca.Resources.IngressPolicy import IngressPolicy from nca.Utils.OutputConfiguration import OutputConfiguration @@ -693,7 +695,6 @@ def exec(self): self.config.name connections = defaultdict(list) res = QueryAnswer(True) - conn_graph = None if self.config.optimized_run != 'true': peers_to_compare = self.config.peer_container.get_all_peers_group() @@ -733,8 +734,6 @@ def exec(self): # collect both peers, even if one of them is not in the subset peers.add(peer1) peers.add(peer2) - #peers2_end = time.time() - #print(f'Original loop: {peer1_cnt+1} / {len(peers_to_compare)} peers, time: {(peers2_end - peers1_start):6.2f} seconds') peers1_end = time.time() print(f'Original loop: time: {(peers1_end - peers1_start):6.2f} seconds') if self.output_config.outputFormat == 'dot': @@ -758,30 +757,23 @@ def exec(self): if all_conns_opt: opt_peers_to_compare = self.config.peer_container.get_all_peers_group() # add all relevant IpBlocks, used in connections - opt_peers_to_compare |= all_conns_opt.project_on_one_dimension('src_peers') \ - | all_conns_opt.project_on_one_dimension('dst_peers') + opt_peers_to_compare |= all_conns_opt.project_on_one_dimension('src_peers') | \ + all_conns_opt.project_on_one_dimension('dst_peers') subset_peers = self.compute_subset(opt_peers_to_compare) src_peers_in_subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, src_peers=subset_peers) dst_peers_in_subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, dst_peers=subset_peers) all_conns_opt &= src_peers_in_subset_conns | dst_peers_in_subset_conns - # conn_graph_opt = ConnectivityGraphOptimized(self.output_config) - # Add connections from peer to itself (except for IPs) - for peer in subset_peers: - if not isinstance(peer, IpBlock): - all_conns_opt |= TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, - src_peers=PeerSet({peer}), - dst_peers=PeerSet({peer}), - exclude_same_src_dst_peers=False) - conn_graph2 = ConnectivityGraph(opt_peers_to_compare, self.config.get_allowed_labels(), self.output_config) - for cube in all_conns_opt: - conn_graph2.add_edges_from_cube_dict(self.config.peer_container, - all_conns_opt.get_cube_dict_with_orig_values(cube)) - # conn_graph_opt.add_edge(all_conns_opt.get_cube_dict(cube)) + if self.output_config.outputFormat == 'dot': + conn_graph2 = ConnectivityGraph(opt_peers_to_compare, self.config.get_allowed_labels(), + self.output_config) + for cube in all_conns_opt: + conn_graph2.add_edges_from_cube_dict(self.config.peer_container, + all_conns_opt.get_cube_dict_with_orig_values(cube)) res.output_explanation = [ - ComputedExplanation(str_explanation=conn_graph.get_connectivity_dot_format_str())] + ComputedExplanation(str_explanation=conn_graph2.get_connectivity_dot_format_str())] if self.config.optimized_run == 'debug': orig_conn_graph = ConnectivityGraph(peers, self.config.get_allowed_labels(), self.output_config) orig_conn_graph.add_edges(connections) @@ -789,7 +781,10 @@ def exec(self): print(f'Opt time: {(opt_end - opt_start):6.2f} seconds') self.compare_orig_to_opt_conn(orig_conn_graph, all_conns_opt) else: - fw_rules2 = conn_graph2.get_minimized_firewall_rules() + cluster_info = ClusterInfo(opt_peers_to_compare, self.config.get_allowed_labels()) + fw_rules2_map = ConnectionSet.tcp_properties_to_fw_rules(all_conns_opt, cluster_info, + self.config.peer_container) + fw_rules2 = MinimizeFWRules(fw_rules2_map, cluster_info, self.output_config, {}) formatted_rules2 = fw_rules2.get_fw_rules_in_required_format() if self.output_config.outputFormat in ['json', 'yaml']: res.output_explanation = [ComputedExplanation(dict_explanation=formatted_rules2)] @@ -798,11 +793,7 @@ def exec(self): opt_end = time.time() print(f'Opt time: {(opt_end - opt_start):6.2f} seconds') if self.config.optimized_run == 'debug': - self.compare_orig_to_opt_conn(conn_graph, all_conns_opt) - # res_opt.output_explanation = conn_graph_opt.get_connectivity_txt_format_str() - # res.output_explanation += "---------------- OPTIMIZED RESULT: -------------\n" +\ - # fw_rules2.get_fw_rules_in_required_format() + \ - # "\n------------------------------------------------\n\n" # TEMP for debug + self.compare_fw_rules(fw_rules, fw_rules2) return res def compare_orig_to_opt_conn(self, orig_conn_graph, opt_props): @@ -812,6 +803,11 @@ def compare_orig_to_opt_conn(self, orig_conn_graph, opt_props): # The following assert exposes the bug in HC set assert not orig_tcp_props.contained_in(opt_props) or not opt_props.contained_in(orig_tcp_props) or orig_tcp_props == opt_props + def compare_fw_rules(self, fw_rules1, fw_rules2): + tcp_props1 = ConnectionSet.fw_rules_to_tcp_properties(fw_rules1, self.config.peer_container) + tcp_props2 = ConnectionSet.fw_rules_to_tcp_properties(fw_rules2, self.config.peer_container) + assert tcp_props1 == tcp_props2 + @staticmethod def filter_istio_edge(peer2, conns): # currently only supporting authorization policies, that do not capture egress rules diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 85a668f77..ba6c3bd19 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -8,7 +8,6 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock, HostEP, PeerSet from nca.CoreDS.TcpLikeProperties import TcpLikeProperties -from nca.CoreDS.PortSet import PortSet from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy from nca.Resources.NetworkPolicy import PolicyConnections, NetworkPolicy @@ -174,8 +173,7 @@ def allowed_connections_optimized(self, peer_container): # exclude IpBlock->IpBlock connections all_ips_peer_set = PeerSet({IpBlock.get_all_ips_block()}) excluded_conns = TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=all_ips_peer_set, - dst_peers=all_ips_peer_set, - exclude_same_src_dst_peers=False) + dst_peers=all_ips_peer_set) res -= excluded_conns return res @@ -291,8 +289,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): non_captured_conns = \ TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=base_peer_set, - dst_peers=non_captured_dst_peers, - exclude_same_src_dst_peers=False) + dst_peers=non_captured_dst_peers) allowed_conn |= non_captured_conns else: captured_src_peers1 = allowed_conn.project_on_one_dimension('src_peers') if allowed_conn else PeerSet() @@ -303,8 +300,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): non_captured_conns = \ TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=non_captured_src_peers, - dst_peers=base_peer_set, - exclude_same_src_dst_peers=False) + dst_peers=base_peer_set) allowed_conn |= non_captured_conns return allowed_conn, denied_conns @@ -338,17 +334,16 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): if non_captured_peers: non_captured_conns = \ TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=base_peer_set, dst_peers=non_captured_peers, - exclude_same_src_dst_peers=False) + src_peers=base_peer_set, dst_peers=non_captured_peers) allowed_conn |= non_captured_conns else: non_captured_conns = \ TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=base_peer_set, dst_peers=base_peer_set, - exclude_same_src_dst_peers=False) + src_peers=base_peer_set, dst_peers=base_peer_set) allowed_conn |= non_captured_conns return allowed_conn, denied_conns + class IngressNetworkLayer(NetworkLayer): def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): @@ -369,16 +364,13 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): if is_ingress: non_captured_conns = \ TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=base_peer_set, dst_peers=base_peer_set, - exclude_same_src_dst_peers=False) + src_peers=base_peer_set, dst_peers=base_peer_set) allowed_conn |= non_captured_conns else: non_captured_peers = base_peer_set - captured if non_captured_peers: non_captured_conns = \ TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=non_captured_peers, dst_peers=base_peer_set, - exclude_same_src_dst_peers=False) + src_peers=non_captured_peers, dst_peers=base_peer_set) allowed_conn |= non_captured_conns return allowed_conn, denied_conns - From 8998470a19e3d2a9ff7d33e9523243a497492f7b Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 24 Jan 2023 18:24:16 +0200 Subject: [PATCH 029/187] Small bug fix Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 6d5d09e2d..2d8c7cc8a 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -557,7 +557,11 @@ def convert_to_tcp_like_properties(self, peer_container): protocols = ProtocolSet() protocols.add_protocol(protocol) this_prop = TcpLikeProperties.make_tcp_like_properties(peer_container, protocols=protocols) - res |= (this_prop & properties) + if bool(properties): + if properties: + res |= this_prop + else: + res |= (this_prop & properties) return res @staticmethod From 1281b2c73f4d6efe656cccc313781fc5099df658 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 24 Jan 2023 20:15:44 +0200 Subject: [PATCH 030/187] Fixed printing peer sets in FWRules. Signed-off-by: Tanya --- nca/FWRules/FWRule.py | 5 +++-- tests/run_all_tests.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nca/FWRules/FWRule.py b/nca/FWRules/FWRule.py index 308d05988..2afa30093 100644 --- a/nca/FWRules/FWRule.py +++ b/nca/FWRules/FWRule.py @@ -399,16 +399,17 @@ def __str__(self): """ :return: string of the represented element """ - return f'pods: {self.get_pod_str()}' + return f'ns: {self.get_ns_str()}, pods: {self.get_pod_str()}' def get_elem_str(self, is_src): """ :param is_src: bool flag to indicate if element is src (True) or dst (False) :return: string of the represented element with src or dst description of fields """ + ns_prefix = 'src_ns: ' if is_src else 'dst_ns: ' pods_prefix = ' src_pods: ' if is_src else ' dst_pods: ' suffix = ' ' if is_src else '' - return pods_prefix + self.get_pod_str() + suffix + return ns_prefix + self.get_ns_str() + pods_prefix + self.get_pod_str() + suffix def __hash__(self): return hash(str(self)) diff --git a/tests/run_all_tests.py b/tests/run_all_tests.py index 85703ca66..ebaf21825 100644 --- a/tests/run_all_tests.py +++ b/tests/run_all_tests.py @@ -64,7 +64,7 @@ def __init__(self, test_dict, cli_tests_base_dir, test_name): class SchemeFile: def __init__(self, scheme_filename): self.test_name = scheme_filename - test_args = ['--scheme', self.test_name] + test_args = ['--scheme', self.test_name, '-opt=debug'] self.args_obj = TestArgs(test_args) def update_arg_at_scheme_file_output_config(self, arg_name, arg_value): From 278b92162d35b6796377f84d706e814155cc61c5 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 29 Jan 2023 15:24:32 +0200 Subject: [PATCH 031/187] More released comparison between original and optimized fw-rules (allowing differences in auto-connections). Signed-off-by: Tanya --- nca/CoreDS/CanonicalIntervalSet.py | 13 +++++++++++++ nca/CoreDS/TcpLikeProperties.py | 9 +++++++++ nca/NetworkConfig/NetworkConfigQuery.py | 10 +++++++++- nca/Parsers/IstioSidecarYamlParser.py | 2 +- 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/nca/CoreDS/CanonicalIntervalSet.py b/nca/CoreDS/CanonicalIntervalSet.py index 8a16a6b47..545267e59 100644 --- a/nca/CoreDS/CanonicalIntervalSet.py +++ b/nca/CoreDS/CanonicalIntervalSet.py @@ -220,6 +220,12 @@ def is_subset(self, other): """ return other.start <= self.start and other.end >= self.end + def is_single_value(self): + """ + :return: Whether the interval contains a single value + """ + return self.start == self.end + def find_interval_left(self, interval): """ find from left to right the last interval which is lower than the input interval, @@ -354,3 +360,10 @@ def get_interval_set_list_numbers_and_ranges(self): else: res.append(f'{interval.start}-{interval.end}') return res + + def is_single_value(self): + """ + :return: Whether the interval set contains a single value + """ + return len(self) == 1 and list(self)[0].is_single_value() + diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index 11d30aff9..35249f29f 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -587,6 +587,15 @@ def make_empty_properties(peer_container=None): def make_all_properties(peer_container=None): return TcpLikeProperties(base_peer_set=peer_container.peer_set.copy() if peer_container else None) + def are_auto_conns(self): + if self.active_dimensions != ['src_peers', 'dst_peers']: + return False + for cube in self: + if cube[0] != cube[1] or not cube[0].is_single_value(): + return False + return True + + ####################################### ICMP-related functions ####################################### @staticmethod diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 54c7c31c6..0b77d4f0c 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -806,7 +806,15 @@ def compare_orig_to_opt_conn(self, orig_conn_graph, opt_props): def compare_fw_rules(self, fw_rules1, fw_rules2): tcp_props1 = ConnectionSet.fw_rules_to_tcp_properties(fw_rules1, self.config.peer_container) tcp_props2 = ConnectionSet.fw_rules_to_tcp_properties(fw_rules2, self.config.peer_container) - assert tcp_props1 == tcp_props2 + if tcp_props1 == tcp_props2: + print("Original and optimized fw-rules are semantically equivalent") + else: + diff_prop = (tcp_props1 - tcp_props2) | (tcp_props2 - tcp_props1) + if diff_prop.are_auto_conns(): + print("Original and optimized fw-rules differ only in auto-connections") + else: + print("Error: original and optimized fw-rules are different") + assert False @staticmethod def filter_istio_edge(peer2, conns): diff --git a/nca/Parsers/IstioSidecarYamlParser.py b/nca/Parsers/IstioSidecarYamlParser.py index 24c71d1a0..b3e3465ec 100644 --- a/nca/Parsers/IstioSidecarYamlParser.py +++ b/nca/Parsers/IstioSidecarYamlParser.py @@ -171,7 +171,7 @@ def parse_policy(self): sidecar_spec = self.policy['spec'] # currently, supported fields in spec are workloadSelector and egress allowed_spec_keys = {'workloadSelector': [0, dict], 'ingress': [3, list], 'egress': [0, list], - 'outboundTrafficPolicy': [3, str]} + 'outboundTrafficPolicy': [3, dict]} self.check_fields_validity(sidecar_spec, 'Sidecar spec', allowed_spec_keys) res_policy.affects_egress = sidecar_spec.get('egress') is not None From c408b49e098c0de64eff85674d9ef6c33f59b21c Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 5 Feb 2023 17:41:06 +0200 Subject: [PATCH 032/187] Splitting istio opt properties to tcp and non-tcp properties. Fixed handling non captured peers in Istio policy. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 13 ++++++-- nca/CoreDS/ProtocolSet.py | 6 ++++ nca/CoreDS/TcpLikeProperties.py | 12 +++++-- nca/NetworkConfig/NetworkConfig.py | 12 ------- nca/NetworkConfig/NetworkConfigQuery.py | 7 ++-- nca/NetworkConfig/NetworkLayer.py | 43 ++++++++++++++----------- nca/NetworkConfig/PeerContainer.py | 9 ++++-- nca/Resources/IstioNetworkPolicy.py | 6 ++-- 8 files changed, 62 insertions(+), 46 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 2d8c7cc8a..50928e2f2 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -557,7 +557,7 @@ def convert_to_tcp_like_properties(self, peer_container): protocols = ProtocolSet() protocols.add_protocol(protocol) this_prop = TcpLikeProperties.make_tcp_like_properties(peer_container, protocols=protocols) - if bool(properties): + if isinstance(properties, bool): if properties: res |= this_prop else: @@ -580,7 +580,14 @@ def get_non_tcp_connections(): # TODO - after moving to the optimized HC set implementation, # get rid of ConnectionSet and move the code below to TcpLikeProperties.py @staticmethod - def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container): + def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container, connectivity_restriction): + ignore_protocols = ProtocolSet() + if connectivity_restriction: + if connectivity_restriction == 'TCP': + ignore_protocols.add_protocol('TCP') + else: #connectivity_restriction == 'non-TCP' + ignore_protocols = ProtocolSet.get_non_tcp_protocols() + fw_rules_map = defaultdict(list) for cube in tcp_props: cube_dict = tcp_props.get_cube_dict_with_orig_values(cube) @@ -598,7 +605,7 @@ def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container): protocols = new_cube_dict.get('protocols') if protocols: new_cube_dict.pop('protocols') - if not protocols and not new_cube_dict: + if not new_cube_dict and (not protocols or protocols == ignore_protocols): conns = ConnectionSet(True) else: conns = ConnectionSet() diff --git a/nca/CoreDS/ProtocolSet.py b/nca/CoreDS/ProtocolSet.py index cd3f8edeb..e44137a44 100644 --- a/nca/CoreDS/ProtocolSet.py +++ b/nca/CoreDS/ProtocolSet.py @@ -22,6 +22,12 @@ def __init__(self, all_protocols=False): if all_protocols: # the whole range self.add_interval(self._whole_range_interval()) + @staticmethod + def get_non_tcp_protocols(): + res = ProtocolSet(True) + res.remove_protocol('TCP') + return res + def __contains__(self, protocol): if isinstance(protocol, str): protocol_num = ProtocolNameResolver.get_protocol_number(protocol) diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index 35249f29f..1620ba592 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -588,10 +588,18 @@ def make_all_properties(peer_container=None): return TcpLikeProperties(base_peer_set=peer_container.peer_set.copy() if peer_container else None) def are_auto_conns(self): - if self.active_dimensions != ['src_peers', 'dst_peers']: + if not {'src_peers', 'dst_peers'}.issubset(set(self.active_dimensions)): return False + src_peers_index = None + dst_peers_index = None + for i, dim in enumerate(self.active_dimensions): + if dim == "src_peers": + src_peers_index = i + elif dim == "dst_peers": + dst_peers_index = i + for cube in self: - if cube[0] != cube[1] or not cube[0].is_single_value(): + if cube[src_peers_index] != cube[dst_peers_index] or not cube[src_peers_index].is_single_value(): return False return True diff --git a/nca/NetworkConfig/NetworkConfig.py b/nca/NetworkConfig/NetworkConfig.py index 7e45c53ba..73f6bf58a 100644 --- a/nca/NetworkConfig/NetworkConfig.py +++ b/nca/NetworkConfig/NetworkConfig.py @@ -270,18 +270,6 @@ def allowed_connections_optimized(self, connectivityFilterIstioEdges, layer_name # all allowed connections: intersection of all allowed connections from all layers conns_res &= conns_per_layer - if self.policies_container.layers.does_contain_single_layer(NetworkLayerName.Istio) \ - and connectivityFilterIstioEdges: - protocols = ProtocolSet() - protocols.add_protocol('TCP') - dst_peers_no_ip = conns_res.project_on_one_dimension('dst_peers') if conns_res else Peer.PeerSet() - dst_peers_no_ip -= Peer.IpBlock.get_all_ips_block_peer_set() - if dst_peers_no_ip: - conns_res &= TcpLikeProperties.make_tcp_like_properties(self.peer_container, dst_peers=dst_peers_no_ip, - protocols=protocols) - else: - conns_res = TcpLikeProperties.make_empty_properties(self.peer_container) - return conns_res def append_policy_to_config(self, policy): diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 078e3d421..fbb15e893 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -758,9 +758,9 @@ def exec(self): opt_end = time.time() print(f'Opt time: {(opt_end - opt_start):6.2f} seconds') if self.config.optimized_run == 'debug': - if fw_rules_tcp and opt_fw_rules_tcp: + if fw_rules_tcp.fw_rules_map and opt_fw_rules_tcp.fw_rules_map: self.compare_fw_rules(fw_rules_tcp, opt_fw_rules_tcp) - if fw_rules_non_tcp and opt_fw_rules_non_tcp: + if fw_rules_non_tcp.fw_rules_map and opt_fw_rules_non_tcp.fw_rules_map: self.compare_fw_rules(fw_rules_non_tcp, opt_fw_rules_non_tcp) else: output_res, opt_fw_rules = self.get_props_output_full(all_conns_opt, opt_peers_to_compare) @@ -942,7 +942,8 @@ def fw_rules_from_props(self, props, peers_to_compare, connectivity_restriction= :rtype: Union[str, dict] """ cluster_info = ClusterInfo(peers_to_compare, self.config.get_allowed_labels()) - fw_rules_map = ConnectionSet.tcp_properties_to_fw_rules(props, cluster_info, self.config.peer_container) + fw_rules_map = ConnectionSet.tcp_properties_to_fw_rules(props, cluster_info, self.config.peer_container, + connectivity_restriction) fw_rules = MinimizeFWRules(fw_rules_map, cluster_info, self.output_config, {}) formatted_rules = fw_rules.get_fw_rules_in_required_format(connectivity_restriction=connectivity_restriction) return formatted_rules, fw_rules diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 513779cd5..05800cb94 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -8,6 +8,7 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock, HostEP, PeerSet from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy from nca.Resources.NetworkPolicy import PolicyConnections, NetworkPolicy @@ -176,8 +177,8 @@ def allowed_connections_optimized(self, peer_container): allowed_ingress_conns, denied_ingres_conns = self._allowed_xgress_conns_optimized(True, peer_container) allowed_egress_conns, denied_egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) res = allowed_ingress_conns & allowed_egress_conns - res -= denied_ingres_conns - res -= denied_egress_conns + # res -= denied_ingres_conns + # res -= denied_egress_conns # exclude IpBlock->IpBlock connections all_ips_peer_set = PeerSet({IpBlock.get_all_ips_block()}) excluded_conns = TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=all_ips_peer_set, @@ -229,11 +230,13 @@ def collect_policies_conns(self, from_peer, to_peer, is_ingress, pass_conns |= policy_conns.pass_conns return allowed_conns, denied_conns, pass_conns, captured_res - def collect_policies_conns_optimized(self, is_ingress): + def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda policy: True): """ Collect all connections (between all relevant peers), considering all layer's policies that capture the relevant peers. :param bool is_ingress: indicates whether to return ingress connections or egress connections + :param captured_func: callable that returns True if the policy satisfies additional conditions required for + considering captured pods instead of applying the default connections. :return: allowed_conns, denied_conns and set of peers to be added to captured peers :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) """ @@ -243,7 +246,6 @@ def collect_policies_conns_optimized(self, is_ingress): for policy in self.policies_list: policy_allowed_conns, policy_denied_conns, policy_captured = \ policy.allowed_connections_optimized(is_ingress) - policy_denied_conns -= allowed_conns #policy_denied_conns -= pass_conns # Preparation for handling of pass policy_allowed_conns -= denied_conns @@ -254,7 +256,8 @@ def collect_policies_conns_optimized(self, is_ingress): allowed_conns |= policy_allowed_conns denied_conns |= policy_denied_conns - captured |= policy_captured + if captured_func(policy): + captured |= policy_captured return allowed_conns, denied_conns, captured @@ -284,8 +287,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): # Note: The below computation of non-captured conns cannot be done during the parse stage, # since before computing non-captured conns we should collect all policies conns # compute non-captured connections - base_peer_set = peer_container.peer_set.copy() - base_peer_set.add(IpBlock.get_all_ips_block()) + base_peer_set = peer_container.get_all_peers_group(True) base_peer_set_no_hep = PeerSet(set([peer for peer in base_peer_set if not isinstance(peer, HostEP)])) if is_ingress: # TODO - probably captured_dst_peers1 and captured_dst_peers2 calculation is redundant (all included in add_to_captured) @@ -315,16 +317,17 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): class IstioNetworkLayer(NetworkLayer): + @staticmethod + def captured_cond_func(policy): + if policy.policy_kind == NetworkPolicy.PolicyType.IstioAuthorizationPolicy: + return policy.action == IstioNetworkPolicy.ActionType.Allow + return True # only for Istio AuthorizationPolicy the captured condition is more refined with 'Allow' policies def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): # in istio applying default-allow if there is no capturing policy with action allow - def captured_cond_func(policy): - if policy.policy_kind == NetworkPolicy.PolicyType.IstioAuthorizationPolicy: - return policy.action == IstioNetworkPolicy.ActionType.Allow - return True # only for Istio AuthorizationPolicy the captured condition is more refined with 'Allow' policies allowed_conns, denied_conns, _, captured_res = self.collect_policies_conns(from_peer, to_peer, is_ingress, - captured_cond_func) + IstioNetworkLayer.captured_cond_func) # for istio initialize non-captured conns with non-TCP connections allowed_non_captured_conns = ConnectionSet.get_non_tcp_connections() if not captured_res: # no allow policies for target @@ -334,21 +337,24 @@ def captured_cond_func(policy): all_allowed_conns=allowed_conns | allowed_non_captured_conns) def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): - allowed_conn, denied_conns, captured = self.collect_policies_conns_optimized(is_ingress) - base_peer_set = peer_container.peer_set.copy() - base_peer_set.add(IpBlock.get_all_ips_block()) + allowed_conn, denied_conns, captured = self.collect_policies_conns_optimized(is_ingress, + IstioNetworkLayer.captured_cond_func) + base_peer_set = peer_container.get_all_peers_group(True) if is_ingress: non_captured_peers = base_peer_set - captured if non_captured_peers: non_captured_conns = \ TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=base_peer_set, dst_peers=non_captured_peers) - allowed_conn |= non_captured_conns + allowed_conn |= (non_captured_conns - denied_conns) else: non_captured_conns = \ TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=base_peer_set, dst_peers=base_peer_set) - allowed_conn |= non_captured_conns + allowed_conn |= (non_captured_conns - denied_conns) + allowed_conn |= TcpLikeProperties.make_tcp_like_properties(peer_container, + protocols=ProtocolSet.get_non_tcp_protocols(), + src_peers=base_peer_set, dst_peers=base_peer_set) return allowed_conn, denied_conns @@ -367,8 +373,7 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): allowed_conn, denied_conns, captured = self.collect_policies_conns_optimized(is_ingress) - base_peer_set = peer_container.peer_set.copy() - base_peer_set.add(IpBlock.get_all_ips_block()) + base_peer_set = peer_container.get_all_peers_group(True) if is_ingress: non_captured_conns = \ TcpLikeProperties.make_tcp_like_properties(peer_container, diff --git a/nca/NetworkConfig/PeerContainer.py b/nca/NetworkConfig/PeerContainer.py index 2310dbe93..5ed79d668 100644 --- a/nca/NetworkConfig/PeerContainer.py +++ b/nca/NetworkConfig/PeerContainer.py @@ -277,9 +277,12 @@ def get_all_peers_group(self, add_external_ips=False, include_globals=True): :return PeerSet: The required set of peers """ res = PeerSet() - for peer in self.peer_set: - if include_globals or not peer.is_global_peer(): - res.add(peer) + if include_globals: + res = self.peer_set.copy() + else: + for peer in self.peer_set: + if not peer.is_global_peer(): + res.add(peer) if add_external_ips: res.add(IpBlock.get_all_ips_block()) return res diff --git a/nca/Resources/IstioNetworkPolicy.py b/nca/Resources/IstioNetworkPolicy.py index 162dbc835..f5fc3fb02 100644 --- a/nca/Resources/IstioNetworkPolicy.py +++ b/nca/Resources/IstioNetworkPolicy.py @@ -104,13 +104,11 @@ def allowed_connections_optimized(self, is_ingress): if is_ingress: allowed = self.optimized_ingress_props.copy() denied = self.optimized_denied_ingress_props.copy() - captured = self.selected_peers if \ - (self.affects_ingress and self.action == IstioNetworkPolicy.ActionType.Allow) else PeerSet() + captured = self.selected_peers else: allowed = self.optimized_egress_props.copy() denied = self.optimized_denied_egress_props.copy() - captured = self.selected_peers if \ - (self.affects_egress and self.action == IstioNetworkPolicy.ActionType.Allow) else PeerSet() + captured = PeerSet() return allowed, denied, captured def referenced_ip_blocks(self): From 4a187f2f6234b6ceda15f4101ac36e8e3877f669 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 5 Feb 2023 18:17:46 +0200 Subject: [PATCH 033/187] Fixed creation TcpLikeProperties with empty methods or protocols. Signed-off-by: Tanya --- nca/CoreDS/TcpLikeProperties.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index 1620ba592..fe07c2c1e 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -79,7 +79,7 @@ def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protoco if dst_peers is not None: cube.append(dst_peers) active_dims.append("dst_peers") - if protocols and not protocols.is_whole_range(): + if protocols is not None and not protocols.is_whole_range(): cube.append(protocols) active_dims.append("protocols") if not source_ports.is_all(): @@ -88,7 +88,7 @@ def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protoco if not dest_ports.is_all(): cube.append(dest_ports.port_set) active_dims.append("dst_ports") - if methods and not methods.is_whole_range(): + if methods is not None and not methods.is_whole_range(): cube.append(methods) active_dims.append("methods") if paths is not None: From 74234aba02f9ab8f73d47485bcbb7a82595e4f8c Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 12 Feb 2023 18:53:20 +0200 Subject: [PATCH 034/187] Optimizing sidecar priorities handling by refinement of sidecar.selected_peers in parse time. Istio sidecar optimized connectivity implementation. Removing unused methods. Better non-captured handling. Signed-off-by: Tanya --- nca/FWRules/MinimizeFWRules.py | 26 ------- nca/NetworkConfig/NetworkLayer.py | 105 +++++++++++++------------- nca/NetworkConfig/PoliciesFinder.py | 12 ++- nca/Parsers/IstioSidecarYamlParser.py | 66 +++++++++++++--- nca/Resources/IstioSidecar.py | 44 ++++++++++- 5 files changed, 160 insertions(+), 93 deletions(-) diff --git a/nca/FWRules/MinimizeFWRules.py b/nca/FWRules/MinimizeFWRules.py index 7d365a61c..568dad37b 100644 --- a/nca/FWRules/MinimizeFWRules.py +++ b/nca/FWRules/MinimizeFWRules.py @@ -659,32 +659,6 @@ def __eq__(self, other): return self.fw_rules_map == other.fw_rules_map and self.cluster_info == other.cluster_info \ and self.output_config == other.output_config and self.results_map == other.results_map - @staticmethod - def same_peers(fw_rules_list1, fw_rules_list2): - # assuming the same lists order - if len(fw_rules_list1) != len(fw_rules_list2): - return False - for index, rule1 in enumerate(fw_rules_list1): - rule2 = fw_rules_list2[index] - if rule1.src != rule2.src or rule1.dst != rule2.dst: - return False - return True - - def unite_fw_rules_with_same_peers(self): - new_fw_rules_map = self.fw_rules_map - self.fw_rules_map = defaultdict(list) - while new_fw_rules_map: - the_conn, the_fw_rules = new_fw_rules_map.popitem() - conns_to_remove = [] - for conn, fw_rules in new_fw_rules_map.items(): - if self.same_peers(fw_rules, the_fw_rules): - the_conn |= conn - conns_to_remove.append(conn) - for r in the_fw_rules: r.conn = the_conn - self.fw_rules_map[the_conn] = the_fw_rules - for conn in conns_to_remove: - new_fw_rules_map.pop(conn) - def get_fw_rules_in_required_format(self, add_txt_header=True, add_csv_header=True, connectivity_restriction=None): """ :param add_txt_header: bool flag to indicate if header of fw-rules query should be added in txt format diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 05800cb94..1f859bc33 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -174,13 +174,16 @@ def allowed_connections_optimized(self, peer_container): :return: all allowed connections :rtype: TcpLikeProperties """ + all_pods = peer_container.get_all_peers_group() + all_ips_peer_set = PeerSet({IpBlock.get_all_ips_block()}) allowed_ingress_conns, denied_ingres_conns = self._allowed_xgress_conns_optimized(True, peer_container) + allowed_ingress_conns |= TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=all_pods, + dst_peers=all_ips_peer_set) allowed_egress_conns, denied_egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) + allowed_egress_conns |= TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=all_ips_peer_set, + dst_peers=all_pods) res = allowed_ingress_conns & allowed_egress_conns - # res -= denied_ingres_conns - # res -= denied_egress_conns # exclude IpBlock->IpBlock connections - all_ips_peer_set = PeerSet({IpBlock.get_all_ips_block()}) excluded_conns = TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=all_ips_peer_set, dst_peers=all_ips_peer_set) res -= excluded_conns @@ -246,18 +249,18 @@ def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda poli for policy in self.policies_list: policy_allowed_conns, policy_denied_conns, policy_captured = \ policy.allowed_connections_optimized(is_ingress) - policy_denied_conns -= allowed_conns - #policy_denied_conns -= pass_conns # Preparation for handling of pass - policy_allowed_conns -= denied_conns - #policy_allowed_conns -= pass_conns # Preparation for handling of pass - #policy_pass_conns -= denied_conns - #policy_pass_conns -= allowed_conns - #pass_conns |= policy_pass_conns - - allowed_conns |= policy_allowed_conns - denied_conns |= policy_denied_conns - if captured_func(policy): - captured |= policy_captured + if policy_captured: # not empty + policy_denied_conns -= allowed_conns + #policy_denied_conns -= pass_conns # Preparation for handling of pass + policy_allowed_conns -= denied_conns + #policy_allowed_conns -= pass_conns # Preparation for handling of pass + #policy_pass_conns -= denied_conns + #policy_pass_conns -= allowed_conns + #pass_conns |= policy_pass_conns + allowed_conns |= policy_allowed_conns + denied_conns |= policy_denied_conns + if captured_func(policy): + captured |= policy_captured return allowed_conns, denied_conns, captured @@ -283,36 +286,26 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): all_allowed_conns=allowed_conns | allowed_non_captured_conns) def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): - allowed_conn, denied_conns, add_to_captured = self.collect_policies_conns_optimized(is_ingress) + allowed_conn, denied_conns, captured = self.collect_policies_conns_optimized(is_ingress) # Note: The below computation of non-captured conns cannot be done during the parse stage, # since before computing non-captured conns we should collect all policies conns # compute non-captured connections - base_peer_set = peer_container.get_all_peers_group(True) - base_peer_set_no_hep = PeerSet(set([peer for peer in base_peer_set if not isinstance(peer, HostEP)])) - if is_ingress: - # TODO - probably captured_dst_peers1 and captured_dst_peers2 calculation is redundant (all included in add_to_captured) - captured_dst_peers1 = allowed_conn.project_on_one_dimension('dst_peers') if allowed_conn else PeerSet() - captured_dst_peers2 = denied_conns.project_on_one_dimension('dst_peers') if denied_conns else PeerSet() - captured_dst_peers = captured_dst_peers1 | captured_dst_peers2 | add_to_captured - non_captured_dst_peers = base_peer_set_no_hep - captured_dst_peers - if non_captured_dst_peers: + base_peer_set_with_ip = peer_container.get_all_peers_group(True) + base_peer_set_no_ip = peer_container.get_all_peers_group() + base_peer_set_no_hep = PeerSet(set([peer for peer in base_peer_set_no_ip if not isinstance(peer, HostEP)])) + non_captured = base_peer_set_no_hep - captured + if non_captured: + if is_ingress: non_captured_conns = \ TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=base_peer_set, - dst_peers=non_captured_dst_peers) - allowed_conn |= non_captured_conns - else: - captured_src_peers1 = allowed_conn.project_on_one_dimension('src_peers') if allowed_conn else PeerSet() - captured_src_peers2 = denied_conns.project_on_one_dimension('src_peers') if denied_conns else PeerSet() - captured_src_peers = captured_src_peers1 | captured_src_peers2 | add_to_captured - non_captured_src_peers = base_peer_set_no_hep - captured_src_peers - if non_captured_src_peers: + src_peers=base_peer_set_with_ip, + dst_peers=non_captured) + else: non_captured_conns = \ TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=non_captured_src_peers, - dst_peers=base_peer_set) - allowed_conn |= non_captured_conns - + src_peers=non_captured, + dst_peers=base_peer_set_with_ip) + allowed_conn |= non_captured_conns return allowed_conn, denied_conns @@ -339,22 +332,25 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): allowed_conn, denied_conns, captured = self.collect_policies_conns_optimized(is_ingress, IstioNetworkLayer.captured_cond_func) - base_peer_set = peer_container.get_all_peers_group(True) - if is_ingress: - non_captured_peers = base_peer_set - captured - if non_captured_peers: + base_peer_set_with_ip = peer_container.get_all_peers_group(True) + base_peer_set_no_ip = peer_container.get_all_peers_group() + non_captured_peers = base_peer_set_no_ip - captured + if non_captured_peers: + if is_ingress: non_captured_conns = \ TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=base_peer_set, dst_peers=non_captured_peers) - allowed_conn |= (non_captured_conns - denied_conns) - else: - non_captured_conns = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=base_peer_set, dst_peers=base_peer_set) + src_peers=base_peer_set_with_ip, + dst_peers=non_captured_peers) + else: + non_captured_conns = \ + TcpLikeProperties.make_tcp_like_properties(peer_container, + src_peers=non_captured_peers, + dst_peers=base_peer_set_with_ip) allowed_conn |= (non_captured_conns - denied_conns) allowed_conn |= TcpLikeProperties.make_tcp_like_properties(peer_container, protocols=ProtocolSet.get_non_tcp_protocols(), - src_peers=base_peer_set, dst_peers=base_peer_set) + src_peers=base_peer_set_with_ip, + dst_peers=base_peer_set_with_ip) return allowed_conn, denied_conns @@ -373,17 +369,20 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): allowed_conn, denied_conns, captured = self.collect_policies_conns_optimized(is_ingress) - base_peer_set = peer_container.get_all_peers_group(True) + base_peer_set_with_ip = peer_container.get_all_peers_group(True) + base_peer_set_no_ip = peer_container.get_all_peers_group() if is_ingress: non_captured_conns = \ TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=base_peer_set, dst_peers=base_peer_set) + src_peers=base_peer_set_with_ip, + dst_peers=base_peer_set_no_ip) allowed_conn |= non_captured_conns else: - non_captured_peers = base_peer_set - captured + non_captured_peers = base_peer_set_no_ip - captured if non_captured_peers: non_captured_conns = \ TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=non_captured_peers, dst_peers=base_peer_set) + src_peers=non_captured_peers, + dst_peers=base_peer_set_with_ip) allowed_conn |= non_captured_conns return allowed_conn, denied_conns diff --git a/nca/NetworkConfig/PoliciesFinder.py b/nca/NetworkConfig/PoliciesFinder.py index d1ef07bdb..8328142e1 100644 --- a/nca/NetworkConfig/PoliciesFinder.py +++ b/nca/NetworkConfig/PoliciesFinder.py @@ -68,6 +68,7 @@ def _add_policy(self, policy): def parse_policies_in_parse_queue(self): istio_traffic_parser = None + istio_sidecar_parser = None for policy, file_name, policy_type in self._parse_queue: if policy_type == NetworkPolicy.PolicyType.CalicoProfile: parsed_element = CalicoPolicyYamlParser(policy, self.peer_container, file_name, self.optimized_run) @@ -82,8 +83,11 @@ def parse_policies_in_parse_queue(self): parsed_element = IstioPolicyYamlParser(policy, self.peer_container, file_name) self._add_policy(parsed_element.parse_policy()) elif policy_type == NetworkPolicy.PolicyType.IstioSidecar: - parsed_element = IstioSidecarYamlParser(policy, self.peer_container, file_name) - self._add_policy(parsed_element.parse_policy()) + if not istio_sidecar_parser: + istio_sidecar_parser = IstioSidecarYamlParser(policy, self.peer_container, file_name) + else: + istio_sidecar_parser.reset(policy, self.peer_container, file_name) + istio_sidecar_parser.parse_policy() elif policy_type == NetworkPolicy.PolicyType.Ingress: parsed_element = IngressPolicyYamlParser(policy, self.peer_container, file_name) self._add_policy(parsed_element.parse_policy()) @@ -106,6 +110,10 @@ def parse_policies_in_parse_queue(self): istio_traffic_policies = istio_traffic_parser.create_istio_traffic_policies() for istio_traffic_policy in istio_traffic_policies: self._add_policy(istio_traffic_policy) + if istio_sidecar_parser: + istio_sidecars = istio_sidecar_parser.get_istio_sidecars() + for istio_sidecar in istio_sidecars: + self._add_policy(istio_sidecar) def parse_yaml_code_for_policy(self, policy_object, file_name): policy_type = NetworkPolicy.get_policy_type_from_dict(policy_object) diff --git a/nca/Parsers/IstioSidecarYamlParser.py b/nca/Parsers/IstioSidecarYamlParser.py index b3e3465ec..29d93ec8d 100644 --- a/nca/Parsers/IstioSidecarYamlParser.py +++ b/nca/Parsers/IstioSidecarYamlParser.py @@ -4,6 +4,7 @@ # import re from nca.CoreDS.Peer import PeerSet +from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from nca.Resources.NetworkPolicy import NetworkPolicy from nca.Resources.IstioSidecar import IstioSidecar, IstioSidecarRule from nca.Resources.IstioTrafficResources import istio_root_namespace @@ -14,6 +15,15 @@ class IstioSidecarYamlParser(IstioGenericYamlParser): """ A parser for Istio Sidecar objects """ + def __init__(self, policy, peer_container, file_name=''): + IstioGenericYamlParser.__init__(self, policy, peer_container, file_name) + self.peers_referenced_by_labels = PeerSet() + self.specific_sidecars = [] + self.default_sidecars = [] + self.global_default_sidecars = [] + + def reset(self, policy, peer_container, file_name=''): + IstioGenericYamlParser.__init__(self, policy, peer_container, file_name) def _validate_and_partition_host_format(self, host): """ @@ -145,18 +155,18 @@ def _check_and_save_sidecar_if_top_priority(self, curr_sidecar): self.namespace.prior_default_sidecar = curr_sidecar return - for peer in curr_sidecar.selected_peers: - if peer.prior_sidecar: # this sidecar is not first one - self.warning(f'Peer "{peer.full_name()}" already has a Sidecar configuration selecting it. Sidecar: ' - f'"{curr_sidecar.full_name()}" will not be considered as connections for this workload') - continue - peer.prior_sidecar = curr_sidecar + prior_referenced_by_label = curr_sidecar.selected_peers & self.peers_referenced_by_labels + if prior_referenced_by_label: + self.warning(f'Peers {", ".join([peer.full_name() for peer in prior_referenced_by_label])}' + f' already have a Sidecar configuration selecting them. Sidecar: ' + f'"{curr_sidecar.full_name()}" will not be considered as connections for these workloads') + curr_sidecar.selected_peers -= self.peers_referenced_by_labels + self.peers_referenced_by_labels |= curr_sidecar.selected_peers def parse_policy(self): """ - Parses the input object to create a IstioSidecar object - :return: a IstioSidecar object with proper PeerSets - :rtype: IstioSidecar + Parses the input object to create a IstioSidecar object and adds the resulting IstioSidecar object + to specific / default sidecar list """ policy_name, policy_ns = self.parse_generic_yaml_objects_fields(self.policy, ['Sidecar'], ['networking.istio.io/v1alpha3', @@ -167,6 +177,7 @@ def parse_policy(self): self.namespace = self.peer_container.get_namespace(policy_ns, warn_if_missing) res_policy = IstioSidecar(policy_name, self.namespace) res_policy.policy_kind = NetworkPolicy.PolicyType.IstioSidecar + res_policy.add_optimized_ingress_props(TcpLikeProperties.make_all_properties(self.peer_container)) sidecar_spec = self.policy['spec'] # currently, supported fields in spec are workloadSelector and egress @@ -192,5 +203,38 @@ def parse_policy(self): res_policy.findings = self.warning_msgs res_policy.referenced_labels = self.referenced_labels - - return res_policy + if res_policy.default_sidecar: + if str(self.namespace) == istio_root_namespace: + self.global_default_sidecars.append(res_policy) + else: + self.default_sidecars.append(res_policy) + else: + self.specific_sidecars.append(res_policy) + + def get_istio_sidecars(self): + # 1st priority: specific sidecars + # their selected_peers were already refined during parse_policy() + res = [] + for sidecar in self.specific_sidecars: + sidecar.create_opt_egress_props(self.peer_container) + res.append(sidecar) + referenced_peers = self.peers_referenced_by_labels.copy() + # 2nd priority: default sidecars + # refine their selected_peers according to sidecar order in the config and previously referenced peers + for sidecar in self.default_sidecars: + if sidecar.selected_peers & referenced_peers: + sidecar.selected_peers -= referenced_peers + referenced_peers |= sidecar.selected_peers + sidecar.create_opt_egress_props(self.peer_container) + res.append(sidecar) + + # the lowest priority - global default sidecars + # refine their selected_peers according to sidecar order in the config and previously referenced peers + for sidecar in self.global_default_sidecars: + if sidecar.selected_peers & referenced_peers: + sidecar.selected_peers -= referenced_peers + referenced_peers |= sidecar.selected_peers + sidecar.create_opt_egress_props(self.peer_container) + res.append(sidecar) + + return res diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index 75bc61d3f..ca180b5e6 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -4,6 +4,8 @@ # from dataclasses import dataclass from nca.CoreDS.ConnectionSet import ConnectionSet +from nca.CoreDS.Peer import IpBlock, PeerSet +from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from .NetworkPolicy import PolicyConnections, NetworkPolicy from .IstioTrafficResources import istio_root_namespace @@ -54,7 +56,8 @@ def allowed_connections(self, from_peer, to_peer, is_ingress): captured = from_peer in self.selected_peers # if not captured, or captured but the sidecar is not in from_peer top priority, don't consider connections - if not captured or (captured and not self._is_sidecar_prior(from_peer)): + # if not captured or (captured and not self._is_sidecar_prior(from_peer)): + if not captured: return PolicyConnections(False) conns = ConnectionSet(True) @@ -68,6 +71,17 @@ def allowed_connections(self, from_peer, to_peer, is_ingress): # or if the sidecar is global and to_peer is not in same namespace of from_peer while rule host's ns is '.' return PolicyConnections(True, allowed_conns=ConnectionSet()) + def allowed_connections_optimized(self, is_ingress): + if is_ingress: + allowed = self.optimized_ingress_props.copy() + denied = self.optimized_denied_ingress_props.copy() + captured = PeerSet() + else: + allowed = self.optimized_egress_props.copy() + denied = self.optimized_denied_egress_props.copy() + captured = self.selected_peers if self.affects_egress else PeerSet() + return allowed, denied, captured + def has_empty_rules(self, config_name=''): """ Checks whether the sidecar contains empty rules (rules that do not select any peers) @@ -135,3 +149,31 @@ def _is_sidecar_prior(self, from_peer): self == self.namespace.prior_default_sidecar: return True return False + + @staticmethod + def combine_peer_sets_by_ns(from_peer_set, to_peer_set, peer_container): + res = [] + from_peer_set_copy = from_peer_set.copy() + while from_peer_set_copy: + peer = list(from_peer_set_copy)[0] + if isinstance(peer, IpBlock): + from_peer_set_copy.remove(peer) + continue + peers_in_curr_ns = peer_container.get_namespace_pods(peer.namespace) + res.append((from_peer_set_copy & peers_in_curr_ns, to_peer_set & peers_in_curr_ns)) + from_peer_set_copy -= peers_in_curr_ns + return res + + def create_opt_egress_props(self, peer_container): + for rule in self.egress_rules: + if self.selected_peers and rule.egress_peer_set: + self.optimized_egress_props = \ + TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=self.selected_peers, + dst_peers=rule.egress_peer_set) + peers_sets_by_ns = self.combine_peer_sets_by_ns(self.selected_peers, rule.special_egress_peer_set, + peer_container) + for (from_peers, to_peers) in peers_sets_by_ns: + if from_peers and to_peers: + self.optimized_egress_props |= \ + TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=from_peers, + dst_peers=to_peers) From 2c4b1af697783269dbdb6fdbb6b95a0c13dc4dd5 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 14 Feb 2023 17:36:25 +0200 Subject: [PATCH 035/187] Adding newline at the end of connectivity test expected results. Signed-off-by: Tanya --- .../expected_output/testcase16-scheme_output.txt | 2 +- ...ise-interferes-different-ranges-writing-additional-port.txt | 2 +- ...26-config-1-k8s-calico-istio-ingress-2_connectivity_map.txt | 2 +- .../testcase26-config-1-k8s-istio-ingress_connectivity_map.txt | 2 +- ...6-semanticDiff-config-1-calico-ingress-config-allow-all.txt | 2 +- .../expected_output/testcase8-semantic-diff-query.txt | 2 +- .../basic_connectivity_csv_query_output.txt | 2 +- .../basic_connectivity_dot_query_output.txt | 3 ++- .../basic_connectivity_md_query_output.txt | 2 +- .../basic_connectivity_yaml_query_output.txt | 2 +- .../basic_semantic_diff_csv_query_output.txt | 2 +- .../basic_semantic_diff_md_query_output.txt | 2 +- .../basic_semantic_diff_query_output.txt | 2 +- .../basic_semantic_diff_yaml_query_output.txt | 2 +- .../different_topologies_semantic_diff_query_output.txt | 2 +- .../calico-testcase20-Eran_gnps_query_output.txt | 2 +- .../calico-testcase20-Eran_gnps_query_output.yaml | 2 +- .../calico-testcase20-np_2_all_outbound_hep_query_output.txt | 2 +- .../calico-testcase20-np_2_all_outbound_hep_query_output.yaml | 2 +- ...calico-testcase20-np_3_outbound_hep_to_wep_query_output.txt | 2 +- ...alico-testcase20-np_3_outbound_hep_to_wep_query_output.yaml | 2 +- ...case20-np_4_outbound_all_namespaceSelector_query_output.txt | 2 +- ...ase20-np_4_outbound_all_namespaceSelector_query_output.yaml | 2 +- .../expected_output/cyclonus-simple-example-scheme_output.txt | 2 +- .../expected_output/cyclonus-simple-example-scheme_output.yaml | 2 +- .../expected_output/istio-allow-all-scheme_output.yaml | 2 +- .../expected_output/istio-allow-nothing-1_query_output.txt | 2 +- .../expected_output/istio-allow-nothing-1_query_output.yaml | 2 +- .../expected_output/istio-allow-nothing-2_query_output.txt | 2 +- .../expected_output/istio-allow-nothing-2_query_output.yaml | 2 +- .../expected_output/istio-allow-nothing-3_query_output.txt | 2 +- .../expected_output/istio-allow-nothing-3_query_output.yaml | 2 +- ...bookinfo-connectivity_test_methods_basic_1_query_output.txt | 2 +- ...ookinfo-connectivity_test_methods_basic_1_query_output.yaml | 2 +- ...bookinfo-connectivity_test_methods_basic_2_query_output.txt | 2 +- ...ookinfo-connectivity_test_methods_basic_2_query_output.yaml | 2 +- ...bookinfo-connectivity_test_methods_paths_1_query_output.txt | 2 +- ...ookinfo-connectivity_test_methods_paths_1_query_output.yaml | 2 +- ...okinfo-connectivity_test_operation_allow_1_query_output.txt | 2 +- ...kinfo-connectivity_test_operation_allow_1_query_output.yaml | 2 +- ...ookinfo-connectivity_test_operation_deny_1_query_output.txt | 2 +- ...okinfo-connectivity_test_operation_deny_1_query_output.yaml | 2 +- .../policies/expected_output/istio-deny-all-scheme_output.yaml | 2 +- .../expected_output/istio-test1-scheme_query1_output.txt | 2 +- .../expected_output/istio-test1-scheme_query1_output.yaml | 2 +- .../expected_output/istio-test1-scheme_query2_output.txt | 2 +- .../expected_output/istio-test1-scheme_query2_output.yaml | 2 +- .../policies/expected_output/poc2-scheme_output.txt | 2 +- .../poc4_scheme_connectivity_map_query_output.txt | 2 +- .../expected_output/semantic_diff_a_to_b_query_output.csv | 2 +- .../expected_output/semantic_diff_a_to_b_query_output.md | 2 +- .../expected_output/semantic_diff_a_to_b_query_output.txt | 2 +- .../expected_output/semantic_diff_a_to_b_query_output.yaml | 2 +- .../semantic_diff_a_to_b_with_ipBlock_query_output.csv | 2 +- .../semantic_diff_a_to_b_with_ipBlock_query_output.md | 2 +- .../semantic_diff_a_to_b_with_ipBlock_query_output.txt | 2 +- .../semantic_diff_a_to_b_with_ipBlock_query_output.yaml | 2 +- .../expected_output/semantic_diff_b_to_a_query_output.csv | 2 +- .../expected_output/semantic_diff_b_to_a_query_output.md | 2 +- .../expected_output/semantic_diff_b_to_a_query_output.yaml | 2 +- .../semantic_diff_disjoint_old1_config_a_query_output.csv | 2 +- .../semantic_diff_disjoint_old1_config_a_query_output.md | 2 +- .../semantic_diff_disjoint_old1_config_a_query_output.txt | 2 +- .../semantic_diff_disjoint_old1_config_a_query_output.yaml | 2 +- .../semantic_diff_ipblocks__np1_np4_query_output.csv | 2 +- .../semantic_diff_ipblocks__np1_np4_query_output.md | 2 +- .../semantic_diff_ipblocks__np1_np4_query_output.txt | 2 +- .../semantic_diff_ipblocks__np1_np4_query_output.yaml | 2 +- ...antic_diff_named_ports_np1_and_np2_by_pods_query_output.txt | 2 +- .../semantic_diff_named_ports_np1_and_np2_query_output.csv | 2 +- .../semantic_diff_named_ports_np1_and_np2_query_output.md | 2 +- .../semantic_diff_named_ports_np1_and_np2_query_output.txt | 2 +- .../semantic_diff_named_ports_np1_and_np2_query_output.yaml | 2 +- .../expected_output/semantic_diff_poc-scheme_output.txt | 2 +- .../policies/expected_output/test10-scheme_output.txt | 2 +- ...vity_map_bookinfo_adding_default_sidecar_after_specific.txt | 2 +- .../connectivity_map_bookinfo_default_sidecar.txt | 2 +- ...nnectivity_map_bookinfo_ignoring_second_default_sidecar.txt | 2 +- .../connectivity_map_bookinfo_multiple_sidecar_overrides.txt | 2 +- .../connectivity_map_bookinfo_productpage_sidecar.txt | 2 +- ...ectivity_map_bookinfo_sidecars_with_different_selectors.txt | 2 +- ..._map_bookinfo_specific_sidecar_overrides_global_sidecar.txt | 2 +- ...y_map_bookinfo_two_sidecars_with_same_workload_selector.txt | 2 +- .../connectivity_map_global_sidecar_from_istio_ref.txt | 2 +- .../connectivity_map_of_onlineboutique_resources.txt | 2 +- .../connectivity_map_online_boutique_frontend_sidecar.txt | 2 +- .../expected_output/new_online_boutique_connectivity_map.txt | 2 +- .../new_online_boutique_synth_res_connectivity_map.txt | 2 +- ..._boutique_synth_res_connectivity_map_with_baseline_rule.txt | 2 +- .../semantic_diff_online_boutique_new_input_vs_synth_res.txt | 2 +- .../expected_output/new_online_boutique_connectivity_map.txt | 2 +- .../orig_online_boutique_synthesis_res_connectivity_map.txt | 2 +- ...iff_online_boutique_new_synthesized_vs_orig_synthesized.txt | 2 +- 93 files changed, 94 insertions(+), 93 deletions(-) diff --git a/tests/calico_testcases/expected_output/testcase16-scheme_output.txt b/tests/calico_testcases/expected_output/testcase16-scheme_output.txt index 94c9b2e13..554889130 100644 --- a/tests/calico_testcases/expected_output/testcase16-scheme_output.txt +++ b/tests/calico_testcases/expected_output/testcase16-scheme_output.txt @@ -4,4 +4,4 @@ src_ns: [default,vendor-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connectio src_ns: [default,vendor-system] src_pods: [*] dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections src_ns: [kube-system] src_pods: [!has(tier)] dst: 0.0.0.0/0 conn: All connections src_ns: [kube-system] src_pods: [!has(tier)] dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections -src_ns: [kube-system] src_pods: [tier=frontend] dst: 64.0.0.0-255.255.255.255 conn: TCP \ No newline at end of file +src_ns: [kube-system] src_pods: [tier=frontend] dst: 64.0.0.0-255.255.255.255 conn: TCP diff --git a/tests/calico_testcases/expected_output/testcase18-scheme-pair-wise-interferes-different-ranges-writing-additional-port.txt b/tests/calico_testcases/expected_output/testcase18-scheme-pair-wise-interferes-different-ranges-writing-additional-port.txt index 30ebdaca1..5313f7fc3 100644 --- a/tests/calico_testcases/expected_output/testcase18-scheme-pair-wise-interferes-different-ranges-writing-additional-port.txt +++ b/tests/calico_testcases/expected_output/testcase18-scheme-pair-wise-interferes-different-ranges-writing-additional-port.txt @@ -1220,4 +1220,4 @@ src: kube-system/vpn-858f6d9777-2bw5m, dst: kube-system/tiller-deploy-5c45c9966b src: kube-system/vpn-858f6d9777-2bw5m, dst: vendor-system/barbar-app-lp6tw, description: np-ports-based/testcase18-different-ranges-writing-slightly-bigger allows communication using protocol UDP while np-ports-based/testcase18-different-ranges-writing1 does not. src: kube-system/vpn-858f6d9777-2bw5m, dst: vendor-system/barbar-app-vsh47, description: np-ports-based/testcase18-different-ranges-writing-slightly-bigger allows communication using protocol UDP while np-ports-based/testcase18-different-ranges-writing1 does not. src: kube-system/vpn-858f6d9777-2bw5m, dst: vendor-system/foofoo-app-r66p2, description: np-ports-based/testcase18-different-ranges-writing-slightly-bigger allows communication using protocol UDP while np-ports-based/testcase18-different-ranges-writing1 does not. -src: kube-system/vpn-858f6d9777-2bw5m, dst: vendor-system/foofoo-app-zv2ch, description: np-ports-based/testcase18-different-ranges-writing-slightly-bigger allows communication using protocol UDP while np-ports-based/testcase18-different-ranges-writing1 does not. \ No newline at end of file +src: kube-system/vpn-858f6d9777-2bw5m, dst: vendor-system/foofoo-app-zv2ch, description: np-ports-based/testcase18-different-ranges-writing-slightly-bigger allows communication using protocol UDP while np-ports-based/testcase18-different-ranges-writing1 does not. diff --git a/tests/calico_testcases/expected_output/testcase26-config-1-k8s-calico-istio-ingress-2_connectivity_map.txt b/tests/calico_testcases/expected_output/testcase26-config-1-k8s-calico-istio-ingress-2_connectivity_map.txt index 62fcb143f..9224bbcdf 100644 --- a/tests/calico_testcases/expected_output/testcase26-config-1-k8s-calico-istio-ingress-2_connectivity_map.txt +++ b/tests/calico_testcases/expected_output/testcase26-config-1-k8s-calico-istio-ingress-2_connectivity_map.txt @@ -8,4 +8,4 @@ src_ns: [istio-system] src_pods: [*] dst_ns: [istio-system] dst_pods: [*] conn: For connections of type non-TCP, final fw rules for query: connectivity-6, config: testcase26-config-1-k8s-calico-istio-ingress-2: src_ns: [default] src_pods: [ratings-v1-b6994bb9] dst_ns: [default] dst_pods: [productpage-v1-6b746f74dc] conn: UDP src_ns: [ingress-nginx] src_pods: [*] dst_ns: [ingress-nginx] dst_pods: [*] conn: All connections -src_ns: [istio-system] src_pods: [*] dst_ns: [istio-system] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [istio-system] src_pods: [*] dst_ns: [istio-system] dst_pods: [*] conn: All connections diff --git a/tests/calico_testcases/expected_output/testcase26-config-1-k8s-istio-ingress_connectivity_map.txt b/tests/calico_testcases/expected_output/testcase26-config-1-k8s-istio-ingress_connectivity_map.txt index e99dd3814..bef7160d1 100644 --- a/tests/calico_testcases/expected_output/testcase26-config-1-k8s-istio-ingress_connectivity_map.txt +++ b/tests/calico_testcases/expected_output/testcase26-config-1-k8s-istio-ingress_connectivity_map.txt @@ -21,4 +21,4 @@ src_ns: [default] src_pods: [app!=ratings] dst: 0.0.0.0/0 conn: All connections src_ns: [default] src_pods: [app!=ratings] dst_ns: [default,ingress-nginx,istio-system] dst_pods: [*] conn: All connections src_ns: [ingress-nginx,istio-system] src_pods: [*] dst_ns: [ingress-nginx] dst_pods: [*] conn: All connections src_ns: [istio-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [istio-system] src_pods: [*] dst_ns: [default,istio-system] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [istio-system] src_pods: [*] dst_ns: [default,istio-system] dst_pods: [*] conn: All connections diff --git a/tests/calico_testcases/expected_output/testcase26-semanticDiff-config-1-calico-ingress-config-allow-all.txt b/tests/calico_testcases/expected_output/testcase26-semanticDiff-config-1-calico-ingress-config-allow-all.txt index 864d43ea2..db83dac22 100644 --- a/tests/calico_testcases/expected_output/testcase26-semanticDiff-config-1-calico-ingress-config-allow-all.txt +++ b/tests/calico_testcases/expected_output/testcase26-semanticDiff-config-1-calico-ingress-config-allow-all.txt @@ -10,4 +10,4 @@ src_ns: [ingress-nginx] src_pods: [*] dst_ns: [default] dst_pods: [app in (ratin Added connections between persistent peers and ipBlocks (based on topology from config: allow-all-config) : src: 0.0.0.0/0 dst_ns: [default] dst_pods: [productpage-v1-6b746f74dc] conn: All connections -src_ns: [ingress-nginx] src_pods: [*] dst: 0.0.0.0/0 conn: All connections \ No newline at end of file +src_ns: [ingress-nginx] src_pods: [*] dst: 0.0.0.0/0 conn: All connections diff --git a/tests/calico_testcases/expected_output/testcase8-semantic-diff-query.txt b/tests/calico_testcases/expected_output/testcase8-semantic-diff-query.txt index af94438a4..8a263c506 100644 --- a/tests/calico_testcases/expected_output/testcase8-semantic-diff-query.txt +++ b/tests/calico_testcases/expected_output/testcase8-semantic-diff-query.txt @@ -7,4 +7,4 @@ src_ns: [kube-system] src_pods: [app=kube-fluentd] dst_ns: [kube-system] dst_pod Added connections between persistent peers and ipBlocks (based on topology from config: global-allow-all) : src: 0.0.0.0/0 dst_ns: [kube-system] dst_pods: [app=kube-fluentd] conn: All connections -src_ns: [kube-system] src_pods: [app=kube-fluentd] dst: 0.0.0.0/0 conn: All connections \ No newline at end of file +src_ns: [kube-system] src_pods: [app=kube-fluentd] dst: 0.0.0.0/0 conn: All connections diff --git a/tests/expected_cmdline_output_files/basic_connectivity_csv_query_output.txt b/tests/expected_cmdline_output_files/basic_connectivity_csv_query_output.txt index 589721248..803b2bbfe 100644 --- a/tests/expected_cmdline_output_files/basic_connectivity_csv_query_output.txt +++ b/tests/expected_cmdline_output_files/basic_connectivity_csv_query_output.txt @@ -2,4 +2,4 @@ ", config: testcase8-networkpolicy1.yaml","","","","","", "","","0.0.0.0/0","[default,kube-system,kube-system-dummy-to-ignore,vendor-system]","[*]","All connections", "","[default,kube-system,kube-system-dummy-to-ignore,vendor-system]","[*]","","0.0.0.0/0","All connections", -"","[default,kube-system,kube-system-dummy-to-ignore,vendor-system]","[*]","[default,kube-system,kube-system-dummy-to-ignore,vendor-system]","[*]","All connections", \ No newline at end of file +"","[default,kube-system,kube-system-dummy-to-ignore,vendor-system]","[*]","[default,kube-system,kube-system-dummy-to-ignore,vendor-system]","[*]","All connections", diff --git a/tests/expected_cmdline_output_files/basic_connectivity_dot_query_output.txt b/tests/expected_cmdline_output_files/basic_connectivity_dot_query_output.txt index 81d1f9437..623325006 100644 --- a/tests/expected_cmdline_output_files/basic_connectivity_dot_query_output.txt +++ b/tests/expected_cmdline_output_files/basic_connectivity_dot_query_output.txt @@ -332,4 +332,5 @@ digraph { "vendor-system/foofoo-app(ReplicaSet)" -> "kube-system/vpn(Deployment-StatefulSet)" [label="All connections" color="gold2" fontcolor="darkgreen"] "vendor-system/foofoo-app(ReplicaSet)" -> "vendor-system/barbar-app(ReplicaSet)" [label="All connections" color="gold2" fontcolor="darkgreen"] "vendor-system/foofoo-app(ReplicaSet)" -> "vendor-system/foofoo-app(ReplicaSet)" [label="All connections" color="gold2" fontcolor="darkgreen"] -} \ No newline at end of file +} + diff --git a/tests/expected_cmdline_output_files/basic_connectivity_md_query_output.txt b/tests/expected_cmdline_output_files/basic_connectivity_md_query_output.txt index a76f5a9e1..47331dbd5 100644 --- a/tests/expected_cmdline_output_files/basic_connectivity_md_query_output.txt +++ b/tests/expected_cmdline_output_files/basic_connectivity_md_query_output.txt @@ -3,4 +3,4 @@ |, config: testcase8-networkpolicy1.yaml|||||| |||0.0.0.0/0|[default,kube-system,kube-system-dummy-to-ignore,vendor-system]|[*]|All connections| ||[default,kube-system,kube-system-dummy-to-ignore,vendor-system]|[*]||0.0.0.0/0|All connections| -||[default,kube-system,kube-system-dummy-to-ignore,vendor-system]|[*]|[default,kube-system,kube-system-dummy-to-ignore,vendor-system]|[*]|All connections| \ No newline at end of file +||[default,kube-system,kube-system-dummy-to-ignore,vendor-system]|[*]|[default,kube-system,kube-system-dummy-to-ignore,vendor-system]|[*]|All connections| diff --git a/tests/expected_cmdline_output_files/basic_connectivity_yaml_query_output.txt b/tests/expected_cmdline_output_files/basic_connectivity_yaml_query_output.txt index b05616cfb..117bd3f1a 100644 --- a/tests/expected_cmdline_output_files/basic_connectivity_yaml_query_output.txt +++ b/tests/expected_cmdline_output_files/basic_connectivity_yaml_query_output.txt @@ -41,4 +41,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/expected_cmdline_output_files/basic_semantic_diff_csv_query_output.txt b/tests/expected_cmdline_output_files/basic_semantic_diff_csv_query_output.txt index 7e7e4c6a1..79804b5da 100644 --- a/tests/expected_cmdline_output_files/basic_semantic_diff_csv_query_output.txt +++ b/tests/expected_cmdline_output_files/basic_semantic_diff_csv_query_output.txt @@ -5,4 +5,4 @@ "","[kube-system,kube-system-dummy-to-ignore]","[*]","[kube-system]","[*]","TCP+UDP 53", "","[vendor-system]","[*]","[kube-system]","[*]","All connections", "Removed connections between persistent peers and ipBlocks","","","","","", -"","","0.0.0.0/0","[kube-system]","[*]","TCP+UDP 53", \ No newline at end of file +"","","0.0.0.0/0","[kube-system]","[*]","TCP+UDP 53", diff --git a/tests/expected_cmdline_output_files/basic_semantic_diff_md_query_output.txt b/tests/expected_cmdline_output_files/basic_semantic_diff_md_query_output.txt index b9091d69b..16abfb3ba 100644 --- a/tests/expected_cmdline_output_files/basic_semantic_diff_md_query_output.txt +++ b/tests/expected_cmdline_output_files/basic_semantic_diff_md_query_output.txt @@ -6,4 +6,4 @@ ||[kube-system,kube-system-dummy-to-ignore]|[*]|[kube-system]|[*]|TCP+UDP 53| ||[vendor-system]|[*]|[kube-system]|[*]|All connections| |Removed connections between persistent peers and ipBlocks|||||| -|||0.0.0.0/0|[kube-system]|[*]|TCP+UDP 53| \ No newline at end of file +|||0.0.0.0/0|[kube-system]|[*]|TCP+UDP 53| diff --git a/tests/expected_cmdline_output_files/basic_semantic_diff_query_output.txt b/tests/expected_cmdline_output_files/basic_semantic_diff_query_output.txt index e744cd51e..8d0d62349 100644 --- a/tests/expected_cmdline_output_files/basic_semantic_diff_query_output.txt +++ b/tests/expected_cmdline_output_files/basic_semantic_diff_query_output.txt @@ -8,4 +8,4 @@ src_ns: [kube-system,kube-system-dummy-to-ignore] src_pods: [*] dst_ns: [kube-sy src_ns: [vendor-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections Removed connections between persistent peers and ipBlocks (based on topology from config: testcase7-networkpolicy2.yaml) : -src: 0.0.0.0/0 dst_ns: [kube-system] dst_pods: [*] conn: TCP+UDP 53 \ No newline at end of file +src: 0.0.0.0/0 dst_ns: [kube-system] dst_pods: [*] conn: TCP+UDP 53 diff --git a/tests/expected_cmdline_output_files/basic_semantic_diff_yaml_query_output.txt b/tests/expected_cmdline_output_files/basic_semantic_diff_yaml_query_output.txt index ced2acb50..b9bd18673 100644 --- a/tests/expected_cmdline_output_files/basic_semantic_diff_yaml_query_output.txt +++ b/tests/expected_cmdline_output_files/basic_semantic_diff_yaml_query_output.txt @@ -66,4 +66,4 @@ - 53 - Protocol: UDP Ports: - - 53 \ No newline at end of file + - 53 diff --git a/tests/expected_cmdline_output_files/different_topologies_semantic_diff_query_output.txt b/tests/expected_cmdline_output_files/different_topologies_semantic_diff_query_output.txt index 6c2314103..0bd296652 100644 --- a/tests/expected_cmdline_output_files/different_topologies_semantic_diff_query_output.txt +++ b/tests/expected_cmdline_output_files/different_topologies_semantic_diff_query_output.txt @@ -33,4 +33,4 @@ src_ns: [default] src_pods: [cog-agents] dst_ns: [default] dst_pods: [cog-agents New connections between added peers and ipBlocks (based on topology from config: policy_b.yaml) : src: 0.0.0.0/0 dst_ns: [default] dst_pods: [app in (app-5,app-6)] conn: All connections -src_ns: [default] src_pods: [app in (app-5,app-6)] dst: 0.0.0.0/0 conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [app in (app-5,app-6)] dst: 0.0.0.0/0 conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-Eran_gnps_query_output.txt b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-Eran_gnps_query_output.txt index de9bb9f75..30dbdeb7e 100644 --- a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-Eran_gnps_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-Eran_gnps_query_output.txt @@ -124,4 +124,4 @@ src_ns: [None] src_pods: [vendor.role=worker_public] dst_ns: [None] dst_pods: [v src_ns: [None] src_pods: [vendor.role=worker_public] dst_ns: [kube-system] dst_pods: [*] conn: All connections src_ns: [kube-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections src_ns: [kube-system] src_pods: [*] dst_ns: [None] dst_pods: [vendor.role=worker_public] conn: TCP+UDP 52311,ICMP,VRRP -src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-Eran_gnps_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-Eran_gnps_query_output.yaml index e4a676df3..841824d94 100644 --- a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-Eran_gnps_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-Eran_gnps_query_output.yaml @@ -1870,4 +1870,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_2_all_outbound_hep_query_output.txt b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_2_all_outbound_hep_query_output.txt index 98ec4ed8b..afb0a5ae6 100644 --- a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_2_all_outbound_hep_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_2_all_outbound_hep_query_output.txt @@ -9,4 +9,4 @@ src_ns: [None] src_pods: [vendor.role=worker_public] dst_ns: [None] dst_pods: [v src_ns: [None] src_pods: [vendor.role=worker_public] dst_ns: [kube-system] dst_pods: [*] conn: All connections src_ns: [kube-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections src_ns: [kube-system] src_pods: [*] dst_ns: [None] dst_pods: [vendor.role=worker_public] conn: All connections -src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_2_all_outbound_hep_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_2_all_outbound_hep_query_output.yaml index df933f5dc..77b6cc464 100644 --- a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_2_all_outbound_hep_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_2_all_outbound_hep_query_output.yaml @@ -99,4 +99,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_3_outbound_hep_to_wep_query_output.txt b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_3_outbound_hep_to_wep_query_output.txt index fc786eddc..891c3f19a 100644 --- a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_3_outbound_hep_to_wep_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_3_outbound_hep_to_wep_query_output.txt @@ -8,4 +8,4 @@ src_ns: [None] src_pods: [vendor.role=worker_public] dst: 203.0.115.0/29 conn: A src_ns: [None] src_pods: [vendor.role=worker_public] dst_ns: [kube-system] dst_pods: [*] conn: All connections src_ns: [kube-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections src_ns: [kube-system] src_pods: [*] dst_ns: [None] dst_pods: [vendor.role=worker_public] conn: All connections -src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_3_outbound_hep_to_wep_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_3_outbound_hep_to_wep_query_output.yaml index a6e9e1b53..d3792c4da 100644 --- a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_3_outbound_hep_to_wep_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_3_outbound_hep_to_wep_query_output.yaml @@ -89,4 +89,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_4_outbound_all_namespaceSelector_query_output.txt b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_4_outbound_all_namespaceSelector_query_output.txt index 5ddedbf7c..cd6f2ebf2 100644 --- a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_4_outbound_all_namespaceSelector_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_4_outbound_all_namespaceSelector_query_output.txt @@ -8,4 +8,4 @@ src_ns: [None] src_pods: [vendor.role=worker_public] dst: 203.0.115.0/29 conn: A src_ns: [None] src_pods: [vendor.role=worker_public] dst_ns: [kube-system] dst_pods: [*] conn: All connections src_ns: [kube-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections src_ns: [kube-system] src_pods: [*] dst_ns: [None] dst_pods: [vendor.role=worker_public] conn: All connections -src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_4_outbound_all_namespaceSelector_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_4_outbound_all_namespaceSelector_query_output.yaml index 65a338c22..b2b6372e6 100644 --- a/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_4_outbound_all_namespaceSelector_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/calico-testcase20-np_4_outbound_all_namespaceSelector_query_output.yaml @@ -89,4 +89,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/cyclonus-simple-example-scheme_output.txt b/tests/fw_rules_tests/policies/expected_output/cyclonus-simple-example-scheme_output.txt index 43aa1a2c9..91e2dcc3c 100644 --- a/tests/fw_rules_tests/policies/expected_output/cyclonus-simple-example-scheme_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/cyclonus-simple-example-scheme_output.txt @@ -2,4 +2,4 @@ final fw rules for query: connectivity_map, config: cyclonus-simple-example: src: 0.0.0.0/0 dst_ns: [y] dst_pods: [b] conn: All connections src: 0.0.0.0/24 dst_ns: [y] dst_pods: [c] conn: All connections src_ns: [y] src_pods: [a] dst_ns: [y] dst_pods: [b] conn: All connections -src_ns: [y] src_pods: [pod!=c] dst: 0.0.0.0/0 conn: All connections \ No newline at end of file +src_ns: [y] src_pods: [pod!=c] dst: 0.0.0.0/0 conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/cyclonus-simple-example-scheme_output.yaml b/tests/fw_rules_tests/policies/expected_output/cyclonus-simple-example-scheme_output.yaml index 489182cc1..5a56fce8e 100644 --- a/tests/fw_rules_tests/policies/expected_output/cyclonus-simple-example-scheme_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/cyclonus-simple-example-scheme_output.yaml @@ -37,4 +37,4 @@ dst_ip_block: - 0.0.0.0/0 connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-allow-all-scheme_output.yaml b/tests/fw_rules_tests/policies/expected_output/istio-allow-all-scheme_output.yaml index a2a76f37b..1b75c0b5e 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-allow-all-scheme_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/istio-allow-all-scheme_output.yaml @@ -72,4 +72,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-1_query_output.txt b/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-1_query_output.txt index 402f37248..a17a73fe3 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-1_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-1_query_output.txt @@ -6,4 +6,4 @@ src_ns: [default,kube-system,vendor-system] src_pods: [*] dst_ns: [kube-system,v For connections of type non-TCP, final fw rules for query: istio-allow-nothing-1, config: istio-allow-nothing-1: src: 0.0.0.0/0 dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections src_ns: [default,kube-system,vendor-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default,kube-system,vendor-system] src_pods: [*] dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default,kube-system,vendor-system] src_pods: [*] dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-1_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-1_query_output.yaml index ffd513136..127ae8438 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-1_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-1_query_output.yaml @@ -70,4 +70,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-2_query_output.txt b/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-2_query_output.txt index 1cd4b859f..522172e49 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-2_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-2_query_output.txt @@ -6,4 +6,4 @@ src_ns: [default,kube-system,vendor-system] src_pods: [*] dst_ns: [kube-system,v For connections of type non-TCP, final fw rules for query: istio-allow-nothing-2, config: istio-allow-nothing-2: src: 0.0.0.0/0 dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections src_ns: [default,kube-system,vendor-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default,kube-system,vendor-system] src_pods: [*] dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default,kube-system,vendor-system] src_pods: [*] dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-2_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-2_query_output.yaml index b33e36352..acec3b403 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-2_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-2_query_output.yaml @@ -70,4 +70,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-3_query_output.txt b/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-3_query_output.txt index 8676e66bd..d44361dd2 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-3_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-3_query_output.txt @@ -4,4 +4,4 @@ src_ns: [default,kube-system,vendor-system] src_pods: [*] dst: 0.0.0.0/0 conn: A For connections of type non-TCP, final fw rules for query: istio-allow-nothing-3, config: istio-allow-nothing-3: src: 0.0.0.0/0 dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections src_ns: [default,kube-system,vendor-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default,kube-system,vendor-system] src_pods: [*] dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default,kube-system,vendor-system] src_pods: [*] dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-3_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-3_query_output.yaml index 691bba786..5e8315215 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-3_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/istio-allow-nothing-3_query_output.yaml @@ -48,4 +48,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_1_query_output.txt b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_1_query_output.txt index 89a30525a..18daf40d8 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_1_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_1_query_output.txt @@ -6,4 +6,4 @@ src_ns: [default] src_pods: [app=productpage] dst_ns: [default] dst_pods: [app=r For connections of type non-TCP, final fw rules for query: connectivity-istio-test-methods-basic-1, config: istio-test-methods-basic-1: src: 0.0.0.0/0 dst_ns: [default] dst_pods: [*] conn: All connections src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_1_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_1_query_output.yaml index ce436c12b..65925aac6 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_1_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_1_query_output.yaml @@ -62,4 +62,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_2_query_output.txt b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_2_query_output.txt index ba8690f7c..6a21a4537 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_2_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_2_query_output.txt @@ -6,4 +6,4 @@ src_ns: [default] src_pods: [app=productpage] dst_ns: [default] dst_pods: [app=r For connections of type non-TCP, final fw rules for query: connectivity-istio-test-methods-basic-2, config: istio-test-methods-basic-2: src: 0.0.0.0/0 dst_ns: [default] dst_pods: [*] conn: All connections src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_2_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_2_query_output.yaml index 517382312..6c81212e7 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_2_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_basic_2_query_output.yaml @@ -62,4 +62,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_paths_1_query_output.txt b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_paths_1_query_output.txt index ecfa9310f..0706666eb 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_paths_1_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_paths_1_query_output.txt @@ -5,4 +5,4 @@ src_ns: [default] src_pods: [app=productpage] dst_ns: [default] dst_pods: [app=d For connections of type non-TCP, final fw rules for query: connectivity-istio-test-methods-paths-1, config: istio-test-methods-paths-1: src: 0.0.0.0/0 dst_ns: [default] dst_pods: [*] conn: All connections src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_paths_1_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_paths_1_query_output.yaml index 639a295f4..4df8e5821 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_paths_1_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_methods_paths_1_query_output.yaml @@ -95,4 +95,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_allow_1_query_output.txt b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_allow_1_query_output.txt index ed177362f..9caa2e709 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_allow_1_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_allow_1_query_output.txt @@ -5,4 +5,4 @@ src_ns: [default] src_pods: [app=productpage] dst_ns: [default] dst_pods: [app=d For connections of type non-TCP, final fw rules for query: connectivity-istio-test-operation-allow-1, config: istio-test-operation-allow-1: src: 0.0.0.0/0 dst_ns: [default] dst_pods: [*] conn: All connections src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_allow_1_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_allow_1_query_output.yaml index 201d5b91e..c392a209f 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_allow_1_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_allow_1_query_output.yaml @@ -53,4 +53,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_deny_1_query_output.txt b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_deny_1_query_output.txt index f4813978d..46b738ffd 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_deny_1_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_deny_1_query_output.txt @@ -9,4 +9,4 @@ src_ns: [default] src_pods: [app=details] dst_ns: [default] dst_pods: [*] conn: For connections of type non-TCP, final fw rules for query: connectivity-istio-test-operation-deny-1, config: istio-test-operation-deny-1: src: 0.0.0.0/0 dst_ns: [default] dst_pods: [*] conn: All connections src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_deny_1_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_deny_1_query_output.yaml index 423c42820..9a4dc0956 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_deny_1_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/istio-bookinfo-connectivity_test_operation_deny_1_query_output.yaml @@ -88,4 +88,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-deny-all-scheme_output.yaml b/tests/fw_rules_tests/policies/expected_output/istio-deny-all-scheme_output.yaml index 089e6d63e..8178dcae5 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-deny-all-scheme_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/istio-deny-all-scheme_output.yaml @@ -70,4 +70,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query1_output.txt b/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query1_output.txt index 71507294a..75198d391 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query1_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query1_output.txt @@ -10,4 +10,4 @@ src_ns: [default,vendor-system] src_pods: [*] dst_ns: [default] dst_pods: [*] co For connections of type non-TCP, final fw rules for query: istio-policy1, config: istio-policy1: src: 0.0.0.0/0 dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections src_ns: [default,kube-system,vendor-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default,kube-system,vendor-system] src_pods: [*] dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default,kube-system,vendor-system] src_pods: [*] dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query1_output.yaml b/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query1_output.yaml index 744b112f1..0f991aa23 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query1_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query1_output.yaml @@ -113,4 +113,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query2_output.txt b/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query2_output.txt index 4b6b62d34..534455a61 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query2_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query2_output.txt @@ -8,4 +8,4 @@ src_ns: [default] src_pods: [app=special_skydive] dst_ns: [default] dst_pods: [* For connections of type non-TCP, final fw rules for query: istio-policy2, config: istio-policy2: src: 0.0.0.0/0 dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections src_ns: [default,kube-system,vendor-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default,kube-system,vendor-system] src_pods: [*] dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default,kube-system,vendor-system] src_pods: [*] dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query2_output.yaml b/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query2_output.yaml index 288d276e9..7494479f1 100644 --- a/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query2_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/istio-test1-scheme_query2_output.yaml @@ -97,4 +97,4 @@ dst_pods: - '*' connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/poc2-scheme_output.txt b/tests/fw_rules_tests/policies/expected_output/poc2-scheme_output.txt index cd05e6a82..8147ff7fc 100644 --- a/tests/fw_rules_tests/policies/expected_output/poc2-scheme_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/poc2-scheme_output.txt @@ -15,4 +15,4 @@ src_ns: [default] src_pods: [frontend] dst_ns: [default] dst_pods: [recommendati src_ns: [default] src_pods: [loadgenerator] dst_ns: [default] dst_pods: [frontend] conn: TCP 8080 src_ns: [kube-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections src_ns: [kube-system] src_pods: [*] dst_ns: [default] dst_pods: [frontend] conn: TCP 8080 -src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/poc4_scheme_connectivity_map_query_output.txt b/tests/fw_rules_tests/policies/expected_output/poc4_scheme_connectivity_map_query_output.txt index c08d6b68c..67f82c94f 100644 --- a/tests/fw_rules_tests/policies/expected_output/poc4_scheme_connectivity_map_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/poc4_scheme_connectivity_map_query_output.txt @@ -15,4 +15,4 @@ src_ns: [default] src_pods: [frontend] dst_ns: [default] dst_pods: [recommendati src_ns: [default] src_pods: [loadgenerator] dst_ns: [default] dst_pods: [frontend] conn: TCP 23,8080 src_ns: [kube-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections src_ns: [kube-system] src_pods: [*] dst_ns: [default] dst_pods: [frontend] conn: TCP 8080 -src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.csv b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.csv index 44448e5c4..976210d96 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.csv +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.csv @@ -24,4 +24,4 @@ "","[default]","[app=app-6]","[default]","[app=app-5]","All connections", "semantic_diff, config1: config_a, config2: config_b, key: New connections between added peers and ipBlocks","","","","","", "","","0.0.0.0/0","[default]","[app in (app-5,app-6)]","All connections", -"","[default]","[app in (app-5,app-6)]","","0.0.0.0/0","All connections", \ No newline at end of file +"","[default]","[app in (app-5,app-6)]","","0.0.0.0/0","All connections", diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.md b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.md index 7545d2fa2..eef3bdce5 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.md +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.md @@ -25,4 +25,4 @@ ||[default]|[app=app-6]|[default]|[app=app-5]|All connections| |semantic_diff, config1: config_a, config2: config_b, key: New connections between added peers and ipBlocks|||||| |||0.0.0.0/0|[default]|[app in (app-5,app-6)]|All connections| -||[default]|[app in (app-5,app-6)]||0.0.0.0/0|All connections| \ No newline at end of file +||[default]|[app in (app-5,app-6)]||0.0.0.0/0|All connections| diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.txt b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.txt index 801cb9040..832c9bdcb 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.txt @@ -34,4 +34,4 @@ src_ns: [default] src_pods: [app=app-6] dst_ns: [default] dst_pods: [app=app-5] New connections between added peers and ipBlocks (based on topology from config: config_b) : src: 0.0.0.0/0 dst_ns: [default] dst_pods: [app in (app-5,app-6)] conn: All connections -src_ns: [default] src_pods: [app in (app-5,app-6)] dst: 0.0.0.0/0 conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [app in (app-5,app-6)] dst: 0.0.0.0/0 conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.yaml index 95b686358..fac5db175 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_query_output.yaml @@ -172,4 +172,4 @@ dst_ip_block: - 0.0.0.0/0 connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.csv b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.csv index 486190974..250c7d81b 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.csv +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.csv @@ -30,4 +30,4 @@ "","[default]","[app=app-6]","[default]","[app=app-5]","All connections", "semantic_diff, config1: config_a_with_ipBlock, config2: config_b_with_ipBlock, key: New connections between added peers and ipBlocks","","","","","", "","","0.0.0.0/0","[default]","[app in (app-5,app-6)]","All connections", -"","[default]","[app in (app-5,app-6)]","","0.0.0.0/0","All connections", \ No newline at end of file +"","[default]","[app in (app-5,app-6)]","","0.0.0.0/0","All connections", diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.md b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.md index 11cae30ed..01a30e28a 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.md +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.md @@ -31,4 +31,4 @@ ||[default]|[app=app-6]|[default]|[app=app-5]|All connections| |semantic_diff, config1: config_a_with_ipBlock, config2: config_b_with_ipBlock, key: New connections between added peers and ipBlocks|||||| |||0.0.0.0/0|[default]|[app in (app-5,app-6)]|All connections| -||[default]|[app in (app-5,app-6)]||0.0.0.0/0|All connections| \ No newline at end of file +||[default]|[app in (app-5,app-6)]||0.0.0.0/0|All connections| diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.txt b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.txt index fe96732d2..6cef15323 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.txt @@ -40,4 +40,4 @@ src_ns: [default] src_pods: [app=app-6] dst_ns: [default] dst_pods: [app=app-5] New connections between added peers and ipBlocks (based on topology from config: config_b_with_ipBlock) : src: 0.0.0.0/0 dst_ns: [default] dst_pods: [app in (app-5,app-6)] conn: All connections -src_ns: [default] src_pods: [app in (app-5,app-6)] dst: 0.0.0.0/0 conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [app in (app-5,app-6)] dst: 0.0.0.0/0 conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.yaml index 144c9027b..3099cc8ff 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_a_to_b_with_ipBlock_query_output.yaml @@ -245,4 +245,4 @@ dst_ip_block: - 0.0.0.0/0 connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_b_to_a_query_output.csv b/tests/fw_rules_tests/policies/expected_output/semantic_diff_b_to_a_query_output.csv index a95b522f2..f6766241e 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_b_to_a_query_output.csv +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_b_to_a_query_output.csv @@ -24,4 +24,4 @@ "","[default]","[app=app-4]","[default]","[app=app-3]","All connections", "semantic_diff, config1: config_b, config2: config_a, key: New connections between added peers and ipBlocks","","","","","", "","","0.0.0.0/0","[default]","[app in (app-3,app-4)]","All connections", -"","[default]","[app in (app-3,app-4)]","","0.0.0.0/0","All connections", \ No newline at end of file +"","[default]","[app in (app-3,app-4)]","","0.0.0.0/0","All connections", diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_b_to_a_query_output.md b/tests/fw_rules_tests/policies/expected_output/semantic_diff_b_to_a_query_output.md index ae7b5b95a..9b9ca5029 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_b_to_a_query_output.md +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_b_to_a_query_output.md @@ -25,4 +25,4 @@ ||[default]|[app=app-4]|[default]|[app=app-3]|All connections| |semantic_diff, config1: config_b, config2: config_a, key: New connections between added peers and ipBlocks|||||| |||0.0.0.0/0|[default]|[app in (app-3,app-4)]|All connections| -||[default]|[app in (app-3,app-4)]||0.0.0.0/0|All connections| \ No newline at end of file +||[default]|[app in (app-3,app-4)]||0.0.0.0/0|All connections| diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_b_to_a_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/semantic_diff_b_to_a_query_output.yaml index 74023edba..33260b8d5 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_b_to_a_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_b_to_a_query_output.yaml @@ -172,4 +172,4 @@ dst_ip_block: - 0.0.0.0/0 connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.csv b/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.csv index ec6491388..13e10ff2d 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.csv +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.csv @@ -9,4 +9,4 @@ "","[default]","[app in (app-1,app-2)]","[default]","[*]","All connections", "semantic_diff, config1: old1, config2: config_a, key: New connections between added peers and ipBlocks","","","","","", "","","0.0.0.0/0","[default]","[app!=app-1]","All connections", -"","[default]","[*]","","0.0.0.0/0","All connections", \ No newline at end of file +"","[default]","[*]","","0.0.0.0/0","All connections", diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.md b/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.md index bceb1e45c..cb140b682 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.md +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.md @@ -10,4 +10,4 @@ ||[default]|[app in (app-1,app-2)]|[default]|[*]|All connections| |semantic_diff, config1: old1, config2: config_a, key: New connections between added peers and ipBlocks|||||| |||0.0.0.0/0|[default]|[app!=app-1]|All connections| -||[default]|[*]||0.0.0.0/0|All connections| \ No newline at end of file +||[default]|[*]||0.0.0.0/0|All connections| diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.txt b/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.txt index 1592b826e..c8972d40b 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.txt @@ -13,4 +13,4 @@ src_ns: [default] src_pods: [app in (app-1,app-2)] dst_ns: [default] dst_pods: [ New connections between added peers and ipBlocks (based on topology from config: config_a) : src: 0.0.0.0/0 dst_ns: [default] dst_pods: [app!=app-1] conn: All connections -src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.yaml index 766434790..a61116e00 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_disjoint_old1_config_a_query_output.yaml @@ -82,4 +82,4 @@ dst_ip_block: - 0.0.0.0/0 connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.csv b/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.csv index 37099268f..b2ec85845 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.csv +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.csv @@ -23,4 +23,4 @@ "","[kube-system]","[tier=frontend]","","49.50.0.3/32","All connections", "","[kube-system]","[tier=frontend]","","49.50.0.5/32","All connections", "","[kube-system]","[tier=frontend]","","49.50.0.7/32","All connections", -"","[kube-system]","[tier=frontend]","","49.50.0.9/32","All connections", \ No newline at end of file +"","[kube-system]","[tier=frontend]","","49.50.0.9/32","All connections", diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.md b/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.md index 31f0bb005..dc33b6245 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.md +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.md @@ -24,4 +24,4 @@ ||[kube-system]|[tier=frontend]||49.50.0.3/32|All connections| ||[kube-system]|[tier=frontend]||49.50.0.5/32|All connections| ||[kube-system]|[tier=frontend]||49.50.0.7/32|All connections| -||[kube-system]|[tier=frontend]||49.50.0.9/32|All connections| \ No newline at end of file +||[kube-system]|[tier=frontend]||49.50.0.9/32|All connections| diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.txt b/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.txt index dbbebc860..8ac650461 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.txt @@ -27,4 +27,4 @@ src_ns: [kube-system] src_pods: [tier=frontend] dst: 49.50.0.17-255.255.255.255 src_ns: [kube-system] src_pods: [tier=frontend] dst: 49.50.0.3/32 conn: All connections src_ns: [kube-system] src_pods: [tier=frontend] dst: 49.50.0.5/32 conn: All connections src_ns: [kube-system] src_pods: [tier=frontend] dst: 49.50.0.7/32 conn: All connections -src_ns: [kube-system] src_pods: [tier=frontend] dst: 49.50.0.9/32 conn: All connections \ No newline at end of file +src_ns: [kube-system] src_pods: [tier=frontend] dst: 49.50.0.9/32 conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.yaml index 92c853db2..1dfd1dea7 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_ipblocks__np1_np4_query_output.yaml @@ -253,4 +253,4 @@ dst_ip_block: - 49.50.0.9/32 connection: - - All connections \ No newline at end of file + - All connections diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_by_pods_query_output.txt b/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_by_pods_query_output.txt index a1d2888d8..401c47f35 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_by_pods_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_by_pods_query_output.txt @@ -6,4 +6,4 @@ src_ns: [default,kube-system,kube-system-dummy-to-ignore,vendor-system] src_pods Added connections between persistent peers and ipBlocks (based on topology from config: np2_named_ports) : src: 0.0.0.0/0 dst_ns: [kube-system-dummy-to-ignore] dst_pods: [kube-dns-amd64-d66bf76db-9s486] conn: TCP 10054 -src: 0.0.0.0/0 dst_ns: [kube-system-dummy-to-ignore] dst_pods: [kube-dns-amd64-d66bf76db-bbvts] conn: TCP 10054 \ No newline at end of file +src: 0.0.0.0/0 dst_ns: [kube-system-dummy-to-ignore] dst_pods: [kube-dns-amd64-d66bf76db-bbvts] conn: TCP 10054 diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.csv b/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.csv index 659bd5d90..b850aa9f0 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.csv +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.csv @@ -2,4 +2,4 @@ "semantic_diff, config1: np1_named_ports, config2: np2_named_ports, key: Added connections between persistent peers","","","","","", "","[default,kube-system,kube-system-dummy-to-ignore,vendor-system]","[*]","[kube-system-dummy-to-ignore]","[kube-dns-amd64-d66bf76db]","TCP 10054", "semantic_diff, config1: np1_named_ports, config2: np2_named_ports, key: Added connections between persistent peers and ipBlocks","","","","","", -"","","0.0.0.0/0","[kube-system-dummy-to-ignore]","[kube-dns-amd64-d66bf76db]","TCP 10054", \ No newline at end of file +"","","0.0.0.0/0","[kube-system-dummy-to-ignore]","[kube-dns-amd64-d66bf76db]","TCP 10054", diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.md b/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.md index bb1a36e43..7a71b913a 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.md +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.md @@ -3,4 +3,4 @@ |semantic_diff, config1: np1_named_ports, config2: np2_named_ports, key: Added connections between persistent peers|||||| ||[default,kube-system,kube-system-dummy-to-ignore,vendor-system]|[*]|[kube-system-dummy-to-ignore]|[kube-dns-amd64-d66bf76db]|TCP 10054| |semantic_diff, config1: np1_named_ports, config2: np2_named_ports, key: Added connections between persistent peers and ipBlocks|||||| -|||0.0.0.0/0|[kube-system-dummy-to-ignore]|[kube-dns-amd64-d66bf76db]|TCP 10054| \ No newline at end of file +|||0.0.0.0/0|[kube-system-dummy-to-ignore]|[kube-dns-amd64-d66bf76db]|TCP 10054| diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.txt b/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.txt index 3e1e822e8..4a597d7a8 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.txt @@ -4,4 +4,4 @@ Added connections between persistent peers (based on topology from config: np2_n src_ns: [default,kube-system,kube-system-dummy-to-ignore,vendor-system] src_pods: [*] dst_ns: [kube-system-dummy-to-ignore] dst_pods: [kube-dns-amd64-d66bf76db] conn: TCP 10054 Added connections between persistent peers and ipBlocks (based on topology from config: np2_named_ports) : -src: 0.0.0.0/0 dst_ns: [kube-system-dummy-to-ignore] dst_pods: [kube-dns-amd64-d66bf76db] conn: TCP 10054 \ No newline at end of file +src: 0.0.0.0/0 dst_ns: [kube-system-dummy-to-ignore] dst_pods: [kube-dns-amd64-d66bf76db] conn: TCP 10054 diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.yaml b/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.yaml index def513370..82d3155eb 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_named_ports_np1_and_np2_query_output.yaml @@ -33,4 +33,4 @@ connection: - Protocol: TCP Ports: - - 10054 \ No newline at end of file + - 10054 diff --git a/tests/fw_rules_tests/policies/expected_output/semantic_diff_poc-scheme_output.txt b/tests/fw_rules_tests/policies/expected_output/semantic_diff_poc-scheme_output.txt index 6e9c37496..1852687c7 100644 --- a/tests/fw_rules_tests/policies/expected_output/semantic_diff_poc-scheme_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/semantic_diff_poc-scheme_output.txt @@ -26,4 +26,4 @@ src_ns: [kube-system] src_pods: [*] dst_ns: [default] dst_pods: [app!=frontend] Removed connections between persistent peers and ipBlocks (based on topology from config: allow_all) : src: 0.0.0.0/0 dst_ns: [default] dst_pods: [*] conn: All but TCP 8080 src: 0.0.0.0/0 dst_ns: [default] dst_pods: [app!=frontend] conn: All connections -src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections diff --git a/tests/fw_rules_tests/policies/expected_output/test10-scheme_output.txt b/tests/fw_rules_tests/policies/expected_output/test10-scheme_output.txt index 9a46fc72b..b4538078f 100644 --- a/tests/fw_rules_tests/policies/expected_output/test10-scheme_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/test10-scheme_output.txt @@ -5,4 +5,4 @@ src_ns: [default,ibm-system-new,kube-system-new,kube-system-new-dummy-to-ignore] src_ns: [default] src_pods: [app=skydive and tier=agent] dst: 0.0.0.0/0 conn: All connections src_ns: [default] src_pods: [app=skydive and tier=agent] dst_ns: [ibm-system-new,kube-system-new,kube-system-new-dummy-to-ignore] dst_pods: [*] conn: All connections src_ns: [ibm-system-new,kube-system-new,kube-system-new-dummy-to-ignore] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [ibm-system-new,kube-system-new,kube-system-new-dummy-to-ignore] src_pods: [*] dst_ns: [ibm-system-new,kube-system-new,kube-system-new-dummy-to-ignore] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [ibm-system-new,kube-system-new,kube-system-new-dummy-to-ignore] src_pods: [*] dst_ns: [ibm-system-new,kube-system-new,kube-system-new-dummy-to-ignore] dst_pods: [*] conn: All connections diff --git a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_adding_default_sidecar_after_specific.txt b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_adding_default_sidecar_after_specific.txt index 9742fde25..7b755bcbf 100644 --- a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_adding_default_sidecar_after_specific.txt +++ b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_adding_default_sidecar_after_specific.txt @@ -6,4 +6,4 @@ src_ns: [default] src_pods: [app=reviews] dst_ns: [default] dst_pods: [ratings-v For connections of type non-TCP, final fw rules for query: connectivity-map-bookinfo-adding-default-sidecar-after-specific, config: adding-default-sidecar-after-specific: src: 0.0.0.0/0 dst_ns: [default] dst_pods: [*] conn: All connections src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections diff --git a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_default_sidecar.txt b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_default_sidecar.txt index ec726d371..fb7cba96e 100644 --- a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_default_sidecar.txt +++ b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_default_sidecar.txt @@ -10,4 +10,4 @@ src_ns: [default] src_pods: [productpage-v1] dst_ns: [default] dst_pods: [*] con For connections of type non-TCP, final fw rules for query: connectivity-map-default-sidecar-1, config: bookinfo-default-sidecar-1: src: 0.0.0.0/0 dst_ns: [default] dst_pods: [*] conn: All connections src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections diff --git a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_ignoring_second_default_sidecar.txt b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_ignoring_second_default_sidecar.txt index 82559391b..deec46f90 100644 --- a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_ignoring_second_default_sidecar.txt +++ b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_ignoring_second_default_sidecar.txt @@ -10,4 +10,4 @@ src_ns: [default] src_pods: [productpage-v1] dst_ns: [default] dst_pods: [*] con For connections of type non-TCP, final fw rules for query: connectivity-map-bookinfo-ignoring-second-default-sidecar-in-same-namespace, config: bookinfo-two-selector-less-sidecars: src: 0.0.0.0/0 dst_ns: [default] dst_pods: [*] conn: All connections src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections diff --git a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_multiple_sidecar_overrides.txt b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_multiple_sidecar_overrides.txt index 0007aba1e..2c963220c 100644 --- a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_multiple_sidecar_overrides.txt +++ b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_multiple_sidecar_overrides.txt @@ -6,4 +6,4 @@ src_ns: [default] src_pods: [productpage-v1] dst_ns: [default] dst_pods: [rating For connections of type non-TCP, final fw rules for query: connectivity-map-bookinfo-multiple-sidecar-overrides, config: multiple-sidecar-overrides: src: 0.0.0.0/0 dst_ns: [default] dst_pods: [*] conn: All connections src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections diff --git a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_productpage_sidecar.txt b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_productpage_sidecar.txt index ea372f324..72fcec3cb 100644 --- a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_productpage_sidecar.txt +++ b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_productpage_sidecar.txt @@ -7,4 +7,4 @@ src_ns: [default] src_pods: [app!=productpage] dst_ns: [default] dst_pods: [*] c For connections of type non-TCP, final fw rules for query: connectivity-map-bookinfo-productpage, config: bookinfo-productpage-sidecar-1: src: 0.0.0.0/0 dst_ns: [default] dst_pods: [*] conn: All connections src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections diff --git a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_sidecars_with_different_selectors.txt b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_sidecars_with_different_selectors.txt index 933ec81df..3e61cd617 100644 --- a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_sidecars_with_different_selectors.txt +++ b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_sidecars_with_different_selectors.txt @@ -8,4 +8,4 @@ src_ns: [default] src_pods: [productpage-v1] dst_ns: [default] dst_pods: [app in For connections of type non-TCP, final fw rules for query: connectivity-map-bookinfo-sidecars-with-different-selectors, config: sidecars-with-different-selectors: src: 0.0.0.0/0 dst_ns: [default] dst_pods: [*] conn: All connections src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections diff --git a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_specific_sidecar_overrides_global_sidecar.txt b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_specific_sidecar_overrides_global_sidecar.txt index 2f355b2ce..b659251ca 100644 --- a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_specific_sidecar_overrides_global_sidecar.txt +++ b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_specific_sidecar_overrides_global_sidecar.txt @@ -6,4 +6,4 @@ src_ns: [default] src_pods: [app!=productpage] dst_ns: [default] dst_pods: [*] c For connections of type non-TCP, final fw rules for query: connectivity-map-bookinfo-specific-sidecar-overrides-istio-global-sidecar, config: sidecar-with-workload-selector-overrides-istio-global-sidecar: src: 0.0.0.0/0 dst_ns: [default] dst_pods: [*] conn: All connections src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections diff --git a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_two_sidecars_with_same_workload_selector.txt b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_two_sidecars_with_same_workload_selector.txt index 628d48682..07fe0629a 100644 --- a/tests/istio_testcases/expected_output/connectivity_map_bookinfo_two_sidecars_with_same_workload_selector.txt +++ b/tests/istio_testcases/expected_output/connectivity_map_bookinfo_two_sidecars_with_same_workload_selector.txt @@ -7,4 +7,4 @@ src_ns: [default] src_pods: [app!=productpage] dst_ns: [default] dst_pods: [*] c For connections of type non-TCP, final fw rules for query: connectivity-map-bookinfo-two-sidecars-with-same-workload-selector, config: two-sidecars-with-same-workload-selector: src: 0.0.0.0/0 dst_ns: [default] dst_pods: [*] conn: All connections src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections diff --git a/tests/istio_testcases/expected_output/connectivity_map_global_sidecar_from_istio_ref.txt b/tests/istio_testcases/expected_output/connectivity_map_global_sidecar_from_istio_ref.txt index fdcf82d8f..1031f7463 100644 --- a/tests/istio_testcases/expected_output/connectivity_map_global_sidecar_from_istio_ref.txt +++ b/tests/istio_testcases/expected_output/connectivity_map_global_sidecar_from_istio_ref.txt @@ -5,4 +5,4 @@ src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connec For connections of type non-TCP, final fw rules for query: connectivity-map-global-sidecar, config: global-sidecar-from-istio-ref: src: 0.0.0.0/0 dst_ns: [default] dst_pods: [*] conn: All connections src_ns: [default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [default] src_pods: [*] dst_ns: [default] dst_pods: [*] conn: All connections diff --git a/tests/istio_testcases/expected_output/connectivity_map_of_onlineboutique_resources.txt b/tests/istio_testcases/expected_output/connectivity_map_of_onlineboutique_resources.txt index 4e84b0022..3e1465bf2 100644 --- a/tests/istio_testcases/expected_output/connectivity_map_of_onlineboutique_resources.txt +++ b/tests/istio_testcases/expected_output/connectivity_map_of_onlineboutique_resources.txt @@ -11,4 +11,4 @@ src_ns: [onlineboutique] src_pods: [frontend] dst_ns: [onlineboutique] dst_pods: src_ns: [onlineboutique] src_pods: [loadgenerator] dst_ns: [onlineboutique] dst_pods: [frontend] conn: TCP {'dst_ports': '8080', 'methods': 'GET, POST'} For connections of type non-TCP, final fw rules for query: connectivity-map-of-onlineboutique, config: onlineboutique-resources: -src_ns: [onlineboutique] src_pods: [*] dst: 0.0.0.0/0 conn: All connections \ No newline at end of file +src_ns: [onlineboutique] src_pods: [*] dst: 0.0.0.0/0 conn: All connections diff --git a/tests/istio_testcases/expected_output/connectivity_map_online_boutique_frontend_sidecar.txt b/tests/istio_testcases/expected_output/connectivity_map_online_boutique_frontend_sidecar.txt index 484ab8b73..138b5cfab 100644 --- a/tests/istio_testcases/expected_output/connectivity_map_online_boutique_frontend_sidecar.txt +++ b/tests/istio_testcases/expected_output/connectivity_map_online_boutique_frontend_sidecar.txt @@ -9,4 +9,4 @@ src_ns: [default] src_pods: [app!=frontend] dst_ns: [asm-ingress,default] dst_po For connections of type non-TCP, final fw rules for query: frontend_sidecar_connectivity_map, config: frontend_sidecar: src: 0.0.0.0/0 dst_ns: [asm-ingress,default] dst_pods: [*] conn: All connections src_ns: [asm-ingress,default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [asm-ingress,default] src_pods: [*] dst_ns: [asm-ingress,default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [asm-ingress,default] src_pods: [*] dst_ns: [asm-ingress,default] dst_pods: [*] conn: All connections diff --git a/tests/istio_testcases/expected_output/new_online_boutique_connectivity_map.txt b/tests/istio_testcases/expected_output/new_online_boutique_connectivity_map.txt index 75a15de2f..1d2ef9ac8 100644 --- a/tests/istio_testcases/expected_output/new_online_boutique_connectivity_map.txt +++ b/tests/istio_testcases/expected_output/new_online_boutique_connectivity_map.txt @@ -20,4 +20,4 @@ src_ns: [default] src_pods: [loadgenerator] dst_ns: [default] dst_pods: [fronten For connections of type non-TCP, final fw rules for query: new_online_boutique_connectivity_map, config: new_online_boutique: src: 0.0.0.0/0 dst_ns: [asm-ingress,default] dst_pods: [*] conn: All connections src_ns: [asm-ingress,default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [asm-ingress,default] src_pods: [*] dst_ns: [asm-ingress,default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [asm-ingress,default] src_pods: [*] dst_ns: [asm-ingress,default] dst_pods: [*] conn: All connections diff --git a/tests/istio_testcases/expected_output/new_online_boutique_synth_res_connectivity_map.txt b/tests/istio_testcases/expected_output/new_online_boutique_synth_res_connectivity_map.txt index 0af50f18c..815c1c10b 100644 --- a/tests/istio_testcases/expected_output/new_online_boutique_synth_res_connectivity_map.txt +++ b/tests/istio_testcases/expected_output/new_online_boutique_synth_res_connectivity_map.txt @@ -16,4 +16,4 @@ src_ns: [default] src_pods: [loadgenerator] dst_ns: [default] dst_pods: [fronten For connections of type non-TCP, final fw rules for query: new_online_boutique_synth_res_connectivity_map, config: new_online_boutique_synthesis_res: src: 0.0.0.0/0 dst_ns: [asm-ingress,default] dst_pods: [*] conn: All connections src_ns: [asm-ingress,default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [asm-ingress,default] src_pods: [*] dst_ns: [asm-ingress,default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [asm-ingress,default] src_pods: [*] dst_ns: [asm-ingress,default] dst_pods: [*] conn: All connections diff --git a/tests/istio_testcases/expected_output/new_online_boutique_synth_res_connectivity_map_with_baseline_rule.txt b/tests/istio_testcases/expected_output/new_online_boutique_synth_res_connectivity_map_with_baseline_rule.txt index d9eaa18c4..c34b01d7f 100644 --- a/tests/istio_testcases/expected_output/new_online_boutique_synth_res_connectivity_map_with_baseline_rule.txt +++ b/tests/istio_testcases/expected_output/new_online_boutique_synth_res_connectivity_map_with_baseline_rule.txt @@ -15,4 +15,4 @@ src_ns: [default] src_pods: [loadgenerator] dst_ns: [default] dst_pods: [fronten For connections of type non-TCP, final fw rules for query: new_online_boutique_synth_res_connectivity_map_with_baseline_rule, config: new_online_boutique_synthesis_res_with_baseline_restrict_access_to_payment_service: src: 0.0.0.0/0 dst_ns: [asm-ingress,default] dst_pods: [*] conn: All connections src_ns: [asm-ingress,default] src_pods: [*] dst: 0.0.0.0/0 conn: All connections -src_ns: [asm-ingress,default] src_pods: [*] dst_ns: [asm-ingress,default] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [asm-ingress,default] src_pods: [*] dst_ns: [asm-ingress,default] dst_pods: [*] conn: All connections diff --git a/tests/istio_testcases/expected_output/semantic_diff_online_boutique_new_input_vs_synth_res.txt b/tests/istio_testcases/expected_output/semantic_diff_online_boutique_new_input_vs_synth_res.txt index c0e2af85a..fc240ed03 100644 --- a/tests/istio_testcases/expected_output/semantic_diff_online_boutique_new_input_vs_synth_res.txt +++ b/tests/istio_testcases/expected_output/semantic_diff_online_boutique_new_input_vs_synth_res.txt @@ -22,4 +22,4 @@ Added connections between persistent peers and ipBlocks (based on topology from src: 0.0.0.0/0 dst_ns: [asm-ingress] dst_pods: [*] conn: TCP 1-8079,8081-65535 Removed connections between persistent peers and ipBlocks (based on topology from config: new_online_boutique) : -src: 0.0.0.0/0 dst_ns: [default] dst_pods: [loadgenerator] conn: TCP \ No newline at end of file +src: 0.0.0.0/0 dst_ns: [default] dst_pods: [loadgenerator] conn: TCP diff --git a/tests/k8s_testcases/expected_output/new_online_boutique_connectivity_map.txt b/tests/k8s_testcases/expected_output/new_online_boutique_connectivity_map.txt index c00a9ce86..50f5ce659 100644 --- a/tests/k8s_testcases/expected_output/new_online_boutique_connectivity_map.txt +++ b/tests/k8s_testcases/expected_output/new_online_boutique_connectivity_map.txt @@ -11,4 +11,4 @@ src_ns: [default] src_pods: [checkoutservice] dst_ns: [default] dst_pods: [payme src_ns: [default] src_pods: [frontend] dst_ns: [default] dst_pods: [adservice] conn: TCP 9555 src_ns: [default] src_pods: [frontend] dst_ns: [default] dst_pods: [checkoutservice] conn: TCP 5050 src_ns: [default] src_pods: [frontend] dst_ns: [default] dst_pods: [recommendationservice] conn: TCP 8080 -src_ns: [default] src_pods: [loadgenerator] dst_ns: [default] dst_pods: [frontend] conn: TCP 8080 \ No newline at end of file +src_ns: [default] src_pods: [loadgenerator] dst_ns: [default] dst_pods: [frontend] conn: TCP 8080 diff --git a/tests/k8s_testcases/expected_output/orig_online_boutique_synthesis_res_connectivity_map.txt b/tests/k8s_testcases/expected_output/orig_online_boutique_synthesis_res_connectivity_map.txt index 0103a0fdb..78abbf069 100644 --- a/tests/k8s_testcases/expected_output/orig_online_boutique_synthesis_res_connectivity_map.txt +++ b/tests/k8s_testcases/expected_output/orig_online_boutique_synthesis_res_connectivity_map.txt @@ -15,4 +15,4 @@ src_ns: [default] src_pods: [frontend] dst_ns: [default] dst_pods: [recommendati src_ns: [default] src_pods: [loadgenerator] dst_ns: [default] dst_pods: [frontend] conn: TCP 8080 src_ns: [kube-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections src_ns: [kube-system] src_pods: [*] dst_ns: [default] dst_pods: [frontend] conn: TCP 8080 -src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections diff --git a/tests/k8s_testcases/expected_output/semantic_diff_online_boutique_new_synthesized_vs_orig_synthesized.txt b/tests/k8s_testcases/expected_output/semantic_diff_online_boutique_new_synthesized_vs_orig_synthesized.txt index fbb40838d..c6aa85ebe 100644 --- a/tests/k8s_testcases/expected_output/semantic_diff_online_boutique_new_synthesized_vs_orig_synthesized.txt +++ b/tests/k8s_testcases/expected_output/semantic_diff_online_boutique_new_synthesized_vs_orig_synthesized.txt @@ -8,4 +8,4 @@ src_ns: [default] src_pods: [cartservice] dst_ns: [kube-system] dst_pods: [*] co src_ns: [kube-system] src_pods: [*] dst_ns: [default] dst_pods: [frontend] conn: TCP 8080 Removed connections between persistent peers and ipBlocks (based on topology from config: orig_online_boutique_synthesis_res) : -src: 0.0.0.0/0 dst_ns: [default] dst_pods: [frontend] conn: TCP 8080 \ No newline at end of file +src: 0.0.0.0/0 dst_ns: [default] dst_pods: [frontend] conn: TCP 8080 From 9ff9d25521207ab38bcefae3b1541536b514889a Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 14 Feb 2023 17:49:39 +0200 Subject: [PATCH 036/187] Handling exclude_ipv6 print in optimized calculation. Signed-off-by: Tanya --- nca/CoreDS/Peer.py | 19 +++++++++++++++++++ nca/NetworkConfig/NetworkConfigQuery.py | 24 ++++++++++-------------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index 4294da19d..12d537bde 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -381,6 +381,12 @@ def add_cidr(self, cidr, exceptions=None): IPNetworkAddress(exception_n.broadcast_address)) self.add_hole(hole) + def remove_cidr(self, cidr): + ipn = ip_network(cidr, False) # strict is False as k8s API shows an example CIDR where host bits are set + hole = CanonicalIntervalSet.Interval(IPNetworkAddress(ipn.network_address), + IPNetworkAddress(ipn.broadcast_address)) + self.add_hole(hole) + def get_peer_set(self): """ :return: get PeerSet from IpBlock (empty set if IpBlock is empty) @@ -660,6 +666,19 @@ def get_peer_set_by_indices(self, peer_interval_set): peer_list.append(self.sorted_peer_list[ind]) return PeerSet(set(peer_list)) + @staticmethod + def remove_ipv6_full_block(peer_set): + res = PeerSet() + for peer in peer_set: + if isinstance(peer, IpBlock): + peer.remove_cidr('::/0') + if peer: + res.add(peer) + else: + res.add(peer) + return res + + def by_full_name(elem): """ diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 1cf94f524..26c5deb7a 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -694,22 +694,17 @@ def exec(self): self.output_config.fullExplanation = True # assign true for this query - it is always ok to compare its results self.output_config.configName = os.path.basename(self.config.name) if self.config.name.startswith('./') else \ self.config.name - peers_to_compare = self.config.peer_container.get_all_peers_group() - - exclude_ipv6 = self.output_config.excludeIPv6Range - ref_ip_blocks = IpBlock.disjoint_ip_blocks(self.config.get_referenced_ip_blocks(exclude_ipv6), - IpBlock.get_all_ips_block_peer_set(exclude_ipv6), - exclude_ipv6) connections = defaultdict(list) res = QueryAnswer(True) fw_rules = None fw_rules_tcp = None fw_rules_non_tcp = None + exclude_ipv6 = self.output_config.excludeIPv6Range if self.config.optimized_run != 'true': peers_to_compare = self.config.peer_container.get_all_peers_group() - - ref_ip_blocks = IpBlock.disjoint_ip_blocks(self.config.get_referenced_ip_blocks(), - IpBlock.get_all_ips_block_peer_set()) + ref_ip_blocks = IpBlock.disjoint_ip_blocks(self.config.get_referenced_ip_blocks(exclude_ipv6), + IpBlock.get_all_ips_block_peer_set(exclude_ipv6), + exclude_ipv6) peers_to_compare |= ref_ip_blocks peers = PeerSet() peers1_start = time.time() @@ -753,12 +748,13 @@ def exec(self): # add all relevant IpBlocks, used in connections opt_peers_to_compare |= all_conns_opt.project_on_one_dimension('src_peers') | \ all_conns_opt.project_on_one_dimension('dst_peers') + if exclude_ipv6: + opt_peers_to_compare = PeerSet.remove_ipv6_full_block(opt_peers_to_compare) + subset_peers = self.compute_subset(opt_peers_to_compare) - src_peers_in_subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, - src_peers=subset_peers) - dst_peers_in_subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, - dst_peers=subset_peers) - all_conns_opt &= src_peers_in_subset_conns | dst_peers_in_subset_conns + subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, + src_peers=subset_peers, dst_peers=subset_peers) + all_conns_opt &= subset_conns if self.config.policies_container.layers.does_contain_layer(NetworkLayerName.Istio): output_res, opt_fw_rules_tcp, opt_fw_rules_non_tcp = self.get_props_output_split_by_tcp(all_conns_opt, opt_peers_to_compare) opt_end = time.time() From eb763fb82cd2f2097509cccad37bc9ff9c83c5b2 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Feb 2023 10:41:36 +0200 Subject: [PATCH 037/187] Fixing initialization of MethodSet in HTTPRoute (None means no methods, MethodSet() means empty method set, which would create empty connections). Signed-off-by: Tanya --- nca/Resources/IstioTrafficResources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/Resources/IstioTrafficResources.py b/nca/Resources/IstioTrafficResources.py index 69968bd30..c175d927f 100644 --- a/nca/Resources/IstioTrafficResources.py +++ b/nca/Resources/IstioTrafficResources.py @@ -94,7 +94,7 @@ class HTTPRoute: def __init__(self): self.uri_dfa = None # self.scheme_dfa = None # not supported yet - self.methods = MethodSet() + self.methods = None # self.authority_dfa = None # not supported yet self.destinations = [] From dabbd17e7315ec94b44117ad7eebad4c79c06cf2 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Feb 2023 12:19:36 +0200 Subject: [PATCH 038/187] Changed output format of ICMP data. Signed-off-by: Tanya --- .../expected_output/calico-testcase14-scheme_output.txt | 2 +- .../expected_output/calico-testcase14-scheme_output.yaml | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/fw_rules_tests/policies/expected_output/calico-testcase14-scheme_output.txt b/tests/fw_rules_tests/policies/expected_output/calico-testcase14-scheme_output.txt index ccb2dd04b..4b9d60721 100644 --- a/tests/fw_rules_tests/policies/expected_output/calico-testcase14-scheme_output.txt +++ b/tests/fw_rules_tests/policies/expected_output/calico-testcase14-scheme_output.txt @@ -1,2 +1,2 @@ final fw rules for query: match-icmp-also-within-default-test, config: match-icmp-also-within-default: -src_ns: [kube-system] src_pods: [app=keepalived-watcher] dst_ns: [kube-system] dst_pods: [app=keepalived-watcher] conn: ICMP (type=100,code=230) +src_ns: [kube-system] src_pods: [app=keepalived-watcher] dst_ns: [kube-system] dst_pods: [app=keepalived-watcher] conn: ICMP {'icmp_type': '100', 'icmp_code': '230'} diff --git a/tests/fw_rules_tests/policies/expected_output/calico-testcase14-scheme_output.yaml b/tests/fw_rules_tests/policies/expected_output/calico-testcase14-scheme_output.yaml index ed3a96376..261801ce3 100644 --- a/tests/fw_rules_tests/policies/expected_output/calico-testcase14-scheme_output.yaml +++ b/tests/fw_rules_tests/policies/expected_output/calico-testcase14-scheme_output.yaml @@ -14,5 +14,8 @@ - app=keepalived-watcher connection: - Protocol: ICMP - Type/Code: - - 100/230 + properties: + - icmp_type: + - 100 + icmp_code: + - 230 \ No newline at end of file From 1388a4c0fb8fd262c4fb1e7c882744a7fb6c284d Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Feb 2023 15:33:14 +0200 Subject: [PATCH 039/187] Making default the original (not optimized) implementation in run_all_tests Signed-off-by: Tanya --- tests/run_all_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run_all_tests.py b/tests/run_all_tests.py index a703efb6b..ca51bb24e 100644 --- a/tests/run_all_tests.py +++ b/tests/run_all_tests.py @@ -63,7 +63,7 @@ def __init__(self, test_dict, cli_tests_base_dir, test_name): class SchemeFile: def __init__(self, scheme_filename): self.test_name = scheme_filename - test_args = ['--scheme', self.test_name, '-opt=debug'] + test_args = ['--scheme', self.test_name, '-opt=false'] self.args_obj = TestArgs(test_args) def update_arg_at_scheme_file_output_config(self, arg_name, arg_value): From 3882e85a397edbc33e3a4bff1bb7ec64a7be035f Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 26 Feb 2023 19:14:36 +0200 Subject: [PATCH 040/187] 1. Merge with master 2. Improved filtering of ipv6 blocks in optimized solution. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 6 ++- nca/CoreDS/Peer.py | 41 +++++++++++------ nca/FWRules/ConnectivityGraph.py | 6 ++- nca/NetworkConfig/NetworkConfigQuery.py | 60 +++++++++++++++---------- 4 files changed, 73 insertions(+), 40 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 8a3bece0e..85e1f61c6 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -580,7 +580,8 @@ def get_non_tcp_connections(): # TODO - after moving to the optimized HC set implementation, # get rid of ConnectionSet and move the code below to TcpLikeProperties.py @staticmethod - def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container, connectivity_restriction): + def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container, ip_blocks_filter, + connectivity_restriction): ignore_protocols = ProtocolSet() if connectivity_restriction: if connectivity_restriction == 'TCP': @@ -602,6 +603,9 @@ def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container, connecti new_cube_dict.pop('dst_peers') else: dst_peers = peer_container.get_all_peers_group(True) + if IpBlock.get_all_ips_block() != ip_blocks_filter: + src_peers.filter_ipv6_blocks(ip_blocks_filter) + dst_peers.filter_ipv6_blocks(ip_blocks_filter) protocols = new_cube_dict.get('protocols') if protocols: new_cube_dict.pop('protocols') diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index 249188e95..ea2f96606 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -329,13 +329,17 @@ def get_ip_range_or_cidr_str(self, range_only=False): return self.get_cidr_list_str() @staticmethod - def get_all_ips_block(exclude_ipv6=False): + def get_all_ips_block(exclude_ipv6=False, exclude_ipv4=False): """ :return: The full range of ipv4 and ipv6 addresses if exclude_ipv6 is False :param bool exclude_ipv6: indicates if to exclude the IPv6 addresses + :param bool exclude_ipv4: indicates if to exclude the IPv4 addresses :rtype: IpBlock """ - res = IpBlock('0.0.0.0/0') + assert not exclude_ipv6 or not exclude_ipv4 + res = IpBlock() + if not exclude_ipv4: + res.add_cidr('0.0.0.0/0') if not exclude_ipv6: res.add_cidr('::/0') return res @@ -348,7 +352,7 @@ def get_all_ips_block_peer_set(exclude_ipv6=False): :rtype: PeerSet """ res = PeerSet() - res.add(IpBlock.get_all_ips_block(exclude_ipv6)) + res.add(IpBlock.get_all_ips_block(exclude_ipv6=exclude_ipv6)) return res def split(self): @@ -668,18 +672,27 @@ def get_peer_set_by_indices(self, peer_interval_set): peer_list.append(self.sorted_peer_list[ind]) return PeerSet(set(peer_list)) - @staticmethod - def remove_ipv6_full_block(peer_set): - res = PeerSet() - for peer in peer_set: + def filter_ipv6_blocks(self, ip_block_filter): + ipv6_cidr = '::/0' + peers_to_remove = [] + peers_to_add = [] + for peer in self: if isinstance(peer, IpBlock): - peer.remove_cidr('::/0') - if peer: - res.add(peer) - else: - res.add(peer) - return res - + peers_to_remove.append(peer) + if IpBlock.get_all_ips_block(exclude_ipv4=True).contained_in(peer): + new_peer = peer.copy() + new_peer.remove_cidr(ipv6_cidr) + if new_peer: + peers_to_add.append(new_peer) + elif peer.overlaps(ip_block_filter): + new_peer = peer.copy() + new_peer &= ip_block_filter + peers_to_add.append(new_peer) + + for peer in peers_to_remove: + self.remove(peer) + for peer in peers_to_add: + self.add(peer) def by_full_name(elem): diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index bc2ec9240..12161cb8a 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -79,7 +79,7 @@ def add_edges(self, connections): """ self.connections_to_peers.update(connections) - def add_edges_from_cube_dict(self, peer_container, cube_dict): + def add_edges_from_cube_dict(self, peer_container, cube_dict, ip_blocks_filter): """ Add edges to the graph according to the give cube :param peer_container: the peer_container containing all possible peers @@ -96,6 +96,10 @@ def add_edges_from_cube_dict(self, peer_container, cube_dict): new_cube_dict.pop('dst_peers') else: dst_peers = peer_container.get_all_peers_group(True) + if IpBlock.get_all_ips_block() != ip_blocks_filter: + src_peers.filter_ipv6_blocks(ip_blocks_filter) + dst_peers.filter_ipv6_blocks(ip_blocks_filter) + protocols = new_cube_dict.get('protocols') if protocols: new_cube_dict.pop('protocols') diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 8fbc718d6..bf3e38257 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -748,12 +748,6 @@ def exec(self): # add all relevant IpBlocks, used in connections opt_peers_to_compare |= all_conns_opt.project_on_one_dimension('src_peers') | \ all_conns_opt.project_on_one_dimension('dst_peers') - if exclude_ipv6: - opt_peers_to_compare = PeerSet.remove_ipv6_full_block(opt_peers_to_compare) - possible_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, - src_peers=opt_peers_to_compare, - dst_peers=opt_peers_to_compare) - all_conns_opt &= possible_conns subset_peers = self.compute_subset(opt_peers_to_compare) subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, @@ -761,20 +755,33 @@ def exec(self): TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, dst_peers=subset_peers) all_conns_opt &= subset_conns + ip_blocks_filter = IpBlock.get_all_ips_block() + if exclude_ipv6: + ip_blocks_filter = IpBlock.get_all_ips_block(exclude_ipv6=True) + ref_ip_blocks = self.config.get_referenced_ip_blocks(exclude_ipv6) + for ip_block in ref_ip_blocks: + ip_blocks_filter |= ip_block + opt_peers_to_compare.filter_ipv6_blocks(ip_blocks_filter) + if self.config.policies_container.layers.does_contain_layer(NetworkLayerName.Istio): - output_res, opt_fw_rules_tcp, opt_fw_rules_non_tcp = self.get_props_output_split_by_tcp(all_conns_opt, opt_peers_to_compare) + output_res, opt_fw_rules_tcp, opt_fw_rules_non_tcp = \ + self.get_props_output_split_by_tcp(all_conns_opt, opt_peers_to_compare, ip_blocks_filter) opt_end = time.time() print(f'Opt time: {(opt_end - opt_start):6.2f} seconds') if self.config.optimized_run == 'debug': - if fw_rules_tcp.fw_rules_map and opt_fw_rules_tcp.fw_rules_map: + if fw_rules_tcp and fw_rules_tcp.fw_rules_map and \ + opt_fw_rules_tcp and opt_fw_rules_tcp.fw_rules_map: self.compare_fw_rules(fw_rules_tcp, opt_fw_rules_tcp) - if fw_rules_non_tcp.fw_rules_map and opt_fw_rules_non_tcp.fw_rules_map: + if fw_rules_non_tcp and fw_rules_non_tcp.fw_rules_map and \ + opt_fw_rules_non_tcp and opt_fw_rules_non_tcp.fw_rules_map: self.compare_fw_rules(fw_rules_non_tcp, opt_fw_rules_non_tcp) else: - output_res, opt_fw_rules = self.get_props_output_full(all_conns_opt, opt_peers_to_compare) + output_res, opt_fw_rules = self.get_props_output_full(all_conns_opt, opt_peers_to_compare, + ip_blocks_filter) opt_end = time.time() print(f'Opt time: {(opt_end - opt_start):6.2f} seconds') - if self.config.optimized_run == 'debug' and fw_rules and opt_fw_rules: + if self.config.optimized_run == 'debug' and fw_rules and fw_rules.fw_rules_map and \ + opt_fw_rules and opt_fw_rules.fw_rules_map: self.compare_fw_rules(fw_rules, opt_fw_rules) if self.output_config.outputFormat in ['json', 'yaml']: @@ -821,7 +828,7 @@ def get_connectivity_output_full(self, connections, peers, peers_to_compare): formatted_rules, fw_rules = self.fw_rules_from_connections_dict(connections, peers_to_compare) return formatted_rules, fw_rules - def get_props_output_full(self, props, peers_to_compare): + def get_props_output_full(self, props, peers_to_compare, ip_blocks_filter): """ get the connectivity map output considering all connections in the output :param TcpLikeProperties props: properties describing allowed connections @@ -829,11 +836,11 @@ def get_props_output_full(self, props, peers_to_compare): :rtype Union[str,dict] """ if self.output_config.outputFormat in ['dot', 'jpg']: - dot_full = self.dot_format_from_props(props, peers_to_compare) + dot_full = self.dot_format_from_props(props, peers_to_compare, ip_blocks_filter) return dot_full, None # TODO - handle 'txt_no_fw_rules' output format # handle other formats - formatted_rules, fw_rules = self.fw_rules_from_props(props, peers_to_compare) + formatted_rules, fw_rules = self.fw_rules_from_props(props, peers_to_compare, ip_blocks_filter) return formatted_rules, fw_rules def get_connectivity_output_split_by_tcp(self, connections, peers, peers_to_compare): @@ -870,7 +877,7 @@ def get_connectivity_output_split_by_tcp(self, connections, peers, peers_to_comp res_str = formatted_rules_tcp + formatted_rules_non_tcp return res_str, fw_rules_tcp, fw_rules_non_tcp - def get_props_output_split_by_tcp(self, props, peers_to_compare): + def get_props_output_split_by_tcp(self, props, peers_to_compare, ip_blocks_filter): """ get the connectivity map output as two parts: TCP and non-TCP :param TcpLikeProperties props: properties describing allowed connections @@ -881,14 +888,19 @@ def get_props_output_split_by_tcp(self, props, peers_to_compare): connectivity_non_tcp_str = 'non-TCP' props_tcp, props_non_tcp = self.convert_props_to_split_by_tcp(props) if self.output_config.outputFormat == 'dot': - dot_tcp = self.dot_format_from_props(props_tcp, peers_to_compare, connectivity_tcp_str) - dot_non_tcp = self.dot_format_from_props(props_non_tcp, peers_to_compare, connectivity_non_tcp_str) + dot_tcp = self.dot_format_from_props(props_tcp, peers_to_compare, ip_blocks_filter, + connectivity_tcp_str) + dot_non_tcp = self.dot_format_from_props(props_non_tcp, peers_to_compare, ip_blocks_filter, + connectivity_non_tcp_str) # concatenate the two graphs into one dot file res_str = dot_tcp + dot_non_tcp return res_str, None, None # handle formats other than dot - formatted_rules_tcp, fw_rules_tcp = self.fw_rules_from_props(props_tcp, peers_to_compare, connectivity_tcp_str) - formatted_rules_non_tcp, fw_rules_non_tcp = self.fw_rules_from_props(props_non_tcp, peers_to_compare, connectivity_non_tcp_str) + formatted_rules_tcp, fw_rules_tcp = self.fw_rules_from_props(props_tcp, peers_to_compare, ip_blocks_filter, + connectivity_tcp_str) + formatted_rules_non_tcp, fw_rules_non_tcp = self.fw_rules_from_props(props_non_tcp, peers_to_compare, + ip_blocks_filter, + connectivity_non_tcp_str) if self.output_config.outputFormat in ['json', 'yaml']: # get a dict object containing the two maps on different keys (TCP_rules and non-TCP_rules) rules = formatted_rules_tcp @@ -935,7 +947,7 @@ def dot_format_from_connections_dict(self, connections, peers, connectivity_rest conn_graph = self._get_conn_graph(connections, peers) return conn_graph.get_connectivity_dot_format_str(connectivity_restriction) - def dot_format_from_props(self, props, peers, connectivity_restriction=None): + def dot_format_from_props(self, props, peers, ip_blocks_filter, connectivity_restriction=None): """ :param TcpLikeProperties props: properties describing allowed connections :param PeerSet peers: the peers to consider for dot output @@ -946,8 +958,8 @@ def dot_format_from_props(self, props, peers, connectivity_restriction=None): """ conn_graph = ConnectivityGraph(peers, self.config.get_allowed_labels(), self.output_config) for cube in props: - conn_graph.add_edges_from_cube_dict(self.config.peer_container, - props.get_cube_dict_with_orig_values(cube)) + conn_graph.add_edges_from_cube_dict(self.config.peer_container, props.get_cube_dict_with_orig_values(cube), + ip_blocks_filter) return conn_graph.get_connectivity_dot_format_str(connectivity_restriction) def fw_rules_from_connections_dict(self, connections, peers_to_compare, connectivity_restriction=None): @@ -964,7 +976,7 @@ def fw_rules_from_connections_dict(self, connections, peers_to_compare, connecti formatted_rules = fw_rules.get_fw_rules_in_required_format(connectivity_restriction=connectivity_restriction) return formatted_rules, fw_rules - def fw_rules_from_props(self, props, peers_to_compare, connectivity_restriction=None): + def fw_rules_from_props(self, props, peers_to_compare, ip_blocks_filter, connectivity_restriction=None): """ :param TcpLikeProperties props: properties describing allowed connections :param PeerSet peers_to_compare: the peers to consider for fw-rules output @@ -975,7 +987,7 @@ def fw_rules_from_props(self, props, peers_to_compare, connectivity_restriction= """ cluster_info = ClusterInfo(peers_to_compare, self.config.get_allowed_labels()) fw_rules_map = ConnectionSet.tcp_properties_to_fw_rules(props, cluster_info, self.config.peer_container, - connectivity_restriction) + ip_blocks_filter, connectivity_restriction) fw_rules = MinimizeFWRules(fw_rules_map, cluster_info, self.output_config, {}) formatted_rules = fw_rules.get_fw_rules_in_required_format(connectivity_restriction=connectivity_restriction) return formatted_rules, fw_rules From 6350ebdc42c9bb5d48a44c0298eeb45fed1a3bff Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 28 Feb 2023 13:42:11 +0200 Subject: [PATCH 041/187] Simplifying and improving make_tcp_like_properties function. Fixing lint errors. Signed-off-by: Tanya --- nca/CoreDS/CanonicalIntervalSet.py | 3 +- nca/CoreDS/ConnectionSet.py | 6 +- nca/CoreDS/TcpLikeProperties.py | 109 +++++++++++------------------ 3 files changed, 43 insertions(+), 75 deletions(-) diff --git a/nca/CoreDS/CanonicalIntervalSet.py b/nca/CoreDS/CanonicalIntervalSet.py index 545267e59..b1dcfa6d8 100644 --- a/nca/CoreDS/CanonicalIntervalSet.py +++ b/nca/CoreDS/CanonicalIntervalSet.py @@ -365,5 +365,4 @@ def is_single_value(self): """ :return: Whether the interval set contains a single value """ - return len(self) == 1 and list(self)[0].is_single_value() - + return len(self) == 1 and list(self)[0].is_single_value() \ No newline at end of file diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 85e1f61c6..995ae9edd 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -441,10 +441,8 @@ def _add_all_connections_of_protocol(self, protocol): :param protocol: the given protocol number :return: None """ - if self.protocol_supports_ports(protocol): + if self.protocol_supports_ports(protocol) or self.protocol_is_icmp(protocol): self.allowed_protocols[protocol] = TcpLikeProperties.make_all_properties() - elif self.protocol_is_icmp(protocol): - self.allowed_protocols[protocol] = ICMPDataSet(add_all=True) else: self.allowed_protocols[protocol] = True @@ -586,7 +584,7 @@ def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container, ip_block if connectivity_restriction: if connectivity_restriction == 'TCP': ignore_protocols.add_protocol('TCP') - else: #connectivity_restriction == 'non-TCP' + else: # connectivity_restriction == 'non-TCP' ignore_protocols = ProtocolSet.get_non_tcp_protocols() fw_rules_map = defaultdict(list) diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index a052a6abc..cbe440904 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -407,6 +407,21 @@ def cube_dict_to_str(cube_dict): res += str(item[0]) + " : " + str(item[1]) return res + @staticmethod + def resolve_named_port(named_port, peer, protocols): + peer_named_ports = peer.get_named_ports() + real_port = peer_named_ports.get(named_port) + if not real_port: + print(f'Warning: Missing named port {named_port} in the pod {peer}. Ignoring the pod') + return None + if real_port[1] not in protocols: + print(f'Warning: Illegal protocol {real_port[1]} in the named port {named_port} ' + f'of the pod {peer}. Ignoring the pod') + return None + real_ports = PortSet() + real_ports.add_port(real_port[0]) + return real_ports + @staticmethod def make_tcp_like_properties(peer_container, src_ports=PortSet(True), dst_ports=PortSet(True), protocols=ProtocolSet(True), src_peers=None, dst_peers=None, @@ -441,65 +456,36 @@ def make_tcp_like_properties(peer_container, src_ports=PortSet(True), dst_ports= dst_peers_interval = None if (not src_ports.named_ports or not src_peers) and (not dst_ports.named_ports or not dst_peers): # Should not resolve named ports - res = TcpLikeProperties(source_ports=src_ports, dest_ports=dst_ports, - protocols=protocols, methods=methods, paths=paths_dfa, hosts=hosts_dfa, - icmp_type=icmp_type, icmp_code=icmp_code, - src_peers=src_peers_interval, dst_peers=dst_peers_interval, - base_peer_set=base_peer_set) - return res + return TcpLikeProperties(source_ports=src_ports, dest_ports=dst_ports, + protocols=protocols, methods=methods, paths=paths_dfa, hosts=hosts_dfa, + icmp_type=icmp_type, icmp_code=icmp_code, + src_peers=src_peers_interval, dst_peers=dst_peers_interval, + base_peer_set=base_peer_set) # Resolving named ports - tcp_properties = None + tcp_properties = TcpLikeProperties.make_empty_properties(peer_container) if src_ports.named_ports and dst_ports.named_ports: - assert src_peers - assert dst_peers - assert not src_ports.port_set - assert not dst_ports.port_set + assert src_peers and dst_peers + assert not src_ports.port_set and not dst_ports.port_set assert len(src_ports.named_ports) == 1 and len(dst_ports.named_ports) == 1 src_named_port = list(src_ports.named_ports)[0] dst_named_port = list(dst_ports.named_ports)[0] - tcp_protocol = ProtocolSet() - tcp_protocol.add_protocol(ProtocolNameResolver.get_protocol_number('TCP')) for src_peer in src_peers: - src_peer_named_ports = src_peer.get_named_ports() - real_src_port = src_peer_named_ports.get(src_named_port) - if not real_src_port: - print(f'Warning: Missing named port {src_named_port} in the pod {src_peer}. Ignoring the pod') + real_src_ports = TcpLikeProperties.resolve_named_port(src_named_port, src_peer, protocols) + if not real_src_ports: continue - if real_src_port[1] not in protocols: - print(f'Warning: Illegal protocol {real_src_port[1]} in the named port {src_named_port} ' - f'of the target pod {src_peer}. Ignoring the pod') - continue - src_peer_in_set = PeerSet() - src_peer_in_set.add(src_peer) - real_src_ports = PortSet() - real_src_ports.add_port(real_src_port[0]) for dst_peer in dst_peers: - dst_peer_named_ports = dst_peer.get_named_ports() - real_dst_port = dst_peer_named_ports.get(dst_named_port) - if not real_dst_port: - print(f'Warning: Missing named port {dst_named_port} in the pod {dst_peer}. Ignoring the pod') - continue - if real_dst_port[1] not in protocols: - print(f'Warning: Illegal protocol {real_dst_port[1]} in the named port {dst_named_port} ' - f'of the target pod {dst_peer}. Ignoring the pod') + real_dst_ports = TcpLikeProperties.resolve_named_port(dst_named_port, dst_peer, protocols) + if not real_dst_ports: continue - dst_peer_in_set = PeerSet() - dst_peer_in_set.add(dst_peer) - real_dst_ports = PortSet() - real_dst_ports.add_port(real_dst_port[0]) - props = TcpLikeProperties(source_ports=real_src_ports, dest_ports=real_dst_ports, - protocols=protocols if protocols else tcp_protocol, methods=methods, + protocols=protocols, methods=methods, paths=paths_dfa, hosts=hosts_dfa, icmp_type=icmp_type, icmp_code=icmp_code, - src_peers=base_peer_set.get_peer_interval_of(src_peer_in_set), - dst_peers=base_peer_set.get_peer_interval_of(dst_peer_in_set), + src_peers=base_peer_set.get_peer_interval_of(PeerSet({src_peer})), + dst_peers=base_peer_set.get_peer_interval_of(PeerSet({dst_peer})), base_peer_set=base_peer_set) - if tcp_properties: - tcp_properties |= props - else: - tcp_properties = props + tcp_properties |= props else: # either only src_ports or only dst_ports contain named ports if src_ports.named_ports: @@ -512,41 +498,26 @@ def make_tcp_like_properties(peer_container, src_ports=PortSet(True), dst_ports= assert not port_set_with_named_ports.port_set assert len(port_set_with_named_ports.named_ports) == 1 port = list(port_set_with_named_ports.named_ports)[0] - tcp_protocol = ProtocolSet() - tcp_protocol.add_protocol(ProtocolNameResolver.get_protocol_number('TCP')) for peer in peers_for_named_ports: - named_ports = peer.get_named_ports() - real_port = named_ports.get(port) - if not real_port: - print(f'Warning: Missing named port {port} in the pod {peer}. Ignoring the pod') - continue - if real_port[1] not in protocols: - print(f'Warning: Illegal protocol {real_port[1]} in the named port {port} of the target pod {peer}.' - f'Ignoring the pod') + real_ports = TcpLikeProperties.resolve_named_port(port, peer, protocols) + if not real_ports: continue - peer_in_set = PeerSet() - peer_in_set.add(peer) - ports = PortSet() - ports.add_port(real_port[0]) if src_ports.named_ports: - props = TcpLikeProperties(source_ports=ports, dest_ports=dst_ports, - protocols=protocols if protocols else tcp_protocol, methods=methods, + props = TcpLikeProperties(source_ports=real_ports, dest_ports=dst_ports, + protocols=protocols, methods=methods, paths=paths_dfa, hosts=hosts_dfa, icmp_type=icmp_type, icmp_code=icmp_code, - src_peers=base_peer_set.get_peer_interval_of(peer_in_set), + src_peers=base_peer_set.get_peer_interval_of(PeerSet({peer})), dst_peers=dst_peers_interval, base_peer_set=base_peer_set) else: - props = TcpLikeProperties(source_ports=src_ports, dest_ports=ports, - protocols=protocols if protocols else tcp_protocol, methods=methods, + props = TcpLikeProperties(source_ports=src_ports, dest_ports=real_ports, + protocols=protocols, methods=methods, paths=paths_dfa, hosts=hosts_dfa, icmp_type=icmp_type, icmp_code=icmp_code, src_peers=src_peers_interval, - dst_peers=base_peer_set.get_peer_interval_of(peer_in_set), + dst_peers=base_peer_set.get_peer_interval_of(PeerSet({peer})), base_peer_set=base_peer_set) - if tcp_properties: - tcp_properties |= props - else: - tcp_properties = props + tcp_properties |= props return tcp_properties @staticmethod From 54a17086142093bf30f71aee86539fe034d4f8aa Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 28 Feb 2023 14:16:05 +0200 Subject: [PATCH 042/187] Fixing lint errors. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 20 ++++-- nca/CoreDS/TcpLikeProperties.py | 4 +- nca/FWRules/ConnectivityGraph.py | 12 ++-- nca/FWRules/MinimizeFWRules.py | 1 - nca/NetworkConfig/NetworkConfig.py | 3 +- nca/NetworkConfig/NetworkConfigQuery.py | 70 ++++++++++++--------- nca/NetworkConfig/NetworkLayer.py | 18 +++--- nca/NetworkConfig/PoliciesFinder.py | 2 +- nca/Parsers/CalicoPolicyYamlParser.py | 6 -- nca/Parsers/GenericIngressLikeYamlParser.py | 1 - nca/Resources/IstioTrafficResources.py | 1 - nca/Resources/NetworkPolicy.py | 8 +-- nca/SchemeRunner.py | 2 +- 13 files changed, 80 insertions(+), 68 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 995ae9edd..ff25219f5 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -6,7 +6,6 @@ from collections import defaultdict from .CanonicalIntervalSet import CanonicalIntervalSet from .TcpLikeProperties import TcpLikeProperties -from .ICMPDataSet import ICMPDataSet from .ProtocolNameResolver import ProtocolNameResolver from .ProtocolSet import ProtocolSet from .Peer import PeerSet, IpBlock @@ -578,8 +577,19 @@ def get_non_tcp_connections(): # TODO - after moving to the optimized HC set implementation, # get rid of ConnectionSet and move the code below to TcpLikeProperties.py @staticmethod - def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container, ip_blocks_filter, + def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container, ip_blocks_mask, connectivity_restriction): + """ + Build FWRules from the given TcpLikeProperties + :param TcpLikeProperties tcp_props: properties describing allowed connections + :param ClusterInfo cluster_info: the cluster info + :param PeerContainer peer_container: the peer container + :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, + whereas all other values should be filtered out in the output + :param Union[str,None] connectivity_restriction: specify if connectivity is restricted to + TCP / non-TCP , or not + :return: FWRules map + """ ignore_protocols = ProtocolSet() if connectivity_restriction: if connectivity_restriction == 'TCP': @@ -601,9 +611,9 @@ def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container, ip_block new_cube_dict.pop('dst_peers') else: dst_peers = peer_container.get_all_peers_group(True) - if IpBlock.get_all_ips_block() != ip_blocks_filter: - src_peers.filter_ipv6_blocks(ip_blocks_filter) - dst_peers.filter_ipv6_blocks(ip_blocks_filter) + if IpBlock.get_all_ips_block() != ip_blocks_mask: + src_peers.filter_ipv6_blocks(ip_blocks_mask) + dst_peers.filter_ipv6_blocks(ip_blocks_mask) protocols = new_cube_dict.get('protocols') if protocols: new_cube_dict.pop('protocols') diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index cbe440904..6fbd8cf91 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -569,9 +569,7 @@ def are_auto_conns(self): return False return True - - ####################################### ICMP-related functions ####################################### - + # ICMP-related functions @staticmethod def make_icmp_properties(peer_container, protocol="", src_peers=None, dst_peers=None, icmp_type=None, icmp_code=None): diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index 12161cb8a..48e17840f 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -7,7 +7,7 @@ import re from collections import defaultdict import networkx -from nca.CoreDS.Peer import Peer, IpBlock, PeerSet, ClusterEP, Pod +from nca.CoreDS.Peer import IpBlock, PeerSet, ClusterEP, Pod from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.ProtocolSet import ProtocolSet from nca.CoreDS.TcpLikeProperties import TcpLikeProperties @@ -79,11 +79,13 @@ def add_edges(self, connections): """ self.connections_to_peers.update(connections) - def add_edges_from_cube_dict(self, peer_container, cube_dict, ip_blocks_filter): + def add_edges_from_cube_dict(self, peer_container, cube_dict, ip_blocks_mask): """ Add edges to the graph according to the give cube :param peer_container: the peer_container containing all possible peers :param dict cube_dict: the given cube in dictionary format + :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, + whereas all other values should be filtered out in the output """ new_cube_dict = cube_dict.copy() src_peers = new_cube_dict.get('src_peers') @@ -96,9 +98,9 @@ def add_edges_from_cube_dict(self, peer_container, cube_dict, ip_blocks_filter): new_cube_dict.pop('dst_peers') else: dst_peers = peer_container.get_all_peers_group(True) - if IpBlock.get_all_ips_block() != ip_blocks_filter: - src_peers.filter_ipv6_blocks(ip_blocks_filter) - dst_peers.filter_ipv6_blocks(ip_blocks_filter) + if IpBlock.get_all_ips_block() != ip_blocks_mask: + src_peers.filter_ipv6_blocks(ip_blocks_mask) + dst_peers.filter_ipv6_blocks(ip_blocks_mask) protocols = new_cube_dict.get('protocols') if protocols: diff --git a/nca/FWRules/MinimizeFWRules.py b/nca/FWRules/MinimizeFWRules.py index 568dad37b..cd03c8529 100644 --- a/nca/FWRules/MinimizeFWRules.py +++ b/nca/FWRules/MinimizeFWRules.py @@ -3,7 +3,6 @@ # SPDX-License-Identifier: Apache2.0 # -from collections import defaultdict from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock, ClusterEP, Pod, HostEP from .FWRule import FWRuleElement, FWRule, PodElement, LabelExpr, PodLabelsElement, IPBlockElement diff --git a/nca/NetworkConfig/NetworkConfig.py b/nca/NetworkConfig/NetworkConfig.py index 75b9b4955..73ad5c787 100644 --- a/nca/NetworkConfig/NetworkConfig.py +++ b/nca/NetworkConfig/NetworkConfig.py @@ -7,7 +7,6 @@ from nca.CoreDS import Peer from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.TcpLikeProperties import TcpLikeProperties -from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.NetworkPolicy import NetworkPolicy from .NetworkLayer import NetworkLayersContainer, NetworkLayerName @@ -269,7 +268,7 @@ def allowed_connections(self, from_peer, to_peer, layer_name=None): return allowed_conns_res, captured_flag_res, allowed_captured_conns_res, denied_conns_res - def allowed_connections_optimized(self, connectivityFilterIstioEdges, layer_name=None): + def allowed_connections_optimized(self, layer_name=None): """ Computes the set of allowed connections between any relevant peers. :param NetworkLayerName layer_name: The name of the layer to use, if requested to use a specific layer only diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index bf3e38257..5f3e254d1 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -13,7 +13,6 @@ from nca.CoreDS.Peer import PeerSet, IpBlock, Pod, Peer from nca.CoreDS.ProtocolSet import ProtocolSet from nca.CoreDS.TcpLikeProperties import TcpLikeProperties -from .NetworkConfig import NetworkConfig from nca.FWRules.ConnectivityGraph import ConnectivityGraph from nca.FWRules.MinimizeFWRules import MinimizeFWRules from nca.FWRules.ClusterInfo import ClusterInfo @@ -59,8 +58,8 @@ def policy_title(policy): def execute_and_compute_output_in_required_format(self, cmd_line_flag=False): """ - calls the exec def of the running query, computes its output to fit the required format and returns query results - and output + calls the exec def of the running query, computes its output to fit the required format and returns query + results and output :param cmd_line_flag: indicates if the query is running from a cmd-line, since it affects computing the numerical result of some of TwoNetworkConfigsQuery queries :return: the numerical result of the query, @@ -403,7 +402,8 @@ def other_policy_containing_allow(self, self_policy, config_with_self_policy, la """ policies_list = self.config.policies_container.layers[layer_name].policies_list for other_policy in policies_list: - if other_policy.get_order() and self_policy.get_order() and other_policy.get_order() < self_policy.get_order(): + if other_policy.get_order() and self_policy.get_order() and \ + other_policy.get_order() < self_policy.get_order(): return None # All other policies have a lower order and cannot contain self_policy if other_policy == self_policy: continue @@ -425,7 +425,8 @@ def other_policy_containing_deny(self, self_policy, config_with_self_policy, lay """ policies_list = self.config.policies_container.layers[layer_name].policies_list for other_policy in policies_list: - if other_policy.get_order() and self_policy.get_order() and other_policy.get_order() < self_policy.get_order(): + if other_policy.get_order() and self_policy.get_order() and \ + other_policy.get_order() < self_policy.get_order(): return None # not checking lower priority for Calico if other_policy == self_policy: continue @@ -462,7 +463,8 @@ def other_rule_containing(self, self_policy, self_rule_index, is_ingress, layer_ """ policies_list = self.config.policies_container.layers[layer_name].policies_list for other_policy in policies_list: - if other_policy.get_order() and self_policy.get_order() and other_policy.get_order() < self_policy.get_order(): + if other_policy.get_order() and self_policy.get_order() and \ + other_policy.get_order() < self_policy.get_order(): return None, None, None # All following policies have a lower order - containment is not interesting if is_ingress: found_index, contradict = other_policy.ingress_rule_containing(self_policy, self_rule_index) @@ -729,7 +731,8 @@ def exec(self): # if Istio is a layer in the network config - produce 2 maps, for TCP and for non-TCP # because Istio policies can only capture TCP connectivity if self.config.policies_container.layers.does_contain_layer(NetworkLayerName.Istio): - output_res, fw_rules_tcp, fw_rules_non_tcp = self.get_connectivity_output_split_by_tcp(connections, peers, peers_to_compare) + output_res, fw_rules_tcp, fw_rules_non_tcp = \ + self.get_connectivity_output_split_by_tcp(connections, peers, peers_to_compare) else: output_res, fw_rules = self.get_connectivity_output_full(connections, peers, peers_to_compare) peers1_end = time.time() @@ -742,18 +745,17 @@ def exec(self): all_conns_opt = TcpLikeProperties.make_empty_properties() opt_start = time.time() if self.config.optimized_run != 'false': - all_conns_opt = self.config.allowed_connections_optimized(self.output_config.connectivityFilterIstioEdges) + all_conns_opt = self.config.allowed_connections_optimized() if all_conns_opt: opt_peers_to_compare = self.config.peer_container.get_all_peers_group() # add all relevant IpBlocks, used in connections opt_peers_to_compare |= all_conns_opt.project_on_one_dimension('src_peers') | \ - all_conns_opt.project_on_one_dimension('dst_peers') + all_conns_opt.project_on_one_dimension('dst_peers') subset_peers = self.compute_subset(opt_peers_to_compare) subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, src_peers=subset_peers) | \ - TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, - dst_peers=subset_peers) + TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, dst_peers=subset_peers) all_conns_opt &= subset_conns ip_blocks_filter = IpBlock.get_all_ips_block() if exclude_ipv6: @@ -795,7 +797,8 @@ def compare_orig_to_opt_conn(self, orig_conn_graph, opt_props): orig_tcp_props = orig_conn_graph.convert_to_tcp_like_properties(self.config.peer_container) assert orig_tcp_props.contained_in(opt_props) and opt_props.contained_in(orig_tcp_props) # workaround for == # The following assert exposes the bug in HC set - assert not orig_tcp_props.contained_in(opt_props) or not opt_props.contained_in(orig_tcp_props) or orig_tcp_props == opt_props + assert not orig_tcp_props.contained_in(opt_props) or not opt_props.contained_in(orig_tcp_props) or \ + orig_tcp_props == opt_props def compare_fw_rules(self, fw_rules1, fw_rules2): tcp_props1 = ConnectionSet.fw_rules_to_tcp_properties(fw_rules1, self.config.peer_container) @@ -828,19 +831,21 @@ def get_connectivity_output_full(self, connections, peers, peers_to_compare): formatted_rules, fw_rules = self.fw_rules_from_connections_dict(connections, peers_to_compare) return formatted_rules, fw_rules - def get_props_output_full(self, props, peers_to_compare, ip_blocks_filter): + def get_props_output_full(self, props, peers_to_compare, ip_blocks_mask): """ get the connectivity map output considering all connections in the output :param TcpLikeProperties props: properties describing allowed connections :param PeerSet peers_to_compare: the peers to consider for dot/fw-rules output + :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, + whereas all other values should be filtered out in the output :rtype Union[str,dict] """ if self.output_config.outputFormat in ['dot', 'jpg']: - dot_full = self.dot_format_from_props(props, peers_to_compare, ip_blocks_filter) + dot_full = self.dot_format_from_props(props, peers_to_compare, ip_blocks_mask) return dot_full, None # TODO - handle 'txt_no_fw_rules' output format # handle other formats - formatted_rules, fw_rules = self.fw_rules_from_props(props, peers_to_compare, ip_blocks_filter) + formatted_rules, fw_rules = self.fw_rules_from_props(props, peers_to_compare, ip_blocks_mask) return formatted_rules, fw_rules def get_connectivity_output_split_by_tcp(self, connections, peers, peers_to_compare): @@ -861,10 +866,10 @@ def get_connectivity_output_split_by_tcp(self, connections, peers, peers_to_comp res_str = dot_tcp + dot_non_tcp return res_str, None, None # handle formats other than dot - formatted_rules_tcp, fw_rules_tcp = self.fw_rules_from_connections_dict(connections_tcp, peers_to_compare, - connectivity_tcp_str) - formatted_rules_non_tcp, fw_rules_non_tcp = self.fw_rules_from_connections_dict(connections_non_tcp, peers_to_compare, - connectivity_non_tcp_str) + formatted_rules_tcp, fw_rules_tcp = \ + self.fw_rules_from_connections_dict(connections_tcp, peers_to_compare, connectivity_tcp_str) + formatted_rules_non_tcp, fw_rules_non_tcp = \ + self.fw_rules_from_connections_dict(connections_non_tcp, peers_to_compare, connectivity_non_tcp_str) if self.output_config.outputFormat in ['json', 'yaml']: # get a dict object containing the two maps on different keys (TCP_rules and non-TCP_rules) rules = formatted_rules_tcp @@ -877,29 +882,31 @@ def get_connectivity_output_split_by_tcp(self, connections, peers, peers_to_comp res_str = formatted_rules_tcp + formatted_rules_non_tcp return res_str, fw_rules_tcp, fw_rules_non_tcp - def get_props_output_split_by_tcp(self, props, peers_to_compare, ip_blocks_filter): + def get_props_output_split_by_tcp(self, props, peers_to_compare, ip_blocks_mask): """ get the connectivity map output as two parts: TCP and non-TCP :param TcpLikeProperties props: properties describing allowed connections :param PeerSet peers_to_compare: the peers to consider for dot/fw-rules output + :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, + whereas all other values should be filtered out in the output :rtype Union[str,dict] """ connectivity_tcp_str = 'TCP' connectivity_non_tcp_str = 'non-TCP' props_tcp, props_non_tcp = self.convert_props_to_split_by_tcp(props) if self.output_config.outputFormat == 'dot': - dot_tcp = self.dot_format_from_props(props_tcp, peers_to_compare, ip_blocks_filter, + dot_tcp = self.dot_format_from_props(props_tcp, peers_to_compare, ip_blocks_mask, connectivity_tcp_str) - dot_non_tcp = self.dot_format_from_props(props_non_tcp, peers_to_compare, ip_blocks_filter, + dot_non_tcp = self.dot_format_from_props(props_non_tcp, peers_to_compare, ip_blocks_mask, connectivity_non_tcp_str) # concatenate the two graphs into one dot file res_str = dot_tcp + dot_non_tcp return res_str, None, None # handle formats other than dot - formatted_rules_tcp, fw_rules_tcp = self.fw_rules_from_props(props_tcp, peers_to_compare, ip_blocks_filter, + formatted_rules_tcp, fw_rules_tcp = self.fw_rules_from_props(props_tcp, peers_to_compare, ip_blocks_mask, connectivity_tcp_str) formatted_rules_non_tcp, fw_rules_non_tcp = self.fw_rules_from_props(props_non_tcp, peers_to_compare, - ip_blocks_filter, + ip_blocks_mask, connectivity_non_tcp_str) if self.output_config.outputFormat in ['json', 'yaml']: # get a dict object containing the two maps on different keys (TCP_rules and non-TCP_rules) @@ -947,10 +954,12 @@ def dot_format_from_connections_dict(self, connections, peers, connectivity_rest conn_graph = self._get_conn_graph(connections, peers) return conn_graph.get_connectivity_dot_format_str(connectivity_restriction) - def dot_format_from_props(self, props, peers, ip_blocks_filter, connectivity_restriction=None): + def dot_format_from_props(self, props, peers, ip_blocks_mask, connectivity_restriction=None): """ :param TcpLikeProperties props: properties describing allowed connections :param PeerSet peers: the peers to consider for dot output + :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, + whereas all other values should be filtered out in the output :param Union[str,None] connectivity_restriction: specify if connectivity is restricted to TCP / non-TCP , or not :rtype str @@ -959,7 +968,7 @@ def dot_format_from_props(self, props, peers, ip_blocks_filter, connectivity_res conn_graph = ConnectivityGraph(peers, self.config.get_allowed_labels(), self.output_config) for cube in props: conn_graph.add_edges_from_cube_dict(self.config.peer_container, props.get_cube_dict_with_orig_values(cube), - ip_blocks_filter) + ip_blocks_mask) return conn_graph.get_connectivity_dot_format_str(connectivity_restriction) def fw_rules_from_connections_dict(self, connections, peers_to_compare, connectivity_restriction=None): @@ -976,10 +985,12 @@ def fw_rules_from_connections_dict(self, connections, peers_to_compare, connecti formatted_rules = fw_rules.get_fw_rules_in_required_format(connectivity_restriction=connectivity_restriction) return formatted_rules, fw_rules - def fw_rules_from_props(self, props, peers_to_compare, ip_blocks_filter, connectivity_restriction=None): + def fw_rules_from_props(self, props, peers_to_compare, ip_blocks_mask, connectivity_restriction=None): """ :param TcpLikeProperties props: properties describing allowed connections :param PeerSet peers_to_compare: the peers to consider for fw-rules output + :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, + whereas all other values should be filtered out in the output :param Union[str,None] connectivity_restriction: specify if connectivity is restricted to TCP / non-TCP , or not :return the connectivity map in fw-rules, considering connectivity_restriction if required @@ -987,7 +998,7 @@ def fw_rules_from_props(self, props, peers_to_compare, ip_blocks_filter, connect """ cluster_info = ClusterInfo(peers_to_compare, self.config.get_allowed_labels()) fw_rules_map = ConnectionSet.tcp_properties_to_fw_rules(props, cluster_info, self.config.peer_container, - ip_blocks_filter, connectivity_restriction) + ip_blocks_mask, connectivity_restriction) fw_rules = MinimizeFWRules(fw_rules_map, cluster_info, self.output_config, {}) formatted_rules = fw_rules.get_fw_rules_in_required_format(connectivity_restriction=connectivity_restriction) return formatted_rules, fw_rules @@ -1036,7 +1047,8 @@ def convert_props_to_split_by_tcp(self, props): """ tcp_protocol = ProtocolSet() tcp_protocol.add_protocol('TCP') - tcp_props = props & TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, protocols=tcp_protocol) + tcp_props = props & TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, + protocols=tcp_protocol) non_tcp_props = props - tcp_props return tcp_props, non_tcp_props diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 1f859bc33..14d8ad81c 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -181,7 +181,7 @@ def allowed_connections_optimized(self, peer_container): dst_peers=all_ips_peer_set) allowed_egress_conns, denied_egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) allowed_egress_conns |= TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=all_ips_peer_set, - dst_peers=all_pods) + dst_peers=all_pods) res = allowed_ingress_conns & allowed_egress_conns # exclude IpBlock->IpBlock connections excluded_conns = TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=all_ips_peer_set, @@ -249,14 +249,14 @@ def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda poli for policy in self.policies_list: policy_allowed_conns, policy_denied_conns, policy_captured = \ policy.allowed_connections_optimized(is_ingress) - if policy_captured: # not empty + if policy_captured: # not empty policy_denied_conns -= allowed_conns - #policy_denied_conns -= pass_conns # Preparation for handling of pass + # policy_denied_conns -= pass_conns # Preparation for handling of pass policy_allowed_conns -= denied_conns - #policy_allowed_conns -= pass_conns # Preparation for handling of pass - #policy_pass_conns -= denied_conns - #policy_pass_conns -= allowed_conns - #pass_conns |= policy_pass_conns + # policy_allowed_conns -= pass_conns # Preparation for handling of pass + # policy_pass_conns -= denied_conns + # policy_pass_conns -= allowed_conns + # pass_conns |= policy_pass_conns allowed_conns |= policy_allowed_conns denied_conns |= policy_denied_conns if captured_func(policy): @@ -330,8 +330,8 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): all_allowed_conns=allowed_conns | allowed_non_captured_conns) def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): - allowed_conn, denied_conns, captured = self.collect_policies_conns_optimized(is_ingress, - IstioNetworkLayer.captured_cond_func) + allowed_conn, denied_conns, captured = \ + self.collect_policies_conns_optimized(is_ingress, IstioNetworkLayer.captured_cond_func) base_peer_set_with_ip = peer_container.get_all_peers_group(True) base_peer_set_no_ip = peer_container.get_all_peers_group() non_captured_peers = base_peer_set_no_ip - captured diff --git a/nca/NetworkConfig/PoliciesFinder.py b/nca/NetworkConfig/PoliciesFinder.py index 4dad1eee5..50600fb39 100644 --- a/nca/NetworkConfig/PoliciesFinder.py +++ b/nca/NetworkConfig/PoliciesFinder.py @@ -25,7 +25,7 @@ def __init__(self, optimized_run='false'): self.policies_container = PoliciesContainer() self._parse_queue = deque() self.peer_container = None - self.optimized_run=optimized_run + self.optimized_run = optimized_run # following missing resources fields are relevant for "livesim" mode, # where certain resources are added to enable the analysis self.missing_istio_gw_pods_with_labels = {} diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index 2390448c3..c42b37b5f 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -380,14 +380,12 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): if err: self.syntax_error(err, not_icmp_data) - #res = ICMPDataSet(icmp_data is None and not_icmp_data is None) res = TcpLikeProperties.make_icmp_properties(self.peer_container) opt_props = TcpLikeProperties.make_empty_properties(self.peer_container) if self.optimized_run != 'false' and src_pods and dst_pods: opt_props = TcpLikeProperties.make_icmp_properties(self.peer_container, protocol=protocol, src_peers=src_pods, dst_peers=dst_pods) if icmp_data is not None: - #res.add_to_set(icmp_type, icmp_code) res = TcpLikeProperties.make_icmp_properties(self.peer_container, icmp_type=icmp_type, icmp_code=icmp_code) if self.optimized_run != 'false' and src_pods and dst_pods: opt_props = TcpLikeProperties.make_icmp_properties(self.peer_container, protocol=protocol, @@ -395,12 +393,9 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): icmp_type=icmp_type, icmp_code=icmp_code) if not_icmp_data is not None: if icmp_type == not_icmp_type and icmp_code == not_icmp_code: - #res = ICMPDataSet() res = TcpLikeProperties.make_empty_properties(self.peer_container) self.warning('icmp and notICMP are conflicting - no traffic will be matched', not_icmp_data) elif icmp_type == not_icmp_type and icmp_code is None: - #tmp = ICMPDataSet() # this is the only case where it makes sense to combine icmp and notICMP - #tmp.add_to_set(not_icmp_type, not_icmp_code) tmp = TcpLikeProperties.make_icmp_properties(self.peer_container, icmp_type=not_icmp_type, icmp_code=not_icmp_code) res -= tmp @@ -413,7 +408,6 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): else: self.warning('notICMP has no effect', not_icmp_data) elif not_icmp_data is not None: - #res.add_all_but_given_pair(not_icmp_type, not_icmp_code) res = TcpLikeProperties.make_all_but_given_icmp_properties(self.peer_container, icmp_type=not_icmp_type, icmp_code=not_icmp_code) diff --git a/nca/Parsers/GenericIngressLikeYamlParser.py b/nca/Parsers/GenericIngressLikeYamlParser.py index 6ecbf39e0..0e8d483ac 100644 --- a/nca/Parsers/GenericIngressLikeYamlParser.py +++ b/nca/Parsers/GenericIngressLikeYamlParser.py @@ -8,7 +8,6 @@ from nca.CoreDS.DimensionsManager import DimensionsManager from nca.CoreDS.Peer import PeerSet from nca.CoreDS.PortSet import PortSet -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from nca.Resources.IngressPolicy import IngressPolicyRule from .GenericYamlParser import GenericYamlParser diff --git a/nca/Resources/IstioTrafficResources.py b/nca/Resources/IstioTrafficResources.py index c175d927f..77bd92941 100644 --- a/nca/Resources/IstioTrafficResources.py +++ b/nca/Resources/IstioTrafficResources.py @@ -7,7 +7,6 @@ from nca.CoreDS.Peer import PeerSet from nca.CoreDS.PortSet import PortSet from nca.CoreDS.MinDFA import MinDFA -from nca.CoreDS.MethodSet import MethodSet from .K8sService import K8sService diff --git a/nca/Resources/NetworkPolicy.py b/nca/Resources/NetworkPolicy.py index bc463b05f..04c1c5021 100644 --- a/nca/Resources/NetworkPolicy.py +++ b/nca/Resources/NetworkPolicy.py @@ -53,10 +53,10 @@ def __init__(self, name, namespace): self.selected_peers = PeerSet() # The peers affected by this policy self.ingress_rules = [] self.egress_rules = [] - self.optimized_ingress_props = TcpLikeProperties.make_empty_properties() # allowed properties in hypercube set format - self.optimized_denied_ingress_props = TcpLikeProperties.make_empty_properties() # denied properties in hypercube set format - self.optimized_egress_props = TcpLikeProperties.make_empty_properties() # allowed properties in hypercube set format - self.optimized_denied_egress_props = TcpLikeProperties.make_empty_properties() # denied properties in hypercube set format + self.optimized_ingress_props = TcpLikeProperties.make_empty_properties() + self.optimized_denied_ingress_props = TcpLikeProperties.make_empty_properties() + self.optimized_egress_props = TcpLikeProperties.make_empty_properties() + self.optimized_denied_egress_props = TcpLikeProperties.make_empty_properties() self.affects_ingress = False # whether the policy affects the ingress of the selected peers self.affects_egress = False # whether the policy affects the egress of the selected peers self.findings = [] # accumulated findings which are relevant only to this policy (emptiness and redundancy) diff --git a/nca/SchemeRunner.py b/nca/SchemeRunner.py index 772cdabe1..6ea9cb36f 100644 --- a/nca/SchemeRunner.py +++ b/nca/SchemeRunner.py @@ -26,7 +26,7 @@ def __init__(self, scheme_file_name, output_format=None, output_path=None, optim self.output_config_from_cli_args['outputFormat'] = output_format if output_path is not None: self.output_config_from_cli_args['outputPath'] = output_path - self.optimized_run=optimized_run + self.optimized_run = optimized_run scanner = TreeScannerFactory.get_scanner(scheme_file_name) for yaml_file in scanner.get_yamls(): From a30fc04ca3e49fb1efd158ec8de2affbb1f66d39 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 28 Feb 2023 14:26:40 +0200 Subject: [PATCH 043/187] Fixing lint errors. Signed-off-by: Tanya --- nca/CoreDS/CanonicalIntervalSet.py | 2 +- nca/NetworkConfig/NetworkConfigQuery.py | 2 +- nca/NetworkConfig/PoliciesFinder.py | 2 +- nca/Parsers/CalicoPolicyYamlParser.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nca/CoreDS/CanonicalIntervalSet.py b/nca/CoreDS/CanonicalIntervalSet.py index b1dcfa6d8..64b3e1121 100644 --- a/nca/CoreDS/CanonicalIntervalSet.py +++ b/nca/CoreDS/CanonicalIntervalSet.py @@ -365,4 +365,4 @@ def is_single_value(self): """ :return: Whether the interval set contains a single value """ - return len(self) == 1 and list(self)[0].is_single_value() \ No newline at end of file + return len(self) == 1 and list(self)[0].is_single_value() diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 5f3e254d1..08231bdad 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -692,7 +692,7 @@ def are_labels_all_included(target_labels, pool_labels): return False return True - def exec(self): + def exec(self): # noqa: C901 self.output_config.fullExplanation = True # assign true for this query - it is always ok to compare its results self.output_config.configName = os.path.basename(self.config.name) if self.config.name.startswith('./') else \ self.config.name diff --git a/nca/NetworkConfig/PoliciesFinder.py b/nca/NetworkConfig/PoliciesFinder.py index 50600fb39..e9aeb48bb 100644 --- a/nca/NetworkConfig/PoliciesFinder.py +++ b/nca/NetworkConfig/PoliciesFinder.py @@ -66,7 +66,7 @@ def _add_policy(self, policy): """ self.policies_container.append_policy(policy) - def parse_policies_in_parse_queue(self): + def parse_policies_in_parse_queue(self): # noqa: C901 istio_traffic_parser = None istio_sidecar_parser = None for policy, file_name, policy_type in self._parse_queue: diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index c42b37b5f..a85edd9aa 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -438,7 +438,7 @@ def _parse_protocol(self, protocol, rule): self.syntax_error('invalid protocol name: ' + protocol, rule) return ProtocolNameResolver.get_protocol_number(protocol) - def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): + def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): # noqa: C901 """ Parse a single ingres/egress rule, producing a CalicoPolicyRule :param dict rule: The rule element to parse From ab7dc478f7121e04012c4a2f4d61e8e8434f7e33 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 28 Feb 2023 17:16:15 +0200 Subject: [PATCH 044/187] Removed unised classes ConnectivityGraphPrototype and ConnectivityGraphOptimized. In creating TcpLikeProperties, methods and protocols cannot be None. Signed-off-by: Tanya Signed-off-by: Tanya --- nca/CoreDS/TcpLikeProperties.py | 4 +- nca/FWRules/ConnectivityGraph.py | 134 ++---------------- .../IstioTrafficResourcesYamlParser.py | 2 + 3 files changed, 18 insertions(+), 122 deletions(-) diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index 6fbd8cf91..6c0936e80 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -73,10 +73,10 @@ def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protoco # create a dict object that holds the values required to build the cube dims_to_values = {"src_peers": {"value": src_peers, "is_all": src_peers is None}, "dst_peers": {"value": dst_peers, "is_all": dst_peers is None}, - "protocols": {"value": protocols, "is_all": protocols is None or protocols.is_whole_range()}, + "protocols": {"value": protocols, "is_all": protocols.is_whole_range()}, "src_ports": {"value": source_ports.port_set, "is_all": source_ports.is_all()}, "dst_ports": {"value": dest_ports.port_set, "is_all": dest_ports.is_all()}, - "methods": {"value": methods, "is_all": methods is None or methods.is_whole_range()}, + "methods": {"value": methods, "is_all": methods.is_whole_range()}, "hosts": {"value": hosts, "is_all": hosts is None}, "paths": {"value": paths, "is_all": paths is None}, "icmp_type": {"value": icmp_type, "is_all": icmp_type is None}, diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index 48e17840f..26e4ce876 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -16,14 +16,26 @@ from .ClusterInfo import ClusterInfo -class ConnectivityGraphPrototype: - def __init__(self, output_config): +class ConnectivityGraph: + """ + Represents a connectivity digraph, that is a set of labeled edges, where the nodes are peers and + the labels on the edges are the allowed connections between two peers. + """ + + def __init__(self, all_peers, allowed_labels, output_config): """ Create a ConnectivityGraph object + :param all_peers: PeerSet with the topology all peers (pods and ip blocks) + :param allowed_labels: the set of allowed labels to be used in generated fw-rules, extracted from policy yamls :param output_config: OutputConfiguration object """ # connections_to_peers holds the connectivity graph self.output_config = output_config + self.connections_to_peers = defaultdict(list) + if self.output_config.fwRulesOverrideAllowedLabels: + allowed_labels = set(label for label in self.output_config.fwRulesOverrideAllowedLabels.split(',')) + self.cluster_info = ClusterInfo(all_peers, allowed_labels) + self.allowed_labels = allowed_labels def _get_peer_name(self, peer): """ @@ -39,28 +51,6 @@ def _get_peer_name(self, peer): return peer.workload_name, False return str(peer), False - -class ConnectivityGraph(ConnectivityGraphPrototype): - """ - Represents a connectivity digraph, that is a set of labeled edges, where the nodes are peers and - the labels on the edges are the allowed connections between two peers. - """ - - def __init__(self, all_peers, allowed_labels, output_config): - """ - Create a ConnectivityGraph object - :param all_peers: PeerSet with the topology all peers (pods and ip blocks) - :param allowed_labels: the set of allowed labels to be used in generated fw-rules, extracted from policy yamls - :param output_config: OutputConfiguration object - """ - # connections_to_peers holds the connectivity graph - super().__init__(output_config) - self.connections_to_peers = defaultdict(list) - if self.output_config.fwRulesOverrideAllowedLabels: - allowed_labels = set(label for label in self.output_config.fwRulesOverrideAllowedLabels.split(',')) - self.cluster_info = ClusterInfo(all_peers, allowed_labels) - self.allowed_labels = allowed_labels - def add_edge(self, source_peer, dest_peer, connections): """ Adding a labeled edge to the graph @@ -462,99 +452,3 @@ def conn_graph_has_fw_rules(self): if not conn: # conn is "no connections": return False return True - - -class ConnectivityGraphOptimized(ConnectivityGraphPrototype): - """ - Represents an optimized connectivity digraph, that is a set of labeled edges, where the nodes are sets of peers - and the labels on the edges are the allowed connections between the two sets of peers. - """ - - def __init__(self, output_config): - """ - Create a ConnectivityGraph object - :param output_config: OutputConfiguration object - """ - super().__init__(output_config) - self.edges = [] # the list of tuples(src_peers, dst_peers, connections) - self.peer_sets = set() # the set of all src/dst PeerSets in the graph - - def get_peer_set_names(self, peer_set): - """ - Convert a given peer_set to a string format for the output - :param peer_set: the given peer_set - :return: the string describing the given peer_set - """ - res_names = '' - res_is_only_ip_block = True - res_is_only_pods = True - for peer in peer_set: - peer_name, is_ip_block = self._get_peer_name(peer) - res_names += ', ' + peer_name if res_names else peer_name - res_is_only_ip_block &= is_ip_block - res_is_only_pods &= not is_ip_block - return res_names, res_is_only_ip_block, res_is_only_pods - - def add_edge(self, cube_dict): - """ - Adding a labeled edge to the graph - :param dict cube_dict: The map from all every dimension to its values - :return: None - """ - new_cube_dict = cube_dict.copy() - src_peers = new_cube_dict.get('src_peers') or PeerSet() - dst_peers = new_cube_dict.get('dst_peers') or PeerSet() - self.peer_sets.add(src_peers) - self.peer_sets.add(dst_peers) - if src_peers: - new_cube_dict.pop('src_peers') - if dst_peers: - new_cube_dict.pop('dst_peers') - self.edges.append((src_peers, dst_peers, new_cube_dict)) - - def get_connectivity_dot_format_str(self): - """ - :return: a string with content of dot format for connectivity graph - """ - output_result = f'// The Connectivity Graph of {self.output_config.configName}\n' - output_result += 'digraph ' + '{\n' - if self.output_config.queryName and self.output_config.configName: - output_result += f'\tHEADER [shape="box" label=< {self.output_config.queryName}/' \ - f'{self.output_config.configName} > fontsize=30 color=webmaroon fontcolor=webmaroon];\n' - peer_set_lines = set() - for peer_set in self.peer_sets: - peer_set_name, is_only_ip_block, is_only_pods = self.get_peer_set_names(peer_set) - peer_color = "red2" if is_only_ip_block else "blue" if is_only_pods else "black" - peer_set_lines.add(f'\t\"{peer_set_name}\" [label=\"{peer_set_name}\" color=\"{peer_color}\" ' - f'fontcolor=\"{peer_color}\"]\n') - - edge_lines = set() - for src_peer_set, dst_peer_set, cube_dict in self.edges: - if src_peer_set != dst_peer_set and cube_dict: - src_peers_names, _, _ = self.get_peer_set_names(src_peer_set) - dst_peers_names, _, _ = self.get_peer_set_names(dst_peer_set) - line = '\t' - line += f'\"{src_peers_names}\"' - line += ' -> ' - line += f'\"{dst_peers_names}\"' - conn_str = str(cube_dict).replace("protocols:", "") - line += f' [label=\"{conn_str}\" color=\"gold2\" fontcolor=\"darkgreen\"]\n' - edge_lines.add(line) - output_result += ''.join(line for line in sorted(list(peer_set_lines))) + \ - ''.join(line for line in sorted(list(edge_lines))) + '}\n\n' - return output_result - - def get_connectivity_txt_format_str(self): - """ - :return: a string with content of txt format for connectivity graph - """ - output_result = '' - for src_peer_set, dst_peer_set, cube_dict in self.edges: - src_peers_names, _, _ = self.get_peer_set_names(src_peer_set) - dst_peers_names, _, _ = self.get_peer_set_names(dst_peer_set) - output_result += "src_pods: [" + src_peers_names + "] " - output_result += "dst_pods: [" + dst_peers_names + "] " - conn_str = str(cube_dict).replace("protocols:", "") - output_result += "conn: " + conn_str if cube_dict else "All connections" - output_result += '\n' - return output_result diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index 32a2219fc..80a8ba817 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -265,6 +265,8 @@ def parse_http_match_request(self, route, parsed_route, vs): methods = self.parse_istio_regex_string(item, 'method', vs.full_name()) if methods: parsed_route.add_methods(methods) + else: + parsed_route.add_methods(MethodSet(True)) def parse_http_route_destinations(self, route, parsed_route, vs): """ From e49ddfb461ae0f67a0006b858d95f31fc8a0a2d1 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 28 Feb 2023 19:09:10 +0200 Subject: [PATCH 045/187] Avoid using creation of TcpLikeProperties directly with init; using make_tcp_like_properties, make_tcp_like_properties_from_dict, make_empty_properties or make_all_properties instead. Use icmp_type and icmp_code full domain intervals instead of None in parameters to TcpLikeProperties creation methods. Removed unused or commented-out code. Fixed project_on_one_dimension to return None (to represent all values) for non-active dimensions. Signed-off-by: Tanya --- nca/CoreDS/TcpLikeProperties.py | 51 ++++++--------------- nca/Parsers/GenericIngressLikeYamlParser.py | 3 +- nca/Parsers/GenericYamlParser.py | 11 +++-- nca/Parsers/IstioPolicyYamlParser.py | 5 +- 4 files changed, 27 insertions(+), 43 deletions(-) diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index 6c0936e80..db612bec6 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -43,7 +43,9 @@ class TcpLikeProperties(CanonicalHyperCubeSet): # TODO: change constructor defaults? either all arguments in "allow all" by default, or "empty" by default def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protocols=ProtocolSet(True), - methods=MethodSet(True), paths=None, hosts=None, icmp_type=None, icmp_code=None, + methods=MethodSet(True), paths=None, hosts=None, + icmp_type=DimensionsManager().get_dimension_domain_by_name('icmp_type'), + icmp_code=DimensionsManager().get_dimension_domain_by_name('icmp_code'), base_peer_set=None, src_peers=None, dst_peers=None, create_empty=False): """ @@ -79,8 +81,8 @@ def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protoco "methods": {"value": methods, "is_all": methods.is_whole_range()}, "hosts": {"value": hosts, "is_all": hosts is None}, "paths": {"value": paths, "is_all": paths is None}, - "icmp_type": {"value": icmp_type, "is_all": icmp_type is None}, - "icmp_code": {"value": icmp_code, "is_all": icmp_code is None}} + "icmp_type": {"value": icmp_type, "is_all": icmp_type == DimensionsManager().get_dimension_domain_by_name('icmp_type')}, + "icmp_code": {"value": icmp_code, "is_all": icmp_code == DimensionsManager().get_dimension_domain_by_name('icmp_code')}} cube, active_dims, has_empty_dim_value = self._get_cube_and_active_dims_from_input_values(dims_to_values) @@ -215,7 +217,6 @@ def get_properties_obj(self): def __eq__(self, other): if isinstance(other, TcpLikeProperties): - # assert not self.base_peer_set or not other.base_peer_set or self.base_peer_set == other.base_peer_set res = super().__eq__(other) and self.named_ports == other.named_ports and \ self.excluded_named_ports == other.excluded_named_ports return res @@ -237,8 +238,6 @@ def __sub__(self, other): return res def __iand__(self, other): - # assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ - # not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.has_named_ports() assert not isinstance(other, TcpLikeProperties) or not other.has_named_ports() super().__iand__(other) @@ -247,8 +246,6 @@ def __iand__(self, other): return self def __ior__(self, other): - # assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ - # not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.excluded_named_ports assert not isinstance(other, TcpLikeProperties) or not other.excluded_named_ports super().__ior__(other) @@ -266,8 +263,6 @@ def __ior__(self, other): return self def __isub__(self, other): - # assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ - # not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.has_named_ports() assert not isinstance(other, TcpLikeProperties) or not other.has_named_ports() super().__isub__(other) @@ -281,8 +276,6 @@ def contained_in(self, other): :return: Whether all (source port, target port) pairs in self also appear in other :rtype: bool """ - # assert not isinstance(other, TcpLikeProperties) or not self.base_peer_set or \ - # not other.base_peer_set or self.base_peer_set == other.base_peer_set assert not self.has_named_ports() assert not other.has_named_ports() return super().contained_in(other) @@ -380,18 +373,10 @@ def project_on_one_dimension(self, dim_name): Build the projection of self to the given dimension :param str dim_name: the given dimension :return: the projection on the given dimension, having that dimension type (either IntervalSet or DFA) + or None if the given dimension is not active """ if dim_name not in self.active_dimensions: - if dim_name == "src_peers" or dim_name == "dst_peers": - return PeerSet() - elif dim_name == "src_ports" or dim_name == "dst_ports": - return PortSet() - elif dim_name == "protocols": - return ProtocolSet() - elif dim_name == "methods": - return MethodSet() - else: - return None + return None res = None for cube in self: cube_dict = self.get_cube_dict_with_orig_values(cube) @@ -400,13 +385,6 @@ def project_on_one_dimension(self, dim_name): res = (res | values) if res else values return res - @staticmethod - def cube_dict_to_str(cube_dict): - res = "" - for item in cube_dict.items(): - res += str(item[0]) + " : " + str(item[1]) - return res - @staticmethod def resolve_named_port(named_port, peer, protocols): peer_named_ports = peer.get_named_ports() @@ -426,7 +404,8 @@ def resolve_named_port(named_port, peer, protocols): def make_tcp_like_properties(peer_container, src_ports=PortSet(True), dst_ports=PortSet(True), protocols=ProtocolSet(True), src_peers=None, dst_peers=None, paths_dfa=None, hosts_dfa=None, methods=MethodSet(True), - icmp_type=None, icmp_code=None): + icmp_type=DimensionsManager().get_dimension_domain_by_name('icmp_type'), + icmp_code=DimensionsManager().get_dimension_domain_by_name('icmp_code')): """ get TcpLikeProperties with TCP allowed connections, corresponding to input properties cube. TcpLikeProperties should not contain named ports: substitute them with corresponding port numbers, per peer @@ -531,14 +510,14 @@ def make_tcp_like_properties_from_dict(peer_container, cube_dict): cube_dict_copy = cube_dict.copy() src_ports = cube_dict_copy.pop("src_ports", PortSet(True)) dst_ports = cube_dict_copy.pop("dst_ports", PortSet(True)) - protocols = cube_dict_copy.pop("protocols", None) + protocols = cube_dict_copy.pop("protocols", ProtocolSet(True)) src_peers = cube_dict_copy.pop("src_peers", None) dst_peers = cube_dict_copy.pop("dst_peers", None) paths_dfa = cube_dict_copy.pop("paths", None) hosts_dfa = cube_dict_copy.pop("hosts", None) - methods = cube_dict_copy.pop("methods", None) - icmp_type = cube_dict_copy.pop("icmp_type", None) - icmp_code = cube_dict_copy.pop("icmp_code", None) + methods = cube_dict_copy.pop("methods", MethodSet(True)) + icmp_type = cube_dict_copy.pop("icmp_type", DimensionsManager().get_dimension_domain_by_name('icmp_type')) + icmp_code = cube_dict_copy.pop("icmp_code", DimensionsManager().get_dimension_domain_by_name('icmp_code')) assert not cube_dict_copy return TcpLikeProperties.make_tcp_like_properties(peer_container, src_ports=src_ports, dst_ports=dst_ports, protocols=protocols, src_peers=src_peers, dst_peers=dst_peers, @@ -578,8 +557,8 @@ def make_icmp_properties(peer_container, protocol="", src_peers=None, dst_peers= icmp_protocol_set.add_protocol(ProtocolNameResolver.get_protocol_number(protocol)) else: icmp_protocol_set = ProtocolSet(True) - icmp_type_interval = None - icmp_code_interval = None + icmp_type_interval = DimensionsManager().get_dimension_domain_by_name('icmp_type') + icmp_code_interval = DimensionsManager().get_dimension_domain_by_name('icmp_code') if icmp_type: icmp_type_interval = CanonicalIntervalSet.get_interval_set(icmp_type, icmp_type) if icmp_code: diff --git a/nca/Parsers/GenericIngressLikeYamlParser.py b/nca/Parsers/GenericIngressLikeYamlParser.py index 0e8d483ac..40d6b1346 100644 --- a/nca/Parsers/GenericIngressLikeYamlParser.py +++ b/nca/Parsers/GenericIngressLikeYamlParser.py @@ -97,7 +97,8 @@ def _make_rules_from_conns(self, tcp_conns): port_set.port_set = ports port_set.named_ports = tcp_conns.named_ports port_set.excluded_named_ports = tcp_conns.excluded_named_ports - new_conns = self._get_connection_set_from_properties(port_set, paths_dfa=paths, hosts_dfa=hosts) + new_conns = self._get_connection_set_from_properties(self.peer_container, port_set, paths_dfa=paths, + hosts_dfa=hosts) if peers_to_conns.get(dst_peer_set): peers_to_conns[dst_peer_set] |= new_conns # optimize conns for the same peers else: diff --git a/nca/Parsers/GenericYamlParser.py b/nca/Parsers/GenericYamlParser.py index 0631b3897..f81ad5138 100644 --- a/nca/Parsers/GenericYamlParser.py +++ b/nca/Parsers/GenericYamlParser.py @@ -227,17 +227,20 @@ def validate_value_in_domain(self, value, dim_name, array, value_name): self.syntax_error(err_message, array) @staticmethod - def _get_connection_set_from_properties(dest_ports, method_set=MethodSet(True), paths_dfa=None, hosts_dfa=None): + def _get_connection_set_from_properties(peer_container, dest_ports, methods=MethodSet(True), paths_dfa=None, + hosts_dfa=None): """ get ConnectionSet with TCP allowed connections, corresponding to input properties cube + :param PeerContainer peer_container: the peer container :param PortSet dest_ports: ports set for dset_ports dimension - :param MethodSet method_set: methods set for methods dimension + :param MethodSet methods: methods set for methods dimension :param MinDFA paths_dfa: MinDFA obj for paths dimension :param MinDFA hosts_dfa: MinDFA obj for hosts dimension :return: ConnectionSet with TCP allowed connections , corresponding to input properties cube """ - tcp_properties = TcpLikeProperties(source_ports=PortSet(True), dest_ports=dest_ports, methods=method_set, - paths=paths_dfa, hosts=hosts_dfa) + tcp_properties = TcpLikeProperties.make_tcp_like_properties(peer_container, dst_ports=dest_ports, + methods=methods, paths_dfa=paths_dfa, + hosts_dfa=hosts_dfa) res = ConnectionSet() res.add_connections('TCP', tcp_properties) return res diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index 4309e1b41..fb87baea7 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -195,7 +195,7 @@ def parse_key_values(self, key, values, not_values): return self.parse_principals(values, not_values) # PeerSet elif key == 'destination.port': dst_ports = self.get_rule_ports(values, not_values) # PortSet - return self._get_connection_set_from_properties(dst_ports) # ConnectionSet + return self._get_connection_set_from_properties(self.peer_container, dst_ports) # ConnectionSet return NotImplemented, False def parse_condition(self, condition): @@ -398,7 +398,8 @@ def parse_operation(self, operation_dict): hosts_dfa = self.parse_regex_dimension_values("hosts", operation.get("hosts"), operation.get("notHosts"), operation) - return self._get_connection_set_from_properties(dst_ports, methods_set, paths_dfa, hosts_dfa) + return self._get_connection_set_from_properties(self.peer_container, dst_ports, methods=methods_set, + paths_dfa=paths_dfa, hosts_dfa=hosts_dfa) def parse_source(self, source_dict): """ From 254412a557bb060efc7d7f3accc9759bb05ac3f2 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 28 Feb 2023 19:17:48 +0200 Subject: [PATCH 046/187] Fixing lint errors. Signed-off-by: Tanya --- nca/CoreDS/TcpLikeProperties.py | 9 ++++++--- nca/Parsers/GenericYamlParser.py | 1 - 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index db612bec6..284e489c3 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -81,8 +81,10 @@ def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protoco "methods": {"value": methods, "is_all": methods.is_whole_range()}, "hosts": {"value": hosts, "is_all": hosts is None}, "paths": {"value": paths, "is_all": paths is None}, - "icmp_type": {"value": icmp_type, "is_all": icmp_type == DimensionsManager().get_dimension_domain_by_name('icmp_type')}, - "icmp_code": {"value": icmp_code, "is_all": icmp_code == DimensionsManager().get_dimension_domain_by_name('icmp_code')}} + "icmp_type": {"value": icmp_type, "is_all": + icmp_type == DimensionsManager().get_dimension_domain_by_name('icmp_type')}, + "icmp_code": {"value": icmp_code, "is_all": + icmp_code == DimensionsManager().get_dimension_domain_by_name('icmp_code')}} cube, active_dims, has_empty_dim_value = self._get_cube_and_active_dims_from_input_values(dims_to_values) @@ -526,7 +528,8 @@ def make_tcp_like_properties_from_dict(peer_container, cube_dict): @staticmethod def make_empty_properties(peer_container=None): - return TcpLikeProperties(base_peer_set=peer_container.peer_set.copy() if peer_container else None, create_empty=True) + return TcpLikeProperties(base_peer_set=peer_container.peer_set.copy() if peer_container else None, + create_empty=True) @staticmethod def make_all_properties(peer_container=None): diff --git a/nca/Parsers/GenericYamlParser.py b/nca/Parsers/GenericYamlParser.py index f81ad5138..a1e5e4e44 100644 --- a/nca/Parsers/GenericYamlParser.py +++ b/nca/Parsers/GenericYamlParser.py @@ -9,7 +9,6 @@ from nca.CoreDS.TcpLikeProperties import TcpLikeProperties from nca.CoreDS.MethodSet import MethodSet from nca.CoreDS.ConnectionSet import ConnectionSet -from nca.CoreDS.PortSet import PortSet from nca.CoreDS.Peer import IpBlock from nca.Utils.NcaLogger import NcaLogger from nca.FileScanners.GenericTreeScanner import ObjectWithLocation From 8ec07174277067903f6913c88e733f6ea6c1658b Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 28 Feb 2023 19:19:45 +0200 Subject: [PATCH 047/187] Fixing lint errors. Signed-off-by: Tanya --- nca/CoreDS/TcpLikeProperties.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/TcpLikeProperties.py index 284e489c3..e5688be63 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/TcpLikeProperties.py @@ -81,10 +81,10 @@ def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protoco "methods": {"value": methods, "is_all": methods.is_whole_range()}, "hosts": {"value": hosts, "is_all": hosts is None}, "paths": {"value": paths, "is_all": paths is None}, - "icmp_type": {"value": icmp_type, "is_all": - icmp_type == DimensionsManager().get_dimension_domain_by_name('icmp_type')}, - "icmp_code": {"value": icmp_code, "is_all": - icmp_code == DimensionsManager().get_dimension_domain_by_name('icmp_code')}} + "icmp_type": {"value": icmp_type, + "is_all": icmp_type == DimensionsManager().get_dimension_domain_by_name('icmp_type')}, + "icmp_code": {"value": icmp_code, + "is_all": icmp_code == DimensionsManager().get_dimension_domain_by_name('icmp_code')}} cube, active_dims, has_empty_dim_value = self._get_cube_and_active_dims_from_input_values(dims_to_values) From 95971c4e0a0ebfa7f1e5da6d0cee00d7482a2148 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 5 Mar 2023 12:21:46 +0200 Subject: [PATCH 048/187] Fixing ConnectionSet.__str__ to be accurate, since it is used in sorting functions. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index ff25219f5..6d4d9a226 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -204,9 +204,6 @@ def __str__(self): protocol_text = 'Protocols: ' for idx, protocol in enumerate(self.allowed_protocols.keys()): - if idx > 5: - protocol_text += ', ...' - break if idx > 0: protocol_text += ', ' protocol_text += ProtocolNameResolver.get_protocol_name(protocol) From 2bba713d5ac00c9a6b4687b3f5612ecc2c78b3f9 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 5 Mar 2023 17:50:51 +0200 Subject: [PATCH 049/187] Fixed excluding unused ipv6 blocks in the optimized solution. Signed-off-by: Tanya --- nca/CoreDS/Peer.py | 16 +++++++--------- nca/NetworkConfig/NetworkConfigQuery.py | 12 ++++++------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index ea2f96606..650cb4f78 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -672,21 +672,19 @@ def get_peer_set_by_indices(self, peer_interval_set): peer_list.append(self.sorted_peer_list[ind]) return PeerSet(set(peer_list)) - def filter_ipv6_blocks(self, ip_block_filter): - ipv6_cidr = '::/0' + def filter_ipv6_blocks(self, ip_blocks_mask): + """ + Update ip blocks in the peer set by keeping only parts overlapping with the given mask. + :param ip_blocks_mask: the mask according to which ip blocks should be updated + """ peers_to_remove = [] peers_to_add = [] for peer in self: if isinstance(peer, IpBlock): peers_to_remove.append(peer) - if IpBlock.get_all_ips_block(exclude_ipv4=True).contained_in(peer): - new_peer = peer.copy() - new_peer.remove_cidr(ipv6_cidr) - if new_peer: - peers_to_add.append(new_peer) - elif peer.overlaps(ip_block_filter): + if peer.overlaps(ip_blocks_mask): new_peer = peer.copy() - new_peer &= ip_block_filter + new_peer &= ip_blocks_mask peers_to_add.append(new_peer) for peer in peers_to_remove: diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index ea835be4c..734c7b0e7 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -757,17 +757,17 @@ def exec(self): # noqa: C901 src_peers=subset_peers) | \ TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, dst_peers=subset_peers) all_conns_opt &= subset_conns - ip_blocks_filter = IpBlock.get_all_ips_block() + ip_blocks_mask = IpBlock.get_all_ips_block() if exclude_ipv6: - ip_blocks_filter = IpBlock.get_all_ips_block(exclude_ipv6=True) + ip_blocks_mask = IpBlock.get_all_ips_block(exclude_ipv6=True) ref_ip_blocks = self.config.get_referenced_ip_blocks(exclude_ipv6) for ip_block in ref_ip_blocks: - ip_blocks_filter |= ip_block - opt_peers_to_compare.filter_ipv6_blocks(ip_blocks_filter) + ip_blocks_mask |= ip_block + opt_peers_to_compare.filter_ipv6_blocks(ip_blocks_mask) if self.config.policies_container.layers.does_contain_layer(NetworkLayerName.Istio): output_res, opt_fw_rules_tcp, opt_fw_rules_non_tcp = \ - self.get_props_output_split_by_tcp(all_conns_opt, opt_peers_to_compare, ip_blocks_filter) + self.get_props_output_split_by_tcp(all_conns_opt, opt_peers_to_compare, ip_blocks_mask) opt_end = time.time() print(f'Opt time: {(opt_end - opt_start):6.2f} seconds') if self.config.optimized_run == 'debug': @@ -779,7 +779,7 @@ def exec(self): # noqa: C901 self.compare_fw_rules(fw_rules_non_tcp, opt_fw_rules_non_tcp) else: output_res, opt_fw_rules = self.get_props_output_full(all_conns_opt, opt_peers_to_compare, - ip_blocks_filter) + ip_blocks_mask) opt_end = time.time() print(f'Opt time: {(opt_end - opt_start):6.2f} seconds') if self.config.optimized_run == 'debug' and fw_rules and fw_rules.fw_rules_map and \ From a346c253bcb50771290a4ef7187a1e48023d88c7 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 5 Mar 2023 18:49:16 +0200 Subject: [PATCH 050/187] Renamed TcpLikeProperties to ConnectivityProperties. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 40 ++--- ...roperties.py => ConnectivityProperties.py} | 150 +++++++++--------- nca/FWRules/ConnectivityGraph.py | 30 ++-- nca/NetworkConfig/NetworkConfig.py | 6 +- nca/NetworkConfig/NetworkConfigQuery.py | 30 ++-- nca/NetworkConfig/NetworkLayer.py | 68 ++++---- nca/Parsers/CalicoPolicyYamlParser.py | 92 +++++------ nca/Parsers/GenericIngressLikeYamlParser.py | 18 +-- nca/Parsers/GenericYamlParser.py | 12 +- nca/Parsers/IngressPolicyYamlParser.py | 40 ++--- nca/Parsers/IstioPolicyYamlParser.py | 28 ++-- nca/Parsers/IstioSidecarYamlParser.py | 4 +- .../IstioTrafficResourcesYamlParser.py | 18 +-- nca/Parsers/K8sPolicyYamlParser.py | 38 ++--- nca/Resources/CalicoNetworkPolicy.py | 6 +- nca/Resources/IngressPolicy.py | 12 +- nca/Resources/IstioNetworkPolicy.py | 6 +- nca/Resources/IstioSidecar.py | 10 +- nca/Resources/K8sNetworkPolicy.py | 8 +- nca/Resources/NetworkPolicy.py | 10 +- ...> testConnectivityPropertiesNamedPorts.py} | 22 +-- 21 files changed, 324 insertions(+), 324 deletions(-) rename nca/CoreDS/{TcpLikeProperties.py => ConnectivityProperties.py} (78%) rename tests/classes_unit_tests/{testTcpPropertiesNamedPorts.py => testConnectivityPropertiesNamedPorts.py} (90%) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 6d4d9a226..7c46167b8 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -5,7 +5,7 @@ from collections import defaultdict from .CanonicalIntervalSet import CanonicalIntervalSet -from .TcpLikeProperties import TcpLikeProperties +from .ConnectivityProperties import ConnectivityProperties from .ProtocolNameResolver import ProtocolNameResolver from .ProtocolSet import ProtocolSet from .Peer import PeerSet, IpBlock @@ -160,7 +160,7 @@ def _get_protocol_with_properties_representation(is_str, protocol_text, properti """ :param bool is_str: should get str representation (True) or list representation (False) :param str protocol_text: str description of protocol - :param Union[bool, TcpLikeProperties, ICMPDataSet] properties: properties object of the protocol + :param Union[bool, ConnectivityProperties, ICMPDataSet] properties: properties object of the protocol :return: representation required for a given pair of protocol and its properties :rtype: Union[dict, str] """ @@ -403,7 +403,7 @@ def add_connections(self, protocol, properties=True): Add connections to the set of connections :param int,str protocol: protocol number of the connections to add :param properties: an object with protocol properties (e.g., ports), if relevant - :type properties: Union[bool, TcpLikeProperties, ICMPDataSet] + :type properties: Union[bool, ConnectivityProperties, ICMPDataSet] :return: None """ if isinstance(protocol, str): @@ -438,7 +438,7 @@ def _add_all_connections_of_protocol(self, protocol): :return: None """ if self.protocol_supports_ports(protocol) or self.protocol_is_icmp(protocol): - self.allowed_protocols[protocol] = TcpLikeProperties.make_all_properties() + self.allowed_protocols[protocol] = ConnectivityProperties.make_all_properties() else: self.allowed_protocols[protocol] = True @@ -542,15 +542,15 @@ def print_diff(self, other, self_name, other_name): return 'No diff.' - def convert_to_tcp_like_properties(self, peer_container): + def convert_to_connectivity_properties(self, peer_container): if self.allow_all: - return TcpLikeProperties.make_all_properties(peer_container) + return ConnectivityProperties.make_all_properties(peer_container) - res = TcpLikeProperties.make_empty_properties(peer_container) + res = ConnectivityProperties.make_empty_properties(peer_container) for protocol, properties in self.allowed_protocols.items(): protocols = ProtocolSet() protocols.add_protocol(protocol) - this_prop = TcpLikeProperties.make_tcp_like_properties(peer_container, protocols=protocols) + this_prop = ConnectivityProperties.make_connectivity_properties(peer_container, protocols=protocols) if isinstance(properties, bool): if properties: res |= this_prop @@ -561,7 +561,7 @@ def convert_to_tcp_like_properties(self, peer_container): @staticmethod def get_all_tcp_connections(): tcp_conns = ConnectionSet() - tcp_conns.add_connections('TCP', TcpLikeProperties.make_all_properties()) + tcp_conns.add_connections('TCP', ConnectivityProperties.make_all_properties()) return tcp_conns @staticmethod @@ -572,13 +572,13 @@ def get_non_tcp_connections(): # return ConnectionSet(True) - ConnectionSet.get_all_TCP_connections() # TODO - after moving to the optimized HC set implementation, - # get rid of ConnectionSet and move the code below to TcpLikeProperties.py + # get rid of ConnectionSet and move the code below to ConnectivityProperties.py @staticmethod def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container, ip_blocks_mask, connectivity_restriction): """ - Build FWRules from the given TcpLikeProperties - :param TcpLikeProperties tcp_props: properties describing allowed connections + Build FWRules from the given ConnectivityProperties + :param ConnectivityProperties tcp_props: properties describing allowed connections :param ClusterInfo cluster_info: the cluster info :param PeerContainer peer_container: the peer container :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, @@ -621,13 +621,13 @@ def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container, ip_block protocol_names = ProtocolSet.get_protocol_names_from_interval_set(protocols) if protocols else ['TCP'] for protocol in protocol_names: if new_cube_dict: - conns.add_connections(protocol, TcpLikeProperties.make_tcp_like_properties_from_dict(peer_container, - new_cube_dict)) + conns.add_connections(protocol, ConnectivityProperties.make_connectivity_properties_from_dict(peer_container, + new_cube_dict)) else: if ConnectionSet.protocol_supports_ports(protocol): - conns.add_connections(protocol, TcpLikeProperties.make_all_properties(peer_container)) + conns.add_connections(protocol, ConnectivityProperties.make_all_properties(peer_container)) elif ConnectionSet.protocol_is_icmp(protocol): - conns.add_connections(protocol, TcpLikeProperties.make_all_properties(peer_container)) + conns.add_connections(protocol, ConnectivityProperties.make_all_properties(peer_container)) else: conns.add_connections(protocol, True) # create FWRules for src_peers and dst_peers @@ -671,13 +671,13 @@ def split_peer_set_to_fw_rule_elements(peer_set, cluster_info): @staticmethod def fw_rules_to_tcp_properties(fw_rules, peer_container): - res = TcpLikeProperties.make_empty_properties(peer_container) + res = ConnectivityProperties.make_empty_properties(peer_container) for fw_rules_list in fw_rules.fw_rules_map.values(): for fw_rule in fw_rules_list: - conn_props = fw_rule.conn.convert_to_tcp_like_properties(peer_container) + conn_props = fw_rule.conn.convert_to_connectivity_properties(peer_container) src_peers = PeerSet(fw_rule.src.get_peer_set(fw_rules.cluster_info)) dst_peers = PeerSet(fw_rule.dst.get_peer_set(fw_rules.cluster_info)) - rule_props = TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=src_peers, - dst_peers=dst_peers) & conn_props + rule_props = ConnectivityProperties.make_connectivity_properties(peer_container, src_peers=src_peers, + dst_peers=dst_peers) & conn_props res |= rule_props return res diff --git a/nca/CoreDS/TcpLikeProperties.py b/nca/CoreDS/ConnectivityProperties.py similarity index 78% rename from nca/CoreDS/TcpLikeProperties.py rename to nca/CoreDS/ConnectivityProperties.py index e5688be63..91d9ff7fd 100644 --- a/nca/CoreDS/TcpLikeProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -13,9 +13,9 @@ from .ProtocolNameResolver import ProtocolNameResolver -class TcpLikeProperties(CanonicalHyperCubeSet): +class ConnectivityProperties(CanonicalHyperCubeSet): """ - A class for holding a set of cubes, each defined over dimensions from TcpLikeProperties.dimensions_list + A class for holding a set of cubes, each defined over dimensions from ConnectivityProperties.dimensions_list For UDP, SCTP protocols, the actual used dimensions are only [source ports, dest ports] for TCP, it may be any of the dimensions from dimensions_list. @@ -26,7 +26,7 @@ class TcpLikeProperties(CanonicalHyperCubeSet): The method convert_named_ports is responsible for applying this late binding, and is called by a policy's method allowed_connections() to get policy's allowed connections, given peers and direction ingress/egress Given a specific dest-peer context, the pod's named ports mapping is known, and used for the named ports conversion. - Some of the operators for TcpLikeProperties are not supported for objects with (included and excluded) named ports. + Some of the operators for ConnectivityProperties are not supported for objects with (included and excluded) named ports. For example, in the general case, the result for (all but port "x") | (all but port 10) has 2 options: (1) if the dest pod has named port "x" mapped to 10 -> the result would be: (all but port 10) (2) otherwise, the result would be: (all ports) @@ -62,7 +62,7 @@ def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protoco :param CanonicalIntervalSet src_peers: the set of source peers :param CanonicalIntervalSet dst_peers: the set of target peers """ - super().__init__(TcpLikeProperties.dimensions_list) + super().__init__(ConnectivityProperties.dimensions_list) assert (not src_peers and not dst_peers) or base_peer_set self.named_ports = {} # a mapping from dst named port (String) to src ports interval set @@ -113,7 +113,7 @@ def _get_cube_and_active_dims_from_input_values(dims_to_values): active_dims = [] has_empty_dim_value = False # add values to cube by required order of dimensions - for dim in TcpLikeProperties.dimensions_list: + for dim in ConnectivityProperties.dimensions_list: dim_val = dims_to_values[dim]["value"] add_to_cube = not dims_to_values[dim]["is_all"] if add_to_cube: @@ -218,7 +218,7 @@ def get_properties_obj(self): return {'properties': cubs_dict_list} def __eq__(self, other): - if isinstance(other, TcpLikeProperties): + if isinstance(other, ConnectivityProperties): res = super().__eq__(other) and self.named_ports == other.named_ports and \ self.excluded_named_ports == other.excluded_named_ports return res @@ -241,17 +241,17 @@ def __sub__(self, other): def __iand__(self, other): assert not self.has_named_ports() - assert not isinstance(other, TcpLikeProperties) or not other.has_named_ports() + assert not isinstance(other, ConnectivityProperties) or not other.has_named_ports() super().__iand__(other) - if isinstance(other, TcpLikeProperties): + if isinstance(other, ConnectivityProperties): self.base_peer_set |= other.base_peer_set return self def __ior__(self, other): assert not self.excluded_named_ports - assert not isinstance(other, TcpLikeProperties) or not other.excluded_named_ports + assert not isinstance(other, ConnectivityProperties) or not other.excluded_named_ports super().__ior__(other) - if isinstance(other, TcpLikeProperties): + if isinstance(other, ConnectivityProperties): self.base_peer_set |= other.base_peer_set res_named_ports = dict({}) for port_name in self.named_ports: @@ -266,15 +266,15 @@ def __ior__(self, other): def __isub__(self, other): assert not self.has_named_ports() - assert not isinstance(other, TcpLikeProperties) or not other.has_named_ports() + assert not isinstance(other, ConnectivityProperties) or not other.has_named_ports() super().__isub__(other) - if isinstance(other, TcpLikeProperties): + if isinstance(other, ConnectivityProperties): self.base_peer_set |= other.base_peer_set return self def contained_in(self, other): """ - :param TcpLikeProperties other: Another PortSetPair + :param ConnectivityProperties other: Another PortSetPair :return: Whether all (source port, target port) pairs in self also appear in other :rtype: bool """ @@ -324,7 +324,7 @@ def convert_named_ports(self, named_ports, protocol): self.add_hole(rectangle, active_dims) def copy(self): - res = TcpLikeProperties() + res = ConnectivityProperties() # from CanonicalHyperCubeSet.copy(): for layer in self.layers: res.layers[self._copy_layer_elem(layer)] = self.layers[layer].copy() @@ -337,7 +337,7 @@ def copy(self): def print_diff(self, other, self_name, other_name): """ - :param TcpLikeProperties other: Another PortSetPair object + :param ConnectivityProperties other: Another PortSetPair object :param str self_name: A name for 'self' :param str other_name: A name for 'other' :return: If self!=other, return a string showing a (source, target) pair that appears in only one of them @@ -403,14 +403,14 @@ def resolve_named_port(named_port, peer, protocols): return real_ports @staticmethod - def make_tcp_like_properties(peer_container, src_ports=PortSet(True), dst_ports=PortSet(True), - protocols=ProtocolSet(True), src_peers=None, dst_peers=None, - paths_dfa=None, hosts_dfa=None, methods=MethodSet(True), - icmp_type=DimensionsManager().get_dimension_domain_by_name('icmp_type'), - icmp_code=DimensionsManager().get_dimension_domain_by_name('icmp_code')): + def make_connectivity_properties(peer_container, src_ports=PortSet(True), dst_ports=PortSet(True), + protocols=ProtocolSet(True), src_peers=None, dst_peers=None, + paths_dfa=None, hosts_dfa=None, methods=MethodSet(True), + icmp_type=DimensionsManager().get_dimension_domain_by_name('icmp_type'), + icmp_code=DimensionsManager().get_dimension_domain_by_name('icmp_code')): """ - get TcpLikeProperties with TCP allowed connections, corresponding to input properties cube. - TcpLikeProperties should not contain named ports: substitute them with corresponding port numbers, per peer + get ConnectivityProperties with allowed connections, corresponding to input properties cube. + ConnectivityProperties should not contain named ports: substitute them with corresponding port numbers, per peer :param PeerContainer peer_container: The set of endpoints and their namespaces :param PortSet src_ports: ports set for src_ports dimension (possibly containing named ports) :param PortSet dst_ports: ports set for dst_ports dimension (possibly containing named ports) @@ -422,7 +422,7 @@ def make_tcp_like_properties(peer_container, src_ports=PortSet(True), dst_ports= :param MethodSet methods: CanonicalIntervalSet obj for methods dimension :param CanonicalIntervalSet icmp_type: ICMP-specific parameter (type dimension) :param CanonicalIntervalSet icmp_code: ICMP-specific parameter (code dimension) - :return: TcpLikeProperties with TCP allowed connections, corresponding to input properties cube + :return: ConnectivityProperties with allowed connections, corresponding to input properties cube """ base_peer_set = peer_container.peer_set.copy() if src_peers: @@ -437,13 +437,13 @@ def make_tcp_like_properties(peer_container, src_ports=PortSet(True), dst_ports= dst_peers_interval = None if (not src_ports.named_ports or not src_peers) and (not dst_ports.named_ports or not dst_peers): # Should not resolve named ports - return TcpLikeProperties(source_ports=src_ports, dest_ports=dst_ports, - protocols=protocols, methods=methods, paths=paths_dfa, hosts=hosts_dfa, - icmp_type=icmp_type, icmp_code=icmp_code, - src_peers=src_peers_interval, dst_peers=dst_peers_interval, - base_peer_set=base_peer_set) + return ConnectivityProperties(source_ports=src_ports, dest_ports=dst_ports, + protocols=protocols, methods=methods, paths=paths_dfa, hosts=hosts_dfa, + icmp_type=icmp_type, icmp_code=icmp_code, + src_peers=src_peers_interval, dst_peers=dst_peers_interval, + base_peer_set=base_peer_set) # Resolving named ports - tcp_properties = TcpLikeProperties.make_empty_properties(peer_container) + conn_properties = ConnectivityProperties.make_empty_properties(peer_container) if src_ports.named_ports and dst_ports.named_ports: assert src_peers and dst_peers assert not src_ports.port_set and not dst_ports.port_set @@ -451,22 +451,22 @@ def make_tcp_like_properties(peer_container, src_ports=PortSet(True), dst_ports= src_named_port = list(src_ports.named_ports)[0] dst_named_port = list(dst_ports.named_ports)[0] for src_peer in src_peers: - real_src_ports = TcpLikeProperties.resolve_named_port(src_named_port, src_peer, protocols) + real_src_ports = ConnectivityProperties.resolve_named_port(src_named_port, src_peer, protocols) if not real_src_ports: continue for dst_peer in dst_peers: - real_dst_ports = TcpLikeProperties.resolve_named_port(dst_named_port, dst_peer, protocols) + real_dst_ports = ConnectivityProperties.resolve_named_port(dst_named_port, dst_peer, protocols) if not real_dst_ports: continue - props = TcpLikeProperties(source_ports=real_src_ports, dest_ports=real_dst_ports, - protocols=protocols, methods=methods, - paths=paths_dfa, hosts=hosts_dfa, - icmp_type=icmp_type, icmp_code=icmp_code, - src_peers=base_peer_set.get_peer_interval_of(PeerSet({src_peer})), - dst_peers=base_peer_set.get_peer_interval_of(PeerSet({dst_peer})), - base_peer_set=base_peer_set) - - tcp_properties |= props + props = ConnectivityProperties(source_ports=real_src_ports, dest_ports=real_dst_ports, + protocols=protocols, methods=methods, + paths=paths_dfa, hosts=hosts_dfa, + icmp_type=icmp_type, icmp_code=icmp_code, + src_peers=base_peer_set.get_peer_interval_of(PeerSet({src_peer})), + dst_peers=base_peer_set.get_peer_interval_of(PeerSet({dst_peer})), + base_peer_set=base_peer_set) + + conn_properties |= props else: # either only src_ports or only dst_ports contain named ports if src_ports.named_ports: @@ -480,34 +480,34 @@ def make_tcp_like_properties(peer_container, src_ports=PortSet(True), dst_ports= assert len(port_set_with_named_ports.named_ports) == 1 port = list(port_set_with_named_ports.named_ports)[0] for peer in peers_for_named_ports: - real_ports = TcpLikeProperties.resolve_named_port(port, peer, protocols) + real_ports = ConnectivityProperties.resolve_named_port(port, peer, protocols) if not real_ports: continue if src_ports.named_ports: - props = TcpLikeProperties(source_ports=real_ports, dest_ports=dst_ports, - protocols=protocols, methods=methods, - paths=paths_dfa, hosts=hosts_dfa, - icmp_type=icmp_type, icmp_code=icmp_code, - src_peers=base_peer_set.get_peer_interval_of(PeerSet({peer})), - dst_peers=dst_peers_interval, base_peer_set=base_peer_set) + props = ConnectivityProperties(source_ports=real_ports, dest_ports=dst_ports, + protocols=protocols, methods=methods, + paths=paths_dfa, hosts=hosts_dfa, + icmp_type=icmp_type, icmp_code=icmp_code, + src_peers=base_peer_set.get_peer_interval_of(PeerSet({peer})), + dst_peers=dst_peers_interval, base_peer_set=base_peer_set) else: - props = TcpLikeProperties(source_ports=src_ports, dest_ports=real_ports, - protocols=protocols, methods=methods, - paths=paths_dfa, hosts=hosts_dfa, - icmp_type=icmp_type, icmp_code=icmp_code, - src_peers=src_peers_interval, - dst_peers=base_peer_set.get_peer_interval_of(PeerSet({peer})), - base_peer_set=base_peer_set) - tcp_properties |= props - return tcp_properties + props = ConnectivityProperties(source_ports=src_ports, dest_ports=real_ports, + protocols=protocols, methods=methods, + paths=paths_dfa, hosts=hosts_dfa, + icmp_type=icmp_type, icmp_code=icmp_code, + src_peers=src_peers_interval, + dst_peers=base_peer_set.get_peer_interval_of(PeerSet({peer})), + base_peer_set=base_peer_set) + conn_properties |= props + return conn_properties @staticmethod - def make_tcp_like_properties_from_dict(peer_container, cube_dict): + def make_connectivity_properties_from_dict(peer_container, cube_dict): """ - Create TcpLikeProperties from the given cube + Create ConnectivityProperties from the given cube :param PeerContainer peer_container: the set of all peers :param dict cube_dict: the given cube represented as a dictionary - :return: TcpLikeProperties + :return: ConnectivityProperties """ cube_dict_copy = cube_dict.copy() src_ports = cube_dict_copy.pop("src_ports", PortSet(True)) @@ -521,19 +521,19 @@ def make_tcp_like_properties_from_dict(peer_container, cube_dict): icmp_type = cube_dict_copy.pop("icmp_type", DimensionsManager().get_dimension_domain_by_name('icmp_type')) icmp_code = cube_dict_copy.pop("icmp_code", DimensionsManager().get_dimension_domain_by_name('icmp_code')) assert not cube_dict_copy - return TcpLikeProperties.make_tcp_like_properties(peer_container, src_ports=src_ports, dst_ports=dst_ports, - protocols=protocols, src_peers=src_peers, dst_peers=dst_peers, - paths_dfa=paths_dfa, hosts_dfa=hosts_dfa, methods=methods, - icmp_type=icmp_type, icmp_code=icmp_code) + return ConnectivityProperties.make_connectivity_properties(peer_container, src_ports=src_ports, dst_ports=dst_ports, + protocols=protocols, src_peers=src_peers, dst_peers=dst_peers, + paths_dfa=paths_dfa, hosts_dfa=hosts_dfa, methods=methods, + icmp_type=icmp_type, icmp_code=icmp_code) @staticmethod def make_empty_properties(peer_container=None): - return TcpLikeProperties(base_peer_set=peer_container.peer_set.copy() if peer_container else None, - create_empty=True) + return ConnectivityProperties(base_peer_set=peer_container.peer_set.copy() if peer_container else None, + create_empty=True) @staticmethod def make_all_properties(peer_container=None): - return TcpLikeProperties(base_peer_set=peer_container.peer_set.copy() if peer_container else None) + return ConnectivityProperties(base_peer_set=peer_container.peer_set.copy() if peer_container else None) def are_auto_conns(self): if not {'src_peers', 'dst_peers'}.issubset(set(self.active_dimensions)): @@ -566,9 +566,9 @@ def make_icmp_properties(peer_container, protocol="", src_peers=None, dst_peers= icmp_type_interval = CanonicalIntervalSet.get_interval_set(icmp_type, icmp_type) if icmp_code: icmp_code_interval = CanonicalIntervalSet.get_interval_set(icmp_code, icmp_code) - return TcpLikeProperties.make_tcp_like_properties(peer_container=peer_container, protocols=icmp_protocol_set, - src_peers=src_peers, dst_peers=dst_peers, - icmp_type=icmp_type_interval, icmp_code=icmp_code_interval) + return ConnectivityProperties.make_connectivity_properties(peer_container=peer_container, protocols=icmp_protocol_set, + src_peers=src_peers, dst_peers=dst_peers, + icmp_type=icmp_type_interval, icmp_code=icmp_code_interval) @staticmethod def make_all_but_given_icmp_properties(peer_container, protocol="", src_peers=None, dst_peers=None, @@ -578,10 +578,10 @@ def make_all_but_given_icmp_properties(peer_container, protocol="", src_peers=No icmp_protocol_set.add_protocol(ProtocolNameResolver.get_protocol_number(protocol)) else: icmp_protocol_set = ProtocolSet(True) - all_icmp_props = TcpLikeProperties.make_tcp_like_properties(peer_container=peer_container, - protocols=icmp_protocol_set, - src_peers=src_peers, dst_peers=dst_peers) - given_icmp_props = TcpLikeProperties.make_icmp_properties(peer_container, protocol=protocol, - src_peers=src_peers, dst_peers=dst_peers, - icmp_type=icmp_type, icmp_code=icmp_code,) + all_icmp_props = ConnectivityProperties.make_connectivity_properties(peer_container=peer_container, + protocols=icmp_protocol_set, + src_peers=src_peers, dst_peers=dst_peers) + given_icmp_props = ConnectivityProperties.make_icmp_properties(peer_container, protocol=protocol, + src_peers=src_peers, dst_peers=dst_peers, + icmp_type=icmp_type, icmp_code=icmp_code, ) return all_icmp_props-given_icmp_props diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index 6c86129b5..48ab98171 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -9,7 +9,7 @@ from nca.CoreDS.Peer import IpBlock, PeerSet, ClusterEP, Pod from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.ProtocolSet import ProtocolSet -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from .DotGraph import DotGraph from .MinimizeFWRules import MinimizeCsFwRules, MinimizeFWRules from .ClusterInfo import ClusterInfo @@ -102,13 +102,13 @@ def add_edges_from_cube_dict(self, peer_container, cube_dict, ip_blocks_mask): protocol_names = ProtocolSet.get_protocol_names_from_interval_set(protocols) if protocols else ['TCP'] for protocol in protocol_names: if new_cube_dict: - conns.add_connections(protocol, TcpLikeProperties.make_tcp_like_properties_from_dict(peer_container, - new_cube_dict)) + conns.add_connections(protocol, ConnectivityProperties.make_connectivity_properties_from_dict(peer_container, + new_cube_dict)) else: if ConnectionSet.protocol_supports_ports(protocol): - conns.add_connections(protocol, TcpLikeProperties.make_all_properties()) + conns.add_connections(protocol, ConnectivityProperties.make_all_properties()) elif ConnectionSet.protocol_is_icmp(protocol): - conns.add_connections(protocol, TcpLikeProperties.make_all_properties()) + conns.add_connections(protocol, ConnectivityProperties.make_all_properties()) else: conns.add_connections(protocol, True) for src_peer in src_peers: @@ -450,26 +450,26 @@ def get_connectivity_dot_format_str(self, connectivity_restriction=None): def convert_to_tcp_like_properties(self, peer_container): """ - Used for testing of the optimized solution: converting connectivity graph back to TcpLikeProperties + Used for testing of the optimized solution: converting connectivity graph back to ConnectivityProperties :param peer_container: The peer container - :return: TcpLikeProperties representing the connectivity graph + :return: ConnectivityProperties representing the connectivity graph """ - res = TcpLikeProperties.make_empty_properties() + res = ConnectivityProperties.make_empty_properties() for item in self.connections_to_peers.items(): if item[0].allow_all: for peer_pair in item[1]: - res |= TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=PeerSet({peer_pair[0]}), - dst_peers=PeerSet({peer_pair[1]})) + res |= ConnectivityProperties.make_connectivity_properties(peer_container, + src_peers=PeerSet({peer_pair[0]}), + dst_peers=PeerSet({peer_pair[1]})) else: for prot in item[0].allowed_protocols.items(): protocols = ProtocolSet() protocols.add_protocol(prot[0]) if isinstance(prot[1], bool): for peer_pair in item[1]: - res |= TcpLikeProperties.make_tcp_like_properties(peer_container, protocols=protocols, - src_peers=PeerSet({peer_pair[0]}), - dst_peers=PeerSet({peer_pair[1]})) + res |= ConnectivityProperties.make_connectivity_properties(peer_container, protocols=protocols, + src_peers=PeerSet({peer_pair[0]}), + dst_peers=PeerSet({peer_pair[1]})) continue for cube in prot[1]: cube_dict = prot[1].get_cube_dict_with_orig_values(cube) @@ -478,7 +478,7 @@ def convert_to_tcp_like_properties(self, peer_container): new_cube_dict = cube_dict.copy() new_cube_dict["src_peers"] = PeerSet({peer_pair[0]}) new_cube_dict["dst_peers"] = PeerSet({peer_pair[1]}) - res |= TcpLikeProperties.make_tcp_like_properties_from_dict(peer_container, new_cube_dict) + res |= ConnectivityProperties.make_connectivity_properties_from_dict(peer_container, new_cube_dict) return res def get_minimized_firewall_rules(self): diff --git a/nca/NetworkConfig/NetworkConfig.py b/nca/NetworkConfig/NetworkConfig.py index 73ad5c787..4c68f05dd 100644 --- a/nca/NetworkConfig/NetworkConfig.py +++ b/nca/NetworkConfig/NetworkConfig.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field from nca.CoreDS import Peer from nca.CoreDS.ConnectionSet import ConnectionSet -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.Resources.NetworkPolicy import NetworkPolicy from .NetworkLayer import NetworkLayersContainer, NetworkLayerName @@ -273,7 +273,7 @@ def allowed_connections_optimized(self, layer_name=None): Computes the set of allowed connections between any relevant peers. :param NetworkLayerName layer_name: The name of the layer to use, if requested to use a specific layer only :return: allowed_conns: all allowed connections for relevant peers. - :rtype: TcpLikeProperties + :rtype: ConnectivityProperties """ if layer_name is not None: if layer_name not in self.policies_container.layers: @@ -282,7 +282,7 @@ def allowed_connections_optimized(self, layer_name=None): else: conns_res = self.policies_container.layers[layer_name].allowed_connections_optimized(self.peer_container) else: - conns_res = TcpLikeProperties.make_all_properties() # all connections + conns_res = ConnectivityProperties.make_all_properties() # all connections for layer, layer_obj in self.policies_container.layers.items(): conns_per_layer = layer_obj.allowed_connections_optimized(self.peer_container) # all allowed connections: intersection of all allowed connections from all layers diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 734c7b0e7..f18c6b6ba 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -12,7 +12,7 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import PeerSet, IpBlock, Pod, Peer from nca.CoreDS.ProtocolSet import ProtocolSet -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.FWRules.ConnectivityGraph import ConnectivityGraph from nca.FWRules.MinimizeFWRules import MinimizeFWRules from nca.FWRules.ClusterInfo import ClusterInfo @@ -742,7 +742,7 @@ def exec(self): # noqa: C901 else: res.output_explanation = [ComputedExplanation(str_explanation=output_res)] - all_conns_opt = TcpLikeProperties.make_empty_properties() + all_conns_opt = ConnectivityProperties.make_empty_properties() opt_start = time.time() if self.config.optimized_run != 'false': all_conns_opt = self.config.allowed_connections_optimized() @@ -753,9 +753,9 @@ def exec(self): # noqa: C901 all_conns_opt.project_on_one_dimension('dst_peers') subset_peers = self.compute_subset(opt_peers_to_compare) - subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, - src_peers=subset_peers) | \ - TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, dst_peers=subset_peers) + subset_conns = ConnectivityProperties.make_connectivity_properties(self.config.peer_container, + src_peers=subset_peers) | \ + ConnectivityProperties.make_connectivity_properties(self.config.peer_container, dst_peers=subset_peers) all_conns_opt &= subset_conns ip_blocks_mask = IpBlock.get_all_ips_block() if exclude_ipv6: @@ -794,7 +794,7 @@ def exec(self): # noqa: C901 def compare_orig_to_opt_conn(self, orig_conn_graph, opt_props): print("Converting orig_conn_graph to tcp_like_properties...") - orig_tcp_props = orig_conn_graph.convert_to_tcp_like_properties(self.config.peer_container) + orig_tcp_props = orig_conn_graph.convert_to_connectivity_properties(self.config.peer_container) assert orig_tcp_props.contained_in(opt_props) and opt_props.contained_in(orig_tcp_props) # workaround for == # The following assert exposes the bug in HC set assert not orig_tcp_props.contained_in(opt_props) or not opt_props.contained_in(orig_tcp_props) or \ @@ -834,7 +834,7 @@ def get_connectivity_output_full(self, connections, peers, peers_to_compare): def get_props_output_full(self, props, peers_to_compare, ip_blocks_mask): """ get the connectivity map output considering all connections in the output - :param TcpLikeProperties props: properties describing allowed connections + :param ConnectivityProperties props: properties describing allowed connections :param PeerSet peers_to_compare: the peers to consider for dot/fw-rules output :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, whereas all other values should be filtered out in the output @@ -895,7 +895,7 @@ def get_connectivity_output_split_by_tcp(self, connections, peers, peers_to_comp def get_props_output_split_by_tcp(self, props, peers_to_compare, ip_blocks_mask): """ get the connectivity map output as two parts: TCP and non-TCP - :param TcpLikeProperties props: properties describing allowed connections + :param ConnectivityProperties props: properties describing allowed connections :param PeerSet peers_to_compare: the peers to consider for dot/fw-rules output :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, whereas all other values should be filtered out in the output @@ -968,7 +968,7 @@ def dot_format_from_connections_dict(self, connections, peers, connectivity_rest def dot_format_from_props(self, props, peers, ip_blocks_mask, connectivity_restriction=None): """ - :param TcpLikeProperties props: properties describing allowed connections + :param ConnectivityProperties props: properties describing allowed connections :param PeerSet peers: the peers to consider for dot output :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, whereas all other values should be filtered out in the output @@ -999,7 +999,7 @@ def fw_rules_from_connections_dict(self, connections, peers_to_compare, connecti def fw_rules_from_props(self, props, peers_to_compare, ip_blocks_mask, connectivity_restriction=None): """ - :param TcpLikeProperties props: properties describing allowed connections + :param ConnectivityProperties props: properties describing allowed connections :param PeerSet peers_to_compare: the peers to consider for fw-rules output :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, whereas all other values should be filtered out in the output @@ -1051,16 +1051,16 @@ def split_to_tcp_and_non_tcp_conns(conns): def convert_props_to_split_by_tcp(self, props): """ - given the TcpLikeProperties properties set, convert it to two properties sets, one for TCP only, and the other + given the ConnectivityProperties properties set, convert it to two properties sets, one for TCP only, and the other for non-TCP only. - :param TcpLikeProperties props: properties describing allowed connections + :param ConnectivityProperties props: properties describing allowed connections :return: a tuple of the two properties sets: first for TCP, second for non-TCP - :rtype: tuple(TcpLikeProperties, TcpLikeProperties) + :rtype: tuple(ConnectivityProperties, ConnectivityProperties) """ tcp_protocol = ProtocolSet() tcp_protocol.add_protocol('TCP') - tcp_props = props & TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, - protocols=tcp_protocol) + tcp_props = props & ConnectivityProperties.make_connectivity_properties(self.config.peer_container, + protocols=tcp_protocol) non_tcp_props = props - tcp_props return tcp_props, non_tcp_props diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 14d8ad81c..848deaebe 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -7,7 +7,7 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock, HostEP, PeerSet -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy from nca.Resources.NetworkPolicy import PolicyConnections, NetworkPolicy @@ -109,7 +109,7 @@ def empty_layer_allowed_connections_optimized(peer_container, layer_name): Get allowed connections between for all relevant peers for an empty layer (no policies). :param PeerContainer peer_container: holds all the peers :param NetworkLayerName layer_name: The empty layer name - :rtype: TcpLikeProperties + :rtype: ConnectivityProperties """ empty_layer_obj = layer_name.create_network_layer([]) return empty_layer_obj.allowed_connections_optimized(peer_container) @@ -172,20 +172,20 @@ def allowed_connections_optimized(self, peer_container): considering all layer's policies (and defaults) :param PeerContainer peer_container: the peer container holding the peers :return: all allowed connections - :rtype: TcpLikeProperties + :rtype: ConnectivityProperties """ all_pods = peer_container.get_all_peers_group() all_ips_peer_set = PeerSet({IpBlock.get_all_ips_block()}) allowed_ingress_conns, denied_ingres_conns = self._allowed_xgress_conns_optimized(True, peer_container) - allowed_ingress_conns |= TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=all_pods, - dst_peers=all_ips_peer_set) + allowed_ingress_conns |= ConnectivityProperties.make_connectivity_properties(peer_container, src_peers=all_pods, + dst_peers=all_ips_peer_set) allowed_egress_conns, denied_egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) - allowed_egress_conns |= TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=all_ips_peer_set, - dst_peers=all_pods) + allowed_egress_conns |= ConnectivityProperties.make_connectivity_properties(peer_container, src_peers=all_ips_peer_set, + dst_peers=all_pods) res = allowed_ingress_conns & allowed_egress_conns # exclude IpBlock->IpBlock connections - excluded_conns = TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=all_ips_peer_set, - dst_peers=all_ips_peer_set) + excluded_conns = ConnectivityProperties.make_connectivity_properties(peer_container, src_peers=all_ips_peer_set, + dst_peers=all_ips_peer_set) res -= excluded_conns return res @@ -241,10 +241,10 @@ def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda poli :param captured_func: callable that returns True if the policy satisfies additional conditions required for considering captured pods instead of applying the default connections. :return: allowed_conns, denied_conns and set of peers to be added to captured peers - :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) + :rtype: tuple (ConnectivityProperties, ConnectivityProperties, PeerSet) """ - allowed_conns = TcpLikeProperties.make_empty_properties() - denied_conns = TcpLikeProperties.make_empty_properties() + allowed_conns = ConnectivityProperties.make_empty_properties() + denied_conns = ConnectivityProperties.make_empty_properties() captured = PeerSet() for policy in self.policies_list: policy_allowed_conns, policy_denied_conns, policy_captured = \ @@ -297,14 +297,14 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): if non_captured: if is_ingress: non_captured_conns = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=base_peer_set_with_ip, - dst_peers=non_captured) + ConnectivityProperties.make_connectivity_properties(peer_container, + src_peers=base_peer_set_with_ip, + dst_peers=non_captured) else: non_captured_conns = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=non_captured, - dst_peers=base_peer_set_with_ip) + ConnectivityProperties.make_connectivity_properties(peer_container, + src_peers=non_captured, + dst_peers=base_peer_set_with_ip) allowed_conn |= non_captured_conns return allowed_conn, denied_conns @@ -338,19 +338,19 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): if non_captured_peers: if is_ingress: non_captured_conns = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=base_peer_set_with_ip, - dst_peers=non_captured_peers) + ConnectivityProperties.make_connectivity_properties(peer_container, + src_peers=base_peer_set_with_ip, + dst_peers=non_captured_peers) else: non_captured_conns = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=non_captured_peers, - dst_peers=base_peer_set_with_ip) + ConnectivityProperties.make_connectivity_properties(peer_container, + src_peers=non_captured_peers, + dst_peers=base_peer_set_with_ip) allowed_conn |= (non_captured_conns - denied_conns) - allowed_conn |= TcpLikeProperties.make_tcp_like_properties(peer_container, - protocols=ProtocolSet.get_non_tcp_protocols(), - src_peers=base_peer_set_with_ip, - dst_peers=base_peer_set_with_ip) + allowed_conn |= ConnectivityProperties.make_connectivity_properties(peer_container, + protocols=ProtocolSet.get_non_tcp_protocols(), + src_peers=base_peer_set_with_ip, + dst_peers=base_peer_set_with_ip) return allowed_conn, denied_conns @@ -373,16 +373,16 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): base_peer_set_no_ip = peer_container.get_all_peers_group() if is_ingress: non_captured_conns = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=base_peer_set_with_ip, - dst_peers=base_peer_set_no_ip) + ConnectivityProperties.make_connectivity_properties(peer_container, + src_peers=base_peer_set_with_ip, + dst_peers=base_peer_set_no_ip) allowed_conn |= non_captured_conns else: non_captured_peers = base_peer_set_no_ip - captured if non_captured_peers: non_captured_conns = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, - src_peers=non_captured_peers, - dst_peers=base_peer_set_with_ip) + ConnectivityProperties.make_connectivity_properties(peer_container, + src_peers=non_captured_peers, + dst_peers=base_peer_set_with_ip) allowed_conn |= non_captured_conns return allowed_conn, denied_conns diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index a85edd9aa..ae998384f 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -7,7 +7,7 @@ from nca.CoreDS.ProtocolNameResolver import ProtocolNameResolver from nca.CoreDS.Peer import PeerSet, IpBlock from nca.CoreDS.PortSet import PortSet -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.ProtocolSet import ProtocolSet from nca.CoreDS.ICMPDataSet import ICMPDataSet from nca.CoreDS.ConnectionSet import ConnectionSet @@ -358,10 +358,10 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): :param: str protocol: the ICMP-like protocol :param PeerSet src_pods: the source pods :param PeerSet dst_pods: the destination pods - :return: a tuple (ICMPDataSet, TcpLikeProperties), + :return: a tuple (ICMPDataSet, ConnectivityProperties), where ICMPDataSet is an object representing the allowed ICMP connections, - TcpLikeProperties is an optimized-format ICMP connections, including src and dst pods. - :rtype: tuple (ICMPDataSet, TcpLikeProperties) + ConnectivityProperties is an optimized-format ICMP connections, including src and dst pods. + :rtype: tuple (ICMPDataSet, ConnectivityProperties) """ icmp_type = icmp_data.get('type') if icmp_data is not None else None icmp_code = icmp_data.get('code') if icmp_data is not None else None @@ -380,42 +380,42 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): if err: self.syntax_error(err, not_icmp_data) - res = TcpLikeProperties.make_icmp_properties(self.peer_container) - opt_props = TcpLikeProperties.make_empty_properties(self.peer_container) + res = ConnectivityProperties.make_icmp_properties(self.peer_container) + opt_props = ConnectivityProperties.make_empty_properties(self.peer_container) if self.optimized_run != 'false' and src_pods and dst_pods: - opt_props = TcpLikeProperties.make_icmp_properties(self.peer_container, protocol=protocol, - src_peers=src_pods, dst_peers=dst_pods) + opt_props = ConnectivityProperties.make_icmp_properties(self.peer_container, protocol=protocol, + src_peers=src_pods, dst_peers=dst_pods) if icmp_data is not None: - res = TcpLikeProperties.make_icmp_properties(self.peer_container, icmp_type=icmp_type, icmp_code=icmp_code) + res = ConnectivityProperties.make_icmp_properties(self.peer_container, icmp_type=icmp_type, icmp_code=icmp_code) if self.optimized_run != 'false' and src_pods and dst_pods: - opt_props = TcpLikeProperties.make_icmp_properties(self.peer_container, protocol=protocol, - src_peers=src_pods, dst_peers=dst_pods, - icmp_type=icmp_type, icmp_code=icmp_code) + opt_props = ConnectivityProperties.make_icmp_properties(self.peer_container, protocol=protocol, + src_peers=src_pods, dst_peers=dst_pods, + icmp_type=icmp_type, icmp_code=icmp_code) if not_icmp_data is not None: if icmp_type == not_icmp_type and icmp_code == not_icmp_code: - res = TcpLikeProperties.make_empty_properties(self.peer_container) + res = ConnectivityProperties.make_empty_properties(self.peer_container) self.warning('icmp and notICMP are conflicting - no traffic will be matched', not_icmp_data) elif icmp_type == not_icmp_type and icmp_code is None: - tmp = TcpLikeProperties.make_icmp_properties(self.peer_container, icmp_type=not_icmp_type, - icmp_code=not_icmp_code) + tmp = ConnectivityProperties.make_icmp_properties(self.peer_container, icmp_type=not_icmp_type, + icmp_code=not_icmp_code) res -= tmp if self.optimized_run != 'false': - tmp_opt_props = TcpLikeProperties.make_icmp_properties(self.peer_container, protocol=protocol, - src_peers=src_pods, dst_peers=dst_pods, - icmp_type=not_icmp_type, - icmp_code=not_icmp_code) + tmp_opt_props = ConnectivityProperties.make_icmp_properties(self.peer_container, protocol=protocol, + src_peers=src_pods, dst_peers=dst_pods, + icmp_type=not_icmp_type, + icmp_code=not_icmp_code) opt_props -= tmp_opt_props else: self.warning('notICMP has no effect', not_icmp_data) elif not_icmp_data is not None: - res = TcpLikeProperties.make_all_but_given_icmp_properties(self.peer_container, - icmp_type=not_icmp_type, - icmp_code=not_icmp_code) + res = ConnectivityProperties.make_all_but_given_icmp_properties(self.peer_container, + icmp_type=not_icmp_type, + icmp_code=not_icmp_code) if self.optimized_run != 'false' and src_pods and dst_pods: - opt_props = TcpLikeProperties.make_all_but_given_icmp_properties(self.peer_container, protocol=protocol, - src_peers=src_pods, dst_peers=dst_pods, - icmp_type=not_icmp_type, - icmp_code=not_icmp_code) + opt_props = ConnectivityProperties.make_all_but_given_icmp_properties(self.peer_container, protocol=protocol, + src_peers=src_pods, dst_peers=dst_pods, + icmp_type=not_icmp_type, + icmp_code=not_icmp_code) return res, opt_props @@ -445,9 +445,9 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): :param bool is_ingress: Whether this is an ingress rule :param PeerSet policy_selected_eps: The endpoints the policy captured :param bool is_profile: Whether the parsed policy is a Profile object - :return: A tuple (CalicoPolicyRule, TcpLikeProperties) with the proper PeerSets, ConnectionSets and Action, - where TcpLikeProperties is an optimized rule format with protocols, src_peers and dst_peers in a HyperCubeSet - :rtype: tuple(CalicoPolicyRule, TcpLikeProperties) + :return: A tuple (CalicoPolicyRule, ConnectivityProperties) with the proper PeerSets, ConnectionSets and Action, + where ConnectivityProperties is an optimized rule format with protocols, src_peers and dst_peers in a HyperCubeSet + :rtype: tuple(CalicoPolicyRule, ConnectivityProperties) """ allowed_keys = {'action': 1, 'protocol': 0, 'notProtocol': 0, 'icmp': 0, 'notICMP': 0, 'ipVersion': 0, 'source': 0, 'destination': 0, 'http': 2} @@ -482,7 +482,7 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): src_res_pods &= policy_selected_eps connections = ConnectionSet() - tcp_props = TcpLikeProperties.make_empty_properties(self.peer_container) + conn_props = ConnectivityProperties.make_empty_properties(self.peer_container) if protocol is not None: protocols = ProtocolSet() protocols.add_protocol(protocol) @@ -493,42 +493,42 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): self.warning('notProtocol field has no effect', rule) else: if protocol_supports_ports: - connections.add_connections(protocol, TcpLikeProperties.make_tcp_like_properties( + connections.add_connections(protocol, ConnectivityProperties.make_connectivity_properties( self.peer_container, src_ports=src_res_ports, dst_ports=dst_res_ports)) if self.optimized_run != 'false' and src_res_pods and dst_res_pods: src_num_port_set = PortSet() src_num_port_set.port_set = src_res_ports.port_set.copy() dst_num_port_set = PortSet() dst_num_port_set.port_set = dst_res_ports.port_set.copy() - tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, - src_ports=src_num_port_set, - dst_ports=dst_num_port_set, - protocols=protocols, - src_peers=src_res_pods, - dst_peers=dst_res_pods) + conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, + src_ports=src_num_port_set, + dst_ports=dst_num_port_set, + protocols=protocols, + src_peers=src_res_pods, + dst_peers=dst_res_pods) elif ConnectionSet.protocol_is_icmp(protocol): - icmp_props, tcp_props = self._parse_icmp(rule.get('icmp'), rule.get('notICMP'), + icmp_props, conn_props = self._parse_icmp(rule.get('icmp'), rule.get('notICMP'), protocol, src_res_pods, dst_res_pods) connections.add_connections(protocol, icmp_props) else: connections.add_connections(protocol, True) if self.optimized_run != 'false' and src_res_pods and dst_res_pods: - tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, protocols=protocols, - src_peers=src_res_pods, - dst_peers=dst_res_pods) + conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, protocols=protocols, + src_peers=src_res_pods, + dst_peers=dst_res_pods) elif not_protocol is not None: connections.add_all_connections() connections.remove_protocol(not_protocol) if self.optimized_run != 'false' and src_res_pods and dst_res_pods: protocols = ProtocolSet(True) protocols.remove_protocol(not_protocol) - tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, protocols=protocols, - src_peers=src_res_pods, dst_peers=dst_res_pods) + conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, protocols=protocols, + src_peers=src_res_pods, dst_peers=dst_res_pods) else: connections.allow_all = True if self.optimized_run != 'false' and src_res_pods and dst_res_pods: - tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, - src_peers=src_res_pods, dst_peers=dst_res_pods) + conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, + src_peers=src_res_pods, dst_peers=dst_res_pods) self._verify_named_ports(rule, dst_res_pods, connections) if not src_res_pods and policy_selected_eps and (is_ingress or not is_profile): @@ -536,7 +536,7 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): if not dst_res_pods and policy_selected_eps and (not is_ingress or not is_profile): self.warning('Rule selects no destination endpoints', rule) - return CalicoPolicyRule(src_res_pods, dst_res_pods, connections, action), tcp_props + return CalicoPolicyRule(src_res_pods, dst_res_pods, connections, action), conn_props def _verify_named_ports(self, rule, rule_eps, rule_conns): """ diff --git a/nca/Parsers/GenericIngressLikeYamlParser.py b/nca/Parsers/GenericIngressLikeYamlParser.py index 40d6b1346..2653ed580 100644 --- a/nca/Parsers/GenericIngressLikeYamlParser.py +++ b/nca/Parsers/GenericIngressLikeYamlParser.py @@ -57,27 +57,27 @@ def parse_regex_host_value(self, regex_value, rule): def _make_allow_rules(self, allowed_conns): """ Make deny rules from the given connections - :param TcpLikeProperties allowed_conns: the given allowed connections + :param ConnectivityProperties allowed_conns: the given allowed connections :return: the list of deny IngressPolicyRules """ return self._make_rules_from_conns(allowed_conns) - def _make_rules_from_conns(self, tcp_conns): + def _make_rules_from_conns(self, conn_props): """ Make IngressPolicyRules from the given connections - :param TcpLikeProperties tcp_conns: the given connections + :param ConnectivityProperties conn_props: the given connections :return: the list of IngressPolicyRules """ peers_to_conns = {} res = [] # extract peers dimension from cubes - for cube in tcp_conns: + for cube in conn_props: ports = None paths = None hosts = None src_peer_set = None dst_peer_set = None - for i, dim in enumerate(tcp_conns.active_dimensions): + for i, dim in enumerate(conn_props.active_dimensions): if dim == "dst_ports": ports = cube[i] elif dim == "paths": @@ -85,9 +85,9 @@ def _make_rules_from_conns(self, tcp_conns): elif dim == "hosts": hosts = cube[i] elif dim == "src_peers": - src_peer_set = tcp_conns.base_peer_set.get_peer_set_by_indices(cube[i]) + src_peer_set = conn_props.base_peer_set.get_peer_set_by_indices(cube[i]) elif dim == "dst_peers": - dst_peer_set = tcp_conns.base_peer_set.get_peer_set_by_indices(cube[i]) + dst_peer_set = conn_props.base_peer_set.get_peer_set_by_indices(cube[i]) else: assert False assert not src_peer_set @@ -95,8 +95,8 @@ def _make_rules_from_conns(self, tcp_conns): dst_peer_set = self.peer_container.peer_set.copy() port_set = PortSet() port_set.port_set = ports - port_set.named_ports = tcp_conns.named_ports - port_set.excluded_named_ports = tcp_conns.excluded_named_ports + port_set.named_ports = conn_props.named_ports + port_set.excluded_named_ports = conn_props.excluded_named_ports new_conns = self._get_connection_set_from_properties(self.peer_container, port_set, paths_dfa=paths, hosts_dfa=hosts) if peers_to_conns.get(dst_peer_set): diff --git a/nca/Parsers/GenericYamlParser.py b/nca/Parsers/GenericYamlParser.py index a1e5e4e44..f5a3a999b 100644 --- a/nca/Parsers/GenericYamlParser.py +++ b/nca/Parsers/GenericYamlParser.py @@ -6,7 +6,7 @@ from sys import stderr from enum import Enum from nca.CoreDS.DimensionsManager import DimensionsManager -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.MethodSet import MethodSet from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock @@ -229,17 +229,17 @@ def validate_value_in_domain(self, value, dim_name, array, value_name): def _get_connection_set_from_properties(peer_container, dest_ports, methods=MethodSet(True), paths_dfa=None, hosts_dfa=None): """ - get ConnectionSet with TCP allowed connections, corresponding to input properties cube + get ConnectionSet with allowed connections, corresponding to input properties cube :param PeerContainer peer_container: the peer container :param PortSet dest_ports: ports set for dset_ports dimension :param MethodSet methods: methods set for methods dimension :param MinDFA paths_dfa: MinDFA obj for paths dimension :param MinDFA hosts_dfa: MinDFA obj for hosts dimension - :return: ConnectionSet with TCP allowed connections , corresponding to input properties cube + :return: ConnectionSet with allowed connections , corresponding to input properties cube """ - tcp_properties = TcpLikeProperties.make_tcp_like_properties(peer_container, dst_ports=dest_ports, - methods=methods, paths_dfa=paths_dfa, - hosts_dfa=hosts_dfa) + tcp_properties = ConnectivityProperties.make_connectivity_properties(peer_container, dst_ports=dest_ports, + methods=methods, paths_dfa=paths_dfa, + hosts_dfa=hosts_dfa) res = ConnectionSet() res.add_connections('TCP', tcp_properties) return res diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index bee71a08d..d44d0d28c 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -8,7 +8,7 @@ from nca.CoreDS.DimensionsManager import DimensionsManager from nca.CoreDS.Peer import PeerSet from nca.CoreDS.PortSet import PortSet -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.IngressPolicy import IngressPolicy from nca.Resources.NetworkPolicy import NetworkPolicy @@ -183,29 +183,29 @@ def _make_default_connections(self, hosts_dfa, paths_dfa=None): Creates default backend connections for given hosts and paths :param MinDFA hosts_dfa: the hosts for the default connections :param MinDFA paths_dfa: the paths for the default connections - :return: TcpLikeProperties containing default connections or None (when no default backend exists) + :return: ConnectivityProperties containing default connections or None (when no default backend exists) """ - default_conns = TcpLikeProperties.make_empty_properties(self.peer_container) + default_conns = ConnectivityProperties.make_empty_properties(self.peer_container) if self.default_backend_peers: if paths_dfa: default_conns = \ - TcpLikeProperties.make_tcp_like_properties(self.peer_container, - dst_ports=self.default_backend_ports, - dst_peers=self.default_backend_peers, - paths_dfa=paths_dfa, hosts_dfa=hosts_dfa) + ConnectivityProperties.make_connectivity_properties(self.peer_container, + dst_ports=self.default_backend_ports, + dst_peers=self.default_backend_peers, + paths_dfa=paths_dfa, hosts_dfa=hosts_dfa) else: default_conns = \ - TcpLikeProperties.make_tcp_like_properties(self.peer_container, - dst_ports=self.default_backend_ports, - dst_peers=self.default_backend_peers, - hosts_dfa=hosts_dfa) + ConnectivityProperties.make_connectivity_properties(self.peer_container, + dst_ports=self.default_backend_ports, + dst_peers=self.default_backend_peers, + hosts_dfa=hosts_dfa) return default_conns def parse_rule(self, rule): """ Parses a single ingress rule, producing a number of IngressPolicyRules (per path). :param dict rule: The rule resource - :return: A tuple containing TcpLikeProperties including allowed connections for the given rule, + :return: A tuple containing ConnectivityProperties including allowed connections for the given rule, and a dfa for hosts """ if rule is None: @@ -215,8 +215,8 @@ def parse_rule(self, rule): self.check_fields_validity(rule, 'ingress rule', allowed_elements) hosts_dfa = self.parse_regex_host_value(rule.get("host"), rule) paths_array = self.get_key_array_and_validate_not_empty(rule.get('http'), 'paths') - allowed_conns = TcpLikeProperties.make_empty_properties(self.peer_container) - default_conns = TcpLikeProperties.make_empty_properties(self.peer_container) + allowed_conns = ConnectivityProperties.make_empty_properties(self.peer_container) + default_conns = ConnectivityProperties.make_empty_properties(self.peer_container) if paths_array is not None: all_paths_dfa = None parsed_paths = [] @@ -228,9 +228,9 @@ def parse_rule(self, rule): parsed_paths_with_dfa = self.segregate_longest_paths_and_make_dfa(parsed_paths) for (_, paths_dfa, _, peers, ports) in parsed_paths_with_dfa: # every path is converted to allowed connections - conns = TcpLikeProperties.make_tcp_like_properties(self.peer_container, dst_ports=ports, - dst_peers=peers, paths_dfa=paths_dfa, - hosts_dfa=hosts_dfa) + conns = ConnectivityProperties.make_connectivity_properties(self.peer_container, dst_ports=ports, + dst_peers=peers, paths_dfa=paths_dfa, + hosts_dfa=hosts_dfa) allowed_conns |= conns if not all_paths_dfa: all_paths_dfa = paths_dfa @@ -271,7 +271,7 @@ def parse_policy(self): if not res_policy.selected_peers: self.missing_k8s_ingress_peers = True self.warning("No ingress-nginx pods found, the Ingress policy will have no effect") - allowed_conns = TcpLikeProperties.make_empty_properties(self.peer_container) + allowed_conns = ConnectivityProperties.make_empty_properties(self.peer_container) all_hosts_dfa = None for ingress_rule in policy_spec.get('rules', []): conns, hosts_dfa = self.parse_rule(ingress_rule) @@ -293,8 +293,8 @@ def parse_policy(self): res_policy.add_rules(self._make_allow_rules(allowed_conns)) protocols = ProtocolSet() protocols.add_protocol('TCP') - allowed_conns &= TcpLikeProperties.make_tcp_like_properties(self.peer_container, protocols=protocols, - src_peers=res_policy.selected_peers) + allowed_conns &= ConnectivityProperties.make_connectivity_properties(self.peer_container, protocols=protocols, + src_peers=res_policy.selected_peers) res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs return res_policy diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index fb87baea7..b8494ef41 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -10,7 +10,7 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.PortSet import PortSet from nca.CoreDS.MethodSet import MethodSet -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy, IstioPolicyRule from nca.Resources.IstioTrafficResources import istio_root_namespace from nca.Resources.NetworkPolicy import NetworkPolicy @@ -465,9 +465,9 @@ def parse_ingress_rule(self, rule, selected_peers): Parse a single ingress rule, producing a IstioPolicyRule. :param dict rule: The dict with the rule fields :param PeerSet selected_peers: The selected peers of the policy - :return: A tuple (IstioPolicyRule, TcpLikeProperties) with the proper PeerSet and ConnectionSet, - where TcpLikeProperties is an optimized rule format in a HyperCubeSet format - :rtype: tuple(IstioPolicyRule, TcpLikeProperties) + :return: A tuple (IstioPolicyRule, ConnectivityProperties) with the proper PeerSet and ConnectionSet, + where ConnectivityProperties is an optimized rule format in a HyperCubeSet format + :rtype: tuple(IstioPolicyRule, ConnectivityProperties) """ if rule is None: self.syntax_error('Authorization policy rule cannot be null. ') @@ -489,16 +489,16 @@ def parse_ingress_rule(self, rule, selected_peers): to_array = self.get_key_array_and_validate_not_empty(rule, 'to') # currently parsing only ports # TODO: extend operations parsing to include other attributes - tcp_props = TcpLikeProperties.make_empty_properties(self.peer_container) + conn_props = ConnectivityProperties.make_empty_properties(self.peer_container) if to_array is not None: connections = ConnectionSet() for operation_dict in to_array: conns = self.parse_operation(operation_dict) connections |= conns - tcp_props |= conns.convert_to_tcp_like_properties(self.peer_container) + conn_props |= conns.convert_to_connectivity_properties(self.peer_container) else: # no 'to' in the rule => all connections allowed connections = ConnectionSet(True) - tcp_props = TcpLikeProperties.make_all_properties(self.peer_container) + conn_props = ConnectivityProperties.make_all_properties(self.peer_container) # condition possible result value: # source-ip (from) , source-namespace (from) [Peerset], destination.port (to) [ConnectionSet] @@ -515,15 +515,15 @@ def parse_ingress_rule(self, rule, selected_peers): condition_conns &= condition_res if not res_peers: self.warning('Rule selects no pods', rule) - condition_props = condition_conns.convert_to_tcp_like_properties(self.peer_container) + condition_props = condition_conns.convert_to_connectivity_properties(self.peer_container) if not res_peers or not selected_peers: - condition_props = TcpLikeProperties.make_empty_properties(self.peer_container) + condition_props = ConnectivityProperties.make_empty_properties(self.peer_container) else: - condition_props &= TcpLikeProperties.make_tcp_like_properties(self.peer_container, src_peers=res_peers, - dst_peers=selected_peers) + condition_props &= ConnectivityProperties.make_connectivity_properties(self.peer_container, src_peers=res_peers, + dst_peers=selected_peers) connections &= condition_conns - tcp_props &= condition_props - return IstioPolicyRule(res_peers, connections), tcp_props + conn_props &= condition_props + return IstioPolicyRule(res_peers, connections), conn_props @staticmethod def parse_policy_action(action): @@ -575,7 +575,7 @@ def parse_policy(self): res_policy.add_ingress_rule(rule) res_policy.add_optimized_ingress_props(optimized_props, res_policy.action == IstioNetworkPolicy.ActionType.Allow) - res_policy.add_optimized_egress_props(TcpLikeProperties.make_all_properties(self.peer_container)) + res_policy.add_optimized_egress_props(ConnectivityProperties.make_all_properties(self.peer_container)) if not res_policy.ingress_rules and res_policy.action == IstioNetworkPolicy.ActionType.Deny: self.syntax_error("DENY action without rules is meaningless as it will never be triggered") diff --git a/nca/Parsers/IstioSidecarYamlParser.py b/nca/Parsers/IstioSidecarYamlParser.py index 29d93ec8d..464fba458 100644 --- a/nca/Parsers/IstioSidecarYamlParser.py +++ b/nca/Parsers/IstioSidecarYamlParser.py @@ -4,7 +4,7 @@ # import re from nca.CoreDS.Peer import PeerSet -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.Resources.NetworkPolicy import NetworkPolicy from nca.Resources.IstioSidecar import IstioSidecar, IstioSidecarRule from nca.Resources.IstioTrafficResources import istio_root_namespace @@ -177,7 +177,7 @@ def parse_policy(self): self.namespace = self.peer_container.get_namespace(policy_ns, warn_if_missing) res_policy = IstioSidecar(policy_name, self.namespace) res_policy.policy_kind = NetworkPolicy.PolicyType.IstioSidecar - res_policy.add_optimized_ingress_props(TcpLikeProperties.make_all_properties(self.peer_container)) + res_policy.add_optimized_ingress_props(ConnectivityProperties.make_all_properties(self.peer_container)) sidecar_spec = self.policy['spec'] # currently, supported fields in spec are workloadSelector and egress diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index 80a8ba817..770d04018 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -8,7 +8,7 @@ from nca.CoreDS.Peer import PeerSet from nca.CoreDS.MethodSet import MethodSet from nca.CoreDS.ProtocolSet import ProtocolSet -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.Resources.IstioTrafficResources import Gateway, VirtualService from nca.Resources.IngressPolicy import IngressPolicy from nca.Resources.NetworkPolicy import NetworkPolicy @@ -336,16 +336,16 @@ def make_allowed_connections(self, vs, host_dfa): Create allowed connections of the given VirtualService :param VirtualService vs: the given VirtualService and the given hosts :param MinDFA host_dfa: the hosts attribute - :return: TcpLikeProperties with TCP allowed connections + :return: ConnectivityProperties with allowed connections """ - allowed_conns = TcpLikeProperties.make_empty_properties(self.peer_container) + allowed_conns = ConnectivityProperties.make_empty_properties(self.peer_container) for http_route in vs.http_routes: for dest in http_route.destinations: conns = \ - TcpLikeProperties.make_tcp_like_properties(self.peer_container, dst_ports=dest.port, - dst_peers=dest.service.target_pods, - paths_dfa=http_route.uri_dfa, hosts_dfa=host_dfa, - methods=http_route.methods) + ConnectivityProperties.make_connectivity_properties(self.peer_container, dst_ports=dest.port, + dst_peers=dest.service.target_pods, + paths_dfa=http_route.uri_dfa, hosts_dfa=host_dfa, + methods=http_route.methods) allowed_conns |= conns return allowed_conns @@ -399,8 +399,8 @@ def create_istio_traffic_policies(self): res_policy.add_rules(self._make_allow_rules(allowed_conns)) protocols = ProtocolSet() protocols.add_protocol('TCP') - allowed_conns &= TcpLikeProperties.make_tcp_like_properties(self.peer_container, protocols=protocols, - src_peers=res_policy.selected_peers) + allowed_conns &= ConnectivityProperties.make_connectivity_properties(self.peer_container, protocols=protocols, + src_peers=res_policy.selected_peers) res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs vs_policies.append(res_policy) diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index ca44fb12b..e46ac43f7 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -7,7 +7,7 @@ from nca.CoreDS import Peer from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.PortSet import PortSet -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.ProtocolNameResolver import ProtocolNameResolver from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.NetworkPolicy import NetworkPolicy @@ -310,9 +310,9 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): :param dict rule: The rule to parse :param str peer_array_key: The key which defined the peer set ('from' for ingress, 'to' for egress) :param Peer.PeerSet policy_selected_pods: The set of pods the policy applies to - :return: A tuple (K8sPolicyRule, TcpLikeProperties) with the proper PeerSet and attributes, where - TcpLikeProperties is an optimized rule format with protocols, src_peers and dst_peers in a HyperCubeSet - :rtype: tuple(K8sPolicyRule, TcpLikeProperties) + :return: A tuple (K8sPolicyRule, ConnectivityProperties) with the proper PeerSet and attributes, where + ConnectivityProperties is an optimized rule format with protocols, src_peers and dst_peers in a HyperCubeSet + :rtype: tuple(K8sPolicyRule, ConnectivityProperties) """ self.check_fields_validity(rule, 'ingress/egress rule', {peer_array_key: [0, list], 'ports': [0, list]}) peer_array = rule.get(peer_array_key, []) @@ -330,7 +330,7 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): src_pods = policy_selected_pods dst_pods = res_pods - res_opt_props = TcpLikeProperties.make_empty_properties(self.peer_container) # TcpLikeProperties + res_opt_props = ConnectivityProperties.make_empty_properties(self.peer_container) # ConnectivityProperties ports_array = rule.get('ports', []) if ports_array: res_conns = ConnectionSet() @@ -338,23 +338,23 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): protocol, dest_port_set = self.parse_port(port) if isinstance(protocol, str): protocol = ProtocolNameResolver.get_protocol_number(protocol) - res_conns.add_connections(protocol, TcpLikeProperties.make_tcp_like_properties( + res_conns.add_connections(protocol, ConnectivityProperties.make_connectivity_properties( self.peer_container, dst_ports=dest_port_set)) # K8s doesn't reason about src ports if self.optimized_run != 'false' and src_pods and dst_pods: protocols = ProtocolSet() protocols.add_protocol(protocol) dest_num_port_set = PortSet() dest_num_port_set.port_set = dest_port_set.port_set.copy() - tcp_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, - dst_ports=dest_num_port_set, - protocols=protocols, - src_peers=src_pods, dst_peers=dst_pods) - res_opt_props |= tcp_props + conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, + dst_ports=dest_num_port_set, + protocols=protocols, + src_peers=src_pods, dst_peers=dst_pods) + res_opt_props |= conn_props else: res_conns = ConnectionSet(True) if self.optimized_run != 'false' and src_pods and dst_pods: - res_opt_props = TcpLikeProperties.make_tcp_like_properties(self.peer_container, - src_peers=src_pods, dst_peers=dst_pods) + res_opt_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, + src_peers=src_pods, dst_peers=dst_pods) if not res_pods: self.warning('Rule selects no pods', rule) @@ -392,9 +392,9 @@ def parse_ingress_rule(self, rule, policy_selected_pods): Also, checking validity of named ports w.r.t. the policy's captured pods :param dict rule: The dict with the rule fields :param Peer.PeerSet policy_selected_pods: The set of pods the policy applies to - :return: A tuple (K8sPolicyRule, TcpLikeProperties) with the proper PeerSet and attributes, where - TcpLikeProperties is an optimized rule format with protocols, src_peers and dst_peers in a HyperCubeSet - :rtype: tuple(K8sPolicyRule, TcpLikeProperties) + :return: A tuple (K8sPolicyRule, ConnectivityProperties) with the proper PeerSet and attributes, where + ConnectivityProperties is an optimized rule format with protocols, src_peers and dst_peers in a HyperCubeSet + :rtype: tuple(K8sPolicyRule, ConnectivityProperties) """ res_rule, res_opt_props = self.parse_ingress_egress_rule(rule, 'from', policy_selected_pods) self.verify_named_ports(rule, policy_selected_pods, res_rule.port_set) @@ -406,9 +406,9 @@ def parse_egress_rule(self, rule, policy_selected_pods): Also, checking validity of named ports w.r.t. the rule's peer set :param dict rule: The dict with the rule fields :param Peer.PeerSet policy_selected_pods: The set of pods the policy applies to - :return: A tuple (K8sPolicyRule, TcpLikeProperties) with the proper PeerSet and attributes, where - TcpLikeProperties is an optimized rule format with protocols, src_peers and dst_peers in a HyperCubeSet - :rtype: tuple(K8sPolicyRule, TcpLikeProperties) + :return: A tuple (K8sPolicyRule, ConnectivityProperties) with the proper PeerSet and attributes, where + ConnectivityProperties is an optimized rule format with protocols, src_peers and dst_peers in a HyperCubeSet + :rtype: tuple(K8sPolicyRule, ConnectivityProperties) """ res_rule, res_opt_props = self.parse_ingress_egress_rule(rule, 'to', policy_selected_pods) self.verify_named_ports(rule, res_rule.peer_set, res_rule.port_set) diff --git a/nca/Resources/CalicoNetworkPolicy.py b/nca/Resources/CalicoNetworkPolicy.py index c6c38b34c..4b4d2215f 100644 --- a/nca/Resources/CalicoNetworkPolicy.py +++ b/nca/Resources/CalicoNetworkPolicy.py @@ -122,10 +122,10 @@ def allowed_connections_optimized(self, is_ingress): """ Evaluate the set of connections this policy allows/denies/passes between any two peers :param bool is_ingress: whether we evaluate ingress rules only or egress rules only - :return: A TcpLikeProperties object containing all allowed connections for relevant peers, - TcpLikeProperties object containing all denied connections, + :return: A ConnectivityProperties object containing all allowed connections for relevant peers, + ConnectivityProperties object containing all denied connections, and the peer set of captured peers by this policy. - :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) + :rtype: tuple (ConnectivityProperties, ConnectivityProperties, PeerSet) """ if is_ingress: allowed = self.optimized_ingress_props.copy() diff --git a/nca/Resources/IngressPolicy.py b/nca/Resources/IngressPolicy.py index 9cff6c23d..c7d9d67a7 100644 --- a/nca/Resources/IngressPolicy.py +++ b/nca/Resources/IngressPolicy.py @@ -5,7 +5,7 @@ from enum import IntEnum from nca.CoreDS.ConnectionSet import ConnectionSet -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.Peer import PeerSet from .NetworkPolicy import PolicyConnections, NetworkPolicy @@ -102,19 +102,19 @@ def allowed_connections_optimized(self, is_ingress): Evaluate the set of connections this ingress resource allows between any two peers :param bool is_ingress: For compatibility with other policies. Will return the set of allowed connections only for is_ingress being False. - :return: A TcpLikeProperties object containing all allowed connections for any peers, - TcpLikeProperties object containing all denied connections, + :return: A ConnectivityProperties object containing all allowed connections for any peers, + ConnectivityProperties object containing all denied connections, and the peer set of captured peers by this policy. - :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) + :rtype: tuple (ConnectivityProperties, ConnectivityProperties, PeerSet) """ if is_ingress: - allowed = TcpLikeProperties.make_empty_properties() + allowed = ConnectivityProperties.make_empty_properties() captured = PeerSet() else: allowed = self.optimized_egress_props.copy() captured = self.selected_peers if self.affects_egress else PeerSet() - return allowed, TcpLikeProperties.make_empty_properties(), captured + return allowed, ConnectivityProperties.make_empty_properties(), captured def has_empty_rules(self, _config_name=''): """ diff --git a/nca/Resources/IstioNetworkPolicy.py b/nca/Resources/IstioNetworkPolicy.py index b7552e4c9..2f902efbf 100644 --- a/nca/Resources/IstioNetworkPolicy.py +++ b/nca/Resources/IstioNetworkPolicy.py @@ -96,10 +96,10 @@ def allowed_connections_optimized(self, is_ingress): """ Evaluate the set of connections this policy allows/denied/passed between any two peers :param bool is_ingress: whether we evaluate ingress rules only or egress rules only - :return: A TcpLikeProperties object containing all allowed connections for relevant peers, - TcpLikeProperties object containing all denied connections, + :return: A ConnectivityProperties object containing all allowed connections for relevant peers, + ConnectivityProperties object containing all denied connections, and the peer set of captured peers by this policy. - :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) + :rtype: tuple (ConnectivityProperties, ConnectivityProperties, PeerSet) """ if is_ingress: allowed = self.optimized_ingress_props.copy() diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index ca180b5e6..add361aa5 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock, PeerSet -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from .NetworkPolicy import PolicyConnections, NetworkPolicy from .IstioTrafficResources import istio_root_namespace @@ -168,12 +168,12 @@ def create_opt_egress_props(self, peer_container): for rule in self.egress_rules: if self.selected_peers and rule.egress_peer_set: self.optimized_egress_props = \ - TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=self.selected_peers, - dst_peers=rule.egress_peer_set) + ConnectivityProperties.make_connectivity_properties(peer_container, src_peers=self.selected_peers, + dst_peers=rule.egress_peer_set) peers_sets_by_ns = self.combine_peer_sets_by_ns(self.selected_peers, rule.special_egress_peer_set, peer_container) for (from_peers, to_peers) in peers_sets_by_ns: if from_peers and to_peers: self.optimized_egress_props |= \ - TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=from_peers, - dst_peers=to_peers) + ConnectivityProperties.make_connectivity_properties(peer_container, src_peers=from_peers, + dst_peers=to_peers) diff --git a/nca/Resources/K8sNetworkPolicy.py b/nca/Resources/K8sNetworkPolicy.py index e5bfefa58..d42426cd4 100644 --- a/nca/Resources/K8sNetworkPolicy.py +++ b/nca/Resources/K8sNetworkPolicy.py @@ -4,7 +4,7 @@ # from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS import Peer -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from .NetworkPolicy import PolicyConnections, NetworkPolicy @@ -68,10 +68,10 @@ def allowed_connections_optimized(self, is_ingress): Return the set of connections this policy allows between any two peers (either ingress or egress). :param bool is_ingress: whether we evaluate ingress rules only or egress rules only - :return: A TcpLikeProperties object containing all allowed connections for relevant peers, + :return: A ConnectivityProperties object containing all allowed connections for relevant peers, None for denied connections (K8s does not have denied), and the peer set of captured peers by this policy. - :rtype: tuple (TcpLikeProperties, TcpLikeProperties, PeerSet) + :rtype: tuple (ConnectivityProperties, ConnectivityProperties, PeerSet) """ if is_ingress: allowed = self.optimized_ingress_props.copy() @@ -79,7 +79,7 @@ def allowed_connections_optimized(self, is_ingress): else: allowed = self.optimized_egress_props.copy() captured = self.selected_peers if self.affects_egress else Peer.PeerSet() - return allowed, TcpLikeProperties.make_empty_properties(), captured + return allowed, ConnectivityProperties.make_empty_properties(), captured def clone_without_rule(self, rule_to_exclude, ingress_rule): """ diff --git a/nca/Resources/NetworkPolicy.py b/nca/Resources/NetworkPolicy.py index 04c1c5021..3b10c0ef1 100644 --- a/nca/Resources/NetworkPolicy.py +++ b/nca/Resources/NetworkPolicy.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import PeerSet -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties class NetworkPolicy: @@ -53,10 +53,10 @@ def __init__(self, name, namespace): self.selected_peers = PeerSet() # The peers affected by this policy self.ingress_rules = [] self.egress_rules = [] - self.optimized_ingress_props = TcpLikeProperties.make_empty_properties() - self.optimized_denied_ingress_props = TcpLikeProperties.make_empty_properties() - self.optimized_egress_props = TcpLikeProperties.make_empty_properties() - self.optimized_denied_egress_props = TcpLikeProperties.make_empty_properties() + self.optimized_ingress_props = ConnectivityProperties.make_empty_properties() + self.optimized_denied_ingress_props = ConnectivityProperties.make_empty_properties() + self.optimized_egress_props = ConnectivityProperties.make_empty_properties() + self.optimized_denied_egress_props = ConnectivityProperties.make_empty_properties() self.affects_ingress = False # whether the policy affects the ingress of the selected peers self.affects_egress = False # whether the policy affects the egress of the selected peers self.findings = [] # accumulated findings which are relevant only to this policy (emptiness and redundancy) diff --git a/tests/classes_unit_tests/testTcpPropertiesNamedPorts.py b/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py similarity index 90% rename from tests/classes_unit_tests/testTcpPropertiesNamedPorts.py rename to tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py index 7e8388f44..ea9cafd53 100644 --- a/tests/classes_unit_tests/testTcpPropertiesNamedPorts.py +++ b/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py @@ -1,7 +1,7 @@ import unittest from nca.CoreDS.CanonicalIntervalSet import CanonicalIntervalSet from nca.CoreDS.PortSet import PortSet -from nca.CoreDS.TcpLikeProperties import TcpLikeProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties class TestNamedPorts(unittest.TestCase): @@ -12,10 +12,10 @@ def test_k8s_flow(self): src_res_ports = PortSet(True) dst_res_ports = PortSet() dst_res_ports.add_port("x") - tcp_properties1 = TcpLikeProperties(src_res_ports, dst_res_ports) + tcp_properties1 = ConnectivityProperties(src_res_ports, dst_res_ports) dst_res_ports2 = PortSet() dst_res_ports2.add_port("y") - tcp_properties2 = TcpLikeProperties(src_res_ports, dst_res_ports2) + tcp_properties2 = ConnectivityProperties(src_res_ports, dst_res_ports2) tcp_properties_res = tcp_properties1 | tcp_properties2 named_ports_dict = {"x": (15, 6), "z": (20, 6), "y": (16, 6)} tcp_properties_res.convert_named_ports(named_ports_dict, 6) @@ -35,7 +35,7 @@ def test_calico_flow_1(self): dst_res_ports.add_port("y") dst_res_ports.add_port("z") dst_res_ports.add_port("w") - tcp_properties = TcpLikeProperties(src_res_ports, dst_res_ports) + tcp_properties = ConnectivityProperties(src_res_ports, dst_res_ports) tcp_properties_2 = tcp_properties.copy() self.assertTrue(tcp_properties.has_named_ports()) @@ -66,7 +66,7 @@ def test_calico_flow_2(self): dst_res_ports = PortSet(True) dst_res_ports -= not_ports src_res_ports.add_port_range(1, 100) - tcp_properties = TcpLikeProperties(src_res_ports, dst_res_ports) + tcp_properties = ConnectivityProperties(src_res_ports, dst_res_ports) tcp_properties_2 = tcp_properties.copy() self.assertTrue(tcp_properties.has_named_ports()) @@ -107,7 +107,7 @@ def test_and(self): b = PortSet(False) b.add_port_range(80, 100) - first = TcpLikeProperties(b, a) + first = ConnectivityProperties(b, a) c = PortSet(False) c.add_port(1) @@ -117,7 +117,7 @@ def test_and(self): d = PortSet(False) d.add_port_range(85, 90) - second = TcpLikeProperties(d, c) + second = ConnectivityProperties(d, c) res = first & second res_src_port_set = CanonicalIntervalSet() @@ -133,7 +133,7 @@ def test_or(self): b = PortSet(False) b.add_port_range(80, 100) - first = TcpLikeProperties(b, a) + first = ConnectivityProperties(b, a) c = PortSet(False) c.add_port(1) @@ -144,7 +144,7 @@ def test_or(self): d = PortSet(False) d.add_port_range(85, 90) - second = TcpLikeProperties(d, c) + second = ConnectivityProperties(d, c) res = first | second @@ -164,7 +164,7 @@ def test_sub(self): b = PortSet(False) b.add_port_range(80, 100) - first = TcpLikeProperties(b, a) + first = ConnectivityProperties(b, a) c = PortSet(False) c.add_port(1) @@ -175,7 +175,7 @@ def test_sub(self): d = PortSet(False) d.add_port_range(85, 90) - second = TcpLikeProperties(d, c) + second = ConnectivityProperties(d, c) res = first - second From 0d2661b0e373a48c45fc1929519c722f07c8aaee Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 5 Mar 2023 19:32:53 +0200 Subject: [PATCH 051/187] Fixing lint errors. Signed-off-by: Tanya --- nca/NetworkConfig/NetworkConfigQuery.py | 2 +- nca/Parsers/CalicoPolicyYamlParser.py | 28 ++++++++++++++----------- nca/Parsers/K8sPolicyYamlParser.py | 7 ++++--- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index f18c6b6ba..e28dd5ae8 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -755,7 +755,7 @@ def exec(self): # noqa: C901 subset_peers = self.compute_subset(opt_peers_to_compare) subset_conns = ConnectivityProperties.make_connectivity_properties(self.config.peer_container, src_peers=subset_peers) | \ - ConnectivityProperties.make_connectivity_properties(self.config.peer_container, dst_peers=subset_peers) + ConnectivityProperties.make_connectivity_properties(self.config.peer_container, dst_peers=subset_peers) all_conns_opt &= subset_conns ip_blocks_mask = IpBlock.get_all_ips_block() if exclude_ipv6: diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index ae998384f..232436d02 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -501,34 +501,38 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): dst_num_port_set = PortSet() dst_num_port_set.port_set = dst_res_ports.port_set.copy() conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, - src_ports=src_num_port_set, - dst_ports=dst_num_port_set, - protocols=protocols, - src_peers=src_res_pods, - dst_peers=dst_res_pods) + src_ports=src_num_port_set, + dst_ports=dst_num_port_set, + protocols=protocols, + src_peers=src_res_pods, + dst_peers=dst_res_pods) elif ConnectionSet.protocol_is_icmp(protocol): icmp_props, conn_props = self._parse_icmp(rule.get('icmp'), rule.get('notICMP'), - protocol, src_res_pods, dst_res_pods) + protocol, src_res_pods, dst_res_pods) connections.add_connections(protocol, icmp_props) else: connections.add_connections(protocol, True) if self.optimized_run != 'false' and src_res_pods and dst_res_pods: - conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, protocols=protocols, - src_peers=src_res_pods, - dst_peers=dst_res_pods) + conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, + protocols=protocols, + src_peers=src_res_pods, + dst_peers=dst_res_pods) elif not_protocol is not None: connections.add_all_connections() connections.remove_protocol(not_protocol) if self.optimized_run != 'false' and src_res_pods and dst_res_pods: protocols = ProtocolSet(True) protocols.remove_protocol(not_protocol) - conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, protocols=protocols, - src_peers=src_res_pods, dst_peers=dst_res_pods) + conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, + protocols=protocols, + src_peers=src_res_pods, + dst_peers=dst_res_pods) else: connections.allow_all = True if self.optimized_run != 'false' and src_res_pods and dst_res_pods: conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, - src_peers=src_res_pods, dst_peers=dst_res_pods) + src_peers=src_res_pods, + dst_peers=dst_res_pods) self._verify_named_ports(rule, dst_res_pods, connections) if not src_res_pods and policy_selected_eps and (is_ingress or not is_profile): diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index e46ac43f7..4d1eca035 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -346,9 +346,10 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): dest_num_port_set = PortSet() dest_num_port_set.port_set = dest_port_set.port_set.copy() conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, - dst_ports=dest_num_port_set, - protocols=protocols, - src_peers=src_pods, dst_peers=dst_pods) + dst_ports=dest_num_port_set, + protocols=protocols, + src_peers=src_pods, + dst_peers=dst_pods) res_opt_props |= conn_props else: res_conns = ConnectionSet(True) From 64daeed24e3862e62fdf148ebd86fc4e8f20f2da Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 5 Mar 2023 19:52:19 +0200 Subject: [PATCH 052/187] Fixing lint errors. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 24 +++--- nca/CoreDS/ConnectivityProperties.py | 52 ++++++------ nca/FWRules/ConnectivityGraph.py | 24 +++--- nca/NetworkConfig/NetworkConfig.py | 2 +- nca/NetworkConfig/NetworkConfigQuery.py | 11 ++- nca/NetworkConfig/NetworkLayer.py | 60 +++++++------- nca/Parsers/CalicoPolicyYamlParser.py | 82 +++++++++---------- nca/Parsers/GenericYamlParser.py | 6 +- nca/Parsers/IngressPolicyYamlParser.py | 34 ++++---- nca/Parsers/IstioPolicyYamlParser.py | 12 +-- nca/Parsers/IstioSidecarYamlParser.py | 2 +- .../IstioTrafficResourcesYamlParser.py | 14 ++-- nca/Parsers/K8sPolicyYamlParser.py | 18 ++-- nca/Resources/IngressPolicy.py | 4 +- nca/Resources/IstioSidecar.py | 8 +- nca/Resources/K8sNetworkPolicy.py | 2 +- nca/Resources/NetworkPolicy.py | 8 +- 17 files changed, 181 insertions(+), 182 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 7c46167b8..f60815f11 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -438,7 +438,7 @@ def _add_all_connections_of_protocol(self, protocol): :return: None """ if self.protocol_supports_ports(protocol) or self.protocol_is_icmp(protocol): - self.allowed_protocols[protocol] = ConnectivityProperties.make_all_properties() + self.allowed_protocols[protocol] = ConnectivityProperties.make_all_props() else: self.allowed_protocols[protocol] = True @@ -544,13 +544,13 @@ def print_diff(self, other, self_name, other_name): def convert_to_connectivity_properties(self, peer_container): if self.allow_all: - return ConnectivityProperties.make_all_properties(peer_container) + return ConnectivityProperties.make_all_props(peer_container) - res = ConnectivityProperties.make_empty_properties(peer_container) + res = ConnectivityProperties.make_empty_props(peer_container) for protocol, properties in self.allowed_protocols.items(): protocols = ProtocolSet() protocols.add_protocol(protocol) - this_prop = ConnectivityProperties.make_connectivity_properties(peer_container, protocols=protocols) + this_prop = ConnectivityProperties.make_conn_props(peer_container, protocols=protocols) if isinstance(properties, bool): if properties: res |= this_prop @@ -561,7 +561,7 @@ def convert_to_connectivity_properties(self, peer_container): @staticmethod def get_all_tcp_connections(): tcp_conns = ConnectionSet() - tcp_conns.add_connections('TCP', ConnectivityProperties.make_all_properties()) + tcp_conns.add_connections('TCP', ConnectivityProperties.make_all_props()) return tcp_conns @staticmethod @@ -621,13 +621,13 @@ def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container, ip_block protocol_names = ProtocolSet.get_protocol_names_from_interval_set(protocols) if protocols else ['TCP'] for protocol in protocol_names: if new_cube_dict: - conns.add_connections(protocol, ConnectivityProperties.make_connectivity_properties_from_dict(peer_container, - new_cube_dict)) + conns.add_connections(protocol, ConnectivityProperties.make_conn_props_from_dict(peer_container, + new_cube_dict)) else: if ConnectionSet.protocol_supports_ports(protocol): - conns.add_connections(protocol, ConnectivityProperties.make_all_properties(peer_container)) + conns.add_connections(protocol, ConnectivityProperties.make_all_props(peer_container)) elif ConnectionSet.protocol_is_icmp(protocol): - conns.add_connections(protocol, ConnectivityProperties.make_all_properties(peer_container)) + conns.add_connections(protocol, ConnectivityProperties.make_all_props(peer_container)) else: conns.add_connections(protocol, True) # create FWRules for src_peers and dst_peers @@ -671,13 +671,13 @@ def split_peer_set_to_fw_rule_elements(peer_set, cluster_info): @staticmethod def fw_rules_to_tcp_properties(fw_rules, peer_container): - res = ConnectivityProperties.make_empty_properties(peer_container) + res = ConnectivityProperties.make_empty_props(peer_container) for fw_rules_list in fw_rules.fw_rules_map.values(): for fw_rule in fw_rules_list: conn_props = fw_rule.conn.convert_to_connectivity_properties(peer_container) src_peers = PeerSet(fw_rule.src.get_peer_set(fw_rules.cluster_info)) dst_peers = PeerSet(fw_rule.dst.get_peer_set(fw_rules.cluster_info)) - rule_props = ConnectivityProperties.make_connectivity_properties(peer_container, src_peers=src_peers, - dst_peers=dst_peers) & conn_props + rule_props = ConnectivityProperties.make_conn_props(peer_container, src_peers=src_peers, + dst_peers=dst_peers) & conn_props res |= rule_props return res diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 91d9ff7fd..18f7cc7d4 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -403,11 +403,11 @@ def resolve_named_port(named_port, peer, protocols): return real_ports @staticmethod - def make_connectivity_properties(peer_container, src_ports=PortSet(True), dst_ports=PortSet(True), - protocols=ProtocolSet(True), src_peers=None, dst_peers=None, - paths_dfa=None, hosts_dfa=None, methods=MethodSet(True), - icmp_type=DimensionsManager().get_dimension_domain_by_name('icmp_type'), - icmp_code=DimensionsManager().get_dimension_domain_by_name('icmp_code')): + def make_conn_props(peer_container, src_ports=PortSet(True), dst_ports=PortSet(True), + protocols=ProtocolSet(True), src_peers=None, dst_peers=None, + paths_dfa=None, hosts_dfa=None, methods=MethodSet(True), + icmp_type=DimensionsManager().get_dimension_domain_by_name('icmp_type'), + icmp_code=DimensionsManager().get_dimension_domain_by_name('icmp_code')): """ get ConnectivityProperties with allowed connections, corresponding to input properties cube. ConnectivityProperties should not contain named ports: substitute them with corresponding port numbers, per peer @@ -443,7 +443,7 @@ def make_connectivity_properties(peer_container, src_ports=PortSet(True), dst_po src_peers=src_peers_interval, dst_peers=dst_peers_interval, base_peer_set=base_peer_set) # Resolving named ports - conn_properties = ConnectivityProperties.make_empty_properties(peer_container) + conn_properties = ConnectivityProperties.make_empty_props(peer_container) if src_ports.named_ports and dst_ports.named_ports: assert src_peers and dst_peers assert not src_ports.port_set and not dst_ports.port_set @@ -502,7 +502,7 @@ def make_connectivity_properties(peer_container, src_ports=PortSet(True), dst_po return conn_properties @staticmethod - def make_connectivity_properties_from_dict(peer_container, cube_dict): + def make_conn_props_from_dict(peer_container, cube_dict): """ Create ConnectivityProperties from the given cube :param PeerContainer peer_container: the set of all peers @@ -521,18 +521,18 @@ def make_connectivity_properties_from_dict(peer_container, cube_dict): icmp_type = cube_dict_copy.pop("icmp_type", DimensionsManager().get_dimension_domain_by_name('icmp_type')) icmp_code = cube_dict_copy.pop("icmp_code", DimensionsManager().get_dimension_domain_by_name('icmp_code')) assert not cube_dict_copy - return ConnectivityProperties.make_connectivity_properties(peer_container, src_ports=src_ports, dst_ports=dst_ports, - protocols=protocols, src_peers=src_peers, dst_peers=dst_peers, - paths_dfa=paths_dfa, hosts_dfa=hosts_dfa, methods=methods, - icmp_type=icmp_type, icmp_code=icmp_code) + return ConnectivityProperties.make_conn_props(peer_container, src_ports=src_ports, dst_ports=dst_ports, + protocols=protocols, src_peers=src_peers, dst_peers=dst_peers, + paths_dfa=paths_dfa, hosts_dfa=hosts_dfa, methods=methods, + icmp_type=icmp_type, icmp_code=icmp_code) @staticmethod - def make_empty_properties(peer_container=None): + def make_empty_props(peer_container=None): return ConnectivityProperties(base_peer_set=peer_container.peer_set.copy() if peer_container else None, create_empty=True) @staticmethod - def make_all_properties(peer_container=None): + def make_all_props(peer_container=None): return ConnectivityProperties(base_peer_set=peer_container.peer_set.copy() if peer_container else None) def are_auto_conns(self): @@ -553,8 +553,8 @@ def are_auto_conns(self): # ICMP-related functions @staticmethod - def make_icmp_properties(peer_container, protocol="", src_peers=None, dst_peers=None, - icmp_type=None, icmp_code=None): + def make_icmp_props(peer_container, protocol="", src_peers=None, dst_peers=None, + icmp_type=None, icmp_code=None): if protocol: icmp_protocol_set = ProtocolSet() icmp_protocol_set.add_protocol(ProtocolNameResolver.get_protocol_number(protocol)) @@ -566,22 +566,22 @@ def make_icmp_properties(peer_container, protocol="", src_peers=None, dst_peers= icmp_type_interval = CanonicalIntervalSet.get_interval_set(icmp_type, icmp_type) if icmp_code: icmp_code_interval = CanonicalIntervalSet.get_interval_set(icmp_code, icmp_code) - return ConnectivityProperties.make_connectivity_properties(peer_container=peer_container, protocols=icmp_protocol_set, - src_peers=src_peers, dst_peers=dst_peers, - icmp_type=icmp_type_interval, icmp_code=icmp_code_interval) + return ConnectivityProperties.make_conn_props(peer_container=peer_container, protocols=icmp_protocol_set, + src_peers=src_peers, dst_peers=dst_peers, + icmp_type=icmp_type_interval, icmp_code=icmp_code_interval) @staticmethod - def make_all_but_given_icmp_properties(peer_container, protocol="", src_peers=None, dst_peers=None, - icmp_type=None, icmp_code=None): + def make_all_but_given_icmp_props(peer_container, protocol="", src_peers=None, dst_peers=None, + icmp_type=None, icmp_code=None): if protocol: icmp_protocol_set = ProtocolSet() icmp_protocol_set.add_protocol(ProtocolNameResolver.get_protocol_number(protocol)) else: icmp_protocol_set = ProtocolSet(True) - all_icmp_props = ConnectivityProperties.make_connectivity_properties(peer_container=peer_container, - protocols=icmp_protocol_set, - src_peers=src_peers, dst_peers=dst_peers) - given_icmp_props = ConnectivityProperties.make_icmp_properties(peer_container, protocol=protocol, - src_peers=src_peers, dst_peers=dst_peers, - icmp_type=icmp_type, icmp_code=icmp_code, ) + all_icmp_props = ConnectivityProperties.make_conn_props(peer_container=peer_container, + protocols=icmp_protocol_set, + src_peers=src_peers, dst_peers=dst_peers) + given_icmp_props = ConnectivityProperties.make_icmp_props(peer_container, protocol=protocol, + src_peers=src_peers, dst_peers=dst_peers, + icmp_type=icmp_type, icmp_code=icmp_code, ) return all_icmp_props-given_icmp_props diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index 48ab98171..d9cde039c 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -102,13 +102,13 @@ def add_edges_from_cube_dict(self, peer_container, cube_dict, ip_blocks_mask): protocol_names = ProtocolSet.get_protocol_names_from_interval_set(protocols) if protocols else ['TCP'] for protocol in protocol_names: if new_cube_dict: - conns.add_connections(protocol, ConnectivityProperties.make_connectivity_properties_from_dict(peer_container, - new_cube_dict)) + conns.add_connections(protocol, ConnectivityProperties.make_conn_props_from_dict(peer_container, + new_cube_dict)) else: if ConnectionSet.protocol_supports_ports(protocol): - conns.add_connections(protocol, ConnectivityProperties.make_all_properties()) + conns.add_connections(protocol, ConnectivityProperties.make_all_props()) elif ConnectionSet.protocol_is_icmp(protocol): - conns.add_connections(protocol, ConnectivityProperties.make_all_properties()) + conns.add_connections(protocol, ConnectivityProperties.make_all_props()) else: conns.add_connections(protocol, True) for src_peer in src_peers: @@ -454,22 +454,22 @@ def convert_to_tcp_like_properties(self, peer_container): :param peer_container: The peer container :return: ConnectivityProperties representing the connectivity graph """ - res = ConnectivityProperties.make_empty_properties() + res = ConnectivityProperties.make_empty_props() for item in self.connections_to_peers.items(): if item[0].allow_all: for peer_pair in item[1]: - res |= ConnectivityProperties.make_connectivity_properties(peer_container, - src_peers=PeerSet({peer_pair[0]}), - dst_peers=PeerSet({peer_pair[1]})) + res |= ConnectivityProperties.make_conn_props(peer_container, + src_peers=PeerSet({peer_pair[0]}), + dst_peers=PeerSet({peer_pair[1]})) else: for prot in item[0].allowed_protocols.items(): protocols = ProtocolSet() protocols.add_protocol(prot[0]) if isinstance(prot[1], bool): for peer_pair in item[1]: - res |= ConnectivityProperties.make_connectivity_properties(peer_container, protocols=protocols, - src_peers=PeerSet({peer_pair[0]}), - dst_peers=PeerSet({peer_pair[1]})) + res |= ConnectivityProperties.make_conn_props(peer_container, protocols=protocols, + src_peers=PeerSet({peer_pair[0]}), + dst_peers=PeerSet({peer_pair[1]})) continue for cube in prot[1]: cube_dict = prot[1].get_cube_dict_with_orig_values(cube) @@ -478,7 +478,7 @@ def convert_to_tcp_like_properties(self, peer_container): new_cube_dict = cube_dict.copy() new_cube_dict["src_peers"] = PeerSet({peer_pair[0]}) new_cube_dict["dst_peers"] = PeerSet({peer_pair[1]}) - res |= ConnectivityProperties.make_connectivity_properties_from_dict(peer_container, new_cube_dict) + res |= ConnectivityProperties.make_conn_props_from_dict(peer_container, new_cube_dict) return res def get_minimized_firewall_rules(self): diff --git a/nca/NetworkConfig/NetworkConfig.py b/nca/NetworkConfig/NetworkConfig.py index 4c68f05dd..0a9c22e76 100644 --- a/nca/NetworkConfig/NetworkConfig.py +++ b/nca/NetworkConfig/NetworkConfig.py @@ -282,7 +282,7 @@ def allowed_connections_optimized(self, layer_name=None): else: conns_res = self.policies_container.layers[layer_name].allowed_connections_optimized(self.peer_container) else: - conns_res = ConnectivityProperties.make_all_properties() # all connections + conns_res = ConnectivityProperties.make_all_props() # all connections for layer, layer_obj in self.policies_container.layers.items(): conns_per_layer = layer_obj.allowed_connections_optimized(self.peer_container) # all allowed connections: intersection of all allowed connections from all layers diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index e28dd5ae8..b86f9e8fc 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -742,7 +742,7 @@ def exec(self): # noqa: C901 else: res.output_explanation = [ComputedExplanation(str_explanation=output_res)] - all_conns_opt = ConnectivityProperties.make_empty_properties() + all_conns_opt = ConnectivityProperties.make_empty_props() opt_start = time.time() if self.config.optimized_run != 'false': all_conns_opt = self.config.allowed_connections_optimized() @@ -753,9 +753,8 @@ def exec(self): # noqa: C901 all_conns_opt.project_on_one_dimension('dst_peers') subset_peers = self.compute_subset(opt_peers_to_compare) - subset_conns = ConnectivityProperties.make_connectivity_properties(self.config.peer_container, - src_peers=subset_peers) | \ - ConnectivityProperties.make_connectivity_properties(self.config.peer_container, dst_peers=subset_peers) + subset_conns = ConnectivityProperties.make_conn_props(self.config.peer_container, src_peers=subset_peers) | \ + ConnectivityProperties.make_conn_props(self.config.peer_container, dst_peers=subset_peers) all_conns_opt &= subset_conns ip_blocks_mask = IpBlock.get_all_ips_block() if exclude_ipv6: @@ -1059,8 +1058,8 @@ def convert_props_to_split_by_tcp(self, props): """ tcp_protocol = ProtocolSet() tcp_protocol.add_protocol('TCP') - tcp_props = props & ConnectivityProperties.make_connectivity_properties(self.config.peer_container, - protocols=tcp_protocol) + tcp_props = props & ConnectivityProperties.make_conn_props(self.config.peer_container, + protocols=tcp_protocol) non_tcp_props = props - tcp_props return tcp_props, non_tcp_props diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 848deaebe..b07740ccc 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -177,15 +177,15 @@ def allowed_connections_optimized(self, peer_container): all_pods = peer_container.get_all_peers_group() all_ips_peer_set = PeerSet({IpBlock.get_all_ips_block()}) allowed_ingress_conns, denied_ingres_conns = self._allowed_xgress_conns_optimized(True, peer_container) - allowed_ingress_conns |= ConnectivityProperties.make_connectivity_properties(peer_container, src_peers=all_pods, - dst_peers=all_ips_peer_set) + allowed_ingress_conns |= ConnectivityProperties.make_conn_props(peer_container, src_peers=all_pods, + dst_peers=all_ips_peer_set) allowed_egress_conns, denied_egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) - allowed_egress_conns |= ConnectivityProperties.make_connectivity_properties(peer_container, src_peers=all_ips_peer_set, - dst_peers=all_pods) + allowed_egress_conns |= ConnectivityProperties.make_conn_props(peer_container, src_peers=all_ips_peer_set, + dst_peers=all_pods) res = allowed_ingress_conns & allowed_egress_conns # exclude IpBlock->IpBlock connections - excluded_conns = ConnectivityProperties.make_connectivity_properties(peer_container, src_peers=all_ips_peer_set, - dst_peers=all_ips_peer_set) + excluded_conns = ConnectivityProperties.make_conn_props(peer_container, src_peers=all_ips_peer_set, + dst_peers=all_ips_peer_set) res -= excluded_conns return res @@ -243,8 +243,8 @@ def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda poli :return: allowed_conns, denied_conns and set of peers to be added to captured peers :rtype: tuple (ConnectivityProperties, ConnectivityProperties, PeerSet) """ - allowed_conns = ConnectivityProperties.make_empty_properties() - denied_conns = ConnectivityProperties.make_empty_properties() + allowed_conns = ConnectivityProperties.make_empty_props() + denied_conns = ConnectivityProperties.make_empty_props() captured = PeerSet() for policy in self.policies_list: policy_allowed_conns, policy_denied_conns, policy_captured = \ @@ -297,14 +297,14 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): if non_captured: if is_ingress: non_captured_conns = \ - ConnectivityProperties.make_connectivity_properties(peer_container, - src_peers=base_peer_set_with_ip, - dst_peers=non_captured) + ConnectivityProperties.make_conn_props(peer_container, + src_peers=base_peer_set_with_ip, + dst_peers=non_captured) else: non_captured_conns = \ - ConnectivityProperties.make_connectivity_properties(peer_container, - src_peers=non_captured, - dst_peers=base_peer_set_with_ip) + ConnectivityProperties.make_conn_props(peer_container, + src_peers=non_captured, + dst_peers=base_peer_set_with_ip) allowed_conn |= non_captured_conns return allowed_conn, denied_conns @@ -338,19 +338,19 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): if non_captured_peers: if is_ingress: non_captured_conns = \ - ConnectivityProperties.make_connectivity_properties(peer_container, - src_peers=base_peer_set_with_ip, - dst_peers=non_captured_peers) + ConnectivityProperties.make_conn_props(peer_container, + src_peers=base_peer_set_with_ip, + dst_peers=non_captured_peers) else: non_captured_conns = \ - ConnectivityProperties.make_connectivity_properties(peer_container, - src_peers=non_captured_peers, - dst_peers=base_peer_set_with_ip) + ConnectivityProperties.make_conn_props(peer_container, + src_peers=non_captured_peers, + dst_peers=base_peer_set_with_ip) allowed_conn |= (non_captured_conns - denied_conns) - allowed_conn |= ConnectivityProperties.make_connectivity_properties(peer_container, - protocols=ProtocolSet.get_non_tcp_protocols(), - src_peers=base_peer_set_with_ip, - dst_peers=base_peer_set_with_ip) + allowed_conn |= ConnectivityProperties.make_conn_props(peer_container, + protocols=ProtocolSet.get_non_tcp_protocols(), + src_peers=base_peer_set_with_ip, + dst_peers=base_peer_set_with_ip) return allowed_conn, denied_conns @@ -373,16 +373,16 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): base_peer_set_no_ip = peer_container.get_all_peers_group() if is_ingress: non_captured_conns = \ - ConnectivityProperties.make_connectivity_properties(peer_container, - src_peers=base_peer_set_with_ip, - dst_peers=base_peer_set_no_ip) + ConnectivityProperties.make_conn_props(peer_container, + src_peers=base_peer_set_with_ip, + dst_peers=base_peer_set_no_ip) allowed_conn |= non_captured_conns else: non_captured_peers = base_peer_set_no_ip - captured if non_captured_peers: non_captured_conns = \ - ConnectivityProperties.make_connectivity_properties(peer_container, - src_peers=non_captured_peers, - dst_peers=base_peer_set_with_ip) + ConnectivityProperties.make_conn_props(peer_container, + src_peers=non_captured_peers, + dst_peers=base_peer_set_with_ip) allowed_conn |= non_captured_conns return allowed_conn, denied_conns diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index 232436d02..562c2a815 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -380,42 +380,42 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): if err: self.syntax_error(err, not_icmp_data) - res = ConnectivityProperties.make_icmp_properties(self.peer_container) - opt_props = ConnectivityProperties.make_empty_properties(self.peer_container) + res = ConnectivityProperties.make_icmp_props(self.peer_container) + opt_props = ConnectivityProperties.make_empty_props(self.peer_container) if self.optimized_run != 'false' and src_pods and dst_pods: - opt_props = ConnectivityProperties.make_icmp_properties(self.peer_container, protocol=protocol, - src_peers=src_pods, dst_peers=dst_pods) + opt_props = ConnectivityProperties.make_icmp_props(self.peer_container, protocol=protocol, + src_peers=src_pods, dst_peers=dst_pods) if icmp_data is not None: - res = ConnectivityProperties.make_icmp_properties(self.peer_container, icmp_type=icmp_type, icmp_code=icmp_code) + res = ConnectivityProperties.make_icmp_props(self.peer_container, icmp_type=icmp_type, icmp_code=icmp_code) if self.optimized_run != 'false' and src_pods and dst_pods: - opt_props = ConnectivityProperties.make_icmp_properties(self.peer_container, protocol=protocol, - src_peers=src_pods, dst_peers=dst_pods, - icmp_type=icmp_type, icmp_code=icmp_code) + opt_props = ConnectivityProperties.make_icmp_props(self.peer_container, protocol=protocol, + src_peers=src_pods, dst_peers=dst_pods, + icmp_type=icmp_type, icmp_code=icmp_code) if not_icmp_data is not None: if icmp_type == not_icmp_type and icmp_code == not_icmp_code: - res = ConnectivityProperties.make_empty_properties(self.peer_container) + res = ConnectivityProperties.make_empty_props(self.peer_container) self.warning('icmp and notICMP are conflicting - no traffic will be matched', not_icmp_data) elif icmp_type == not_icmp_type and icmp_code is None: - tmp = ConnectivityProperties.make_icmp_properties(self.peer_container, icmp_type=not_icmp_type, - icmp_code=not_icmp_code) + tmp = ConnectivityProperties.make_icmp_props(self.peer_container, icmp_type=not_icmp_type, + icmp_code=not_icmp_code) res -= tmp if self.optimized_run != 'false': - tmp_opt_props = ConnectivityProperties.make_icmp_properties(self.peer_container, protocol=protocol, - src_peers=src_pods, dst_peers=dst_pods, - icmp_type=not_icmp_type, - icmp_code=not_icmp_code) + tmp_opt_props = ConnectivityProperties.make_icmp_props(self.peer_container, protocol=protocol, + src_peers=src_pods, dst_peers=dst_pods, + icmp_type=not_icmp_type, + icmp_code=not_icmp_code) opt_props -= tmp_opt_props else: self.warning('notICMP has no effect', not_icmp_data) elif not_icmp_data is not None: - res = ConnectivityProperties.make_all_but_given_icmp_properties(self.peer_container, - icmp_type=not_icmp_type, - icmp_code=not_icmp_code) + res = ConnectivityProperties.make_all_but_given_icmp_props(self.peer_container, + icmp_type=not_icmp_type, + icmp_code=not_icmp_code) if self.optimized_run != 'false' and src_pods and dst_pods: - opt_props = ConnectivityProperties.make_all_but_given_icmp_properties(self.peer_container, protocol=protocol, - src_peers=src_pods, dst_peers=dst_pods, - icmp_type=not_icmp_type, - icmp_code=not_icmp_code) + opt_props = ConnectivityProperties.make_all_but_given_icmp_props(self.peer_container, protocol=protocol, + src_peers=src_pods, dst_peers=dst_pods, + icmp_type=not_icmp_type, + icmp_code=not_icmp_code) return res, opt_props @@ -482,7 +482,7 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): src_res_pods &= policy_selected_eps connections = ConnectionSet() - conn_props = ConnectivityProperties.make_empty_properties(self.peer_container) + conn_props = ConnectivityProperties.make_empty_props(self.peer_container) if protocol is not None: protocols = ProtocolSet() protocols.add_protocol(protocol) @@ -493,19 +493,19 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): self.warning('notProtocol field has no effect', rule) else: if protocol_supports_ports: - connections.add_connections(protocol, ConnectivityProperties.make_connectivity_properties( + connections.add_connections(protocol, ConnectivityProperties.make_conn_props( self.peer_container, src_ports=src_res_ports, dst_ports=dst_res_ports)) if self.optimized_run != 'false' and src_res_pods and dst_res_pods: src_num_port_set = PortSet() src_num_port_set.port_set = src_res_ports.port_set.copy() dst_num_port_set = PortSet() dst_num_port_set.port_set = dst_res_ports.port_set.copy() - conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, - src_ports=src_num_port_set, - dst_ports=dst_num_port_set, - protocols=protocols, - src_peers=src_res_pods, - dst_peers=dst_res_pods) + conn_props = ConnectivityProperties.make_conn_props(self.peer_container, + src_ports=src_num_port_set, + dst_ports=dst_num_port_set, + protocols=protocols, + src_peers=src_res_pods, + dst_peers=dst_res_pods) elif ConnectionSet.protocol_is_icmp(protocol): icmp_props, conn_props = self._parse_icmp(rule.get('icmp'), rule.get('notICMP'), protocol, src_res_pods, dst_res_pods) @@ -513,26 +513,26 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): else: connections.add_connections(protocol, True) if self.optimized_run != 'false' and src_res_pods and dst_res_pods: - conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, - protocols=protocols, - src_peers=src_res_pods, - dst_peers=dst_res_pods) + conn_props = ConnectivityProperties.make_conn_props(self.peer_container, + protocols=protocols, + src_peers=src_res_pods, + dst_peers=dst_res_pods) elif not_protocol is not None: connections.add_all_connections() connections.remove_protocol(not_protocol) if self.optimized_run != 'false' and src_res_pods and dst_res_pods: protocols = ProtocolSet(True) protocols.remove_protocol(not_protocol) - conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, - protocols=protocols, - src_peers=src_res_pods, - dst_peers=dst_res_pods) + conn_props = ConnectivityProperties.make_conn_props(self.peer_container, + protocols=protocols, + src_peers=src_res_pods, + dst_peers=dst_res_pods) else: connections.allow_all = True if self.optimized_run != 'false' and src_res_pods and dst_res_pods: - conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, - src_peers=src_res_pods, - dst_peers=dst_res_pods) + conn_props = ConnectivityProperties.make_conn_props(self.peer_container, + src_peers=src_res_pods, + dst_peers=dst_res_pods) self._verify_named_ports(rule, dst_res_pods, connections) if not src_res_pods and policy_selected_eps and (is_ingress or not is_profile): diff --git a/nca/Parsers/GenericYamlParser.py b/nca/Parsers/GenericYamlParser.py index f5a3a999b..1336a06f8 100644 --- a/nca/Parsers/GenericYamlParser.py +++ b/nca/Parsers/GenericYamlParser.py @@ -237,9 +237,9 @@ def _get_connection_set_from_properties(peer_container, dest_ports, methods=Meth :param MinDFA hosts_dfa: MinDFA obj for hosts dimension :return: ConnectionSet with allowed connections , corresponding to input properties cube """ - tcp_properties = ConnectivityProperties.make_connectivity_properties(peer_container, dst_ports=dest_ports, - methods=methods, paths_dfa=paths_dfa, - hosts_dfa=hosts_dfa) + tcp_properties = ConnectivityProperties.make_conn_props(peer_container, dst_ports=dest_ports, + methods=methods, paths_dfa=paths_dfa, + hosts_dfa=hosts_dfa) res = ConnectionSet() res.add_connections('TCP', tcp_properties) return res diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index d44d0d28c..0fcee265b 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -185,20 +185,20 @@ def _make_default_connections(self, hosts_dfa, paths_dfa=None): :param MinDFA paths_dfa: the paths for the default connections :return: ConnectivityProperties containing default connections or None (when no default backend exists) """ - default_conns = ConnectivityProperties.make_empty_properties(self.peer_container) + default_conns = ConnectivityProperties.make_empty_props(self.peer_container) if self.default_backend_peers: if paths_dfa: default_conns = \ - ConnectivityProperties.make_connectivity_properties(self.peer_container, - dst_ports=self.default_backend_ports, - dst_peers=self.default_backend_peers, - paths_dfa=paths_dfa, hosts_dfa=hosts_dfa) + ConnectivityProperties.make_conn_props(self.peer_container, + dst_ports=self.default_backend_ports, + dst_peers=self.default_backend_peers, + paths_dfa=paths_dfa, hosts_dfa=hosts_dfa) else: default_conns = \ - ConnectivityProperties.make_connectivity_properties(self.peer_container, - dst_ports=self.default_backend_ports, - dst_peers=self.default_backend_peers, - hosts_dfa=hosts_dfa) + ConnectivityProperties.make_conn_props(self.peer_container, + dst_ports=self.default_backend_ports, + dst_peers=self.default_backend_peers, + hosts_dfa=hosts_dfa) return default_conns def parse_rule(self, rule): @@ -215,8 +215,8 @@ def parse_rule(self, rule): self.check_fields_validity(rule, 'ingress rule', allowed_elements) hosts_dfa = self.parse_regex_host_value(rule.get("host"), rule) paths_array = self.get_key_array_and_validate_not_empty(rule.get('http'), 'paths') - allowed_conns = ConnectivityProperties.make_empty_properties(self.peer_container) - default_conns = ConnectivityProperties.make_empty_properties(self.peer_container) + allowed_conns = ConnectivityProperties.make_empty_props(self.peer_container) + default_conns = ConnectivityProperties.make_empty_props(self.peer_container) if paths_array is not None: all_paths_dfa = None parsed_paths = [] @@ -228,9 +228,9 @@ def parse_rule(self, rule): parsed_paths_with_dfa = self.segregate_longest_paths_and_make_dfa(parsed_paths) for (_, paths_dfa, _, peers, ports) in parsed_paths_with_dfa: # every path is converted to allowed connections - conns = ConnectivityProperties.make_connectivity_properties(self.peer_container, dst_ports=ports, - dst_peers=peers, paths_dfa=paths_dfa, - hosts_dfa=hosts_dfa) + conns = ConnectivityProperties.make_conn_props(self.peer_container, dst_ports=ports, + dst_peers=peers, paths_dfa=paths_dfa, + hosts_dfa=hosts_dfa) allowed_conns |= conns if not all_paths_dfa: all_paths_dfa = paths_dfa @@ -271,7 +271,7 @@ def parse_policy(self): if not res_policy.selected_peers: self.missing_k8s_ingress_peers = True self.warning("No ingress-nginx pods found, the Ingress policy will have no effect") - allowed_conns = ConnectivityProperties.make_empty_properties(self.peer_container) + allowed_conns = ConnectivityProperties.make_empty_props(self.peer_container) all_hosts_dfa = None for ingress_rule in policy_spec.get('rules', []): conns, hosts_dfa = self.parse_rule(ingress_rule) @@ -293,8 +293,8 @@ def parse_policy(self): res_policy.add_rules(self._make_allow_rules(allowed_conns)) protocols = ProtocolSet() protocols.add_protocol('TCP') - allowed_conns &= ConnectivityProperties.make_connectivity_properties(self.peer_container, protocols=protocols, - src_peers=res_policy.selected_peers) + allowed_conns &= ConnectivityProperties.make_conn_props(self.peer_container, protocols=protocols, + src_peers=res_policy.selected_peers) res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs return res_policy diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index b8494ef41..78c7afbf6 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -489,7 +489,7 @@ def parse_ingress_rule(self, rule, selected_peers): to_array = self.get_key_array_and_validate_not_empty(rule, 'to') # currently parsing only ports # TODO: extend operations parsing to include other attributes - conn_props = ConnectivityProperties.make_empty_properties(self.peer_container) + conn_props = ConnectivityProperties.make_empty_props(self.peer_container) if to_array is not None: connections = ConnectionSet() for operation_dict in to_array: @@ -498,7 +498,7 @@ def parse_ingress_rule(self, rule, selected_peers): conn_props |= conns.convert_to_connectivity_properties(self.peer_container) else: # no 'to' in the rule => all connections allowed connections = ConnectionSet(True) - conn_props = ConnectivityProperties.make_all_properties(self.peer_container) + conn_props = ConnectivityProperties.make_all_props(self.peer_container) # condition possible result value: # source-ip (from) , source-namespace (from) [Peerset], destination.port (to) [ConnectionSet] @@ -517,10 +517,10 @@ def parse_ingress_rule(self, rule, selected_peers): self.warning('Rule selects no pods', rule) condition_props = condition_conns.convert_to_connectivity_properties(self.peer_container) if not res_peers or not selected_peers: - condition_props = ConnectivityProperties.make_empty_properties(self.peer_container) + condition_props = ConnectivityProperties.make_empty_props(self.peer_container) else: - condition_props &= ConnectivityProperties.make_connectivity_properties(self.peer_container, src_peers=res_peers, - dst_peers=selected_peers) + condition_props &= ConnectivityProperties.make_conn_props(self.peer_container, src_peers=res_peers, + dst_peers=selected_peers) connections &= condition_conns conn_props &= condition_props return IstioPolicyRule(res_peers, connections), conn_props @@ -575,7 +575,7 @@ def parse_policy(self): res_policy.add_ingress_rule(rule) res_policy.add_optimized_ingress_props(optimized_props, res_policy.action == IstioNetworkPolicy.ActionType.Allow) - res_policy.add_optimized_egress_props(ConnectivityProperties.make_all_properties(self.peer_container)) + res_policy.add_optimized_egress_props(ConnectivityProperties.make_all_props(self.peer_container)) if not res_policy.ingress_rules and res_policy.action == IstioNetworkPolicy.ActionType.Deny: self.syntax_error("DENY action without rules is meaningless as it will never be triggered") diff --git a/nca/Parsers/IstioSidecarYamlParser.py b/nca/Parsers/IstioSidecarYamlParser.py index 464fba458..5b5bd65d3 100644 --- a/nca/Parsers/IstioSidecarYamlParser.py +++ b/nca/Parsers/IstioSidecarYamlParser.py @@ -177,7 +177,7 @@ def parse_policy(self): self.namespace = self.peer_container.get_namespace(policy_ns, warn_if_missing) res_policy = IstioSidecar(policy_name, self.namespace) res_policy.policy_kind = NetworkPolicy.PolicyType.IstioSidecar - res_policy.add_optimized_ingress_props(ConnectivityProperties.make_all_properties(self.peer_container)) + res_policy.add_optimized_ingress_props(ConnectivityProperties.make_all_props(self.peer_container)) sidecar_spec = self.policy['spec'] # currently, supported fields in spec are workloadSelector and egress diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index 770d04018..ad2514bb2 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -338,14 +338,14 @@ def make_allowed_connections(self, vs, host_dfa): :param MinDFA host_dfa: the hosts attribute :return: ConnectivityProperties with allowed connections """ - allowed_conns = ConnectivityProperties.make_empty_properties(self.peer_container) + allowed_conns = ConnectivityProperties.make_empty_props(self.peer_container) for http_route in vs.http_routes: for dest in http_route.destinations: conns = \ - ConnectivityProperties.make_connectivity_properties(self.peer_container, dst_ports=dest.port, - dst_peers=dest.service.target_pods, - paths_dfa=http_route.uri_dfa, hosts_dfa=host_dfa, - methods=http_route.methods) + ConnectivityProperties.make_conn_props(self.peer_container, dst_ports=dest.port, + dst_peers=dest.service.target_pods, + paths_dfa=http_route.uri_dfa, hosts_dfa=host_dfa, + methods=http_route.methods) allowed_conns |= conns return allowed_conns @@ -399,8 +399,8 @@ def create_istio_traffic_policies(self): res_policy.add_rules(self._make_allow_rules(allowed_conns)) protocols = ProtocolSet() protocols.add_protocol('TCP') - allowed_conns &= ConnectivityProperties.make_connectivity_properties(self.peer_container, protocols=protocols, - src_peers=res_policy.selected_peers) + allowed_conns &= ConnectivityProperties.make_conn_props(self.peer_container, protocols=protocols, + src_peers=res_policy.selected_peers) res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs vs_policies.append(res_policy) diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index 4d1eca035..86f3e5c6e 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -330,7 +330,7 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): src_pods = policy_selected_pods dst_pods = res_pods - res_opt_props = ConnectivityProperties.make_empty_properties(self.peer_container) # ConnectivityProperties + res_opt_props = ConnectivityProperties.make_empty_props(self.peer_container) # ConnectivityProperties ports_array = rule.get('ports', []) if ports_array: res_conns = ConnectionSet() @@ -338,24 +338,24 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): protocol, dest_port_set = self.parse_port(port) if isinstance(protocol, str): protocol = ProtocolNameResolver.get_protocol_number(protocol) - res_conns.add_connections(protocol, ConnectivityProperties.make_connectivity_properties( + res_conns.add_connections(protocol, ConnectivityProperties.make_conn_props( self.peer_container, dst_ports=dest_port_set)) # K8s doesn't reason about src ports if self.optimized_run != 'false' and src_pods and dst_pods: protocols = ProtocolSet() protocols.add_protocol(protocol) dest_num_port_set = PortSet() dest_num_port_set.port_set = dest_port_set.port_set.copy() - conn_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, - dst_ports=dest_num_port_set, - protocols=protocols, - src_peers=src_pods, - dst_peers=dst_pods) + conn_props = ConnectivityProperties.make_conn_props(self.peer_container, + dst_ports=dest_num_port_set, + protocols=protocols, + src_peers=src_pods, + dst_peers=dst_pods) res_opt_props |= conn_props else: res_conns = ConnectionSet(True) if self.optimized_run != 'false' and src_pods and dst_pods: - res_opt_props = ConnectivityProperties.make_connectivity_properties(self.peer_container, - src_peers=src_pods, dst_peers=dst_pods) + res_opt_props = ConnectivityProperties.make_conn_props(self.peer_container, + src_peers=src_pods, dst_peers=dst_pods) if not res_pods: self.warning('Rule selects no pods', rule) diff --git a/nca/Resources/IngressPolicy.py b/nca/Resources/IngressPolicy.py index c7d9d67a7..92229b4ce 100644 --- a/nca/Resources/IngressPolicy.py +++ b/nca/Resources/IngressPolicy.py @@ -109,12 +109,12 @@ def allowed_connections_optimized(self, is_ingress): """ if is_ingress: - allowed = ConnectivityProperties.make_empty_properties() + allowed = ConnectivityProperties.make_empty_props() captured = PeerSet() else: allowed = self.optimized_egress_props.copy() captured = self.selected_peers if self.affects_egress else PeerSet() - return allowed, ConnectivityProperties.make_empty_properties(), captured + return allowed, ConnectivityProperties.make_empty_props(), captured def has_empty_rules(self, _config_name=''): """ diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index add361aa5..1896e2d5b 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -168,12 +168,12 @@ def create_opt_egress_props(self, peer_container): for rule in self.egress_rules: if self.selected_peers and rule.egress_peer_set: self.optimized_egress_props = \ - ConnectivityProperties.make_connectivity_properties(peer_container, src_peers=self.selected_peers, - dst_peers=rule.egress_peer_set) + ConnectivityProperties.make_conn_props(peer_container, src_peers=self.selected_peers, + dst_peers=rule.egress_peer_set) peers_sets_by_ns = self.combine_peer_sets_by_ns(self.selected_peers, rule.special_egress_peer_set, peer_container) for (from_peers, to_peers) in peers_sets_by_ns: if from_peers and to_peers: self.optimized_egress_props |= \ - ConnectivityProperties.make_connectivity_properties(peer_container, src_peers=from_peers, - dst_peers=to_peers) + ConnectivityProperties.make_conn_props(peer_container, src_peers=from_peers, + dst_peers=to_peers) diff --git a/nca/Resources/K8sNetworkPolicy.py b/nca/Resources/K8sNetworkPolicy.py index d42426cd4..ac5cc3303 100644 --- a/nca/Resources/K8sNetworkPolicy.py +++ b/nca/Resources/K8sNetworkPolicy.py @@ -79,7 +79,7 @@ def allowed_connections_optimized(self, is_ingress): else: allowed = self.optimized_egress_props.copy() captured = self.selected_peers if self.affects_egress else Peer.PeerSet() - return allowed, ConnectivityProperties.make_empty_properties(), captured + return allowed, ConnectivityProperties.make_empty_props(), captured def clone_without_rule(self, rule_to_exclude, ingress_rule): """ diff --git a/nca/Resources/NetworkPolicy.py b/nca/Resources/NetworkPolicy.py index 3b10c0ef1..24d46ac42 100644 --- a/nca/Resources/NetworkPolicy.py +++ b/nca/Resources/NetworkPolicy.py @@ -53,10 +53,10 @@ def __init__(self, name, namespace): self.selected_peers = PeerSet() # The peers affected by this policy self.ingress_rules = [] self.egress_rules = [] - self.optimized_ingress_props = ConnectivityProperties.make_empty_properties() - self.optimized_denied_ingress_props = ConnectivityProperties.make_empty_properties() - self.optimized_egress_props = ConnectivityProperties.make_empty_properties() - self.optimized_denied_egress_props = ConnectivityProperties.make_empty_properties() + self.optimized_ingress_props = ConnectivityProperties.make_empty_props() + self.optimized_denied_ingress_props = ConnectivityProperties.make_empty_props() + self.optimized_egress_props = ConnectivityProperties.make_empty_props() + self.optimized_denied_egress_props = ConnectivityProperties.make_empty_props() self.affects_ingress = False # whether the policy affects the ingress of the selected peers self.affects_egress = False # whether the policy affects the egress of the selected peers self.findings = [] # accumulated findings which are relevant only to this policy (emptiness and redundancy) From 51b38d56709460597a5df8c1e7feffd4b45815d7 Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Mon, 6 Mar 2023 11:13:23 +0200 Subject: [PATCH 053/187] track expl data Signed-off-by: Shmulik Froimovich --- nca/NetworkConfig/NetworkLayer.py | 6 ++- nca/NetworkConfig/ResourcesHandler.py | 8 ++- nca/NetworkConfig/TopologyObjectsFinder.py | 2 + nca/Utils/ExplTracker.py | 58 ++++++++++++++++++++++ 4 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 nca/Utils/ExplTracker.py diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 1f859bc33..cae55944e 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -11,7 +11,7 @@ from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy from nca.Resources.NetworkPolicy import PolicyConnections, NetworkPolicy - +from nca.Utils.ExplTracker import ExplTracker # TODO: add a layer for connectivity based on service type (culsterIP / LB / NodePort)? / containers ports? @@ -261,7 +261,9 @@ def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda poli denied_conns |= policy_denied_conns if captured_func(policy): captured |= policy_captured - + # Track the peers that were affected by this policy + for peer in captured: + ExplTracker().add_peer_policy(peer, policy) return allowed_conns, denied_conns, captured diff --git a/nca/NetworkConfig/ResourcesHandler.py b/nca/NetworkConfig/ResourcesHandler.py index 27a4c60ad..38c6159e8 100644 --- a/nca/NetworkConfig/ResourcesHandler.py +++ b/nca/NetworkConfig/ResourcesHandler.py @@ -12,7 +12,7 @@ from .PoliciesFinder import PoliciesFinder from .TopologyObjectsFinder import PodsFinder, NamespacesFinder, ServicesFinder from .PeerContainer import PeerContainer - +from nca.Utils.ExplTracker import ExplTracker class ResourceType(Enum): Unknown = 0 @@ -408,11 +408,17 @@ def _parse_resources_path(self, resource_list, resource_flags): if ResourceType.Namespaces in resource_flags: self.ns_finder.parse_yaml_code_for_ns(res_code) if ResourceType.Pods in resource_flags: + if res_code: + # Track filenames and content + ExplTracker().add_item(yaml_file.path, res_code, res_code.get('metadata').get('name')) self.pods_finder.namespaces_finder = self.ns_finder self.pods_finder.add_eps_from_yaml(res_code) self.services_finder.namespaces_finder = self.ns_finder self.services_finder.parse_yaml_code_for_service(res_code, yaml_file) if ResourceType.Policies in resource_flags: + if res_code: + # Track filenames and content + ExplTracker().add_item(yaml_file.path, res_code, res_code.get('metadata').get('name')) self.policies_finder.parse_yaml_code_for_policy(res_code, yaml_file.path) self.policies_finder.parse_policies_in_parse_queue() diff --git a/nca/NetworkConfig/TopologyObjectsFinder.py b/nca/NetworkConfig/TopologyObjectsFinder.py index c346506a3..9f010c92c 100644 --- a/nca/NetworkConfig/TopologyObjectsFinder.py +++ b/nca/NetworkConfig/TopologyObjectsFinder.py @@ -10,6 +10,7 @@ from nca.Resources.K8sNamespace import K8sNamespace from nca.Parsers.K8sServiceYamlParser import K8sServiceYamlParser from nca.Utils.NcaLogger import NcaLogger +from nca.Utils.ExplTracker import ExplTracker class PodsFinder: @@ -160,6 +161,7 @@ def _add_pod_from_workload_yaml(self, workload_resource): for port in container.get('ports') or []: pod.add_named_port(port.get('name'), port.get('containerPort'), port.get('protocol', 'TCP')) self._add_peer(pod) + ExplTracker().add_item('', workload_resource, pod.name) def _add_networkset_from_yaml(self, networkset_object): """ diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py new file mode 100644 index 000000000..5427d86d0 --- /dev/null +++ b/nca/Utils/ExplTracker.py @@ -0,0 +1,58 @@ +# +# Copyright 2022 - IBM Inc. All rights reserved +# SPDX-License-Identifier: Apache2.0 +# + +from nca.Utils.NcaLogger import Singleton + + +class ExplDescriptor: + def __init__(self, name, config_code, config_file): + self.name = name + self.config_code = config_code + self.config_file = config_file + + def __get__(self, obj, type=None) -> object: + return self.config_code, self.config_file + + def __set__(self, obj, value) -> None: + raise AttributeError("Cannot change values, ExplDescriptor is read only") + + def get_name(self): + return self.name + + +class ExplPeer(ExplDescriptor): + pass + + +class ExplPolicy(ExplDescriptor): + pass + + +class ExplTracker(metaclass=Singleton): + """ + The Explainability Tracker is used for tracking the elements and their configuration + so it will be able to specify which configurations are responsible for each peer and each connection + or lack of connection between them. + """ + + def __init__(self): + self.ExplDescriptorContainer = {} + self.ExplPeerToPolicyContainer = {} + + def get_path_from_deployment(self, content): + return self.ExplDescriptorContainer[content.get('metadata').get('name')].get('path') + + def add_item(self, path, content, name): + if path == '': + path = self.get_path_from_deployment(content) + self.ExplDescriptorContainer[name] = {'path': path, 'content': content} + + def add_peer_policy(self, peer, policy): + peer_name = peer.name + policy_name = policy.name + if not self.ExplPeerToPolicyContainer.get(peer_name): + self.ExplPeerToPolicyContainer[peer_name] = set() + self.ExplPeerToPolicyContainer[peer_name].add(policy_name) + From 464f004f32510a9d881b2784b56f1f15448a471e Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Tue, 7 Mar 2023 12:05:23 +0200 Subject: [PATCH 054/187] explain connectivity Signed-off-by: Shmulik Froimovich --- nca/NetworkConfig/NetworkConfigQueryRunner.py | 9 +++-- nca/NetworkConfig/ResourcesHandler.py | 14 +++++-- nca/NetworkConfig/TopologyObjectsFinder.py | 2 +- nca/Utils/ExplTracker.py | 38 +++++++++++++++++-- nca/Utils/NcaLogger.py | 13 +------ nca/Utils/Utils.py | 17 +++++++++ nca/nca_cli.py | 8 ++++ 7 files changed, 79 insertions(+), 22 deletions(-) create mode 100644 nca/Utils/Utils.py diff --git a/nca/NetworkConfig/NetworkConfigQueryRunner.py b/nca/NetworkConfig/NetworkConfigQueryRunner.py index 32d114a61..b3fed139c 100644 --- a/nca/NetworkConfig/NetworkConfigQueryRunner.py +++ b/nca/NetworkConfig/NetworkConfigQueryRunner.py @@ -11,6 +11,7 @@ from nca.Resources.NetworkPolicy import NetworkPolicy from .NetworkConfig import NetworkConfig from . import NetworkConfigQuery +from nca.Utils.ExplTracker import ExplTracker @dataclass @@ -35,12 +36,13 @@ def update(self, iteration_results): self.query_iterations_output.append(iteration_results[1]) self.num_not_executed += iteration_results[2] - def compute_final_results(self, output_format): + def compute_final_results(self, output_format, expl_nodes): """ extracts the final query results from self variables from self.query_iterations_output computes the final str output of the query, other results returned as is from query_result. :param str output_format: the output format to form the final output + :param list str: expl_nodes: the nodes for explaining the connectivity if output format is json, dumps the output list into one-top-leveled string if output format is yaml, dumps the output list into str of a list of yaml objects otherwise, writes the output list items split by \n @@ -54,7 +56,8 @@ def compute_final_results(self, output_format): sort_keys=False) else: output = '\n'.join(self.query_iterations_output) - return self.numerical_result, output, self.num_not_executed + expl_out = '\n'.join(ExplTracker().explain(expl_nodes)) + return self.numerical_result, output+expl_out, self.num_not_executed class NetworkConfigQueryRunner: @@ -169,7 +172,7 @@ def _run_query_for_each_config(self): query_result = QueryResult() for config in self.configs_array: query_result.update(self._execute_one_config_query(self.query_name, self._get_config(config))) - return query_result.compute_final_results(self.output_configuration.outputFormat) + return query_result.compute_final_results(self.output_configuration.outputFormat, self.output_configuration.expl) def _run_query_on_configs_vs_base_config(self, cmd_line_flag): query_result = QueryResult() diff --git a/nca/NetworkConfig/ResourcesHandler.py b/nca/NetworkConfig/ResourcesHandler.py index 38c6159e8..6bf9ad260 100644 --- a/nca/NetworkConfig/ResourcesHandler.py +++ b/nca/NetworkConfig/ResourcesHandler.py @@ -396,7 +396,7 @@ def _parse_resources_path(self, resource_list, resource_flags): elif resource_item == 'istio': self._handle_istio_inputs(resource_flags) else: - fast_load = ResourceType.Policies not in resource_flags + fast_load = (ResourceType.Policies not in resource_flags) and not ExplTracker().is_active() resource_scanner = TreeScannerFactory.get_scanner(resource_item, fast_load=fast_load) if resource_scanner is None: continue @@ -410,7 +410,11 @@ def _parse_resources_path(self, resource_list, resource_flags): if ResourceType.Pods in resource_flags: if res_code: # Track filenames and content - ExplTracker().add_item(yaml_file.path, res_code, res_code.get('metadata').get('name')) + ExplTracker().add_item(yaml_file.path, + res_code, + res_code.get('metadata').get('name'), + res_code.line_number + ) self.pods_finder.namespaces_finder = self.ns_finder self.pods_finder.add_eps_from_yaml(res_code) self.services_finder.namespaces_finder = self.ns_finder @@ -418,7 +422,11 @@ def _parse_resources_path(self, resource_list, resource_flags): if ResourceType.Policies in resource_flags: if res_code: # Track filenames and content - ExplTracker().add_item(yaml_file.path, res_code, res_code.get('metadata').get('name')) + ExplTracker().add_item(yaml_file.path, + res_code, + res_code.get('metadata').get('name'), + res_code.line_number + ) self.policies_finder.parse_yaml_code_for_policy(res_code, yaml_file.path) self.policies_finder.parse_policies_in_parse_queue() diff --git a/nca/NetworkConfig/TopologyObjectsFinder.py b/nca/NetworkConfig/TopologyObjectsFinder.py index 9f010c92c..e5ce12bfc 100644 --- a/nca/NetworkConfig/TopologyObjectsFinder.py +++ b/nca/NetworkConfig/TopologyObjectsFinder.py @@ -161,7 +161,7 @@ def _add_pod_from_workload_yaml(self, workload_resource): for port in container.get('ports') or []: pod.add_named_port(port.get('name'), port.get('containerPort'), port.get('protocol', 'TCP')) self._add_peer(pod) - ExplTracker().add_item('', workload_resource, pod.name) + ExplTracker().add_item('', workload_resource, pod.name, workload_resource.line_number) def _add_networkset_from_yaml(self, networkset_object): """ diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py index 5427d86d0..b91477895 100644 --- a/nca/Utils/ExplTracker.py +++ b/nca/Utils/ExplTracker.py @@ -3,7 +3,8 @@ # SPDX-License-Identifier: Apache2.0 # -from nca.Utils.NcaLogger import Singleton +from nca.Utils.Utils import Singleton +from nca.Utils.NcaLogger import NcaLogger class ExplDescriptor: @@ -40,14 +41,21 @@ class ExplTracker(metaclass=Singleton): def __init__(self): self.ExplDescriptorContainer = {} self.ExplPeerToPolicyContainer = {} + self._is_active = False + + def activate(self): + self._is_active = True + + def is_active(self): + return self._is_active def get_path_from_deployment(self, content): return self.ExplDescriptorContainer[content.get('metadata').get('name')].get('path') - def add_item(self, path, content, name): + def add_item(self, path, content, name, ln): if path == '': path = self.get_path_from_deployment(content) - self.ExplDescriptorContainer[name] = {'path': path, 'content': content} + self.ExplDescriptorContainer[name] = {'path': path, 'line': ln} def add_peer_policy(self, peer, policy): peer_name = peer.name @@ -56,3 +64,27 @@ def add_peer_policy(self, peer, policy): self.ExplPeerToPolicyContainer[peer_name] = set() self.ExplPeerToPolicyContainer[peer_name].add(policy_name) + def explain(self, nodes): + if len(nodes) < 1: + return + elif len(nodes) > 2: + NcaLogger().log_message(f'Explainability error: only 1 or 2 nodes are allowed for explainability query,' + f' found {len(nodes)} ', level='E') + return + results = {} + for node in nodes: + if not self.ExplDescriptorContainer.get(node): + NcaLogger().log_message(f'Explainability error: {node} was not found in the connectivity results', level='E') + return + results[node] = self.ExplDescriptorContainer.get(node) + for policy in self.ExplPeerToPolicyContainer.get(node): + results[policy] = self.ExplDescriptorContainer.get(policy) + out = [] + if len(nodes) == 1: + out.append(f'Configurations affecting node {nodes[0]}: \n') + else: + out.append(f'Configurations affecting the connectivity between {nodes[0]} and {nodes[1]}:') + for name in results.keys(): + out.append(f'{name} - line {results.get(name).get("line")} in file {results.get(name).get("path")}') + + return out diff --git a/nca/Utils/NcaLogger.py b/nca/Utils/NcaLogger.py index 162a67712..77958579c 100644 --- a/nca/Utils/NcaLogger.py +++ b/nca/Utils/NcaLogger.py @@ -3,18 +3,7 @@ # SPDX-License-Identifier: Apache2.0 # import sys - - -class Singleton(type): - """ - A metaclass implementing singleton for NcaLogger - """ - _instances = {} - - def __call__(cls, *args, **kwargs): - if cls not in cls._instances: - cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) - return cls._instances[cls] +from nca.Utils.Utils import Singleton class NcaLogger(metaclass=Singleton): diff --git a/nca/Utils/Utils.py b/nca/Utils/Utils.py new file mode 100644 index 000000000..5e8e6408b --- /dev/null +++ b/nca/Utils/Utils.py @@ -0,0 +1,17 @@ +# +# Copyright 2022 - IBM Inc. All rights reserved +# SPDX-License-Identifier: Apache2.0 +# + + +class Singleton(type): + """ + A metaclass implementing singleton for NcaLogger + """ + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] + diff --git a/nca/nca_cli.py b/nca/nca_cli.py index 5df602c77..257a6723f 100644 --- a/nca/nca_cli.py +++ b/nca/nca_cli.py @@ -14,6 +14,7 @@ from nca.NetworkConfig.NetworkConfigQueryRunner import NetworkConfigQueryRunner from nca.NetworkConfig.ResourcesHandler import ResourcesHandler from nca.SchemeRunner import SchemeRunner +from nca.Utils.ExplTracker import ExplTracker def _valid_path(path_location, allow_ghe=False, allowed_platforms=None): @@ -151,6 +152,7 @@ def run_args(args): 'prURL': args.pr_url or None, 'outputEndpoints': args.output_endpoints, 'subset': {}, + 'expl': [], 'excludeIPv6Range': not args.print_ipv6}) expected_output = None # default values are for sanity query @@ -178,6 +180,10 @@ def run_args(args): all_labels.append(lbl_dict) output_config['subset'].update({'label_subset': all_labels}) + if args.explain is not None: + output_config['expl'] = args.explain.split(',') + ExplTracker().activate() + if args.equiv is not None: np_list = args.equiv if args.equiv != [''] else None query_name = 'twoWayContainment' @@ -294,6 +300,8 @@ def nca_main(argv=None): help='A file/GHE url/cluster-type to read pod list from (may be specified multiple times)') parser.add_argument('--resource_list', '-r', type=_resource_list_valid_path, action='append', help='Network policies entries or Filesystem or GHE location of base network resources ') + parser.add_argument('--explain', '-expl', type=str, + help='A node or 2 nodes (a connection), to explain the configurations affecting them') parser.add_argument('--deployment_subset', '-ds', type=str, help='A list of deployment names to subset the query by') parser.add_argument('--namespace_subset', '-nss', type=str, From b5c65861bc601fcd419baf34ad001ec486fe9e59 Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Tue, 7 Mar 2023 12:19:06 +0200 Subject: [PATCH 055/187] explain connectivity Signed-off-by: Shmulik Froimovich --- nca/Utils/ExplTracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py index b91477895..384d2a666 100644 --- a/nca/Utils/ExplTracker.py +++ b/nca/Utils/ExplTracker.py @@ -85,6 +85,6 @@ def explain(self, nodes): else: out.append(f'Configurations affecting the connectivity between {nodes[0]} and {nodes[1]}:') for name in results.keys(): - out.append(f'{name} - line {results.get(name).get("line")} in file {results.get(name).get("path")}') + out.append(f'{name}: line {results.get(name).get("line")} in file {results.get(name).get("path")}') return out From f8243de421f6daca0556a300172333b3c1de0e3a Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 7 Mar 2023 18:04:52 +0200 Subject: [PATCH 056/187] Simplified and cleaned interfaces. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 22 +- nca/CoreDS/ConnectivityProperties.py | 265 ++++++++++-------- nca/CoreDS/Peer.py | 13 +- nca/NetworkConfig/NetworkConfigQuery.py | 8 +- nca/Parsers/CalicoPolicyYamlParser.py | 6 +- nca/Parsers/IngressPolicyYamlParser.py | 8 +- nca/Parsers/IstioPolicyYamlParser.py | 8 +- nca/Parsers/IstioSidecarYamlParser.py | 2 +- .../IstioTrafficResourcesYamlParser.py | 2 +- nca/Parsers/K8sPolicyYamlParser.py | 2 +- 10 files changed, 193 insertions(+), 143 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index f60815f11..8535b5184 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -544,9 +544,9 @@ def print_diff(self, other, self_name, other_name): def convert_to_connectivity_properties(self, peer_container): if self.allow_all: - return ConnectivityProperties.make_all_props(peer_container) + return ConnectivityProperties.make_all_props() - res = ConnectivityProperties.make_empty_props(peer_container) + res = ConnectivityProperties.make_empty_props() for protocol, properties in self.allowed_protocols.items(): protocols = ProtocolSet() protocols.add_protocol(protocol) @@ -574,11 +574,11 @@ def get_non_tcp_connections(): # TODO - after moving to the optimized HC set implementation, # get rid of ConnectionSet and move the code below to ConnectivityProperties.py @staticmethod - def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container, ip_blocks_mask, - connectivity_restriction): + def conn_props_to_fw_rules(conn_props, cluster_info, peer_container, ip_blocks_mask, + connectivity_restriction): """ Build FWRules from the given ConnectivityProperties - :param ConnectivityProperties tcp_props: properties describing allowed connections + :param ConnectivityProperties conn_props: properties describing allowed connections :param ClusterInfo cluster_info: the cluster info :param PeerContainer peer_container: the peer container :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, @@ -595,8 +595,8 @@ def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container, ip_block ignore_protocols = ProtocolSet.get_non_tcp_protocols() fw_rules_map = defaultdict(list) - for cube in tcp_props: - cube_dict = tcp_props.get_cube_dict_with_orig_values(cube) + for cube in conn_props: + cube_dict = conn_props.get_cube_dict_with_orig_values(cube) new_cube_dict = cube_dict.copy() src_peers = new_cube_dict.get('src_peers') if src_peers: @@ -625,9 +625,9 @@ def tcp_properties_to_fw_rules(tcp_props, cluster_info, peer_container, ip_block new_cube_dict)) else: if ConnectionSet.protocol_supports_ports(protocol): - conns.add_connections(protocol, ConnectivityProperties.make_all_props(peer_container)) + conns.add_connections(protocol, ConnectivityProperties.make_all_props()) elif ConnectionSet.protocol_is_icmp(protocol): - conns.add_connections(protocol, ConnectivityProperties.make_all_props(peer_container)) + conns.add_connections(protocol, ConnectivityProperties.make_all_props()) else: conns.add_connections(protocol, True) # create FWRules for src_peers and dst_peers @@ -670,8 +670,8 @@ def split_peer_set_to_fw_rule_elements(peer_set, cluster_info): return res @staticmethod - def fw_rules_to_tcp_properties(fw_rules, peer_container): - res = ConnectivityProperties.make_empty_props(peer_container) + def fw_rules_to_conn_props(fw_rules, peer_container): + res = ConnectivityProperties.make_empty_props() for fw_rules_list in fw_rules.fw_rules_map.values(): for fw_rule in fw_rules_list: conn_props = fw_rule.conn.convert_to_connectivity_properties(peer_container) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 18f7cc7d4..95ff880a4 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -11,13 +11,15 @@ from .ProtocolSet import ProtocolSet from .Peer import PeerSet from .ProtocolNameResolver import ProtocolNameResolver +from .MinDFA import MinDFA class ConnectivityProperties(CanonicalHyperCubeSet): """ A class for holding a set of cubes, each defined over dimensions from ConnectivityProperties.dimensions_list - For UDP, SCTP protocols, the actual used dimensions are only [source ports, dest ports] - for TCP, it may be any of the dimensions from dimensions_list. + For UDP, SCTP protocols, the actual used dimensions are only [src_peers, dst_peers, source ports, dest ports], + for TCP, it may be any of the dimensions from dimensions_list, except for icmp_type and icmp_code, + for icmp data the actual used dimensions are only [src_peers, dst_peers, icmp_type, icmp_code]. Also, including support for (included and excluded) named ports (relevant for dest ports only). @@ -44,26 +46,30 @@ class ConnectivityProperties(CanonicalHyperCubeSet): # TODO: change constructor defaults? either all arguments in "allow all" by default, or "empty" by default def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protocols=ProtocolSet(True), methods=MethodSet(True), paths=None, hosts=None, - icmp_type=DimensionsManager().get_dimension_domain_by_name('icmp_type'), - icmp_code=DimensionsManager().get_dimension_domain_by_name('icmp_code'), - base_peer_set=None, src_peers=None, dst_peers=None, + icmp_type_interval=DimensionsManager().get_dimension_domain_by_name('icmp_type'), + icmp_code_interval=DimensionsManager().get_dimension_domain_by_name('icmp_code'), + base_peer_set=None, + src_peers_interval=DimensionsManager().get_dimension_domain_by_name('src_peers'), + dst_peers_interval=DimensionsManager().get_dimension_domain_by_name('dst_peers'), create_empty=False): """ - This will create all cubes made of the input arguments ranges/regex values. + This will create connectivity properties made of all cubes made of the input arguments ranges/regex values. + This includes tcp properties, non-tcp properties, icmp data properties. :param PortSet source_ports: The set of source ports (as a set of intervals/ranges) :param PortSet dest_ports: The set of target ports (as a set of intervals/ranges) :param ProtocolSet protocols: the set of eligible protocols :param MethodSet methods: the set of http request methods :param MinDFA paths: The dfa of http request paths :param MinDFA hosts: The dfa of http request hosts - :param CanonicalIntervalSet icmp_type: ICMP-specific parameter (type dimension) - :param CanonicalIntervalSet icmp_code: ICMP-specific parameter (code dimension) + :param CanonicalIntervalSet icmp_type_interval: ICMP-specific parameter (type dimension) + :param CanonicalIntervalSet icmp_code_interval: ICMP-specific parameter (code dimension) :param PeerSet base_peer_set: the base peer set which is referenced by the indices in 'peers' - :param CanonicalIntervalSet src_peers: the set of source peers - :param CanonicalIntervalSet dst_peers: the set of target peers + :param CanonicalIntervalSet src_peers_interval: the set of source peers + :param CanonicalIntervalSet dst_peers_interval: the set of target peers """ super().__init__(ConnectivityProperties.dimensions_list) - assert (not src_peers and not dst_peers) or base_peer_set + assert (src_peers_interval != DimensionsManager().get_dimension_domain_by_name('src_peers') and + dst_peers_interval != DimensionsManager().get_dimension_domain_by_name('dst_peers')) or base_peer_set self.named_ports = {} # a mapping from dst named port (String) to src ports interval set self.excluded_named_ports = {} # a mapping from dst named port (String) to src ports interval set @@ -73,18 +79,20 @@ def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protoco # create the cube from input arguments # create a dict object that holds the values required to build the cube - dims_to_values = {"src_peers": {"value": src_peers, "is_all": src_peers is None}, - "dst_peers": {"value": dst_peers, "is_all": dst_peers is None}, + dims_to_values = {"src_peers": {"value": src_peers_interval, + "is_all": src_peers_interval == DimensionsManager().get_dimension_domain_by_name('src_peers')}, + "dst_peers": {"value": dst_peers_interval, + "is_all": dst_peers_interval == DimensionsManager().get_dimension_domain_by_name('dst_peers')}, "protocols": {"value": protocols, "is_all": protocols.is_whole_range()}, "src_ports": {"value": source_ports.port_set, "is_all": source_ports.is_all()}, "dst_ports": {"value": dest_ports.port_set, "is_all": dest_ports.is_all()}, "methods": {"value": methods, "is_all": methods.is_whole_range()}, "hosts": {"value": hosts, "is_all": hosts is None}, "paths": {"value": paths, "is_all": paths is None}, - "icmp_type": {"value": icmp_type, - "is_all": icmp_type == DimensionsManager().get_dimension_domain_by_name('icmp_type')}, - "icmp_code": {"value": icmp_code, - "is_all": icmp_code == DimensionsManager().get_dimension_domain_by_name('icmp_code')}} + "icmp_type": {"value": icmp_type_interval, + "is_all": icmp_type_interval == DimensionsManager().get_dimension_domain_by_name('icmp_type')}, + "icmp_code": {"value": icmp_code_interval, + "is_all": icmp_code_interval == DimensionsManager().get_dimension_domain_by_name('icmp_code')}} cube, active_dims, has_empty_dim_value = self._get_cube_and_active_dims_from_input_values(dims_to_values) @@ -202,6 +210,38 @@ def get_cube_dict_with_orig_values(self, cube): cube_dict[dim] = values return cube_dict + @staticmethod + def get_empty_dim_having_orig_type(dim): + if dim == "src_peers" or dim == "dst_peers": + return PeerSet() + if dim == "protocols": + return ProtocolSet() + if dim == "src_ports" or dim == "dst_ports": + return PortSet() + if dim == "methods": + return MethodSet() + if dim == "hosts" or dim == "paths": + return MinDFA.dfa_from_regex("") # empty DFA + if dim == "icmp_type" or dim == "icmp_code": + return CanonicalIntervalSet() + return None # unknown dimension + + @staticmethod + def get_full_dim_having_orig_type(dim): + if dim == "src_peers" or dim == "dst_peers": + return None # representing PeerSet with all possible peers + if dim == "protocols": + return ProtocolSet(True) + if dim == "src_ports" or dim == "dst_ports": + return PortSet(True) + if dim == "methods": + return MethodSet(True) + if dim == "hosts" or dim == "paths": + return DimensionsManager().get_dimension_domain_by_name(dim) + if dim == "icmp_type" or dim == "icmp_code": + return DimensionsManager().get_dimension_domain_by_name(dim) + return None # unknown dimension + def get_properties_obj(self): """ get an object for a yaml representation of the protocol's properties @@ -274,7 +314,7 @@ def __isub__(self, other): def contained_in(self, other): """ - :param ConnectivityProperties other: Another PortSetPair + :param ConnectivityProperties other: another connectivity properties :return: Whether all (source port, target port) pairs in self also appear in other :rtype: bool """ @@ -337,7 +377,7 @@ def copy(self): def print_diff(self, other, self_name, other_name): """ - :param ConnectivityProperties other: Another PortSetPair object + :param ConnectivityProperties other: another connectivity properties object :param str self_name: A name for 'self' :param str other_name: A name for 'other' :return: If self!=other, return a string showing a (source, target) pair that appears in only one of them @@ -374,17 +414,17 @@ def project_on_one_dimension(self, dim_name): """ Build the projection of self to the given dimension :param str dim_name: the given dimension - :return: the projection on the given dimension, having that dimension type (either IntervalSet or DFA) - or None if the given dimension is not active + :return: the projection on the given dimension, having that dimension type. + or empty dimension value if the given dimension is not active """ + res = self.get_empty_dim_having_orig_type(dim_name) if dim_name not in self.active_dimensions: - return None - res = None + return res for cube in self: cube_dict = self.get_cube_dict_with_orig_values(cube) values = cube_dict.get(dim_name) if values: - res = (res | values) if res else values + res |= values return res @staticmethod @@ -406,11 +446,15 @@ def resolve_named_port(named_port, peer, protocols): def make_conn_props(peer_container, src_ports=PortSet(True), dst_ports=PortSet(True), protocols=ProtocolSet(True), src_peers=None, dst_peers=None, paths_dfa=None, hosts_dfa=None, methods=MethodSet(True), - icmp_type=DimensionsManager().get_dimension_domain_by_name('icmp_type'), - icmp_code=DimensionsManager().get_dimension_domain_by_name('icmp_code')): + icmp_type_interval=DimensionsManager().get_dimension_domain_by_name('icmp_type'), + icmp_code_interval=DimensionsManager().get_dimension_domain_by_name('icmp_code')): """ get ConnectivityProperties with allowed connections, corresponding to input properties cube. - ConnectivityProperties should not contain named ports: substitute them with corresponding port numbers, per peer + In the optimized solution, the resulting ConnectivityProperties should not contain named ports: + they are substituted with corresponding port numbers, per peer + In the original solution, the resulting ConnectivityProperties may contain named ports; + they cannot yet be resolved, since there are not dst_peer in the original solution; + they will be resolved by convert_named_ports call during query runs. :param PeerContainer peer_container: The set of endpoints and their namespaces :param PortSet src_ports: ports set for src_ports dimension (possibly containing named ports) :param PortSet dst_ports: ports set for dst_ports dimension (possibly containing named ports) @@ -420,85 +464,51 @@ def make_conn_props(peer_container, src_ports=PortSet(True), dst_ports=PortSet(T :param MinDFA paths_dfa: MinDFA obj for paths dimension :param MinDFA hosts_dfa: MinDFA obj for hosts dimension :param MethodSet methods: CanonicalIntervalSet obj for methods dimension - :param CanonicalIntervalSet icmp_type: ICMP-specific parameter (type dimension) - :param CanonicalIntervalSet icmp_code: ICMP-specific parameter (code dimension) + :param CanonicalIntervalSet icmp_type_interval: ICMP-specific parameter (type dimension) + :param CanonicalIntervalSet icmp_code_interval: ICMP-specific parameter (code dimension) :return: ConnectivityProperties with allowed connections, corresponding to input properties cube """ base_peer_set = peer_container.peer_set.copy() if src_peers: base_peer_set |= src_peers + if dst_peers: + base_peer_set |= dst_peers + if src_peers: src_peers_interval = base_peer_set.get_peer_interval_of(src_peers) else: - src_peers_interval = None + src_peers_interval = DimensionsManager().get_dimension_domain_by_name('src_peers') if dst_peers: - base_peer_set |= dst_peers dst_peers_interval = base_peer_set.get_peer_interval_of(dst_peers) else: - dst_peers_interval = None - if (not src_ports.named_ports or not src_peers) and (not dst_ports.named_ports or not dst_peers): + dst_peers_interval = DimensionsManager().get_dimension_domain_by_name('dst_peers') + + assert not src_ports.named_ports + if not dst_ports.named_ports or not dst_peers: # Should not resolve named ports return ConnectivityProperties(source_ports=src_ports, dest_ports=dst_ports, protocols=protocols, methods=methods, paths=paths_dfa, hosts=hosts_dfa, - icmp_type=icmp_type, icmp_code=icmp_code, - src_peers=src_peers_interval, dst_peers=dst_peers_interval, + icmp_type_interval=icmp_type_interval, icmp_code_interval=icmp_code_interval, + src_peers_interval=src_peers_interval, dst_peers_interval=dst_peers_interval, base_peer_set=base_peer_set) - # Resolving named ports - conn_properties = ConnectivityProperties.make_empty_props(peer_container) - if src_ports.named_ports and dst_ports.named_ports: - assert src_peers and dst_peers - assert not src_ports.port_set and not dst_ports.port_set - assert len(src_ports.named_ports) == 1 and len(dst_ports.named_ports) == 1 - src_named_port = list(src_ports.named_ports)[0] - dst_named_port = list(dst_ports.named_ports)[0] - for src_peer in src_peers: - real_src_ports = ConnectivityProperties.resolve_named_port(src_named_port, src_peer, protocols) - if not real_src_ports: - continue - for dst_peer in dst_peers: - real_dst_ports = ConnectivityProperties.resolve_named_port(dst_named_port, dst_peer, protocols) - if not real_dst_ports: - continue - props = ConnectivityProperties(source_ports=real_src_ports, dest_ports=real_dst_ports, - protocols=protocols, methods=methods, - paths=paths_dfa, hosts=hosts_dfa, - icmp_type=icmp_type, icmp_code=icmp_code, - src_peers=base_peer_set.get_peer_interval_of(PeerSet({src_peer})), - dst_peers=base_peer_set.get_peer_interval_of(PeerSet({dst_peer})), - base_peer_set=base_peer_set) - - conn_properties |= props - else: - # either only src_ports or only dst_ports contain named ports - if src_ports.named_ports: - port_set_with_named_ports = src_ports - peers_for_named_ports = src_peers - else: - port_set_with_named_ports = dst_ports - peers_for_named_ports = dst_peers - assert peers_for_named_ports - assert not port_set_with_named_ports.port_set - assert len(port_set_with_named_ports.named_ports) == 1 - port = list(port_set_with_named_ports.named_ports)[0] - for peer in peers_for_named_ports: - real_ports = ConnectivityProperties.resolve_named_port(port, peer, protocols) - if not real_ports: - continue - if src_ports.named_ports: - props = ConnectivityProperties(source_ports=real_ports, dest_ports=dst_ports, - protocols=protocols, methods=methods, - paths=paths_dfa, hosts=hosts_dfa, - icmp_type=icmp_type, icmp_code=icmp_code, - src_peers=base_peer_set.get_peer_interval_of(PeerSet({peer})), - dst_peers=dst_peers_interval, base_peer_set=base_peer_set) - else: - props = ConnectivityProperties(source_ports=src_ports, dest_ports=real_ports, - protocols=protocols, methods=methods, - paths=paths_dfa, hosts=hosts_dfa, - icmp_type=icmp_type, icmp_code=icmp_code, - src_peers=src_peers_interval, - dst_peers=base_peer_set.get_peer_interval_of(PeerSet({peer})), - base_peer_set=base_peer_set) - conn_properties |= props + # Resolving dst named ports + conn_properties = ConnectivityProperties.make_empty_props() + assert dst_peers + assert not dst_ports.port_set + assert not dst_ports.excluded_named_ports + assert len(dst_ports.named_ports) == 1 + port = list(dst_ports.named_ports)[0] + for peer in dst_peers: + real_ports = ConnectivityProperties.resolve_named_port(port, peer, protocols) + if not real_ports: + continue + props = ConnectivityProperties(source_ports=src_ports, dest_ports=real_ports, + protocols=protocols, methods=methods, + paths=paths_dfa, hosts=hosts_dfa, + icmp_type_interval=icmp_type_interval, icmp_code_interval=icmp_code_interval, + src_peers_interval=src_peers_interval, + dst_peers_interval=base_peer_set.get_peer_interval_of(PeerSet({peer})), + base_peer_set=base_peer_set) + conn_properties |= props return conn_properties @staticmethod @@ -510,32 +520,44 @@ def make_conn_props_from_dict(peer_container, cube_dict): :return: ConnectivityProperties """ cube_dict_copy = cube_dict.copy() - src_ports = cube_dict_copy.pop("src_ports", PortSet(True)) - dst_ports = cube_dict_copy.pop("dst_ports", PortSet(True)) - protocols = cube_dict_copy.pop("protocols", ProtocolSet(True)) - src_peers = cube_dict_copy.pop("src_peers", None) - dst_peers = cube_dict_copy.pop("dst_peers", None) - paths_dfa = cube_dict_copy.pop("paths", None) - hosts_dfa = cube_dict_copy.pop("hosts", None) - methods = cube_dict_copy.pop("methods", MethodSet(True)) - icmp_type = cube_dict_copy.pop("icmp_type", DimensionsManager().get_dimension_domain_by_name('icmp_type')) - icmp_code = cube_dict_copy.pop("icmp_code", DimensionsManager().get_dimension_domain_by_name('icmp_code')) + src_ports = cube_dict_copy.pop("src_ports", ConnectivityProperties.get_full_dim_having_orig_type("src_ports")) + dst_ports = cube_dict_copy.pop("dst_ports", ConnectivityProperties.get_full_dim_having_orig_type("dst_ports")) + protocols = cube_dict_copy.pop("protocols", ConnectivityProperties.get_full_dim_having_orig_type("protocols")) + src_peers = cube_dict_copy.pop("src_peers", ConnectivityProperties.get_full_dim_having_orig_type("src_peers")) + dst_peers = cube_dict_copy.pop("dst_peers", ConnectivityProperties.get_full_dim_having_orig_type("dst_peers")) + paths_dfa = cube_dict_copy.pop("paths", ConnectivityProperties.get_full_dim_having_orig_type("paths")) + hosts_dfa = cube_dict_copy.pop("hosts", ConnectivityProperties.get_full_dim_having_orig_type("hosts")) + methods = cube_dict_copy.pop("methods", ConnectivityProperties.get_full_dim_having_orig_type("methods")) + icmp_type_interval = cube_dict_copy.pop("icmp_type", ConnectivityProperties.get_full_dim_having_orig_type("icmp_type")) + icmp_code_interval = cube_dict_copy.pop("icmp_code", ConnectivityProperties.get_full_dim_having_orig_type("icmp_code")) assert not cube_dict_copy return ConnectivityProperties.make_conn_props(peer_container, src_ports=src_ports, dst_ports=dst_ports, protocols=protocols, src_peers=src_peers, dst_peers=dst_peers, paths_dfa=paths_dfa, hosts_dfa=hosts_dfa, methods=methods, - icmp_type=icmp_type, icmp_code=icmp_code) + icmp_type_interval=icmp_type_interval, + icmp_code_interval=icmp_code_interval) @staticmethod - def make_empty_props(peer_container=None): - return ConnectivityProperties(base_peer_set=peer_container.peer_set.copy() if peer_container else None, - create_empty=True) + def make_empty_props(): + """ + Returns empty connectivity properties, representing logical False + :return: ConnectivityProperties + """ + return ConnectivityProperties(create_empty=True) @staticmethod - def make_all_props(peer_container=None): - return ConnectivityProperties(base_peer_set=peer_container.peer_set.copy() if peer_container else None) + def make_all_props(): + """ + Returns all connectivity properties, representing logical True + :return: ConnectivityProperties + """ + return ConnectivityProperties() def are_auto_conns(self): + """ + :return: True iff the given connections are connections from peers to themselves, + i.e., they include only pairs of identical src and dst peers. + """ if not {'src_peers', 'dst_peers'}.issubset(set(self.active_dimensions)): return False src_peers_index = None @@ -555,6 +577,17 @@ def are_auto_conns(self): @staticmethod def make_icmp_props(peer_container, protocol="", src_peers=None, dst_peers=None, icmp_type=None, icmp_code=None): + """ + This function is for icmp data only. It creates icmp properties from the given src/dst peers, protocol, + icmp type and icmp code. + :param peer_container: the peer container as a base for al src/dst peers. + :param protocol: the protocol + :param src_peers: source peers + :param dst_peers: destination peers + :param icmp_type: icmp type + :param icmp_code: icmp code + :return: the resulting ConnectivityProperties + """ if protocol: icmp_protocol_set = ProtocolSet() icmp_protocol_set.add_protocol(ProtocolNameResolver.get_protocol_number(protocol)) @@ -568,11 +601,23 @@ def make_icmp_props(peer_container, protocol="", src_peers=None, dst_peers=None, icmp_code_interval = CanonicalIntervalSet.get_interval_set(icmp_code, icmp_code) return ConnectivityProperties.make_conn_props(peer_container=peer_container, protocols=icmp_protocol_set, src_peers=src_peers, dst_peers=dst_peers, - icmp_type=icmp_type_interval, icmp_code=icmp_code_interval) + icmp_type_interval=icmp_type_interval, + icmp_code_interval=icmp_code_interval) @staticmethod def make_all_but_given_icmp_props(peer_container, protocol="", src_peers=None, dst_peers=None, icmp_type=None, icmp_code=None): + """ + This function is for icmp data only. It creates icmp properties representing the negation of + the properties having the given src/dst peers, protocol, icmp type and icmp code. + :param peer_container: the peer container as a base for al src/dst peers. + :param protocol: the protocol + :param src_peers: source peers + :param dst_peers: destination peers + :param icmp_type: icmp type + :param icmp_code: icmp code + :return: the resulting ConnectivityProperties + """ if protocol: icmp_protocol_set = ProtocolSet() icmp_protocol_set.add_protocol(ProtocolNameResolver.get_protocol_number(protocol)) @@ -584,4 +629,4 @@ def make_all_but_given_icmp_props(peer_container, protocol="", src_peers=None, d given_icmp_props = ConnectivityProperties.make_icmp_props(peer_container, protocol=protocol, src_peers=src_peers, dst_peers=dst_peers, icmp_type=icmp_type, icmp_code=icmp_code, ) - return all_icmp_props-given_icmp_props + return all_icmp_props - given_icmp_props diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index 650cb4f78..bfe661389 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -480,19 +480,20 @@ class PeerSet(set): ipv4_highest_number = int(ip_network('0.0.0.0/0').broadcast_address) ipv6_highest_number = int(ip_network('::/0').broadcast_address) - pod_highest_number = 9999 # assuming maximum 10,000 pods + max_num_of_pods = 10000 gap_width = 5 # the gap is needed to avoid mixed-type intervals union min_ipv4_index = 0 max_ipv4_index = min_ipv4_index + ipv4_highest_number min_ipv6_index = max_ipv4_index + gap_width max_ipv6_index = min_ipv6_index + ipv6_highest_number min_pod_index = max_ipv6_index + gap_width - max_pod_index = min_pod_index + pod_highest_number + max_pod_index = min_pod_index + max_num_of_pods - 1 def __init__(self, peer_set=None): super().__init__(peer_set or set()) self.sorted_peer_list = [] # for converting PeerSet to CanonicalIntervalSet self.last_size_when_updated_sorted_peer_list = 0 + assert len(self.get_set_without_ip_block()) <= self.max_num_of_pods def __contains__(self, item): if isinstance(item, IpBlock): # a special check here because an IpBlock may be contained in another IpBlock @@ -541,10 +542,14 @@ def __and__(self, other): return res def __ior__(self, other): - return PeerSet(super().__ior__(other)) + res = PeerSet(super().__ior__(other)) + assert len(res.get_set_without_ip_block()) <= self.max_num_of_pods + return res def __or__(self, other): - return PeerSet(super().__or__(other)) + res = PeerSet(super().__or__(other)) + assert len(res.get_set_without_ip_block()) <= self.max_num_of_pods + return res def __isub__(self, other): # subtraction on IpBlocks diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index b86f9e8fc..1a94d89d6 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -800,8 +800,8 @@ def compare_orig_to_opt_conn(self, orig_conn_graph, opt_props): orig_tcp_props == opt_props def compare_fw_rules(self, fw_rules1, fw_rules2): - tcp_props1 = ConnectionSet.fw_rules_to_tcp_properties(fw_rules1, self.config.peer_container) - tcp_props2 = ConnectionSet.fw_rules_to_tcp_properties(fw_rules2, self.config.peer_container) + tcp_props1 = ConnectionSet.fw_rules_to_conn_props(fw_rules1, self.config.peer_container) + tcp_props2 = ConnectionSet.fw_rules_to_conn_props(fw_rules2, self.config.peer_container) if tcp_props1 == tcp_props2: print("Original and optimized fw-rules are semantically equivalent") else: @@ -1008,8 +1008,8 @@ def fw_rules_from_props(self, props, peers_to_compare, ip_blocks_mask, connectiv :rtype: Union[str, dict] """ cluster_info = ClusterInfo(peers_to_compare, self.config.get_allowed_labels()) - fw_rules_map = ConnectionSet.tcp_properties_to_fw_rules(props, cluster_info, self.config.peer_container, - ip_blocks_mask, connectivity_restriction) + fw_rules_map = ConnectionSet.conn_props_to_fw_rules(props, cluster_info, self.config.peer_container, + ip_blocks_mask, connectivity_restriction) fw_rules = MinimizeFWRules(fw_rules_map, cluster_info, self.output_config, {}) formatted_rules = fw_rules.get_fw_rules_in_required_format(connectivity_restriction=connectivity_restriction) return formatted_rules, fw_rules diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index 562c2a815..386903b64 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -381,7 +381,7 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): self.syntax_error(err, not_icmp_data) res = ConnectivityProperties.make_icmp_props(self.peer_container) - opt_props = ConnectivityProperties.make_empty_props(self.peer_container) + opt_props = ConnectivityProperties.make_empty_props() if self.optimized_run != 'false' and src_pods and dst_pods: opt_props = ConnectivityProperties.make_icmp_props(self.peer_container, protocol=protocol, src_peers=src_pods, dst_peers=dst_pods) @@ -393,7 +393,7 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): icmp_type=icmp_type, icmp_code=icmp_code) if not_icmp_data is not None: if icmp_type == not_icmp_type and icmp_code == not_icmp_code: - res = ConnectivityProperties.make_empty_props(self.peer_container) + res = ConnectivityProperties.make_empty_props() self.warning('icmp and notICMP are conflicting - no traffic will be matched', not_icmp_data) elif icmp_type == not_icmp_type and icmp_code is None: tmp = ConnectivityProperties.make_icmp_props(self.peer_container, icmp_type=not_icmp_type, @@ -482,7 +482,7 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): src_res_pods &= policy_selected_eps connections = ConnectionSet() - conn_props = ConnectivityProperties.make_empty_props(self.peer_container) + conn_props = ConnectivityProperties.make_empty_props() if protocol is not None: protocols = ProtocolSet() protocols.add_protocol(protocol) diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index 0fcee265b..1c3193df4 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -185,7 +185,7 @@ def _make_default_connections(self, hosts_dfa, paths_dfa=None): :param MinDFA paths_dfa: the paths for the default connections :return: ConnectivityProperties containing default connections or None (when no default backend exists) """ - default_conns = ConnectivityProperties.make_empty_props(self.peer_container) + default_conns = ConnectivityProperties.make_empty_props() if self.default_backend_peers: if paths_dfa: default_conns = \ @@ -215,8 +215,8 @@ def parse_rule(self, rule): self.check_fields_validity(rule, 'ingress rule', allowed_elements) hosts_dfa = self.parse_regex_host_value(rule.get("host"), rule) paths_array = self.get_key_array_and_validate_not_empty(rule.get('http'), 'paths') - allowed_conns = ConnectivityProperties.make_empty_props(self.peer_container) - default_conns = ConnectivityProperties.make_empty_props(self.peer_container) + allowed_conns = ConnectivityProperties.make_empty_props() + default_conns = ConnectivityProperties.make_empty_props() if paths_array is not None: all_paths_dfa = None parsed_paths = [] @@ -271,7 +271,7 @@ def parse_policy(self): if not res_policy.selected_peers: self.missing_k8s_ingress_peers = True self.warning("No ingress-nginx pods found, the Ingress policy will have no effect") - allowed_conns = ConnectivityProperties.make_empty_props(self.peer_container) + allowed_conns = ConnectivityProperties.make_empty_props() all_hosts_dfa = None for ingress_rule in policy_spec.get('rules', []): conns, hosts_dfa = self.parse_rule(ingress_rule) diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index 78c7afbf6..aac0f4cb2 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -489,7 +489,7 @@ def parse_ingress_rule(self, rule, selected_peers): to_array = self.get_key_array_and_validate_not_empty(rule, 'to') # currently parsing only ports # TODO: extend operations parsing to include other attributes - conn_props = ConnectivityProperties.make_empty_props(self.peer_container) + conn_props = ConnectivityProperties.make_empty_props() if to_array is not None: connections = ConnectionSet() for operation_dict in to_array: @@ -498,7 +498,7 @@ def parse_ingress_rule(self, rule, selected_peers): conn_props |= conns.convert_to_connectivity_properties(self.peer_container) else: # no 'to' in the rule => all connections allowed connections = ConnectionSet(True) - conn_props = ConnectivityProperties.make_all_props(self.peer_container) + conn_props = ConnectivityProperties.make_all_props() # condition possible result value: # source-ip (from) , source-namespace (from) [Peerset], destination.port (to) [ConnectionSet] @@ -517,7 +517,7 @@ def parse_ingress_rule(self, rule, selected_peers): self.warning('Rule selects no pods', rule) condition_props = condition_conns.convert_to_connectivity_properties(self.peer_container) if not res_peers or not selected_peers: - condition_props = ConnectivityProperties.make_empty_props(self.peer_container) + condition_props = ConnectivityProperties.make_empty_props() else: condition_props &= ConnectivityProperties.make_conn_props(self.peer_container, src_peers=res_peers, dst_peers=selected_peers) @@ -575,7 +575,7 @@ def parse_policy(self): res_policy.add_ingress_rule(rule) res_policy.add_optimized_ingress_props(optimized_props, res_policy.action == IstioNetworkPolicy.ActionType.Allow) - res_policy.add_optimized_egress_props(ConnectivityProperties.make_all_props(self.peer_container)) + res_policy.add_optimized_egress_props(ConnectivityProperties.make_all_props()) if not res_policy.ingress_rules and res_policy.action == IstioNetworkPolicy.ActionType.Deny: self.syntax_error("DENY action without rules is meaningless as it will never be triggered") diff --git a/nca/Parsers/IstioSidecarYamlParser.py b/nca/Parsers/IstioSidecarYamlParser.py index 5b5bd65d3..1bf0d9238 100644 --- a/nca/Parsers/IstioSidecarYamlParser.py +++ b/nca/Parsers/IstioSidecarYamlParser.py @@ -177,7 +177,7 @@ def parse_policy(self): self.namespace = self.peer_container.get_namespace(policy_ns, warn_if_missing) res_policy = IstioSidecar(policy_name, self.namespace) res_policy.policy_kind = NetworkPolicy.PolicyType.IstioSidecar - res_policy.add_optimized_ingress_props(ConnectivityProperties.make_all_props(self.peer_container)) + res_policy.add_optimized_ingress_props(ConnectivityProperties.make_all_props()) sidecar_spec = self.policy['spec'] # currently, supported fields in spec are workloadSelector and egress diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index ad2514bb2..36400ed1e 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -338,7 +338,7 @@ def make_allowed_connections(self, vs, host_dfa): :param MinDFA host_dfa: the hosts attribute :return: ConnectivityProperties with allowed connections """ - allowed_conns = ConnectivityProperties.make_empty_props(self.peer_container) + allowed_conns = ConnectivityProperties.make_empty_props() for http_route in vs.http_routes: for dest in http_route.destinations: conns = \ diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index 86f3e5c6e..81f2a2086 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -330,7 +330,7 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): src_pods = policy_selected_pods dst_pods = res_pods - res_opt_props = ConnectivityProperties.make_empty_props(self.peer_container) # ConnectivityProperties + res_opt_props = ConnectivityProperties.make_empty_props() ports_array = rule.get('ports', []) if ports_array: res_conns = ConnectionSet() From d77050d5dec821ad8b7e5527e141d24b7abadeb2 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 7 Mar 2023 18:19:01 +0200 Subject: [PATCH 057/187] Fixed lint errors. Removed assertion which is no longer correct. Signed-off-by: Tanya --- nca/CoreDS/ConnectivityProperties.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 95ff880a4..558cecf43 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -68,8 +68,6 @@ def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protoco :param CanonicalIntervalSet dst_peers_interval: the set of target peers """ super().__init__(ConnectivityProperties.dimensions_list) - assert (src_peers_interval != DimensionsManager().get_dimension_domain_by_name('src_peers') and - dst_peers_interval != DimensionsManager().get_dimension_domain_by_name('dst_peers')) or base_peer_set self.named_ports = {} # a mapping from dst named port (String) to src ports interval set self.excluded_named_ports = {} # a mapping from dst named port (String) to src ports interval set @@ -80,9 +78,11 @@ def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protoco # create the cube from input arguments # create a dict object that holds the values required to build the cube dims_to_values = {"src_peers": {"value": src_peers_interval, - "is_all": src_peers_interval == DimensionsManager().get_dimension_domain_by_name('src_peers')}, + "is_all": src_peers_interval == + DimensionsManager().get_dimension_domain_by_name('src_peers')}, "dst_peers": {"value": dst_peers_interval, - "is_all": dst_peers_interval == DimensionsManager().get_dimension_domain_by_name('dst_peers')}, + "is_all": dst_peers_interval == + DimensionsManager().get_dimension_domain_by_name('dst_peers')}, "protocols": {"value": protocols, "is_all": protocols.is_whole_range()}, "src_ports": {"value": source_ports.port_set, "is_all": source_ports.is_all()}, "dst_ports": {"value": dest_ports.port_set, "is_all": dest_ports.is_all()}, @@ -90,9 +90,11 @@ def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protoco "hosts": {"value": hosts, "is_all": hosts is None}, "paths": {"value": paths, "is_all": paths is None}, "icmp_type": {"value": icmp_type_interval, - "is_all": icmp_type_interval == DimensionsManager().get_dimension_domain_by_name('icmp_type')}, + "is_all": icmp_type_interval == + DimensionsManager().get_dimension_domain_by_name('icmp_type')}, "icmp_code": {"value": icmp_code_interval, - "is_all": icmp_code_interval == DimensionsManager().get_dimension_domain_by_name('icmp_code')}} + "is_all": icmp_code_interval == + DimensionsManager().get_dimension_domain_by_name('icmp_code')}} cube, active_dims, has_empty_dim_value = self._get_cube_and_active_dims_from_input_values(dims_to_values) From 8977f439ed1ad50415437a7e48a4ac6d34482dca Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 7 Mar 2023 18:22:13 +0200 Subject: [PATCH 058/187] Fixed lint errors. Signed-off-by: Tanya --- nca/CoreDS/ConnectivityProperties.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 558cecf43..4f3b8022f 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -79,10 +79,10 @@ def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protoco # create a dict object that holds the values required to build the cube dims_to_values = {"src_peers": {"value": src_peers_interval, "is_all": src_peers_interval == - DimensionsManager().get_dimension_domain_by_name('src_peers')}, + DimensionsManager().get_dimension_domain_by_name('src_peers')}, "dst_peers": {"value": dst_peers_interval, "is_all": dst_peers_interval == - DimensionsManager().get_dimension_domain_by_name('dst_peers')}, + DimensionsManager().get_dimension_domain_by_name('dst_peers')}, "protocols": {"value": protocols, "is_all": protocols.is_whole_range()}, "src_ports": {"value": source_ports.port_set, "is_all": source_ports.is_all()}, "dst_ports": {"value": dest_ports.port_set, "is_all": dest_ports.is_all()}, @@ -91,10 +91,10 @@ def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protoco "paths": {"value": paths, "is_all": paths is None}, "icmp_type": {"value": icmp_type_interval, "is_all": icmp_type_interval == - DimensionsManager().get_dimension_domain_by_name('icmp_type')}, + DimensionsManager().get_dimension_domain_by_name('icmp_type')}, "icmp_code": {"value": icmp_code_interval, "is_all": icmp_code_interval == - DimensionsManager().get_dimension_domain_by_name('icmp_code')}} + DimensionsManager().get_dimension_domain_by_name('icmp_code')}} cube, active_dims, has_empty_dim_value = self._get_cube_and_active_dims_from_input_values(dims_to_values) From dae1bb1592ab210631aa90369de291fa0a6effa2 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 14 Mar 2023 16:21:55 +0200 Subject: [PATCH 059/187] Added a new class ConnectivityCube that manages forth and back translations of all dimensions of ConnectivityProperties, (translations between input format and internal format). Consequently, changed API of ConnectivityProperties methods to use the above new class. Improved documentation. Signed-off-by: Tanya --- nca/CoreDS/CanonicalIntervalSet.py | 4 + nca/CoreDS/ConnectionSet.py | 41 +- nca/CoreDS/ConnectivityProperties.py | 548 ++++++++---------- nca/CoreDS/PortSet.py | 3 + nca/FWRules/ConnectivityGraph.py | 68 +-- nca/NetworkConfig/NetworkConfigQuery.py | 18 +- nca/NetworkConfig/NetworkLayer.py | 69 ++- nca/Parsers/CalicoPolicyYamlParser.py | 122 ++-- nca/Parsers/GenericYamlParser.py | 13 +- nca/Parsers/IngressPolicyYamlParser.py | 35 +- nca/Parsers/IstioPolicyYamlParser.py | 8 +- .../IstioTrafficResourcesYamlParser.py | 19 +- nca/Parsers/K8sPolicyYamlParser.py | 27 +- nca/Resources/IstioSidecar.py | 15 +- 14 files changed, 473 insertions(+), 517 deletions(-) diff --git a/nca/CoreDS/CanonicalIntervalSet.py b/nca/CoreDS/CanonicalIntervalSet.py index 64b3e1121..6e035070c 100644 --- a/nca/CoreDS/CanonicalIntervalSet.py +++ b/nca/CoreDS/CanonicalIntervalSet.py @@ -366,3 +366,7 @@ def is_single_value(self): :return: Whether the interval set contains a single value """ return len(self) == 1 and list(self)[0].is_single_value() + + def validate_and_get_single_value(self): + assert self.is_single_value() + return list(self)[0].start diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 8535b5184..4fa76600a 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -5,7 +5,7 @@ from collections import defaultdict from .CanonicalIntervalSet import CanonicalIntervalSet -from .ConnectivityProperties import ConnectivityProperties +from .ConnectivityProperties import ConnectivityProperties, ConnectivityCube from .ProtocolNameResolver import ProtocolNameResolver from .ProtocolSet import ProtocolSet from .Peer import PeerSet, IpBlock @@ -550,7 +550,9 @@ def convert_to_connectivity_properties(self, peer_container): for protocol, properties in self.allowed_protocols.items(): protocols = ProtocolSet() protocols.add_protocol(protocol) - this_prop = ConnectivityProperties.make_conn_props(peer_container, protocols=protocols) + conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) + conn_cube.set_dim("protocols", protocols) + this_prop = ConnectivityProperties.make_conn_props(conn_cube) if isinstance(properties, bool): if properties: res |= this_prop @@ -596,33 +598,28 @@ def conn_props_to_fw_rules(conn_props, cluster_info, peer_container, ip_blocks_m fw_rules_map = defaultdict(list) for cube in conn_props: - cube_dict = conn_props.get_cube_dict_with_orig_values(cube) - new_cube_dict = cube_dict.copy() - src_peers = new_cube_dict.get('src_peers') - if src_peers: - new_cube_dict.pop('src_peers') - else: + conn_cube = conn_props.get_connectivity_cube(cube) + src_peers = conn_cube.get_dim("src_peers") + if not src_peers: src_peers = peer_container.get_all_peers_group(True) - dst_peers = new_cube_dict.get('dst_peers') - if dst_peers: - new_cube_dict.pop('dst_peers') - else: + conn_cube.unset_dim("src_peers") + dst_peers = conn_cube.get_dim("dst_peers") + if not dst_peers: dst_peers = peer_container.get_all_peers_group(True) + conn_cube.unset_dim("dst_peers") if IpBlock.get_all_ips_block() != ip_blocks_mask: src_peers.filter_ipv6_blocks(ip_blocks_mask) dst_peers.filter_ipv6_blocks(ip_blocks_mask) - protocols = new_cube_dict.get('protocols') - if protocols: - new_cube_dict.pop('protocols') - if not new_cube_dict and (not protocols or protocols == ignore_protocols): + protocols = conn_cube.get_dim("protocols") + conn_cube.unset_dim("protocols") + if not conn_cube.has_active_dim() and (not protocols or protocols == ignore_protocols): conns = ConnectionSet(True) else: conns = ConnectionSet() protocol_names = ProtocolSet.get_protocol_names_from_interval_set(protocols) if protocols else ['TCP'] for protocol in protocol_names: - if new_cube_dict: - conns.add_connections(protocol, ConnectivityProperties.make_conn_props_from_dict(peer_container, - new_cube_dict)) + if conn_cube.has_active_dim(): + conns.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) else: if ConnectionSet.protocol_supports_ports(protocol): conns.add_connections(protocol, ConnectivityProperties.make_all_props()) @@ -677,7 +674,9 @@ def fw_rules_to_conn_props(fw_rules, peer_container): conn_props = fw_rule.conn.convert_to_connectivity_properties(peer_container) src_peers = PeerSet(fw_rule.src.get_peer_set(fw_rules.cluster_info)) dst_peers = PeerSet(fw_rule.dst.get_peer_set(fw_rules.cluster_info)) - rule_props = ConnectivityProperties.make_conn_props(peer_container, src_peers=src_peers, - dst_peers=dst_peers) & conn_props + conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) + conn_cube.set_dim("src_peers", src_peers) + conn_cube.set_dim("dst_peers", dst_peers) + rule_props = ConnectivityProperties.make_conn_props(conn_cube) & conn_props res |= rule_props return res diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 4f3b8022f..1f57ee580 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -14,13 +14,181 @@ from .MinDFA import MinDFA +class ConnectivityCube(dict): + """ + This class manages forth and back translations of all dimensions of ConnectivityProperties + (translations between input format and internal format). + It is used as an input interface for ConnectivityProperties methods. + """ + + dimensions_list = ["src_peers", "dst_peers", "protocols", "src_ports", "dst_ports", "methods", "hosts", "paths", + "icmp_type", "icmp_code"] + internal_empty_dim_values = { + "src_peers": CanonicalIntervalSet(), + "dst_peers": CanonicalIntervalSet(), + "protocols": ProtocolSet(), + "src_ports": CanonicalIntervalSet(), + "dst_ports": CanonicalIntervalSet(), + "methods": MethodSet(), + "hosts": MinDFA.dfa_from_regex(""), + "paths": MinDFA.dfa_from_regex(""), + "icmp_type": CanonicalIntervalSet(), + "icmp_code": CanonicalIntervalSet() + } + external_empty_dim_values = { + "src_peers": PeerSet(), + "dst_peers": PeerSet(), + "protocols": ProtocolSet(), + "src_ports": PortSet(), + "dst_ports": PortSet(), + "methods": MethodSet(), + "hosts": MinDFA.dfa_from_regex(""), + "paths": MinDFA.dfa_from_regex(""), + "icmp_type": None, + "icmp_code": None + } + + def __init__(self, base_peer_set): + """ + :param PeerSet base_peer_set: the set of all possible peers, which will be referenced by the indices + in 'src_peers' and 'dst_peers' + """ + super().__init__() + self.named_ports = set() # used only in the original solution + self.excluded_named_ports = set() # used only in the original solution + self.base_peer_set = base_peer_set + for dim in self.dimensions_list: + self[dim] = DimensionsManager().get_dimension_domain_by_name(dim) + + def copy(self): + res = ConnectivityCube(self.base_peer_set.copy()) + for dim_name, dim_value in self.items(): + if isinstance(self[dim_name], MinDFA): + res.set_dim_directly(dim_name, dim_value) + else: + res.set_dim_directly(dim_name, dim_value.copy()) + return res + + @staticmethod + def get_empty_dim(dim_name): + return ConnectivityCube.external_empty_dim_values.get(dim_name) + + def is_empty_dim(self, dim_name): + assert dim_name in self.dimensions_list + if dim_name == "dst_ports": # can have named ports in original solution + return self.get(dim_name) == self.internal_empty_dim_values.get(dim_name) and \ + not self.named_ports and not self.excluded_named_ports + return self.get(dim_name) == self.internal_empty_dim_values.get(dim_name) + + def is_full_dim(self, dim_name): + assert dim_name in self.dimensions_list + return self.get(dim_name) == DimensionsManager().get_dimension_domain_by_name(dim_name) + + def is_active_dim(self, dim_name): + return not self.is_full_dim(dim_name) + + def set_dim_directly(self, dim_name, dim_value): + assert dim_name in self.dimensions_list + self[dim_name] = dim_value + + def set_dim(self, dim_name, dim_value): + assert dim_name in self.dimensions_list + if dim_value is None: + return + if dim_name == "src_peers" or dim_name == "dst_peers": + # translate PeerSet to CanonicalIntervalSet + self[dim_name] = self.base_peer_set.get_peer_interval_of(dim_value) + elif dim_name == "src_ports" or dim_name == "dst_ports": + # extract port_set from PortSet + self[dim_name] = dim_value.port_set + if dim_name == "dst_ports": + self.named_ports = dim_value.named_ports + self.excluded_named_ports = dim_value.excluded_named_ports + elif dim_name == "icmp_type" or dim_name == "icmp_code": + # translate int to CanonicalIntervalSet + self[dim_name] = CanonicalIntervalSet.get_interval_set(dim_value, dim_value) + else: # the rest of dimensions do not need a translation + self[dim_name] = dim_value + + def unset_dim(self, dim_name): + assert dim_name in self.dimensions_list + self[dim_name] = DimensionsManager().get_dimension_domain_by_name(dim_name) + + def get_dim(self, dim_name): + assert dim_name in self.dimensions_list + if dim_name == "src_peers" or dim_name == "dst_peers": + if self.is_active_dim(dim_name): + # translate CanonicalIntervalSet back to PeerSet + return self.base_peer_set.get_peer_set_by_indices(self[dim_name]) + else: + return None + elif dim_name == "src_ports" or dim_name == "dst_ports": + res = PortSet() + res.add_ports(self[dim_name]) + if dim_name == "dst_ports": + res.named_ports = self.named_ports + res.excluded_named_ports = self.excluded_named_ports + return res + elif dim_name == "icmp_type" or dim_name == "icmp_code": + if self.is_active_dim(dim_name): + # translate CanonicalIntervalSet back to int + return self[dim_name].validate_and_get_single_value() + else: + return None + else: # the rest of dimensions do not need a translation + if isinstance(self[dim_name], MinDFA): + return self[dim_name] + else: + return self[dim_name].copy() # TODO - do we need this copy? + + def has_active_dim(self): + for dim in self.dimensions_list: + if self[dim] != DimensionsManager().get_dimension_domain_by_name(dim): + return True + return False + + def get_ordered_cube_and_active_dims(self): + """ + Translate the connectivity cube to an ordered cube, and compute its active dimensions + :return: tuple with: (1) cube values (2) active dimensions (3) bool indication if some dimension is empty + """ + cube = [] + active_dims = [] + has_empty_dim_value = False + # add values to cube by required order of dimensions + for dim in self.dimensions_list: + if self[dim] != DimensionsManager().get_dimension_domain_by_name(dim): + if isinstance(self[dim], MinDFA): + cube.append(self[dim]) + else: + cube.append(self[dim].copy()) # TODO - do we need this copy? + active_dims.append(dim) + has_empty_dim_value |= self.is_empty_dim(dim) + return cube, active_dims, has_empty_dim_value + + class ConnectivityProperties(CanonicalHyperCubeSet): """ - A class for holding a set of cubes, each defined over dimensions from ConnectivityProperties.dimensions_list - For UDP, SCTP protocols, the actual used dimensions are only [src_peers, dst_peers, source ports, dest ports], + A class for holding a set of cubes, each defined over dimensions from ConnectivityCube.dimensions_list + For UDP, SCTP protocols, the actual used dimensions are only [src_peers, dst_peers, src_ports, dst_ports], for TCP, it may be any of the dimensions from dimensions_list, except for icmp_type and icmp_code, for icmp data the actual used dimensions are only [src_peers, dst_peers, icmp_type, icmp_code]. + The usage of this class in the original solution: + In the original solution ConnectivityProperties do not hold src_peers, dst_peers and protocols dimensions. + First, ConnectivityProperties are built at parse time. Since peers are not a part of ConnectivityProperties, + the named ports cannot be resolved at parse time, and so are kept in named_ports and excluded_named_ports, + as explained below. + Second, at the query time, ConnectivityProperties is calculated for every pair of peers, and the named ports + are resolved. The pairs of peers and the protocols are keps in ConnectionSet class, together with + the resulting ConnectivityProperties. + + The usage of this class in the optimized solution: + In the optimized solution ConnectivityProperties potentially hold all the dimensions, including sets + of source peers and destination peers. The connectivity properties are built at the parse time for every policy. + The named ports are resolved during the construction, therefore in the optimized solution named_ports and + excluded_named_ports fields are not used. + Also, including support for (included and excluded) named ports (relevant for dest ports only). The representation with named ports is considered a mid-representation, and is required due to the late binding @@ -40,98 +208,42 @@ class ConnectivityProperties(CanonicalHyperCubeSet): (2) calico: +ve and -ve named ports, no src named ports, and no use of operators between these objects. """ - dimensions_list = ["src_peers", "dst_peers", "protocols", "src_ports", "dst_ports", "methods", "hosts", "paths", - "icmp_type", "icmp_code"] - # TODO: change constructor defaults? either all arguments in "allow all" by default, or "empty" by default - def __init__(self, source_ports=PortSet(True), dest_ports=PortSet(True), protocols=ProtocolSet(True), - methods=MethodSet(True), paths=None, hosts=None, - icmp_type_interval=DimensionsManager().get_dimension_domain_by_name('icmp_type'), - icmp_code_interval=DimensionsManager().get_dimension_domain_by_name('icmp_code'), - base_peer_set=None, - src_peers_interval=DimensionsManager().get_dimension_domain_by_name('src_peers'), - dst_peers_interval=DimensionsManager().get_dimension_domain_by_name('dst_peers'), - create_empty=False): + def __init__(self, conn_cube, create_empty=False): """ - This will create connectivity properties made of all cubes made of the input arguments ranges/regex values. + This will create connectivity properties made of the given connectivity cube. This includes tcp properties, non-tcp properties, icmp data properties. - :param PortSet source_ports: The set of source ports (as a set of intervals/ranges) - :param PortSet dest_ports: The set of target ports (as a set of intervals/ranges) - :param ProtocolSet protocols: the set of eligible protocols - :param MethodSet methods: the set of http request methods - :param MinDFA paths: The dfa of http request paths - :param MinDFA hosts: The dfa of http request hosts - :param CanonicalIntervalSet icmp_type_interval: ICMP-specific parameter (type dimension) - :param CanonicalIntervalSet icmp_code_interval: ICMP-specific parameter (code dimension) - :param PeerSet base_peer_set: the base peer set which is referenced by the indices in 'peers' - :param CanonicalIntervalSet src_peers_interval: the set of source peers - :param CanonicalIntervalSet dst_peers_interval: the set of target peers + :param ConnectivityCube conn_cube: the input connectivity cube including all dimension values, + whereas missing dimensions are represented by their default values (representing all possible values). + :param bool create_empty: whether to create an empty properties (representing logical False) """ - super().__init__(ConnectivityProperties.dimensions_list) + super().__init__(ConnectivityCube.dimensions_list) self.named_ports = {} # a mapping from dst named port (String) to src ports interval set self.excluded_named_ports = {} # a mapping from dst named port (String) to src ports interval set - self.base_peer_set = base_peer_set if base_peer_set else PeerSet() + self.base_peer_set = conn_cube.base_peer_set.copy() if create_empty: return - # create the cube from input arguments - # create a dict object that holds the values required to build the cube - dims_to_values = {"src_peers": {"value": src_peers_interval, - "is_all": src_peers_interval == - DimensionsManager().get_dimension_domain_by_name('src_peers')}, - "dst_peers": {"value": dst_peers_interval, - "is_all": dst_peers_interval == - DimensionsManager().get_dimension_domain_by_name('dst_peers')}, - "protocols": {"value": protocols, "is_all": protocols.is_whole_range()}, - "src_ports": {"value": source_ports.port_set, "is_all": source_ports.is_all()}, - "dst_ports": {"value": dest_ports.port_set, "is_all": dest_ports.is_all()}, - "methods": {"value": methods, "is_all": methods.is_whole_range()}, - "hosts": {"value": hosts, "is_all": hosts is None}, - "paths": {"value": paths, "is_all": paths is None}, - "icmp_type": {"value": icmp_type_interval, - "is_all": icmp_type_interval == - DimensionsManager().get_dimension_domain_by_name('icmp_type')}, - "icmp_code": {"value": icmp_code_interval, - "is_all": icmp_code_interval == - DimensionsManager().get_dimension_domain_by_name('icmp_code')}} - - cube, active_dims, has_empty_dim_value = self._get_cube_and_active_dims_from_input_values(dims_to_values) + cube, active_dims, has_empty_dim_value = conn_cube.get_ordered_cube_and_active_dims() + if has_empty_dim_value: + return if not active_dims: self.set_all() - elif not has_empty_dim_value: + else: self.add_cube(cube, active_dims) - # assuming named ports are only in dest, not src + # assuming named ports may be only in dst, not in src + src_ports = conn_cube.get_dim("src_ports") + dst_ports = conn_cube.get_dim("dst_ports") + assert not src_ports.named_ports and not src_ports.excluded_named_ports all_ports = PortSet.all_ports_interval.copy() - for port_name in dest_ports.named_ports: - self.named_ports[port_name] = source_ports.port_set - for port_name in dest_ports.excluded_named_ports: - # self.excluded_named_ports[port_name] = all_ports - source_ports.port_set + for port_name in dst_ports.named_ports: + self.named_ports[port_name] = src_ports.port_set + for port_name in dst_ports.excluded_named_ports: self.excluded_named_ports[port_name] = all_ports - @staticmethod - def _get_cube_and_active_dims_from_input_values(dims_to_values): - """ - Given initial values, get the matching cube and its active dimensions - :param dict dims_to_values: map from dimension name to values properties - :rtype tuple(list, list, bool) - :return: tuple with: (1) cube values (2) active dimensions (3) bool indication if some dimension is empty - """ - cube = [] - active_dims = [] - has_empty_dim_value = False - # add values to cube by required order of dimensions - for dim in ConnectivityProperties.dimensions_list: - dim_val = dims_to_values[dim]["value"] - add_to_cube = not dims_to_values[dim]["is_all"] - if add_to_cube: - cube.append(dim_val) - active_dims.append(dim) - has_empty_dim_value |= not dim_val - return cube, active_dims, has_empty_dim_value - def __bool__(self): return super().__bool__() or bool(self.named_ports) @@ -153,9 +265,24 @@ def __str__(self): def __hash__(self): return super().__hash__() + def get_connectivity_cube(self, cube): + """ + translate the ordered cube to ConnectivityCube format + :param list cube: the values of the ordered input cube + :return: the cube in ConnectivityCube format + :rtype: ConnectivityCube + """ + res = ConnectivityCube(self.base_peer_set) + for i, dim in enumerate(self.active_dimensions): + if isinstance(cube[i], MinDFA): + res.set_dim_directly(dim, cube[i]) + else: + res.set_dim_directly(dim, cube[i].copy()) + return res + def get_cube_dict(self, cube, is_txt=False): """ - represent the properties cube as dict objet, for output generation as yaml/txt format + represent the properties cube as dict object, for output generation as yaml/txt format :param list cube: the values of the input cube :param bool is_txt: flag indicating if output is for txt or yaml format :return: the cube properties in dict representation @@ -182,68 +309,6 @@ def get_cube_dict(self, cube, is_txt=False): cube_dict[dim] = values_list return cube_dict - def get_cube_dict_with_orig_values(self, cube): - """ - represent the properties cube as dict object, where the values are the original values - with which the cube was built (i.e., MethodSet, PeerSet, etc.) - :param list cube: the values of the input cube - :return: the cube properties in dict representation - :rtype: dict - """ - cube_dict = {} - for i, dim in enumerate(self.active_dimensions): - dim_values = cube[i] - dim_domain = DimensionsManager().get_dimension_domain_by_name(dim) - if dim_domain == dim_values: - continue # skip dimensions with all values allowed in a cube - if dim == 'src_ports' or dim == 'dst_ports': - values = PortSet() - values.port_set = dim_values.copy() - elif dim == 'protocols': - values = ProtocolSet() - values.set_protocols(dim_values) - elif dim == 'methods': - values = MethodSet() - values.set_methods(dim_values) - elif dim == "src_peers" or dim == "dst_peers": - values = self.base_peer_set.get_peer_set_by_indices(dim_values) - else: - values = dim_values - cube_dict[dim] = values - return cube_dict - - @staticmethod - def get_empty_dim_having_orig_type(dim): - if dim == "src_peers" or dim == "dst_peers": - return PeerSet() - if dim == "protocols": - return ProtocolSet() - if dim == "src_ports" or dim == "dst_ports": - return PortSet() - if dim == "methods": - return MethodSet() - if dim == "hosts" or dim == "paths": - return MinDFA.dfa_from_regex("") # empty DFA - if dim == "icmp_type" or dim == "icmp_code": - return CanonicalIntervalSet() - return None # unknown dimension - - @staticmethod - def get_full_dim_having_orig_type(dim): - if dim == "src_peers" or dim == "dst_peers": - return None # representing PeerSet with all possible peers - if dim == "protocols": - return ProtocolSet(True) - if dim == "src_ports" or dim == "dst_ports": - return PortSet(True) - if dim == "methods": - return MethodSet(True) - if dim == "hosts" or dim == "paths": - return DimensionsManager().get_dimension_domain_by_name(dim) - if dim == "icmp_type" or dim == "icmp_code": - return DimensionsManager().get_dimension_domain_by_name(dim) - return None # unknown dimension - def get_properties_obj(self): """ get an object for a yaml representation of the protocol's properties @@ -366,15 +431,13 @@ def convert_named_ports(self, named_ports, protocol): self.add_hole(rectangle, active_dims) def copy(self): - res = ConnectivityProperties() - # from CanonicalHyperCubeSet.copy(): + res = ConnectivityProperties(ConnectivityCube(self.base_peer_set)) for layer in self.layers: res.layers[self._copy_layer_elem(layer)] = self.layers[layer].copy() res.active_dimensions = self.active_dimensions.copy() res.named_ports = self.named_ports.copy() res.excluded_named_ports = self.excluded_named_ports.copy() - res.base_peer_set = self.base_peer_set.copy() return res def print_diff(self, other, self_name, other_name): @@ -419,133 +482,90 @@ def project_on_one_dimension(self, dim_name): :return: the projection on the given dimension, having that dimension type. or empty dimension value if the given dimension is not active """ - res = self.get_empty_dim_having_orig_type(dim_name) + res = ConnectivityCube.get_empty_dim(dim_name) if dim_name not in self.active_dimensions: return res for cube in self: - cube_dict = self.get_cube_dict_with_orig_values(cube) - values = cube_dict.get(dim_name) + conn_cube = self.get_connectivity_cube(cube) + values = conn_cube.get_dim(dim_name) if values: res |= values + return res @staticmethod - def resolve_named_port(named_port, peer, protocols): + def resolve_named_ports(named_ports, peer, protocols): peer_named_ports = peer.get_named_ports() - real_port = peer_named_ports.get(named_port) - if not real_port: - print(f'Warning: Missing named port {named_port} in the pod {peer}. Ignoring the pod') - return None - if real_port[1] not in protocols: - print(f'Warning: Illegal protocol {real_port[1]} in the named port {named_port} ' - f'of the pod {peer}. Ignoring the pod') - return None real_ports = PortSet() - real_ports.add_port(real_port[0]) + for named_port in named_ports: + real_port = peer_named_ports.get(named_port) + if not real_port: + print(f'Warning: Missing named port {named_port} in the pod {peer}. Ignoring the pod') + continue + if real_port[1] not in protocols: + print(f'Warning: Illegal protocol {real_port[1]} in the named port {named_port} ' + f'of the pod {peer}. Ignoring the pod') + continue + real_ports.add_port(real_port[0]) return real_ports @staticmethod - def make_conn_props(peer_container, src_ports=PortSet(True), dst_ports=PortSet(True), - protocols=ProtocolSet(True), src_peers=None, dst_peers=None, - paths_dfa=None, hosts_dfa=None, methods=MethodSet(True), - icmp_type_interval=DimensionsManager().get_dimension_domain_by_name('icmp_type'), - icmp_code_interval=DimensionsManager().get_dimension_domain_by_name('icmp_code')): + def make_conn_props(conn_cube): """ - get ConnectivityProperties with allowed connections, corresponding to input properties cube. + This will create connectivity properties made of the given connectivity cube. + This includes tcp properties, non-tcp properties, icmp data properties. + If possible (i.e., in original solution, when dst_peers are supported), the named ports will be resolved. + In the optimized solution, the resulting ConnectivityProperties should not contain named ports: they are substituted with corresponding port numbers, per peer In the original solution, the resulting ConnectivityProperties may contain named ports; - they cannot yet be resolved, since there are not dst_peer in the original solution; + they cannot yet be resolved, since dst peers are not provided at this stage the original solution; they will be resolved by convert_named_ports call during query runs. - :param PeerContainer peer_container: The set of endpoints and their namespaces - :param PortSet src_ports: ports set for src_ports dimension (possibly containing named ports) - :param PortSet dst_ports: ports set for dst_ports dimension (possibly containing named ports) - :param ProtocolSet protocols: CanonicalIntervalSet obj for protocols dimension - :param PeerSet src_peers: the set of source peers - :param PeerSet dst_peers: the set of target peers - :param MinDFA paths_dfa: MinDFA obj for paths dimension - :param MinDFA hosts_dfa: MinDFA obj for hosts dimension - :param MethodSet methods: CanonicalIntervalSet obj for methods dimension - :param CanonicalIntervalSet icmp_type_interval: ICMP-specific parameter (type dimension) - :param CanonicalIntervalSet icmp_code_interval: ICMP-specific parameter (code dimension) - :return: ConnectivityProperties with allowed connections, corresponding to input properties cube + + :param ConnectivityCube conn_cube: the input connectivity cube including all dimension values, + whereas missing dimensions are represented by their default values (representing all possible values). """ - base_peer_set = peer_container.peer_set.copy() - if src_peers: - base_peer_set |= src_peers - if dst_peers: - base_peer_set |= dst_peers - if src_peers: - src_peers_interval = base_peer_set.get_peer_interval_of(src_peers) - else: - src_peers_interval = DimensionsManager().get_dimension_domain_by_name('src_peers') - if dst_peers: - dst_peers_interval = base_peer_set.get_peer_interval_of(dst_peers) - else: - dst_peers_interval = DimensionsManager().get_dimension_domain_by_name('dst_peers') - assert not src_ports.named_ports - if not dst_ports.named_ports or not dst_peers: + src_ports = conn_cube.get_dim("src_ports") + dst_ports = conn_cube.get_dim("dst_ports") + dst_peers = conn_cube.get_dim("dst_peers") + assert not src_ports.named_ports and not src_ports.excluded_named_ports + if (not dst_ports.named_ports and not dst_ports.excluded_named_ports ) or not dst_peers: # Should not resolve named ports - return ConnectivityProperties(source_ports=src_ports, dest_ports=dst_ports, - protocols=protocols, methods=methods, paths=paths_dfa, hosts=hosts_dfa, - icmp_type_interval=icmp_type_interval, icmp_code_interval=icmp_code_interval, - src_peers_interval=src_peers_interval, dst_peers_interval=dst_peers_interval, - base_peer_set=base_peer_set) + return ConnectivityProperties(conn_cube) + + # Initialize conn_properties + if dst_ports.port_set: + dst_ports_no_named_ports = dst_ports.copy() + dst_ports_no_named_ports.named_ports = set() + dst_ports_no_named_ports.excluded_named_ports = set() + conn_cube.set_dim("dst_ports", dst_ports_no_named_ports) + conn_properties = ConnectivityProperties(conn_cube) + else: + conn_properties = ConnectivityProperties.make_empty_props() + # Resolving dst named ports - conn_properties = ConnectivityProperties.make_empty_props() + protocols = conn_cube.get_dim("protocols") assert dst_peers - assert not dst_ports.port_set - assert not dst_ports.excluded_named_ports - assert len(dst_ports.named_ports) == 1 - port = list(dst_ports.named_ports)[0] for peer in dst_peers: - real_ports = ConnectivityProperties.resolve_named_port(port, peer, protocols) + real_ports = ConnectivityProperties.resolve_named_ports(dst_ports.named_ports, peer, protocols) + excluded_real_ports = ConnectivityProperties.resolve_named_ports(dst_ports.excluded_named_ports, peer, protocols) + real_ports -= excluded_real_ports if not real_ports: continue - props = ConnectivityProperties(source_ports=src_ports, dest_ports=real_ports, - protocols=protocols, methods=methods, - paths=paths_dfa, hosts=hosts_dfa, - icmp_type_interval=icmp_type_interval, icmp_code_interval=icmp_code_interval, - src_peers_interval=src_peers_interval, - dst_peers_interval=base_peer_set.get_peer_interval_of(PeerSet({peer})), - base_peer_set=base_peer_set) + conn_cube.set_dim("dst_ports", real_ports) + conn_cube.set_dim("dst_peers", PeerSet({peer})) + props = ConnectivityProperties(conn_cube) conn_properties |= props return conn_properties - @staticmethod - def make_conn_props_from_dict(peer_container, cube_dict): - """ - Create ConnectivityProperties from the given cube - :param PeerContainer peer_container: the set of all peers - :param dict cube_dict: the given cube represented as a dictionary - :return: ConnectivityProperties - """ - cube_dict_copy = cube_dict.copy() - src_ports = cube_dict_copy.pop("src_ports", ConnectivityProperties.get_full_dim_having_orig_type("src_ports")) - dst_ports = cube_dict_copy.pop("dst_ports", ConnectivityProperties.get_full_dim_having_orig_type("dst_ports")) - protocols = cube_dict_copy.pop("protocols", ConnectivityProperties.get_full_dim_having_orig_type("protocols")) - src_peers = cube_dict_copy.pop("src_peers", ConnectivityProperties.get_full_dim_having_orig_type("src_peers")) - dst_peers = cube_dict_copy.pop("dst_peers", ConnectivityProperties.get_full_dim_having_orig_type("dst_peers")) - paths_dfa = cube_dict_copy.pop("paths", ConnectivityProperties.get_full_dim_having_orig_type("paths")) - hosts_dfa = cube_dict_copy.pop("hosts", ConnectivityProperties.get_full_dim_having_orig_type("hosts")) - methods = cube_dict_copy.pop("methods", ConnectivityProperties.get_full_dim_having_orig_type("methods")) - icmp_type_interval = cube_dict_copy.pop("icmp_type", ConnectivityProperties.get_full_dim_having_orig_type("icmp_type")) - icmp_code_interval = cube_dict_copy.pop("icmp_code", ConnectivityProperties.get_full_dim_having_orig_type("icmp_code")) - assert not cube_dict_copy - return ConnectivityProperties.make_conn_props(peer_container, src_ports=src_ports, dst_ports=dst_ports, - protocols=protocols, src_peers=src_peers, dst_peers=dst_peers, - paths_dfa=paths_dfa, hosts_dfa=hosts_dfa, methods=methods, - icmp_type_interval=icmp_type_interval, - icmp_code_interval=icmp_code_interval) - @staticmethod def make_empty_props(): """ Returns empty connectivity properties, representing logical False :return: ConnectivityProperties """ - return ConnectivityProperties(create_empty=True) + return ConnectivityProperties(ConnectivityCube(PeerSet()), create_empty=True) @staticmethod def make_all_props(): @@ -553,7 +573,7 @@ def make_all_props(): Returns all connectivity properties, representing logical True :return: ConnectivityProperties """ - return ConnectivityProperties() + return ConnectivityProperties(ConnectivityCube(PeerSet())) def are_auto_conns(self): """ @@ -574,61 +594,3 @@ def are_auto_conns(self): if cube[src_peers_index] != cube[dst_peers_index] or not cube[src_peers_index].is_single_value(): return False return True - - # ICMP-related functions - @staticmethod - def make_icmp_props(peer_container, protocol="", src_peers=None, dst_peers=None, - icmp_type=None, icmp_code=None): - """ - This function is for icmp data only. It creates icmp properties from the given src/dst peers, protocol, - icmp type and icmp code. - :param peer_container: the peer container as a base for al src/dst peers. - :param protocol: the protocol - :param src_peers: source peers - :param dst_peers: destination peers - :param icmp_type: icmp type - :param icmp_code: icmp code - :return: the resulting ConnectivityProperties - """ - if protocol: - icmp_protocol_set = ProtocolSet() - icmp_protocol_set.add_protocol(ProtocolNameResolver.get_protocol_number(protocol)) - else: - icmp_protocol_set = ProtocolSet(True) - icmp_type_interval = DimensionsManager().get_dimension_domain_by_name('icmp_type') - icmp_code_interval = DimensionsManager().get_dimension_domain_by_name('icmp_code') - if icmp_type: - icmp_type_interval = CanonicalIntervalSet.get_interval_set(icmp_type, icmp_type) - if icmp_code: - icmp_code_interval = CanonicalIntervalSet.get_interval_set(icmp_code, icmp_code) - return ConnectivityProperties.make_conn_props(peer_container=peer_container, protocols=icmp_protocol_set, - src_peers=src_peers, dst_peers=dst_peers, - icmp_type_interval=icmp_type_interval, - icmp_code_interval=icmp_code_interval) - - @staticmethod - def make_all_but_given_icmp_props(peer_container, protocol="", src_peers=None, dst_peers=None, - icmp_type=None, icmp_code=None): - """ - This function is for icmp data only. It creates icmp properties representing the negation of - the properties having the given src/dst peers, protocol, icmp type and icmp code. - :param peer_container: the peer container as a base for al src/dst peers. - :param protocol: the protocol - :param src_peers: source peers - :param dst_peers: destination peers - :param icmp_type: icmp type - :param icmp_code: icmp code - :return: the resulting ConnectivityProperties - """ - if protocol: - icmp_protocol_set = ProtocolSet() - icmp_protocol_set.add_protocol(ProtocolNameResolver.get_protocol_number(protocol)) - else: - icmp_protocol_set = ProtocolSet(True) - all_icmp_props = ConnectivityProperties.make_conn_props(peer_container=peer_container, - protocols=icmp_protocol_set, - src_peers=src_peers, dst_peers=dst_peers) - given_icmp_props = ConnectivityProperties.make_icmp_props(peer_container, protocol=protocol, - src_peers=src_peers, dst_peers=dst_peers, - icmp_type=icmp_type, icmp_code=icmp_code, ) - return all_icmp_props - given_icmp_props diff --git a/nca/CoreDS/PortSet.py b/nca/CoreDS/PortSet.py index 938bc1dbb..1aeb07a61 100644 --- a/nca/CoreDS/PortSet.py +++ b/nca/CoreDS/PortSet.py @@ -56,6 +56,9 @@ def add_port(self, port): interval = CanonicalIntervalSet.Interval(port, port) self.port_set.add_interval(interval) + def add_ports(self, port_set): + self.port_set |= port_set + def remove_port(self, port): if isinstance(port, str): self.named_ports.discard(port) diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index d9cde039c..5d092f271 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -6,7 +6,7 @@ import itertools from collections import defaultdict import networkx -from nca.CoreDS.Peer import IpBlock, PeerSet, ClusterEP, Pod +from nca.CoreDS.Peer import IpBlock, ClusterEP, Pod from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.ProtocolSet import ProtocolSet from nca.CoreDS.ConnectivityProperties import ConnectivityProperties @@ -68,42 +68,31 @@ def add_edges(self, connections): """ self.connections_to_peers.update(connections) - def add_edges_from_cube_dict(self, peer_container, cube_dict, ip_blocks_mask): + def add_edges_from_cube_dict(self, conn_cube, ip_blocks_mask): """ Add edges to the graph according to the give cube - :param peer_container: the peer_container containing all possible peers - :param dict cube_dict: the given cube in dictionary format + :param ConnectivityCube conn_cube: the given cube :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, whereas all other values should be filtered out in the output """ - new_cube_dict = cube_dict.copy() - src_peers = new_cube_dict.get('src_peers') - if src_peers: - new_cube_dict.pop('src_peers') - else: - src_peers = peer_container.get_all_peers_group(True) - dst_peers = new_cube_dict.get('dst_peers') - if dst_peers: - new_cube_dict.pop('dst_peers') - else: - dst_peers = peer_container.get_all_peers_group(True) + src_peers = conn_cube.get_dim("src_peers") + conn_cube.unset_dim("src_peers") + dst_peers = conn_cube.get_dim("dst_peers") + conn_cube.unset_dim("dst_peers") if IpBlock.get_all_ips_block() != ip_blocks_mask: src_peers.filter_ipv6_blocks(ip_blocks_mask) dst_peers.filter_ipv6_blocks(ip_blocks_mask) + protocols = conn_cube.get_dim("protocols") + conn_cube.unset_dim("protocols") - protocols = new_cube_dict.get('protocols') - if protocols: - new_cube_dict.pop('protocols') - - if not protocols and not new_cube_dict: + if not protocols and not conn_cube.has_active_dim(): conns = ConnectionSet(True) else: conns = ConnectionSet() protocol_names = ProtocolSet.get_protocol_names_from_interval_set(protocols) if protocols else ['TCP'] for protocol in protocol_names: - if new_cube_dict: - conns.add_connections(protocol, ConnectivityProperties.make_conn_props_from_dict(peer_container, - new_cube_dict)) + if conn_cube.has_active_dim(): + conns.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) else: if ConnectionSet.protocol_supports_ports(protocol): conns.add_connections(protocol, ConnectivityProperties.make_all_props()) @@ -448,39 +437,6 @@ def get_connectivity_dot_format_str(self, connectivity_restriction=None): dot_graph.add_edge(src_name=edge[0][0], dst_name=edge[1][0], label=conn_str, is_dir=False) return dot_graph.to_str() - def convert_to_tcp_like_properties(self, peer_container): - """ - Used for testing of the optimized solution: converting connectivity graph back to ConnectivityProperties - :param peer_container: The peer container - :return: ConnectivityProperties representing the connectivity graph - """ - res = ConnectivityProperties.make_empty_props() - for item in self.connections_to_peers.items(): - if item[0].allow_all: - for peer_pair in item[1]: - res |= ConnectivityProperties.make_conn_props(peer_container, - src_peers=PeerSet({peer_pair[0]}), - dst_peers=PeerSet({peer_pair[1]})) - else: - for prot in item[0].allowed_protocols.items(): - protocols = ProtocolSet() - protocols.add_protocol(prot[0]) - if isinstance(prot[1], bool): - for peer_pair in item[1]: - res |= ConnectivityProperties.make_conn_props(peer_container, protocols=protocols, - src_peers=PeerSet({peer_pair[0]}), - dst_peers=PeerSet({peer_pair[1]})) - continue - for cube in prot[1]: - cube_dict = prot[1].get_cube_dict_with_orig_values(cube) - cube_dict["protocols"] = protocols - for peer_pair in item[1]: - new_cube_dict = cube_dict.copy() - new_cube_dict["src_peers"] = PeerSet({peer_pair[0]}) - new_cube_dict["dst_peers"] = PeerSet({peer_pair[1]}) - res |= ConnectivityProperties.make_conn_props_from_dict(peer_container, new_cube_dict) - return res - def get_minimized_firewall_rules(self): """ computes and returns minimized firewall rules from original connectivity graph diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 1a94d89d6..bb41a6a11 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -12,7 +12,7 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import PeerSet, IpBlock, Pod, Peer from nca.CoreDS.ProtocolSet import ProtocolSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube from nca.FWRules.ConnectivityGraph import ConnectivityGraph from nca.FWRules.MinimizeFWRules import MinimizeFWRules from nca.FWRules.ClusterInfo import ClusterInfo @@ -753,8 +753,12 @@ def exec(self): # noqa: C901 all_conns_opt.project_on_one_dimension('dst_peers') subset_peers = self.compute_subset(opt_peers_to_compare) - subset_conns = ConnectivityProperties.make_conn_props(self.config.peer_container, src_peers=subset_peers) | \ - ConnectivityProperties.make_conn_props(self.config.peer_container, dst_peers=subset_peers) + src_peers_conn_cube = ConnectivityCube(self.config.peer_container.get_all_peers_group()) + dst_peers_conn_cube = ConnectivityCube(self.config.peer_container.get_all_peers_group()) + src_peers_conn_cube.set_dim("src_peers", subset_peers) + dst_peers_conn_cube.set_dim("dst_peers", subset_peers) + subset_conns = ConnectivityProperties.make_conn_props(src_peers_conn_cube) | \ + ConnectivityProperties.make_conn_props(dst_peers_conn_cube) all_conns_opt &= subset_conns ip_blocks_mask = IpBlock.get_all_ips_block() if exclude_ipv6: @@ -978,8 +982,7 @@ def dot_format_from_props(self, props, peers, ip_blocks_mask, connectivity_restr """ conn_graph = ConnectivityGraph(peers, self.config.get_allowed_labels(), self.output_config) for cube in props: - conn_graph.add_edges_from_cube_dict(self.config.peer_container, props.get_cube_dict_with_orig_values(cube), - ip_blocks_mask) + conn_graph.add_edges_from_cube_dict(props.get_connectivity_cube(cube), ip_blocks_mask) return conn_graph.get_connectivity_dot_format_str(connectivity_restriction) def fw_rules_from_connections_dict(self, connections, peers_to_compare, connectivity_restriction=None): @@ -1058,8 +1061,9 @@ def convert_props_to_split_by_tcp(self, props): """ tcp_protocol = ProtocolSet() tcp_protocol.add_protocol('TCP') - tcp_props = props & ConnectivityProperties.make_conn_props(self.config.peer_container, - protocols=tcp_protocol) + conn_cube = ConnectivityCube(self.config.peer_container.get_all_peers_group()) + conn_cube.set_dim("protocols", tcp_protocol) + tcp_props = props & ConnectivityProperties.make_conn_props(conn_cube) non_tcp_props = props - tcp_props return tcp_props, non_tcp_props diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index b07740ccc..4dc2b8b99 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -7,7 +7,7 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock, HostEP, PeerSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy from nca.Resources.NetworkPolicy import PolicyConnections, NetworkPolicy @@ -176,16 +176,20 @@ def allowed_connections_optimized(self, peer_container): """ all_pods = peer_container.get_all_peers_group() all_ips_peer_set = PeerSet({IpBlock.get_all_ips_block()}) + conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) + conn_cube.set_dim("src_peers", all_pods) + conn_cube.set_dim("dst_peers", all_ips_peer_set) allowed_ingress_conns, denied_ingres_conns = self._allowed_xgress_conns_optimized(True, peer_container) - allowed_ingress_conns |= ConnectivityProperties.make_conn_props(peer_container, src_peers=all_pods, - dst_peers=all_ips_peer_set) + allowed_ingress_conns |= ConnectivityProperties.make_conn_props(conn_cube) allowed_egress_conns, denied_egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) - allowed_egress_conns |= ConnectivityProperties.make_conn_props(peer_container, src_peers=all_ips_peer_set, - dst_peers=all_pods) + conn_cube.set_dim("src_peers", all_ips_peer_set) + conn_cube.set_dim("dst_peers", all_pods) + allowed_egress_conns |= ConnectivityProperties.make_conn_props(conn_cube) res = allowed_ingress_conns & allowed_egress_conns # exclude IpBlock->IpBlock connections - excluded_conns = ConnectivityProperties.make_conn_props(peer_container, src_peers=all_ips_peer_set, - dst_peers=all_ips_peer_set) + conn_cube.set_dim("src_peers", all_ips_peer_set) + conn_cube.set_dim("dst_peers", all_ips_peer_set) + excluded_conns = ConnectivityProperties.make_conn_props(conn_cube) res -= excluded_conns return res @@ -295,16 +299,14 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): base_peer_set_no_hep = PeerSet(set([peer for peer in base_peer_set_no_ip if not isinstance(peer, HostEP)])) non_captured = base_peer_set_no_hep - captured if non_captured: + conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) if is_ingress: - non_captured_conns = \ - ConnectivityProperties.make_conn_props(peer_container, - src_peers=base_peer_set_with_ip, - dst_peers=non_captured) + conn_cube.set_dim("src_peers", base_peer_set_with_ip) + conn_cube.set_dim("dst_peers", non_captured) else: - non_captured_conns = \ - ConnectivityProperties.make_conn_props(peer_container, - src_peers=non_captured, - dst_peers=base_peer_set_with_ip) + conn_cube.set_dim("src_peers", non_captured) + conn_cube.set_dim("dst_peers", base_peer_set_with_ip) + non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conn |= non_captured_conns return allowed_conn, denied_conns @@ -335,22 +337,20 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): base_peer_set_with_ip = peer_container.get_all_peers_group(True) base_peer_set_no_ip = peer_container.get_all_peers_group() non_captured_peers = base_peer_set_no_ip - captured + conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) if non_captured_peers: if is_ingress: - non_captured_conns = \ - ConnectivityProperties.make_conn_props(peer_container, - src_peers=base_peer_set_with_ip, - dst_peers=non_captured_peers) + conn_cube.set_dim("src_peers", base_peer_set_with_ip) + conn_cube.set_dim("dst_peers", non_captured_peers) else: - non_captured_conns = \ - ConnectivityProperties.make_conn_props(peer_container, - src_peers=non_captured_peers, - dst_peers=base_peer_set_with_ip) + conn_cube.set_dim("src_peers", non_captured_peers) + conn_cube.set_dim("dst_peers", base_peer_set_with_ip) + non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conn |= (non_captured_conns - denied_conns) - allowed_conn |= ConnectivityProperties.make_conn_props(peer_container, - protocols=ProtocolSet.get_non_tcp_protocols(), - src_peers=base_peer_set_with_ip, - dst_peers=base_peer_set_with_ip) + conn_cube.set_dim("src_peers", base_peer_set_with_ip) + conn_cube.set_dim("dst_peers", base_peer_set_with_ip) + conn_cube.set_dim("protocols", ProtocolSet.get_non_tcp_protocols()) + allowed_conn |= ConnectivityProperties.make_conn_props(conn_cube) return allowed_conn, denied_conns @@ -371,18 +371,17 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): allowed_conn, denied_conns, captured = self.collect_policies_conns_optimized(is_ingress) base_peer_set_with_ip = peer_container.get_all_peers_group(True) base_peer_set_no_ip = peer_container.get_all_peers_group() + conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) if is_ingress: - non_captured_conns = \ - ConnectivityProperties.make_conn_props(peer_container, - src_peers=base_peer_set_with_ip, - dst_peers=base_peer_set_no_ip) + conn_cube.set_dim("src_peers", base_peer_set_with_ip) + conn_cube.set_dim("dst_peers", base_peer_set_no_ip) + non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conn |= non_captured_conns else: non_captured_peers = base_peer_set_no_ip - captured if non_captured_peers: - non_captured_conns = \ - ConnectivityProperties.make_conn_props(peer_container, - src_peers=non_captured_peers, - dst_peers=base_peer_set_with_ip) + conn_cube.set_dim("src_peers", non_captured_peers) + conn_cube.set_dim("dst_peers", base_peer_set_with_ip) + non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conn |= non_captured_conns return allowed_conn, denied_conns diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index 386903b64..684d4df30 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -7,7 +7,7 @@ from nca.CoreDS.ProtocolNameResolver import ProtocolNameResolver from nca.CoreDS.Peer import PeerSet, IpBlock from nca.CoreDS.PortSet import PortSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube from nca.CoreDS.ProtocolSet import ProtocolSet from nca.CoreDS.ICMPDataSet import ICMPDataSet from nca.CoreDS.ConnectionSet import ConnectionSet @@ -358,10 +358,10 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): :param: str protocol: the ICMP-like protocol :param PeerSet src_pods: the source pods :param PeerSet dst_pods: the destination pods - :return: a tuple (ICMPDataSet, ConnectivityProperties), - where ICMPDataSet is an object representing the allowed ICMP connections, - ConnectivityProperties is an optimized-format ICMP connections, including src and dst pods. - :rtype: tuple (ICMPDataSet, ConnectivityProperties) + :return: a tuple (ConnectivityProperties, ConnectivityProperties), + where the first ConnectivityProperties is an original-format ICMP connections, + and the second ConnectivityProperties is an optimized-format ICMP connections, including src and dst pods. + :rtype: tuple (ConnectivityProperties, ConnectivityProperties) """ icmp_type = icmp_data.get('type') if icmp_data is not None else None icmp_code = icmp_data.get('code') if icmp_data is not None else None @@ -380,43 +380,56 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): if err: self.syntax_error(err, not_icmp_data) - res = ConnectivityProperties.make_icmp_props(self.peer_container) + protocols = ProtocolSet() + protocols.add_protocol(protocol) + base_peer_set = self.peer_container.get_all_peers_group() + conn_cube = ConnectivityCube(base_peer_set) + if icmp_type: + conn_cube.set_dim("icmp_type", icmp_type) + if icmp_code: + conn_cube.set_dim("icmp_code", icmp_code) + not_conn_cube = ConnectivityCube(base_peer_set) + if not_icmp_type: + not_conn_cube.set_dim("icmp_type", not_icmp_type) + if not_icmp_code: + not_conn_cube.set_dim("icmp_code", not_icmp_code) + opt_conn_cube = conn_cube.copy() + opt_not_conn_cube = not_conn_cube.copy() + if self.optimized_run != 'false': + opt_conn_cube.set_dim("src_peers", src_pods) + opt_conn_cube.set_dim("dst_peers", dst_pods) + opt_conn_cube.set_dim("protocols", protocols) + opt_not_conn_cube.set_dim("src_peers", src_pods) + opt_not_conn_cube.set_dim("dst_peers", dst_pods) + opt_not_conn_cube.set_dim("protocols", protocols) + + res = ConnectivityProperties.make_empty_props() opt_props = ConnectivityProperties.make_empty_props() - if self.optimized_run != 'false' and src_pods and dst_pods: - opt_props = ConnectivityProperties.make_icmp_props(self.peer_container, protocol=protocol, - src_peers=src_pods, dst_peers=dst_pods) if icmp_data is not None: - res = ConnectivityProperties.make_icmp_props(self.peer_container, icmp_type=icmp_type, icmp_code=icmp_code) - if self.optimized_run != 'false' and src_pods and dst_pods: - opt_props = ConnectivityProperties.make_icmp_props(self.peer_container, protocol=protocol, - src_peers=src_pods, dst_peers=dst_pods, - icmp_type=icmp_type, icmp_code=icmp_code) + res = ConnectivityProperties.make_conn_props(conn_cube) + if self.optimized_run != 'false': + opt_props = ConnectivityProperties.make_conn_props(opt_conn_cube) if not_icmp_data is not None: if icmp_type == not_icmp_type and icmp_code == not_icmp_code: res = ConnectivityProperties.make_empty_props() self.warning('icmp and notICMP are conflicting - no traffic will be matched', not_icmp_data) elif icmp_type == not_icmp_type and icmp_code is None: - tmp = ConnectivityProperties.make_icmp_props(self.peer_container, icmp_type=not_icmp_type, - icmp_code=not_icmp_code) + tmp = ConnectivityProperties.make_conn_props(not_conn_cube) res -= tmp if self.optimized_run != 'false': - tmp_opt_props = ConnectivityProperties.make_icmp_props(self.peer_container, protocol=protocol, - src_peers=src_pods, dst_peers=dst_pods, - icmp_type=not_icmp_type, - icmp_code=not_icmp_code) + tmp_opt_props = ConnectivityProperties.make_conn_props(opt_not_conn_cube) opt_props -= tmp_opt_props else: self.warning('notICMP has no effect', not_icmp_data) elif not_icmp_data is not None: - res = ConnectivityProperties.make_all_but_given_icmp_props(self.peer_container, - icmp_type=not_icmp_type, - icmp_code=not_icmp_code) - if self.optimized_run != 'false' and src_pods and dst_pods: - opt_props = ConnectivityProperties.make_all_but_given_icmp_props(self.peer_container, protocol=protocol, - src_peers=src_pods, dst_peers=dst_pods, - icmp_type=not_icmp_type, - icmp_code=not_icmp_code) - + res = ConnectivityProperties.make_conn_props(conn_cube) - \ + ConnectivityProperties.make_conn_props(not_conn_cube) + if self.optimized_run != 'false': + opt_props = ConnectivityProperties.make_conn_props(opt_conn_cube) - \ + ConnectivityProperties.make_conn_props(opt_not_conn_cube) + else: # no icmp_data or no_icmp_data; only protocol + res = ConnectivityProperties.make_conn_props(conn_cube) + opt_props = ConnectivityProperties.make_conn_props(opt_conn_cube) return res, opt_props def _parse_protocol(self, protocol, rule): @@ -493,46 +506,45 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): self.warning('notProtocol field has no effect', rule) else: if protocol_supports_ports: - connections.add_connections(protocol, ConnectivityProperties.make_conn_props( - self.peer_container, src_ports=src_res_ports, dst_ports=dst_res_ports)) - if self.optimized_run != 'false' and src_res_pods and dst_res_pods: - src_num_port_set = PortSet() - src_num_port_set.port_set = src_res_ports.port_set.copy() - dst_num_port_set = PortSet() - dst_num_port_set.port_set = dst_res_ports.port_set.copy() - conn_props = ConnectivityProperties.make_conn_props(self.peer_container, - src_ports=src_num_port_set, - dst_ports=dst_num_port_set, - protocols=protocols, - src_peers=src_res_pods, - dst_peers=dst_res_pods) + conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) + conn_cube.set_dim("src_ports", src_res_ports) + conn_cube.set_dim("dst_ports", dst_res_ports) + connections.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) + if self.optimized_run != 'false': + conn_cube.set_dim("protocols", protocols) + conn_cube.set_dim("src_peers", src_res_pods) + conn_cube.set_dim("dst_peers", dst_res_pods) + conn_props = ConnectivityProperties.make_conn_props(conn_cube) elif ConnectionSet.protocol_is_icmp(protocol): icmp_props, conn_props = self._parse_icmp(rule.get('icmp'), rule.get('notICMP'), protocol, src_res_pods, dst_res_pods) connections.add_connections(protocol, icmp_props) else: connections.add_connections(protocol, True) - if self.optimized_run != 'false' and src_res_pods and dst_res_pods: - conn_props = ConnectivityProperties.make_conn_props(self.peer_container, - protocols=protocols, - src_peers=src_res_pods, - dst_peers=dst_res_pods) + if self.optimized_run != 'false': + conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) + conn_cube.set_dim("protocols", protocols) + conn_cube.set_dim("src_peers", src_res_pods) + conn_cube.set_dim("dst_peers", dst_res_pods) + conn_props = ConnectivityProperties.make_conn_props(conn_cube) elif not_protocol is not None: connections.add_all_connections() connections.remove_protocol(not_protocol) if self.optimized_run != 'false' and src_res_pods and dst_res_pods: protocols = ProtocolSet(True) protocols.remove_protocol(not_protocol) - conn_props = ConnectivityProperties.make_conn_props(self.peer_container, - protocols=protocols, - src_peers=src_res_pods, - dst_peers=dst_res_pods) + conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) + conn_cube.set_dim("protocols", protocols) + conn_cube.set_dim("src_peers", src_res_pods) + conn_cube.set_dim("dst_peers", dst_res_pods) + conn_props = ConnectivityProperties.make_conn_props(conn_cube) else: connections.allow_all = True - if self.optimized_run != 'false' and src_res_pods and dst_res_pods: - conn_props = ConnectivityProperties.make_conn_props(self.peer_container, - src_peers=src_res_pods, - dst_peers=dst_res_pods) + if self.optimized_run != 'false': + conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) + conn_cube.set_dim("src_peers", src_res_pods) + conn_cube.set_dim("dst_peers", dst_res_pods) + conn_props = ConnectivityProperties.make_conn_props(conn_cube) self._verify_named_ports(rule, dst_res_pods, connections) if not src_res_pods and policy_selected_eps and (is_ingress or not is_profile): diff --git a/nca/Parsers/GenericYamlParser.py b/nca/Parsers/GenericYamlParser.py index 1336a06f8..7389f0fc1 100644 --- a/nca/Parsers/GenericYamlParser.py +++ b/nca/Parsers/GenericYamlParser.py @@ -6,7 +6,7 @@ from sys import stderr from enum import Enum from nca.CoreDS.DimensionsManager import DimensionsManager -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube from nca.CoreDS.MethodSet import MethodSet from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock @@ -237,9 +237,14 @@ def _get_connection_set_from_properties(peer_container, dest_ports, methods=Meth :param MinDFA hosts_dfa: MinDFA obj for hosts dimension :return: ConnectionSet with allowed connections , corresponding to input properties cube """ - tcp_properties = ConnectivityProperties.make_conn_props(peer_container, dst_ports=dest_ports, - methods=methods, paths_dfa=paths_dfa, - hosts_dfa=hosts_dfa) + conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) + conn_cube.set_dim("dst_ports", dest_ports) + conn_cube.set_dim("methods", methods) + if paths_dfa: + conn_cube.set_dim("paths", paths_dfa) + if hosts_dfa: + conn_cube.set_dim("hosts", hosts_dfa) + tcp_properties = ConnectivityProperties.make_conn_props(conn_cube) res = ConnectionSet() res.add_connections('TCP', tcp_properties) return res diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index 1c3193df4..deca1de3f 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -8,7 +8,7 @@ from nca.CoreDS.DimensionsManager import DimensionsManager from nca.CoreDS.Peer import PeerSet from nca.CoreDS.PortSet import PortSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.IngressPolicy import IngressPolicy from nca.Resources.NetworkPolicy import NetworkPolicy @@ -187,18 +187,16 @@ def _make_default_connections(self, hosts_dfa, paths_dfa=None): """ default_conns = ConnectivityProperties.make_empty_props() if self.default_backend_peers: + conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) + conn_cube.set_dim("dst_ports", self.default_backend_ports) + conn_cube.set_dim("dst_peers", self.default_backend_peers) + if hosts_dfa: + conn_cube.set_dim("hosts", hosts_dfa) if paths_dfa: - default_conns = \ - ConnectivityProperties.make_conn_props(self.peer_container, - dst_ports=self.default_backend_ports, - dst_peers=self.default_backend_peers, - paths_dfa=paths_dfa, hosts_dfa=hosts_dfa) + conn_cube.set_dim("paths", paths_dfa) + default_conns = ConnectivityProperties.make_conn_props(conn_cube) else: - default_conns = \ - ConnectivityProperties.make_conn_props(self.peer_container, - dst_ports=self.default_backend_ports, - dst_peers=self.default_backend_peers, - hosts_dfa=hosts_dfa) + default_conns = ConnectivityProperties.make_conn_props(conn_cube) return default_conns def parse_rule(self, rule): @@ -226,11 +224,14 @@ def parse_rule(self, rule): parsed_paths.append(path_resources) if parsed_paths: parsed_paths_with_dfa = self.segregate_longest_paths_and_make_dfa(parsed_paths) + conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) + conn_cube.set_dim("hosts", hosts_dfa) for (_, paths_dfa, _, peers, ports) in parsed_paths_with_dfa: # every path is converted to allowed connections - conns = ConnectivityProperties.make_conn_props(self.peer_container, dst_ports=ports, - dst_peers=peers, paths_dfa=paths_dfa, - hosts_dfa=hosts_dfa) + conn_cube.set_dim("dst_ports", ports) + conn_cube.set_dim("dst_peers", peers) + conn_cube.set_dim("paths", paths_dfa) + conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conns |= conns if not all_paths_dfa: all_paths_dfa = paths_dfa @@ -293,8 +294,10 @@ def parse_policy(self): res_policy.add_rules(self._make_allow_rules(allowed_conns)) protocols = ProtocolSet() protocols.add_protocol('TCP') - allowed_conns &= ConnectivityProperties.make_conn_props(self.peer_container, protocols=protocols, - src_peers=res_policy.selected_peers) + conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) + conn_cube.set_dim("protocols", protocols) + conn_cube.set_dim("src_peers", res_policy.selected_peers) + allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs return res_policy diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index aac0f4cb2..eb0b46416 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -10,7 +10,7 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.PortSet import PortSet from nca.CoreDS.MethodSet import MethodSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy, IstioPolicyRule from nca.Resources.IstioTrafficResources import istio_root_namespace from nca.Resources.NetworkPolicy import NetworkPolicy @@ -519,8 +519,10 @@ def parse_ingress_rule(self, rule, selected_peers): if not res_peers or not selected_peers: condition_props = ConnectivityProperties.make_empty_props() else: - condition_props &= ConnectivityProperties.make_conn_props(self.peer_container, src_peers=res_peers, - dst_peers=selected_peers) + conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) + conn_cube.set_dim("src_peers", res_peers) + conn_cube.set_dim("dst_peers", selected_peers) + condition_props &= ConnectivityProperties.make_conn_props(conn_cube) connections &= condition_conns conn_props &= condition_props return IstioPolicyRule(res_peers, connections), conn_props diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index 36400ed1e..eb91edf5f 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -8,7 +8,7 @@ from nca.CoreDS.Peer import PeerSet from nca.CoreDS.MethodSet import MethodSet from nca.CoreDS.ProtocolSet import ProtocolSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube from nca.Resources.IstioTrafficResources import Gateway, VirtualService from nca.Resources.IngressPolicy import IngressPolicy from nca.Resources.NetworkPolicy import NetworkPolicy @@ -340,12 +340,15 @@ def make_allowed_connections(self, vs, host_dfa): """ allowed_conns = ConnectivityProperties.make_empty_props() for http_route in vs.http_routes: + conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) + conn_cube.set_dim("paths", http_route.uri_dfa) + conn_cube.set_dim("hosts", host_dfa) + conn_cube.set_dim("methods", http_route.methods) for dest in http_route.destinations: + conn_cube.set_dim("dst_ports", dest.port) + conn_cube.set_dim("dst_peers", dest.service.target_pods) conns = \ - ConnectivityProperties.make_conn_props(self.peer_container, dst_ports=dest.port, - dst_peers=dest.service.target_pods, - paths_dfa=http_route.uri_dfa, hosts_dfa=host_dfa, - methods=http_route.methods) + ConnectivityProperties.make_conn_props(conn_cube) allowed_conns |= conns return allowed_conns @@ -399,8 +402,10 @@ def create_istio_traffic_policies(self): res_policy.add_rules(self._make_allow_rules(allowed_conns)) protocols = ProtocolSet() protocols.add_protocol('TCP') - allowed_conns &= ConnectivityProperties.make_conn_props(self.peer_container, protocols=protocols, - src_peers=res_policy.selected_peers) + conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) + conn_cube.set_dim("protocols", protocols) + conn_cube.set_dim("src_peers", res_policy.selected_peers) + allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs vs_policies.append(res_policy) diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index 81f2a2086..acf94bfd7 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -7,7 +7,7 @@ from nca.CoreDS import Peer from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.PortSet import PortSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube from nca.CoreDS.ProtocolNameResolver import ProtocolNameResolver from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.NetworkPolicy import NetworkPolicy @@ -338,24 +338,25 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): protocol, dest_port_set = self.parse_port(port) if isinstance(protocol, str): protocol = ProtocolNameResolver.get_protocol_number(protocol) - res_conns.add_connections(protocol, ConnectivityProperties.make_conn_props( - self.peer_container, dst_ports=dest_port_set)) # K8s doesn't reason about src ports + conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) + conn_cube.set_dim("dst_ports", dest_port_set) + # K8s doesn't reason about src ports + res_conns.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) if self.optimized_run != 'false' and src_pods and dst_pods: protocols = ProtocolSet() protocols.add_protocol(protocol) - dest_num_port_set = PortSet() - dest_num_port_set.port_set = dest_port_set.port_set.copy() - conn_props = ConnectivityProperties.make_conn_props(self.peer_container, - dst_ports=dest_num_port_set, - protocols=protocols, - src_peers=src_pods, - dst_peers=dst_pods) + conn_cube.set_dim("protocols", protocols) + conn_cube.set_dim("src_peers", src_pods) + conn_cube.set_dim("dst_peers", dst_pods) + conn_props = ConnectivityProperties.make_conn_props(conn_cube) res_opt_props |= conn_props else: res_conns = ConnectionSet(True) - if self.optimized_run != 'false' and src_pods and dst_pods: - res_opt_props = ConnectivityProperties.make_conn_props(self.peer_container, - src_peers=src_pods, dst_peers=dst_pods) + if self.optimized_run != 'false': + conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) + conn_cube.set_dim("src_peers", src_pods) + conn_cube.set_dim("dst_peers", dst_pods) + res_opt_props = ConnectivityProperties.make_conn_props(conn_cube) if not res_pods: self.warning('Rule selects no pods', rule) diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index 1896e2d5b..4edc67298 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock, PeerSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube from .NetworkPolicy import PolicyConnections, NetworkPolicy from .IstioTrafficResources import istio_root_namespace @@ -166,14 +166,15 @@ def combine_peer_sets_by_ns(from_peer_set, to_peer_set, peer_container): def create_opt_egress_props(self, peer_container): for rule in self.egress_rules: + conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) if self.selected_peers and rule.egress_peer_set: - self.optimized_egress_props = \ - ConnectivityProperties.make_conn_props(peer_container, src_peers=self.selected_peers, - dst_peers=rule.egress_peer_set) + conn_cube.set_dim("src_peers", self.selected_peers) + conn_cube.set_dim("dst_peers", rule.egress_peer_set) + self.optimized_egress_props = ConnectivityProperties.make_conn_props(conn_cube) peers_sets_by_ns = self.combine_peer_sets_by_ns(self.selected_peers, rule.special_egress_peer_set, peer_container) for (from_peers, to_peers) in peers_sets_by_ns: if from_peers and to_peers: - self.optimized_egress_props |= \ - ConnectivityProperties.make_conn_props(peer_container, src_peers=from_peers, - dst_peers=to_peers) + conn_cube.set_dim("src_peers", from_peers) + conn_cube.set_dim("dst_peers", to_peers) + self.optimized_egress_props |= ConnectivityProperties.make_conn_props(conn_cube) From ad299fbe8c2fadb335bdfe5b6bf7c547f31e70f1 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 14 Mar 2023 17:56:23 +0200 Subject: [PATCH 060/187] Added set_dims method to set multiple dimensions at once. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 3 +- nca/CoreDS/ConnectivityProperties.py | 7 ++-- nca/NetworkConfig/NetworkLayer.py | 32 +++++++------------ nca/Parsers/CalicoPolicyYamlParser.py | 26 ++++----------- nca/Parsers/GenericYamlParser.py | 3 +- nca/Parsers/IngressPolicyYamlParser.py | 10 ++---- nca/Parsers/IstioPolicyYamlParser.py | 3 +- .../IstioTrafficResourcesYamlParser.py | 10 ++---- nca/Parsers/K8sPolicyYamlParser.py | 7 ++-- nca/Resources/IstioSidecar.py | 6 ++-- 10 files changed, 36 insertions(+), 71 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 4fa76600a..25fae8988 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -675,8 +675,7 @@ def fw_rules_to_conn_props(fw_rules, peer_container): src_peers = PeerSet(fw_rule.src.get_peer_set(fw_rules.cluster_info)) dst_peers = PeerSet(fw_rule.dst.get_peer_set(fw_rules.cluster_info)) conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) - conn_cube.set_dim("src_peers", src_peers) - conn_cube.set_dim("dst_peers", dst_peers) + conn_cube.set_dims({"src_peers": src_peers, "dst_peers": dst_peers}) rule_props = ConnectivityProperties.make_conn_props(conn_cube) & conn_props res |= rule_props return res diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 1f57ee580..8b0de9995 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -110,6 +110,10 @@ def set_dim(self, dim_name, dim_value): else: # the rest of dimensions do not need a translation self[dim_name] = dim_value + def set_dims(self, dims): + for dim_name, dim_value in dims.items(): + self.set_dim(dim_name, dim_value) + def unset_dim(self, dim_name): assert dim_name in self.dimensions_list self[dim_name] = DimensionsManager().get_dimension_domain_by_name(dim_name) @@ -553,8 +557,7 @@ def make_conn_props(conn_cube): real_ports -= excluded_real_ports if not real_ports: continue - conn_cube.set_dim("dst_ports", real_ports) - conn_cube.set_dim("dst_peers", PeerSet({peer})) + conn_cube.set_dims({"dst_ports": real_ports, "dst_peers": PeerSet({peer})}) props = ConnectivityProperties(conn_cube) conn_properties |= props return conn_properties diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 4dc2b8b99..5752b0393 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -177,18 +177,15 @@ def allowed_connections_optimized(self, peer_container): all_pods = peer_container.get_all_peers_group() all_ips_peer_set = PeerSet({IpBlock.get_all_ips_block()}) conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) - conn_cube.set_dim("src_peers", all_pods) - conn_cube.set_dim("dst_peers", all_ips_peer_set) + conn_cube.set_dims({"src_peers": all_pods, "dst_peers": all_ips_peer_set}) allowed_ingress_conns, denied_ingres_conns = self._allowed_xgress_conns_optimized(True, peer_container) allowed_ingress_conns |= ConnectivityProperties.make_conn_props(conn_cube) allowed_egress_conns, denied_egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) - conn_cube.set_dim("src_peers", all_ips_peer_set) - conn_cube.set_dim("dst_peers", all_pods) + conn_cube.set_dims({"src_peers": all_ips_peer_set, "dst_peers": all_pods}) allowed_egress_conns |= ConnectivityProperties.make_conn_props(conn_cube) res = allowed_ingress_conns & allowed_egress_conns # exclude IpBlock->IpBlock connections - conn_cube.set_dim("src_peers", all_ips_peer_set) - conn_cube.set_dim("dst_peers", all_ips_peer_set) + conn_cube.set_dims({"src_peers": all_ips_peer_set, "dst_peers": all_ips_peer_set}) excluded_conns = ConnectivityProperties.make_conn_props(conn_cube) res -= excluded_conns return res @@ -301,11 +298,9 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): if non_captured: conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) if is_ingress: - conn_cube.set_dim("src_peers", base_peer_set_with_ip) - conn_cube.set_dim("dst_peers", non_captured) + conn_cube.set_dims({"src_peers": base_peer_set_with_ip, "dst_peers": non_captured}) else: - conn_cube.set_dim("src_peers", non_captured) - conn_cube.set_dim("dst_peers", base_peer_set_with_ip) + conn_cube.set_dims({"src_peers": non_captured, "dst_peers": base_peer_set_with_ip}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conn |= non_captured_conns return allowed_conn, denied_conns @@ -340,16 +335,13 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) if non_captured_peers: if is_ingress: - conn_cube.set_dim("src_peers", base_peer_set_with_ip) - conn_cube.set_dim("dst_peers", non_captured_peers) + conn_cube.set_dims({"src_peers": base_peer_set_with_ip, "dst_peers": non_captured_peers}) else: - conn_cube.set_dim("src_peers", non_captured_peers) - conn_cube.set_dim("dst_peers", base_peer_set_with_ip) + conn_cube.set_dims({"src_peers": non_captured_peers, "dst_peers": base_peer_set_with_ip}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conn |= (non_captured_conns - denied_conns) - conn_cube.set_dim("src_peers", base_peer_set_with_ip) - conn_cube.set_dim("dst_peers", base_peer_set_with_ip) - conn_cube.set_dim("protocols", ProtocolSet.get_non_tcp_protocols()) + conn_cube.set_dims({"src_peers": base_peer_set_with_ip, "dst_peers": base_peer_set_with_ip, + "protocols": ProtocolSet.get_non_tcp_protocols()}) allowed_conn |= ConnectivityProperties.make_conn_props(conn_cube) return allowed_conn, denied_conns @@ -373,15 +365,13 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): base_peer_set_no_ip = peer_container.get_all_peers_group() conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) if is_ingress: - conn_cube.set_dim("src_peers", base_peer_set_with_ip) - conn_cube.set_dim("dst_peers", base_peer_set_no_ip) + conn_cube.set_dims({"src_peers": base_peer_set_with_ip, "dst_peers": base_peer_set_no_ip}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conn |= non_captured_conns else: non_captured_peers = base_peer_set_no_ip - captured if non_captured_peers: - conn_cube.set_dim("src_peers", non_captured_peers) - conn_cube.set_dim("dst_peers", base_peer_set_with_ip) + conn_cube.set_dims({"src_peers": non_captured_peers, "dst_peers": base_peer_set_with_ip}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conn |= non_captured_conns return allowed_conn, denied_conns diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index 684d4df30..df8d7c88a 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -396,12 +396,8 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): opt_conn_cube = conn_cube.copy() opt_not_conn_cube = not_conn_cube.copy() if self.optimized_run != 'false': - opt_conn_cube.set_dim("src_peers", src_pods) - opt_conn_cube.set_dim("dst_peers", dst_pods) - opt_conn_cube.set_dim("protocols", protocols) - opt_not_conn_cube.set_dim("src_peers", src_pods) - opt_not_conn_cube.set_dim("dst_peers", dst_pods) - opt_not_conn_cube.set_dim("protocols", protocols) + opt_conn_cube.set_dims({"src_peers": src_pods, "dst_peers": dst_pods, "protocols": protocols}) + opt_not_conn_cube.set_dims({"src_peers": src_pods, "dst_peers": dst_pods, "protocols": protocols}) res = ConnectivityProperties.make_empty_props() opt_props = ConnectivityProperties.make_empty_props() @@ -507,13 +503,10 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): else: if protocol_supports_ports: conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dim("src_ports", src_res_ports) - conn_cube.set_dim("dst_ports", dst_res_ports) + conn_cube.set_dims({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) connections.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) if self.optimized_run != 'false': - conn_cube.set_dim("protocols", protocols) - conn_cube.set_dim("src_peers", src_res_pods) - conn_cube.set_dim("dst_peers", dst_res_pods) + conn_cube.set_dims({"protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) elif ConnectionSet.protocol_is_icmp(protocol): icmp_props, conn_props = self._parse_icmp(rule.get('icmp'), rule.get('notICMP'), @@ -523,9 +516,7 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): connections.add_connections(protocol, True) if self.optimized_run != 'false': conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dim("protocols", protocols) - conn_cube.set_dim("src_peers", src_res_pods) - conn_cube.set_dim("dst_peers", dst_res_pods) + conn_cube.set_dims({"protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) elif not_protocol is not None: connections.add_all_connections() @@ -534,16 +525,13 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): protocols = ProtocolSet(True) protocols.remove_protocol(not_protocol) conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dim("protocols", protocols) - conn_cube.set_dim("src_peers", src_res_pods) - conn_cube.set_dim("dst_peers", dst_res_pods) + conn_cube.set_dims({"protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) else: connections.allow_all = True if self.optimized_run != 'false': conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dim("src_peers", src_res_pods) - conn_cube.set_dim("dst_peers", dst_res_pods) + conn_cube.set_dims({"src_peers": src_res_pods, "dst_peers": dst_res_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) self._verify_named_ports(rule, dst_res_pods, connections) diff --git a/nca/Parsers/GenericYamlParser.py b/nca/Parsers/GenericYamlParser.py index 7389f0fc1..ada07b6f4 100644 --- a/nca/Parsers/GenericYamlParser.py +++ b/nca/Parsers/GenericYamlParser.py @@ -238,8 +238,7 @@ def _get_connection_set_from_properties(peer_container, dest_ports, methods=Meth :return: ConnectionSet with allowed connections , corresponding to input properties cube """ conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) - conn_cube.set_dim("dst_ports", dest_ports) - conn_cube.set_dim("methods", methods) + conn_cube.set_dims({"dst_ports": dest_ports, "methods": methods}) if paths_dfa: conn_cube.set_dim("paths", paths_dfa) if hosts_dfa: diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index deca1de3f..a45516890 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -188,8 +188,7 @@ def _make_default_connections(self, hosts_dfa, paths_dfa=None): default_conns = ConnectivityProperties.make_empty_props() if self.default_backend_peers: conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dim("dst_ports", self.default_backend_ports) - conn_cube.set_dim("dst_peers", self.default_backend_peers) + conn_cube.set_dims({"dst_ports": self.default_backend_ports, "dst_peers": self.default_backend_peers}) if hosts_dfa: conn_cube.set_dim("hosts", hosts_dfa) if paths_dfa: @@ -228,9 +227,7 @@ def parse_rule(self, rule): conn_cube.set_dim("hosts", hosts_dfa) for (_, paths_dfa, _, peers, ports) in parsed_paths_with_dfa: # every path is converted to allowed connections - conn_cube.set_dim("dst_ports", ports) - conn_cube.set_dim("dst_peers", peers) - conn_cube.set_dim("paths", paths_dfa) + conn_cube.set_dims({"dst_ports": ports, "dst_peers": peers, "paths": paths_dfa}) conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conns |= conns if not all_paths_dfa: @@ -295,8 +292,7 @@ def parse_policy(self): protocols = ProtocolSet() protocols.add_protocol('TCP') conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dim("protocols", protocols) - conn_cube.set_dim("src_peers", res_policy.selected_peers) + conn_cube.set_dims({"protocols": protocols, "src_peers": res_policy.selected_peers}) allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index eb0b46416..1c5031a5f 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -520,8 +520,7 @@ def parse_ingress_rule(self, rule, selected_peers): condition_props = ConnectivityProperties.make_empty_props() else: conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dim("src_peers", res_peers) - conn_cube.set_dim("dst_peers", selected_peers) + conn_cube.set_dims({"src_peers": res_peers, "dst_peers": selected_peers}) condition_props &= ConnectivityProperties.make_conn_props(conn_cube) connections &= condition_conns conn_props &= condition_props diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index eb91edf5f..fbcd8b844 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -341,12 +341,9 @@ def make_allowed_connections(self, vs, host_dfa): allowed_conns = ConnectivityProperties.make_empty_props() for http_route in vs.http_routes: conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dim("paths", http_route.uri_dfa) - conn_cube.set_dim("hosts", host_dfa) - conn_cube.set_dim("methods", http_route.methods) + conn_cube.set_dims({"paths": http_route.uri_dfa, "hosts": host_dfa, "methods": http_route.methods}) for dest in http_route.destinations: - conn_cube.set_dim("dst_ports", dest.port) - conn_cube.set_dim("dst_peers", dest.service.target_pods) + conn_cube.set_dims({"dst_ports": dest.port, "dst_peers": dest.service.target_pods}) conns = \ ConnectivityProperties.make_conn_props(conn_cube) allowed_conns |= conns @@ -403,8 +400,7 @@ def create_istio_traffic_policies(self): protocols = ProtocolSet() protocols.add_protocol('TCP') conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dim("protocols", protocols) - conn_cube.set_dim("src_peers", res_policy.selected_peers) + conn_cube.set_dims({"protocols": protocols, "src_peers": res_policy.selected_peers}) allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index acf94bfd7..4c4abb72b 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -345,17 +345,14 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): if self.optimized_run != 'false' and src_pods and dst_pods: protocols = ProtocolSet() protocols.add_protocol(protocol) - conn_cube.set_dim("protocols", protocols) - conn_cube.set_dim("src_peers", src_pods) - conn_cube.set_dim("dst_peers", dst_pods) + conn_cube.set_dims({"protocols": protocols, "src_peers": src_pods, "dst_peers": dst_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) res_opt_props |= conn_props else: res_conns = ConnectionSet(True) if self.optimized_run != 'false': conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dim("src_peers", src_pods) - conn_cube.set_dim("dst_peers", dst_pods) + conn_cube.set_dims({"src_peers": src_pods, "dst_peers": dst_pods}) res_opt_props = ConnectivityProperties.make_conn_props(conn_cube) if not res_pods: self.warning('Rule selects no pods', rule) diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index 4edc67298..bf5c1f0ae 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -168,13 +168,11 @@ def create_opt_egress_props(self, peer_container): for rule in self.egress_rules: conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) if self.selected_peers and rule.egress_peer_set: - conn_cube.set_dim("src_peers", self.selected_peers) - conn_cube.set_dim("dst_peers", rule.egress_peer_set) + conn_cube.set_dims({"src_peers": self.selected_peers, "dst_peers": rule.egress_peer_set}) self.optimized_egress_props = ConnectivityProperties.make_conn_props(conn_cube) peers_sets_by_ns = self.combine_peer_sets_by_ns(self.selected_peers, rule.special_egress_peer_set, peer_container) for (from_peers, to_peers) in peers_sets_by_ns: if from_peers and to_peers: - conn_cube.set_dim("src_peers", from_peers) - conn_cube.set_dim("dst_peers", to_peers) + conn_cube.set_dims({"src_peers": from_peers, "dst_peers": to_peers}) self.optimized_egress_props |= ConnectivityProperties.make_conn_props(conn_cube) From a1c56549a80f60f930d5cd552319d27456fe2aca Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 14 Mar 2023 18:12:28 +0200 Subject: [PATCH 061/187] Added get_protocol_set_with_single_protocol function to ProtocolSet. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 3 +-- nca/CoreDS/ProtocolSet.py | 6 ++++++ nca/NetworkConfig/NetworkConfigQuery.py | 3 +-- nca/Parsers/CalicoPolicyYamlParser.py | 6 ++---- nca/Parsers/IngressPolicyYamlParser.py | 3 +-- nca/Parsers/IstioTrafficResourcesYamlParser.py | 3 +-- nca/Parsers/K8sPolicyYamlParser.py | 3 +-- 7 files changed, 13 insertions(+), 14 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 25fae8988..cae5a50ac 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -548,8 +548,7 @@ def convert_to_connectivity_properties(self, peer_container): res = ConnectivityProperties.make_empty_props() for protocol, properties in self.allowed_protocols.items(): - protocols = ProtocolSet() - protocols.add_protocol(protocol) + protocols = ProtocolSet.get_protocol_set_with_single_protocol(protocol) conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) conn_cube.set_dim("protocols", protocols) this_prop = ConnectivityProperties.make_conn_props(conn_cube) diff --git a/nca/CoreDS/ProtocolSet.py b/nca/CoreDS/ProtocolSet.py index e44137a44..99006018b 100644 --- a/nca/CoreDS/ProtocolSet.py +++ b/nca/CoreDS/ProtocolSet.py @@ -28,6 +28,12 @@ def get_non_tcp_protocols(): res.remove_protocol('TCP') return res + @staticmethod + def get_protocol_set_with_single_protocol(protocol): + res = ProtocolSet() + res.add_protocol(protocol) + return res + def __contains__(self, protocol): if isinstance(protocol, str): protocol_num = ProtocolNameResolver.get_protocol_number(protocol) diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index bb41a6a11..53f4f58c8 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -1059,8 +1059,7 @@ def convert_props_to_split_by_tcp(self, props): :return: a tuple of the two properties sets: first for TCP, second for non-TCP :rtype: tuple(ConnectivityProperties, ConnectivityProperties) """ - tcp_protocol = ProtocolSet() - tcp_protocol.add_protocol('TCP') + tcp_protocol = ProtocolSet.get_protocol_set_with_single_protocol('TCP') conn_cube = ConnectivityCube(self.config.peer_container.get_all_peers_group()) conn_cube.set_dim("protocols", tcp_protocol) tcp_props = props & ConnectivityProperties.make_conn_props(conn_cube) diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index df8d7c88a..ba69c6219 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -380,8 +380,7 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): if err: self.syntax_error(err, not_icmp_data) - protocols = ProtocolSet() - protocols.add_protocol(protocol) + protocols = ProtocolSet.get_protocol_set_with_single_protocol(protocol) base_peer_set = self.peer_container.get_all_peers_group() conn_cube = ConnectivityCube(base_peer_set) if icmp_type: @@ -493,8 +492,7 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): connections = ConnectionSet() conn_props = ConnectivityProperties.make_empty_props() if protocol is not None: - protocols = ProtocolSet() - protocols.add_protocol(protocol) + protocols = ProtocolSet.get_protocol_set_with_single_protocol(protocol) if not_protocol is not None: if protocol == not_protocol: self.warning('Protocol and notProtocol are conflicting, no traffic will be matched', rule) diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index a45516890..1dc4711b8 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -289,8 +289,7 @@ def parse_policy(self): # then no connections rules to add (Ingress policy has no effect) if allowed_conns: res_policy.add_rules(self._make_allow_rules(allowed_conns)) - protocols = ProtocolSet() - protocols.add_protocol('TCP') + protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) conn_cube.set_dims({"protocols": protocols, "src_peers": res_policy.selected_peers}) allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index fbcd8b844..4d18ea305 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -397,8 +397,7 @@ def create_istio_traffic_policies(self): allowed_conns = self.make_allowed_connections(vs, host_dfa) if allowed_conns: res_policy.add_rules(self._make_allow_rules(allowed_conns)) - protocols = ProtocolSet() - protocols.add_protocol('TCP') + protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) conn_cube.set_dims({"protocols": protocols, "src_peers": res_policy.selected_peers}) allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index 4c4abb72b..059a962d9 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -343,8 +343,7 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): # K8s doesn't reason about src ports res_conns.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) if self.optimized_run != 'false' and src_pods and dst_pods: - protocols = ProtocolSet() - protocols.add_protocol(protocol) + protocols = ProtocolSet.get_protocol_set_with_single_protocol(protocol) conn_cube.set_dims({"protocols": protocols, "src_peers": src_pods, "dst_peers": dst_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) res_opt_props |= conn_props From ca0cab9d0bc5b782584c78c1c8f77c79b3b3e979 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 19 Mar 2023 10:14:15 +0200 Subject: [PATCH 062/187] Fixing lint errors. Signed-off-by: Tanya --- nca/CoreDS/ConnectivityProperties.py | 2 +- nca/Parsers/CalicoPolicyYamlParser.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 8b0de9995..2cddc1f52 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -534,7 +534,7 @@ def make_conn_props(conn_cube): dst_ports = conn_cube.get_dim("dst_ports") dst_peers = conn_cube.get_dim("dst_peers") assert not src_ports.named_ports and not src_ports.excluded_named_ports - if (not dst_ports.named_ports and not dst_ports.excluded_named_ports ) or not dst_peers: + if (not dst_ports.named_ports and not dst_ports.excluded_named_ports) or not dst_peers: # Should not resolve named ports return ConnectivityProperties(conn_cube) diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index ba69c6219..9810b9415 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -350,7 +350,7 @@ def _parse_entity_rule(self, entity_rule, protocol_supports_ports): return self._get_rule_peers(entity_rule), self._get_rule_ports(entity_rule, protocol_supports_ports) - def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): + def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): # noqa: C901 """ Parse the icmp and notICMP parts of a rule :param dict icmp_data: @@ -398,7 +398,6 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): opt_conn_cube.set_dims({"src_peers": src_pods, "dst_peers": dst_pods, "protocols": protocols}) opt_not_conn_cube.set_dims({"src_peers": src_pods, "dst_peers": dst_pods, "protocols": protocols}) - res = ConnectivityProperties.make_empty_props() opt_props = ConnectivityProperties.make_empty_props() if icmp_data is not None: res = ConnectivityProperties.make_conn_props(conn_cube) @@ -424,7 +423,8 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): ConnectivityProperties.make_conn_props(opt_not_conn_cube) else: # no icmp_data or no_icmp_data; only protocol res = ConnectivityProperties.make_conn_props(conn_cube) - opt_props = ConnectivityProperties.make_conn_props(opt_conn_cube) + if self.optimized_run != 'false': + opt_props = ConnectivityProperties.make_conn_props(opt_conn_cube) return res, opt_props def _parse_protocol(self, protocol, rule): From 85ae2ea1d3cc0903a8b4633262914387b45a0866 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 19 Mar 2023 10:25:58 +0200 Subject: [PATCH 063/187] Fixed connectivity properties unit tests to match the new API. Signed-off-by: Tanya --- .../testConnectivityPropertiesNamedPorts.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py b/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py index ea9cafd53..d50fca841 100644 --- a/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py +++ b/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py @@ -1,7 +1,8 @@ import unittest from nca.CoreDS.CanonicalIntervalSet import CanonicalIntervalSet from nca.CoreDS.PortSet import PortSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties +from nca.CoreDS.Peer import PeerSet +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube class TestNamedPorts(unittest.TestCase): @@ -12,16 +13,19 @@ def test_k8s_flow(self): src_res_ports = PortSet(True) dst_res_ports = PortSet() dst_res_ports.add_port("x") - tcp_properties1 = ConnectivityProperties(src_res_ports, dst_res_ports) + conn_cube = ConnectivityCube(PeerSet()) + conn_cube.set_dims({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) + tcp_properties1 = ConnectivityProperties(conn_cube) dst_res_ports2 = PortSet() dst_res_ports2.add_port("y") - tcp_properties2 = ConnectivityProperties(src_res_ports, dst_res_ports2) + conn_cube.set_dim("dst_ports", dst_res_ports2) + tcp_properties2 = ConnectivityProperties(conn_cube) tcp_properties_res = tcp_properties1 | tcp_properties2 named_ports_dict = {"x": (15, 6), "z": (20, 6), "y": (16, 6)} tcp_properties_res.convert_named_ports(named_ports_dict, 6) - #print(tcp_properties_res) + # print(tcp_properties_res) cubes_list = tcp_properties_res._get_cubes_list_from_layers() - expected_res_cubes = [[CanonicalIntervalSet.get_interval_set(15,16)]] + expected_res_cubes = [[CanonicalIntervalSet.get_interval_set(15, 16)]] self.assertEqual(expected_res_cubes, cubes_list) def test_calico_flow_1(self): @@ -35,7 +39,9 @@ def test_calico_flow_1(self): dst_res_ports.add_port("y") dst_res_ports.add_port("z") dst_res_ports.add_port("w") - tcp_properties = ConnectivityProperties(src_res_ports, dst_res_ports) + conn_cube = ConnectivityCube(PeerSet()) + conn_cube.set_dims({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) + tcp_properties = ConnectivityProperties(conn_cube) tcp_properties_2 = tcp_properties.copy() self.assertTrue(tcp_properties.has_named_ports()) @@ -66,7 +72,9 @@ def test_calico_flow_2(self): dst_res_ports = PortSet(True) dst_res_ports -= not_ports src_res_ports.add_port_range(1, 100) - tcp_properties = ConnectivityProperties(src_res_ports, dst_res_ports) + conn_cube = ConnectivityCube(PeerSet()) + conn_cube.set_dims({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) + tcp_properties = ConnectivityProperties(conn_cube) tcp_properties_2 = tcp_properties.copy() self.assertTrue(tcp_properties.has_named_ports()) From e72081f48b5e3a5c3e3ff2bfd1c59135f83c1dd0 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 19 Mar 2023 12:46:23 +0200 Subject: [PATCH 064/187] Aligned get_cube_dict to return str for all dimensions. Fixed resolving named ports for excluded_named_ports in the optimized solution. Signed-off-by: Tanya --- nca/CoreDS/ConnectivityProperties.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 2cddc1f52..5c0bec8d7 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -303,6 +303,7 @@ def get_cube_dict(self, cube, is_txt=False): values_list = str(dim_values) elif dim == "src_peers" or dim == "dst_peers": values_list = self.base_peer_set.get_peer_set_by_indices(dim_values) + values_list = ','.join(str(peer.full_name()) for peer in values_list) elif dim_type == DimensionsManager.DimensionType.IntervalSet: values_list = dim_values.get_interval_set_list_numbers_and_ranges() if is_txt: @@ -553,13 +554,13 @@ def make_conn_props(conn_cube): assert dst_peers for peer in dst_peers: real_ports = ConnectivityProperties.resolve_named_ports(dst_ports.named_ports, peer, protocols) + if real_ports: + conn_cube.set_dims({"dst_ports": real_ports, "dst_peers": PeerSet({peer})}) + conn_properties |= ConnectivityProperties(conn_cube) excluded_real_ports = ConnectivityProperties.resolve_named_ports(dst_ports.excluded_named_ports, peer, protocols) - real_ports -= excluded_real_ports - if not real_ports: - continue - conn_cube.set_dims({"dst_ports": real_ports, "dst_peers": PeerSet({peer})}) - props = ConnectivityProperties(conn_cube) - conn_properties |= props + if excluded_real_ports: + conn_cube.set_dims({"dst_ports": excluded_real_ports, "dst_peers": PeerSet({peer})}) + conn_properties -= ConnectivityProperties(conn_cube) return conn_properties @staticmethod From 458c3732119f3110a0a8fabfbfba2904d97c303d Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 19 Mar 2023 13:22:56 +0200 Subject: [PATCH 065/187] Removed unused ICMPDAtaSet class and its unit tests. Optimized ConnectivityProperties properties creation for semantic True (all full dimensions). Removed unused methods. Signed-off-by: Tanya --- nca/CoreDS/ConnectivityProperties.py | 9 +- nca/CoreDS/ICMPDataSet.py | 171 -------------------- nca/NetworkConfig/NetworkConfigQuery.py | 16 +- nca/Parsers/CalicoPolicyYamlParser.py | 27 +++- tests/classes_unit_tests/testICMPDataSet.py | 32 ---- 5 files changed, 34 insertions(+), 221 deletions(-) delete mode 100644 nca/CoreDS/ICMPDataSet.py delete mode 100644 tests/classes_unit_tests/testICMPDataSet.py diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 5c0bec8d7..ecdce2013 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -213,7 +213,7 @@ class ConnectivityProperties(CanonicalHyperCubeSet): """ # TODO: change constructor defaults? either all arguments in "allow all" by default, or "empty" by default - def __init__(self, conn_cube, create_empty=False): + def __init__(self, conn_cube, create_empty=False, create_all=False): """ This will create connectivity properties made of the given connectivity cube. This includes tcp properties, non-tcp properties, icmp data properties. @@ -222,12 +222,15 @@ def __init__(self, conn_cube, create_empty=False): :param bool create_empty: whether to create an empty properties (representing logical False) """ super().__init__(ConnectivityCube.dimensions_list) - + assert not create_empty or not create_all self.named_ports = {} # a mapping from dst named port (String) to src ports interval set self.excluded_named_ports = {} # a mapping from dst named port (String) to src ports interval set self.base_peer_set = conn_cube.base_peer_set.copy() if create_empty: return + if create_all: # optimization + self.set_all() + return cube, active_dims, has_empty_dim_value = conn_cube.get_ordered_cube_and_active_dims() if has_empty_dim_value: @@ -577,7 +580,7 @@ def make_all_props(): Returns all connectivity properties, representing logical True :return: ConnectivityProperties """ - return ConnectivityProperties(ConnectivityCube(PeerSet())) + return ConnectivityProperties(ConnectivityCube(PeerSet()), create_all=True) def are_auto_conns(self): """ diff --git a/nca/CoreDS/ICMPDataSet.py b/nca/CoreDS/ICMPDataSet.py deleted file mode 100644 index bd67ff47e..000000000 --- a/nca/CoreDS/ICMPDataSet.py +++ /dev/null @@ -1,171 +0,0 @@ -# -# Copyright 2020- IBM Inc. All rights reserved -# SPDX-License-Identifier: Apache2.0 -# - -import copy -from .CanonicalHyperCubeSet import CanonicalHyperCubeSet -from .CanonicalIntervalSet import CanonicalIntervalSet -from .DimensionsManager import DimensionsManager - - -class ICMPDataSet(CanonicalHyperCubeSet): - """ - A class holding the set of allowed ICMP connections. Each such connection has a type and code properties. - The class uses the CanonicalHyperCubeSet to compactly represent a set of (type,code) pairs. - """ - - dimensions_list = ["icmp_type", "icmp_code"] - - def __init__(self, add_all=False): - super().__init__(ICMPDataSet.dimensions_list, add_all) - - def __str__(self): - if not self: - return 'no types' - if self.is_all(): - return '' - cubes_list = self._get_cubes_list_from_layers() - return ",".join(self._get_icmp_cube_str(cube) for cube in cubes_list) - - def _get_icmp_cube_str(self, cube): - """ - get string representation for icmp properties cube - :param list cube: a single cube in self - :return: string representation for the input cube - """ - components_str_list = [] - for dim_index, dim_values in enumerate(cube): - dim_name = (self.active_dimensions[dim_index]).replace("icmp_", "") - components_str_list.append(f'{dim_name}={dim_values}') - return f"({','.join(c for c in components_str_list)})" - - def get_properties_obj(self): - """ - get an object for a yaml representation of the protocol's properties - """ - if self.is_all(): - return {} - cubes_list = [] - res_obj = {'Type/Code': cubes_list} - for properties_cube in iter(self): - # for a cube with only one dimension, the missing (inactive) dimension is icmp_code - if len(properties_cube) == 1: - properties_cube.append(DimensionsManager().get_dimension_domain_by_name("icmp_code")) - cube_str = '/'.join(str(dim_val) for dim_val in properties_cube) - cubes_list.append(cube_str) - return res_obj - - def get_cube_dict_with_orig_values(self, cube): - """ - represent the properties cube as dict object, where the values are the original values - with which the cube was built (i.e., icmp_type and icmp_code) - :param list cube: the values of the input cube - :return: the cube properties in dict representation - :rtype: dict - """ - cube_dict = {} - for i, dim in enumerate(self.active_dimensions): - dim_values = cube[i] - dim_domain = DimensionsManager().get_dimension_domain_by_name(dim) - if dim_domain == dim_values: - continue # skip dimensions with all values allowed in a cube - cube_dict[dim] = dim_values - return cube_dict - - def copy(self): - new_copy = copy.copy(self) - return new_copy - - @staticmethod - def check_code_type_validity(icmp_type, icmp_code): - """ - Checks that the type,code pair is a valid combination for an ICMP connection - :param int icmp_type: Connection type - :param int icmp_code: Connection code - :return: A string with an error if the pair is invalid. An empty string otherwise - :rtype: str - """ - if icmp_code is not None and icmp_type is None: - return 'ICMP code cannot be specified without a type' - - is_valid, err_message = DimensionsManager().validate_value_by_domain(icmp_type, 'icmp_type', 'ICMP type') - if not is_valid: - return err_message - if icmp_code is not None: - is_valid, err_message = DimensionsManager().validate_value_by_domain(icmp_code, 'icmp_code', 'ICMP code') - if not is_valid: - return err_message - return '' - - @staticmethod - def _get_properties_cube(icmp_type, icmp_code): - """ - assuming the icmp_type is not None, return the relevant icmp properties cube: - if icmp code is None -> res cube is [icmp_type] - if icmp code is not None -> res cube is [icmp_type, icmp_code] - For a cube with a missing dimension, all its values apply, thus a cube of [icmp_type] - is equivalent to a cube of [icmp_type, all-icmp-code-domain] - :param icmp_type: int : the icmp type value - :param icmp_code: int or None : the icmp code value - :return: list[CanonicalIntervalSet]: result cube - """ - properties_cube = [CanonicalIntervalSet.get_interval_set(icmp_type, icmp_type)] - if icmp_code is not None: - properties_cube.append(CanonicalIntervalSet.get_interval_set(icmp_code, icmp_code)) - return properties_cube - - def add_to_set(self, icmp_type, icmp_code): - """ - Add a new connection to the set of allowed connection - :param int icmp_type: connection type - :param int icmp_code: connection code - :return: None - """ - if icmp_type is None: - self.add_all() - return - - self.add_cube(self._get_properties_cube(icmp_type, icmp_code)) - - def add_all_but_given_pair(self, icmp_type, icmp_code): - """ - Add all possible ICMP connections except for the given (type,code) pair - :param int icmp_type: connection type - :param int icmp_code: connection code - :return: None - """ - if icmp_type is None: - self.clear() # all but everything == nothing - return - - self.add_all() - self.add_hole(self._get_properties_cube(icmp_type, icmp_code)) - - def add_all(self): - """ - Add all possible ICMP connections to the set - :return: None - """ - self.set_all() - - def print_diff(self, other, self_name, other_name): - """ - Print the diff between two sets of ICMP connections - :param ICMPDataSet other: The set of ICMP connections to compare against - :param self_name: the name of the self set of connections - :param other_name: the name of the other set of connections - :return: a string showing one diff in connections (if exists). - :rtype: str - """ - self_does_not = ' while ' + self_name + ' does not.' - other_does_not = ' while ' + other_name + ' does not.' - self_minus_other = self - other - other_minus_self = other - self - if self_minus_other: - item = self_minus_other.get_first_item() - return self_name + ' allows code ' + str(item[1]) + ' for type ' + str(item[0]) + other_does_not - if other_minus_self: - item = other_minus_self.get_first_item() - return other_name + ' allows code ' + str(item[1]) + ' for type ' + str(item[0]) + self_does_not - return 'No diff.' diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 53f4f58c8..c02b31d93 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -795,21 +795,13 @@ def exec(self): # noqa: C901 res.output_explanation = [ComputedExplanation(str_explanation=output_res)] return res - def compare_orig_to_opt_conn(self, orig_conn_graph, opt_props): - print("Converting orig_conn_graph to tcp_like_properties...") - orig_tcp_props = orig_conn_graph.convert_to_connectivity_properties(self.config.peer_container) - assert orig_tcp_props.contained_in(opt_props) and opt_props.contained_in(orig_tcp_props) # workaround for == - # The following assert exposes the bug in HC set - assert not orig_tcp_props.contained_in(opt_props) or not opt_props.contained_in(orig_tcp_props) or \ - orig_tcp_props == opt_props - def compare_fw_rules(self, fw_rules1, fw_rules2): - tcp_props1 = ConnectionSet.fw_rules_to_conn_props(fw_rules1, self.config.peer_container) - tcp_props2 = ConnectionSet.fw_rules_to_conn_props(fw_rules2, self.config.peer_container) - if tcp_props1 == tcp_props2: + conn_props1 = ConnectionSet.fw_rules_to_conn_props(fw_rules1, self.config.peer_container) + conn_props2 = ConnectionSet.fw_rules_to_conn_props(fw_rules2, self.config.peer_container) + if conn_props1 == conn_props2: print("Original and optimized fw-rules are semantically equivalent") else: - diff_prop = (tcp_props1 - tcp_props2) | (tcp_props2 - tcp_props1) + diff_prop = (conn_props1 - conn_props2) | (conn_props2 - conn_props1) if diff_prop.are_auto_conns(): print("Original and optimized fw-rules differ only in auto-connections") else: diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index 9810b9415..cd7e3bd61 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -9,7 +9,7 @@ from nca.CoreDS.PortSet import PortSet from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube from nca.CoreDS.ProtocolSet import ProtocolSet -from nca.CoreDS.ICMPDataSet import ICMPDataSet +from nca.CoreDS.DimensionsManager import DimensionsManager from nca.CoreDS.ConnectionSet import ConnectionSet from nca.Resources.NetworkPolicy import NetworkPolicy from nca.Resources.CalicoNetworkPolicy import CalicoNetworkPolicy, CalicoPolicyRule @@ -350,6 +350,27 @@ def _parse_entity_rule(self, entity_rule, protocol_supports_ports): return self._get_rule_peers(entity_rule), self._get_rule_ports(entity_rule, protocol_supports_ports) + @staticmethod + def check_icmp_code_type_validity(icmp_type, icmp_code): + """ + Checks that the type,code pair is a valid combination for an ICMP connection + :param int icmp_type: Connection type + :param int icmp_code: Connection code + :return: A string with an error if the pair is invalid. An empty string otherwise + :rtype: str + """ + if icmp_code is not None and icmp_type is None: + return 'ICMP code cannot be specified without a type' + + is_valid, err_message = DimensionsManager().validate_value_by_domain(icmp_type, 'icmp_type', 'ICMP type') + if not is_valid: + return err_message + if icmp_code is not None: + is_valid, err_message = DimensionsManager().validate_value_by_domain(icmp_code, 'icmp_code', 'ICMP code') + if not is_valid: + return err_message + return '' + def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): # noqa: C901 """ Parse the icmp and notICMP parts of a rule @@ -371,12 +392,12 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): allowed_keys = {'type': 0, 'code': 0} if icmp_data is not None: self.check_fields_validity(icmp_data, 'ICMP', allowed_keys) - err = ICMPDataSet.check_code_type_validity(icmp_type, icmp_code) + err = self.check_icmp_code_type_validity(icmp_type, icmp_code) if err: self.syntax_error(err, icmp_data) if not_icmp_data is not None: self.check_fields_validity(not_icmp_data, 'notICMP', allowed_keys) - err = ICMPDataSet.check_code_type_validity(not_icmp_type, not_icmp_code) + err = self.check_icmp_code_type_validity(not_icmp_type, not_icmp_code) if err: self.syntax_error(err, not_icmp_data) diff --git a/tests/classes_unit_tests/testICMPDataSet.py b/tests/classes_unit_tests/testICMPDataSet.py deleted file mode 100644 index 93705264d..000000000 --- a/tests/classes_unit_tests/testICMPDataSet.py +++ /dev/null @@ -1,32 +0,0 @@ -import unittest -from nca.CoreDS.ICMPDataSet import ICMPDataSet - - -class TestICMPDataSet(unittest.TestCase): - def test_basic(self): - x = ICMPDataSet() - y = ICMPDataSet() - w = ICMPDataSet() - f = ICMPDataSet() - f.add_to_set(100,230) - print(f) - z = f.get_properties_obj() - print(z) - - x. add_all_but_given_pair(20, None) - y.add_all() - w. add_all_but_given_pair(20, 50) - diff_str = x.print_diff(y, "x", "y") - print(diff_str) - - z = x.get_properties_obj() - print(z) - z = y.get_properties_obj() - print(z) - z = w.get_properties_obj() - print(z) - print(x) - print(y) - print(w) - - From 4fdeed57b82bcf42d3d516db10adce798f491fc5 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 19 Mar 2023 16:55:18 +0200 Subject: [PATCH 066/187] Cleaner code using the new ConnectivityCube API. Signed-off-by: Tanya --- nca/Parsers/GenericIngressLikeYamlParser.py | 41 +++++++-------------- nca/Parsers/GenericYamlParser.py | 23 ------------ nca/Parsers/IstioPolicyYamlParser.py | 37 ++++++++++--------- 3 files changed, 34 insertions(+), 67 deletions(-) diff --git a/nca/Parsers/GenericIngressLikeYamlParser.py b/nca/Parsers/GenericIngressLikeYamlParser.py index 2653ed580..5f2bf8868 100644 --- a/nca/Parsers/GenericIngressLikeYamlParser.py +++ b/nca/Parsers/GenericIngressLikeYamlParser.py @@ -8,6 +8,8 @@ from nca.CoreDS.DimensionsManager import DimensionsManager from nca.CoreDS.Peer import PeerSet from nca.CoreDS.PortSet import PortSet +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties +from nca.CoreDS.ConnectionSet import ConnectionSet from nca.Resources.IngressPolicy import IngressPolicyRule from .GenericYamlParser import GenericYamlParser @@ -62,43 +64,28 @@ def _make_allow_rules(self, allowed_conns): """ return self._make_rules_from_conns(allowed_conns) - def _make_rules_from_conns(self, conn_props): + @staticmethod + def _make_rules_from_conns(conn_props): """ Make IngressPolicyRules from the given connections :param ConnectivityProperties conn_props: the given connections :return: the list of IngressPolicyRules """ + assert not conn_props.named_ports + assert not conn_props.excluded_named_ports peers_to_conns = {} res = [] # extract peers dimension from cubes for cube in conn_props: - ports = None - paths = None - hosts = None - src_peer_set = None - dst_peer_set = None - for i, dim in enumerate(conn_props.active_dimensions): - if dim == "dst_ports": - ports = cube[i] - elif dim == "paths": - paths = cube[i] - elif dim == "hosts": - hosts = cube[i] - elif dim == "src_peers": - src_peer_set = conn_props.base_peer_set.get_peer_set_by_indices(cube[i]) - elif dim == "dst_peers": - dst_peer_set = conn_props.base_peer_set.get_peer_set_by_indices(cube[i]) - else: - assert False + conn_cube = conn_props.get_connectivity_cube(cube) + src_peer_set = conn_cube.get_dim("src_peers") + conn_cube.unset_dim("src_peers") + dst_peer_set = conn_cube.get_dim("dst_peers") + conn_cube.unset_dim("dst_peers") assert not src_peer_set - if not dst_peer_set: - dst_peer_set = self.peer_container.peer_set.copy() - port_set = PortSet() - port_set.port_set = ports - port_set.named_ports = conn_props.named_ports - port_set.excluded_named_ports = conn_props.excluded_named_ports - new_conns = self._get_connection_set_from_properties(self.peer_container, port_set, paths_dfa=paths, - hosts_dfa=hosts) + new_props = ConnectivityProperties.make_conn_props(conn_cube) + new_conns = ConnectionSet() + new_conns.add_connections('TCP', new_props) if peers_to_conns.get(dst_peer_set): peers_to_conns[dst_peer_set] |= new_conns # optimize conns for the same peers else: diff --git a/nca/Parsers/GenericYamlParser.py b/nca/Parsers/GenericYamlParser.py index ada07b6f4..1b5671deb 100644 --- a/nca/Parsers/GenericYamlParser.py +++ b/nca/Parsers/GenericYamlParser.py @@ -225,29 +225,6 @@ def validate_value_in_domain(self, value, dim_name, array, value_name): if not is_valid: self.syntax_error(err_message, array) - @staticmethod - def _get_connection_set_from_properties(peer_container, dest_ports, methods=MethodSet(True), paths_dfa=None, - hosts_dfa=None): - """ - get ConnectionSet with allowed connections, corresponding to input properties cube - :param PeerContainer peer_container: the peer container - :param PortSet dest_ports: ports set for dset_ports dimension - :param MethodSet methods: methods set for methods dimension - :param MinDFA paths_dfa: MinDFA obj for paths dimension - :param MinDFA hosts_dfa: MinDFA obj for hosts dimension - :return: ConnectionSet with allowed connections , corresponding to input properties cube - """ - conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) - conn_cube.set_dims({"dst_ports": dest_ports, "methods": methods}) - if paths_dfa: - conn_cube.set_dim("paths", paths_dfa) - if hosts_dfa: - conn_cube.set_dim("hosts", hosts_dfa) - tcp_properties = ConnectivityProperties.make_conn_props(conn_cube) - res = ConnectionSet() - res.add_connections('TCP', tcp_properties) - return res - def check_and_update_has_ipv6_addresses(self, peers): """ checks if the peer list has ipv6 addresses diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index 1c5031a5f..389158532 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -185,7 +185,7 @@ def parse_key_values(self, key, values, not_values): :param str key: the specified key str :param list values: a list of strings with values for this key :param list not_values: a list of strings with negative values for this key - :return: PeerSet or ConnectionSet (depends on the key) with allowed values + :return: PeerSet or ConnectivityProperties (depends on the key) with allowed values """ if key == 'source.ip': return self.parse_ip_block(values, not_values) # PeerSet @@ -195,14 +195,16 @@ def parse_key_values(self, key, values, not_values): return self.parse_principals(values, not_values) # PeerSet elif key == 'destination.port': dst_ports = self.get_rule_ports(values, not_values) # PortSet - return self._get_connection_set_from_properties(self.peer_container, dst_ports) # ConnectionSet + conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) + conn_cube.set_dim("dst_ports", dst_ports) + return ConnectivityProperties.make_conn_props(conn_cube) # ConnectivityProperties return NotImplemented, False def parse_condition(self, condition): """ parse a condition component in a rule :param dict condition: the condition to parse - :return: PeerSet or ConnectionSet (depends on the key) with allowed values + :return: PeerSet or ConnectivityProperties (depends on the key) with allowed values """ allowed_elements = {'key': [1, str], 'values': [0, list], 'notValues': [0, list]} allowed_key_values = {'key': ['source.ip', 'source.namespace', 'source.principal', 'destination.port']} @@ -219,7 +221,7 @@ def parse_condition(self, condition): if not values and not not_values: self.syntax_error('error parsing condition: at least one of values or not_values must be set. ', condition) - return self.parse_key_values(key, values, not_values) # PeerSet or ConnectionSet + return self.parse_key_values(key, values, not_values) # PeerSet or ConnectivityProperties # TODO: avoid code duplication with Calico version... def _parse_port(self, port, array): @@ -372,7 +374,7 @@ def parse_operation(self, operation_dict): """ parse an operation component in a rule :param dict operation_dict: the operation to parse - :return: ConnectionSet object with allowed connections + :return: ConnectivityProperties object with allowed connections """ to_allowed_elements = {'operation': [1, dict]} @@ -397,9 +399,9 @@ def parse_operation(self, operation_dict): operation) hosts_dfa = self.parse_regex_dimension_values("hosts", operation.get("hosts"), operation.get("notHosts"), operation) - - return self._get_connection_set_from_properties(self.peer_container, dst_ports, methods=methods_set, - paths_dfa=paths_dfa, hosts_dfa=hosts_dfa) + conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) + conn_cube.set_dims({"dst_ports": dst_ports, "methods": methods_set, "paths": paths_dfa, 'hosts': hosts_dfa}) + return ConnectivityProperties.make_conn_props(conn_cube) def parse_source(self, source_dict): """ @@ -491,31 +493,32 @@ def parse_ingress_rule(self, rule, selected_peers): # TODO: extend operations parsing to include other attributes conn_props = ConnectivityProperties.make_empty_props() if to_array is not None: - connections = ConnectionSet() for operation_dict in to_array: - conns = self.parse_operation(operation_dict) - connections |= conns - conn_props |= conns.convert_to_connectivity_properties(self.peer_container) + conn_props |= self.parse_operation(operation_dict) + connections = ConnectionSet() + connections.add_connections('TCP', conn_props) else: # no 'to' in the rule => all connections allowed connections = ConnectionSet(True) conn_props = ConnectivityProperties.make_all_props() # condition possible result value: - # source-ip (from) , source-namespace (from) [Peerset], destination.port (to) [ConnectionSet] - # should update either res_pods or connections according to the condition + # source-ip (from) , source-namespace (from) [Peerset], destination.port (to) [ConnectivityProperties] + # should update either res_pods or condition_props according to the condition condition_array = rule.get('when') # this array can be empty (unlike 'to' and 'from') # the combined condition ("AND" of all conditions) should be applied condition_conns = ConnectionSet(True) + condition_props = ConnectivityProperties.make_all_props() if condition_array is not None: for condition in condition_array: condition_res = self.parse_condition(condition) if isinstance(condition_res, PeerSet): res_peers &= condition_res - elif isinstance(condition_res, ConnectionSet): - condition_conns &= condition_res + elif isinstance(condition_res, ConnectivityProperties): + condition_props &= condition_res + condition_conns = ConnectionSet() + condition_conns.add_connections('TCP', condition_props) if not res_peers: self.warning('Rule selects no pods', rule) - condition_props = condition_conns.convert_to_connectivity_properties(self.peer_container) if not res_peers or not selected_peers: condition_props = ConnectivityProperties.make_empty_props() else: From 13fcaffab7cf93077de48631cc9fa4e284405172 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 19 Mar 2023 17:54:14 +0200 Subject: [PATCH 067/187] Added missing copy() in ConnectionSet. Fixed lint errors. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 2 +- nca/Parsers/GenericYamlParser.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index cae5a50ac..0b06bae32 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -415,7 +415,7 @@ def add_connections(self, protocol, properties=True): if protocol in self.allowed_protocols: self.allowed_protocols[protocol] |= properties else: - self.allowed_protocols[protocol] = properties + self.allowed_protocols[protocol] = properties.copy() def remove_protocol(self, protocol): """ diff --git a/nca/Parsers/GenericYamlParser.py b/nca/Parsers/GenericYamlParser.py index 1b5671deb..b7f5db646 100644 --- a/nca/Parsers/GenericYamlParser.py +++ b/nca/Parsers/GenericYamlParser.py @@ -6,9 +6,6 @@ from sys import stderr from enum import Enum from nca.CoreDS.DimensionsManager import DimensionsManager -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube -from nca.CoreDS.MethodSet import MethodSet -from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock from nca.Utils.NcaLogger import NcaLogger from nca.FileScanners.GenericTreeScanner import ObjectWithLocation From d58118c55ae9d2918e6475ad8b02fb26c955f197 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 19 Mar 2023 17:54:14 +0200 Subject: [PATCH 068/187] Added missing copy() in ConnectionSet. Fixed lint errors. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 2 +- nca/Parsers/GenericYamlParser.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index cae5a50ac..3d99a0f48 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -415,7 +415,7 @@ def add_connections(self, protocol, properties=True): if protocol in self.allowed_protocols: self.allowed_protocols[protocol] |= properties else: - self.allowed_protocols[protocol] = properties + self.allowed_protocols[protocol] = properties if isinstance(properties, bool) else properties.copy() def remove_protocol(self, protocol): """ diff --git a/nca/Parsers/GenericYamlParser.py b/nca/Parsers/GenericYamlParser.py index 1b5671deb..b7f5db646 100644 --- a/nca/Parsers/GenericYamlParser.py +++ b/nca/Parsers/GenericYamlParser.py @@ -6,9 +6,6 @@ from sys import stderr from enum import Enum from nca.CoreDS.DimensionsManager import DimensionsManager -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube -from nca.CoreDS.MethodSet import MethodSet -from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock from nca.Utils.NcaLogger import NcaLogger from nca.FileScanners.GenericTreeScanner import ObjectWithLocation From 52b3ec9f89299a4f390e640e9b5989107b6f38c7 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 19 Mar 2023 18:59:11 +0200 Subject: [PATCH 069/187] When running with -opt=debug, printing the original results of ConnectivityMap query. Signed-off-by: Tanya --- nca/NetworkConfig/NetworkConfigQuery.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index c02b31d93..b67343424 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -788,11 +788,11 @@ def exec(self): # noqa: C901 if self.config.optimized_run == 'debug' and fw_rules and fw_rules.fw_rules_map and \ opt_fw_rules and opt_fw_rules.fw_rules_map: self.compare_fw_rules(fw_rules, opt_fw_rules) - - if self.output_config.outputFormat in ['json', 'yaml']: - res.output_explanation = [ComputedExplanation(dict_explanation=output_res)] - else: - res.output_explanation = [ComputedExplanation(str_explanation=output_res)] + if self.config.optimized_run == 'true': + if self.output_config.outputFormat in ['json', 'yaml']: + res.output_explanation = [ComputedExplanation(dict_explanation=output_res)] + else: + res.output_explanation = [ComputedExplanation(str_explanation=output_res)] return res def compare_fw_rules(self, fw_rules1, fw_rules2): From e38aff184de3126043bb947ba4f2f6edbd0c23a6 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 12:09:55 +0200 Subject: [PATCH 070/187] Made cleaner interface of ConectivityCube class, using __setitem__, __getitem__ and update functions. Made cleaner interface of creating empty/full/by cube ConnectivityProperties. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 10 +-- nca/CoreDS/ConnectivityProperties.py | 80 ++++++++++--------- nca/FWRules/ConnectivityGraph.py | 6 +- nca/NetworkConfig/NetworkConfigQuery.py | 6 +- nca/NetworkConfig/NetworkLayer.py | 20 ++--- nca/Parsers/CalicoPolicyYamlParser.py | 22 ++--- nca/Parsers/GenericIngressLikeYamlParser.py | 4 +- nca/Parsers/IngressPolicyYamlParser.py | 12 +-- nca/Parsers/IstioPolicyYamlParser.py | 6 +- .../IstioTrafficResourcesYamlParser.py | 6 +- nca/Parsers/K8sPolicyYamlParser.py | 6 +- nca/Resources/IstioSidecar.py | 4 +- .../testConnectivityPropertiesNamedPorts.py | 16 ++-- 13 files changed, 101 insertions(+), 97 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 3d99a0f48..d82612a45 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -550,7 +550,7 @@ def convert_to_connectivity_properties(self, peer_container): for protocol, properties in self.allowed_protocols.items(): protocols = ProtocolSet.get_protocol_set_with_single_protocol(protocol) conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) - conn_cube.set_dim("protocols", protocols) + conn_cube["protocols"] = protocols this_prop = ConnectivityProperties.make_conn_props(conn_cube) if isinstance(properties, bool): if properties: @@ -598,18 +598,18 @@ def conn_props_to_fw_rules(conn_props, cluster_info, peer_container, ip_blocks_m fw_rules_map = defaultdict(list) for cube in conn_props: conn_cube = conn_props.get_connectivity_cube(cube) - src_peers = conn_cube.get_dim("src_peers") + src_peers = conn_cube["src_peers"] if not src_peers: src_peers = peer_container.get_all_peers_group(True) conn_cube.unset_dim("src_peers") - dst_peers = conn_cube.get_dim("dst_peers") + dst_peers = conn_cube["dst_peers"] if not dst_peers: dst_peers = peer_container.get_all_peers_group(True) conn_cube.unset_dim("dst_peers") if IpBlock.get_all_ips_block() != ip_blocks_mask: src_peers.filter_ipv6_blocks(ip_blocks_mask) dst_peers.filter_ipv6_blocks(ip_blocks_mask) - protocols = conn_cube.get_dim("protocols") + protocols = conn_cube["protocols"] conn_cube.unset_dim("protocols") if not conn_cube.has_active_dim() and (not protocols or protocols == ignore_protocols): conns = ConnectionSet(True) @@ -674,7 +674,7 @@ def fw_rules_to_conn_props(fw_rules, peer_container): src_peers = PeerSet(fw_rule.src.get_peer_set(fw_rules.cluster_info)) dst_peers = PeerSet(fw_rule.dst.get_peer_set(fw_rules.cluster_info)) conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) - conn_cube.set_dims({"src_peers": src_peers, "dst_peers": dst_peers}) + conn_cube.update({"src_peers": src_peers, "dst_peers": dst_peers}) rule_props = ConnectivityProperties.make_conn_props(conn_cube) & conn_props res |= rule_props return res diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index ecdce2013..9c6919baf 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -91,7 +91,7 @@ def set_dim_directly(self, dim_name, dim_value): assert dim_name in self.dimensions_list self[dim_name] = dim_value - def set_dim(self, dim_name, dim_value): + def __setitem__(self, dim_name, dim_value): assert dim_name in self.dimensions_list if dim_value is None: return @@ -110,15 +110,15 @@ def set_dim(self, dim_name, dim_value): else: # the rest of dimensions do not need a translation self[dim_name] = dim_value - def set_dims(self, dims): + def update(self, dims=None, **f): for dim_name, dim_value in dims.items(): - self.set_dim(dim_name, dim_value) + self[dim_name] = dim_value def unset_dim(self, dim_name): assert dim_name in self.dimensions_list self[dim_name] = DimensionsManager().get_dimension_domain_by_name(dim_name) - def get_dim(self, dim_name): + def __getitem__(self, dim_name): assert dim_name in self.dimensions_list if dim_name == "src_peers" or dim_name == "dst_peers": if self.is_active_dim(dim_name): @@ -212,44 +212,48 @@ class ConnectivityProperties(CanonicalHyperCubeSet): (2) calico: +ve and -ve named ports, no src named ports, and no use of operators between these objects. """ - # TODO: change constructor defaults? either all arguments in "allow all" by default, or "empty" by default - def __init__(self, conn_cube, create_empty=False, create_all=False): + def __init__(self, create_all=False): """ - This will create connectivity properties made of the given connectivity cube. - This includes tcp properties, non-tcp properties, icmp data properties. - :param ConnectivityCube conn_cube: the input connectivity cube including all dimension values, - whereas missing dimensions are represented by their default values (representing all possible values). - :param bool create_empty: whether to create an empty properties (representing logical False) + This will create empty or full connectivity properties, depending on create_all flag. + :param create_all: whether to create full connectivity properties. """ super().__init__(ConnectivityCube.dimensions_list) - assert not create_empty or not create_all self.named_ports = {} # a mapping from dst named port (String) to src ports interval set self.excluded_named_ports = {} # a mapping from dst named port (String) to src ports interval set - self.base_peer_set = conn_cube.base_peer_set.copy() - if create_empty: - return - if create_all: # optimization + self.base_peer_set = PeerSet() + if create_all: self.set_all() - return + + @staticmethod + def create_props_from_cube(conn_cube): + """ + This will create connectivity properties made of the given connectivity cube. + This includes tcp properties, non-tcp properties, icmp data properties. + :param ConnectivityCube conn_cube: the input connectivity cube including all dimension values, + whereas missing dimensions are represented by their default values (representing all possible values). + """ + res = ConnectivityProperties() + res.base_peer_set = conn_cube.base_peer_set.copy() cube, active_dims, has_empty_dim_value = conn_cube.get_ordered_cube_and_active_dims() if has_empty_dim_value: return if not active_dims: - self.set_all() + res.set_all() else: - self.add_cube(cube, active_dims) + res.add_cube(cube, active_dims) # assuming named ports may be only in dst, not in src - src_ports = conn_cube.get_dim("src_ports") - dst_ports = conn_cube.get_dim("dst_ports") + src_ports = conn_cube["src_ports"] + dst_ports = conn_cube["dst_ports"] assert not src_ports.named_ports and not src_ports.excluded_named_ports all_ports = PortSet.all_ports_interval.copy() for port_name in dst_ports.named_ports: - self.named_ports[port_name] = src_ports.port_set + res.named_ports[port_name] = src_ports.port_set for port_name in dst_ports.excluded_named_ports: - self.excluded_named_ports[port_name] = all_ports + res.excluded_named_ports[port_name] = all_ports + return res def __bool__(self): return super().__bool__() or bool(self.named_ports) @@ -439,7 +443,7 @@ def convert_named_ports(self, named_ports, protocol): self.add_hole(rectangle, active_dims) def copy(self): - res = ConnectivityProperties(ConnectivityCube(self.base_peer_set)) + res = ConnectivityProperties.create_props_from_cube(ConnectivityCube(self.base_peer_set)) for layer in self.layers: res.layers[self._copy_layer_elem(layer)] = self.layers[layer].copy() res.active_dimensions = self.active_dimensions.copy() @@ -495,7 +499,7 @@ def project_on_one_dimension(self, dim_name): return res for cube in self: conn_cube = self.get_connectivity_cube(cube) - values = conn_cube.get_dim(dim_name) + values = conn_cube[dim_name] if values: res |= values @@ -534,36 +538,36 @@ def make_conn_props(conn_cube): whereas missing dimensions are represented by their default values (representing all possible values). """ - src_ports = conn_cube.get_dim("src_ports") - dst_ports = conn_cube.get_dim("dst_ports") - dst_peers = conn_cube.get_dim("dst_peers") + src_ports = conn_cube["src_ports"] + dst_ports = conn_cube["dst_ports"] + dst_peers = conn_cube["dst_peers"] assert not src_ports.named_ports and not src_ports.excluded_named_ports if (not dst_ports.named_ports and not dst_ports.excluded_named_ports) or not dst_peers: # Should not resolve named ports - return ConnectivityProperties(conn_cube) + return ConnectivityProperties.create_props_from_cube(conn_cube) # Initialize conn_properties if dst_ports.port_set: dst_ports_no_named_ports = dst_ports.copy() dst_ports_no_named_ports.named_ports = set() dst_ports_no_named_ports.excluded_named_ports = set() - conn_cube.set_dim("dst_ports", dst_ports_no_named_ports) - conn_properties = ConnectivityProperties(conn_cube) + conn_cube["dst_ports"] = dst_ports_no_named_ports + conn_properties = ConnectivityProperties.create_props_from_cube(conn_cube) else: conn_properties = ConnectivityProperties.make_empty_props() # Resolving dst named ports - protocols = conn_cube.get_dim("protocols") + protocols = conn_cube["protocols"] assert dst_peers for peer in dst_peers: real_ports = ConnectivityProperties.resolve_named_ports(dst_ports.named_ports, peer, protocols) if real_ports: - conn_cube.set_dims({"dst_ports": real_ports, "dst_peers": PeerSet({peer})}) - conn_properties |= ConnectivityProperties(conn_cube) + conn_cube.update({"dst_ports": real_ports, "dst_peers": PeerSet({peer})}) + conn_properties |= ConnectivityProperties.create_props_from_cube(conn_cube) excluded_real_ports = ConnectivityProperties.resolve_named_ports(dst_ports.excluded_named_ports, peer, protocols) if excluded_real_ports: - conn_cube.set_dims({"dst_ports": excluded_real_ports, "dst_peers": PeerSet({peer})}) - conn_properties -= ConnectivityProperties(conn_cube) + conn_cube.update({"dst_ports": excluded_real_ports, "dst_peers": PeerSet({peer})}) + conn_properties -= ConnectivityProperties.create_props_from_cube(conn_cube) return conn_properties @staticmethod @@ -572,7 +576,7 @@ def make_empty_props(): Returns empty connectivity properties, representing logical False :return: ConnectivityProperties """ - return ConnectivityProperties(ConnectivityCube(PeerSet()), create_empty=True) + return ConnectivityProperties() @staticmethod def make_all_props(): @@ -580,7 +584,7 @@ def make_all_props(): Returns all connectivity properties, representing logical True :return: ConnectivityProperties """ - return ConnectivityProperties(ConnectivityCube(PeerSet()), create_all=True) + return ConnectivityProperties(True) def are_auto_conns(self): """ diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index 5d092f271..a0e3465bd 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -75,14 +75,14 @@ def add_edges_from_cube_dict(self, conn_cube, ip_blocks_mask): :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, whereas all other values should be filtered out in the output """ - src_peers = conn_cube.get_dim("src_peers") + src_peers = conn_cube["src_peers"] conn_cube.unset_dim("src_peers") - dst_peers = conn_cube.get_dim("dst_peers") + dst_peers = conn_cube["dst_peers"] conn_cube.unset_dim("dst_peers") if IpBlock.get_all_ips_block() != ip_blocks_mask: src_peers.filter_ipv6_blocks(ip_blocks_mask) dst_peers.filter_ipv6_blocks(ip_blocks_mask) - protocols = conn_cube.get_dim("protocols") + protocols = conn_cube["protocols"] conn_cube.unset_dim("protocols") if not protocols and not conn_cube.has_active_dim(): diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index b67343424..584779602 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -755,8 +755,8 @@ def exec(self): # noqa: C901 subset_peers = self.compute_subset(opt_peers_to_compare) src_peers_conn_cube = ConnectivityCube(self.config.peer_container.get_all_peers_group()) dst_peers_conn_cube = ConnectivityCube(self.config.peer_container.get_all_peers_group()) - src_peers_conn_cube.set_dim("src_peers", subset_peers) - dst_peers_conn_cube.set_dim("dst_peers", subset_peers) + src_peers_conn_cube["src_peers"] = subset_peers + dst_peers_conn_cube["dst_peers"] = subset_peers subset_conns = ConnectivityProperties.make_conn_props(src_peers_conn_cube) | \ ConnectivityProperties.make_conn_props(dst_peers_conn_cube) all_conns_opt &= subset_conns @@ -1053,7 +1053,7 @@ def convert_props_to_split_by_tcp(self, props): """ tcp_protocol = ProtocolSet.get_protocol_set_with_single_protocol('TCP') conn_cube = ConnectivityCube(self.config.peer_container.get_all_peers_group()) - conn_cube.set_dim("protocols", tcp_protocol) + conn_cube["protocols"] = tcp_protocol tcp_props = props & ConnectivityProperties.make_conn_props(conn_cube) non_tcp_props = props - tcp_props return tcp_props, non_tcp_props diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 5752b0393..1765f0d86 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -177,15 +177,15 @@ def allowed_connections_optimized(self, peer_container): all_pods = peer_container.get_all_peers_group() all_ips_peer_set = PeerSet({IpBlock.get_all_ips_block()}) conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) - conn_cube.set_dims({"src_peers": all_pods, "dst_peers": all_ips_peer_set}) + conn_cube.update({"src_peers": all_pods, "dst_peers": all_ips_peer_set}) allowed_ingress_conns, denied_ingres_conns = self._allowed_xgress_conns_optimized(True, peer_container) allowed_ingress_conns |= ConnectivityProperties.make_conn_props(conn_cube) allowed_egress_conns, denied_egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) - conn_cube.set_dims({"src_peers": all_ips_peer_set, "dst_peers": all_pods}) + conn_cube.update({"src_peers": all_ips_peer_set, "dst_peers": all_pods}) allowed_egress_conns |= ConnectivityProperties.make_conn_props(conn_cube) res = allowed_ingress_conns & allowed_egress_conns # exclude IpBlock->IpBlock connections - conn_cube.set_dims({"src_peers": all_ips_peer_set, "dst_peers": all_ips_peer_set}) + conn_cube.update({"src_peers": all_ips_peer_set, "dst_peers": all_ips_peer_set}) excluded_conns = ConnectivityProperties.make_conn_props(conn_cube) res -= excluded_conns return res @@ -298,9 +298,9 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): if non_captured: conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) if is_ingress: - conn_cube.set_dims({"src_peers": base_peer_set_with_ip, "dst_peers": non_captured}) + conn_cube.update({"src_peers": base_peer_set_with_ip, "dst_peers": non_captured}) else: - conn_cube.set_dims({"src_peers": non_captured, "dst_peers": base_peer_set_with_ip}) + conn_cube.update({"src_peers": non_captured, "dst_peers": base_peer_set_with_ip}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conn |= non_captured_conns return allowed_conn, denied_conns @@ -335,12 +335,12 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) if non_captured_peers: if is_ingress: - conn_cube.set_dims({"src_peers": base_peer_set_with_ip, "dst_peers": non_captured_peers}) + conn_cube.update({"src_peers": base_peer_set_with_ip, "dst_peers": non_captured_peers}) else: - conn_cube.set_dims({"src_peers": non_captured_peers, "dst_peers": base_peer_set_with_ip}) + conn_cube.update({"src_peers": non_captured_peers, "dst_peers": base_peer_set_with_ip}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conn |= (non_captured_conns - denied_conns) - conn_cube.set_dims({"src_peers": base_peer_set_with_ip, "dst_peers": base_peer_set_with_ip, + conn_cube.update({"src_peers": base_peer_set_with_ip, "dst_peers": base_peer_set_with_ip, "protocols": ProtocolSet.get_non_tcp_protocols()}) allowed_conn |= ConnectivityProperties.make_conn_props(conn_cube) return allowed_conn, denied_conns @@ -365,13 +365,13 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): base_peer_set_no_ip = peer_container.get_all_peers_group() conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) if is_ingress: - conn_cube.set_dims({"src_peers": base_peer_set_with_ip, "dst_peers": base_peer_set_no_ip}) + conn_cube.update({"src_peers": base_peer_set_with_ip, "dst_peers": base_peer_set_no_ip}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conn |= non_captured_conns else: non_captured_peers = base_peer_set_no_ip - captured if non_captured_peers: - conn_cube.set_dims({"src_peers": non_captured_peers, "dst_peers": base_peer_set_with_ip}) + conn_cube.update({"src_peers": non_captured_peers, "dst_peers": base_peer_set_with_ip}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conn |= non_captured_conns return allowed_conn, denied_conns diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index cd7e3bd61..720980a98 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -405,19 +405,19 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): base_peer_set = self.peer_container.get_all_peers_group() conn_cube = ConnectivityCube(base_peer_set) if icmp_type: - conn_cube.set_dim("icmp_type", icmp_type) + conn_cube["icmp_type"] = icmp_type if icmp_code: - conn_cube.set_dim("icmp_code", icmp_code) + conn_cube["icmp_code"] = icmp_code not_conn_cube = ConnectivityCube(base_peer_set) if not_icmp_type: - not_conn_cube.set_dim("icmp_type", not_icmp_type) + not_conn_cube["icmp_type"] = not_icmp_type if not_icmp_code: - not_conn_cube.set_dim("icmp_code", not_icmp_code) + not_conn_cube["icmp_code"] = not_icmp_code opt_conn_cube = conn_cube.copy() opt_not_conn_cube = not_conn_cube.copy() if self.optimized_run != 'false': - opt_conn_cube.set_dims({"src_peers": src_pods, "dst_peers": dst_pods, "protocols": protocols}) - opt_not_conn_cube.set_dims({"src_peers": src_pods, "dst_peers": dst_pods, "protocols": protocols}) + opt_conn_cube.update({"src_peers": src_pods, "dst_peers": dst_pods, "protocols": protocols}) + opt_not_conn_cube.update({"src_peers": src_pods, "dst_peers": dst_pods, "protocols": protocols}) opt_props = ConnectivityProperties.make_empty_props() if icmp_data is not None: @@ -522,10 +522,10 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): else: if protocol_supports_ports: conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dims({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) + conn_cube.update({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) connections.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) if self.optimized_run != 'false': - conn_cube.set_dims({"protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) + conn_cube.update({"protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) elif ConnectionSet.protocol_is_icmp(protocol): icmp_props, conn_props = self._parse_icmp(rule.get('icmp'), rule.get('notICMP'), @@ -535,7 +535,7 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): connections.add_connections(protocol, True) if self.optimized_run != 'false': conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dims({"protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) + conn_cube.update({"protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) elif not_protocol is not None: connections.add_all_connections() @@ -544,13 +544,13 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): protocols = ProtocolSet(True) protocols.remove_protocol(not_protocol) conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dims({"protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) + conn_cube.update({"protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) else: connections.allow_all = True if self.optimized_run != 'false': conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dims({"src_peers": src_res_pods, "dst_peers": dst_res_pods}) + conn_cube.update({"src_peers": src_res_pods, "dst_peers": dst_res_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) self._verify_named_ports(rule, dst_res_pods, connections) diff --git a/nca/Parsers/GenericIngressLikeYamlParser.py b/nca/Parsers/GenericIngressLikeYamlParser.py index 5f2bf8868..7565c193d 100644 --- a/nca/Parsers/GenericIngressLikeYamlParser.py +++ b/nca/Parsers/GenericIngressLikeYamlParser.py @@ -78,9 +78,9 @@ def _make_rules_from_conns(conn_props): # extract peers dimension from cubes for cube in conn_props: conn_cube = conn_props.get_connectivity_cube(cube) - src_peer_set = conn_cube.get_dim("src_peers") + src_peer_set = conn_cube["src_peers"] conn_cube.unset_dim("src_peers") - dst_peer_set = conn_cube.get_dim("dst_peers") + dst_peer_set = conn_cube["dst_peers"] conn_cube.unset_dim("dst_peers") assert not src_peer_set new_props = ConnectivityProperties.make_conn_props(conn_cube) diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index 1dc4711b8..dd3ba6cb1 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -188,11 +188,11 @@ def _make_default_connections(self, hosts_dfa, paths_dfa=None): default_conns = ConnectivityProperties.make_empty_props() if self.default_backend_peers: conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dims({"dst_ports": self.default_backend_ports, "dst_peers": self.default_backend_peers}) + conn_cube.update({"dst_ports": self.default_backend_ports, "dst_peers": self.default_backend_peers}) if hosts_dfa: - conn_cube.set_dim("hosts", hosts_dfa) + conn_cube["hosts"] = hosts_dfa if paths_dfa: - conn_cube.set_dim("paths", paths_dfa) + conn_cube["paths"] = paths_dfa default_conns = ConnectivityProperties.make_conn_props(conn_cube) else: default_conns = ConnectivityProperties.make_conn_props(conn_cube) @@ -224,10 +224,10 @@ def parse_rule(self, rule): if parsed_paths: parsed_paths_with_dfa = self.segregate_longest_paths_and_make_dfa(parsed_paths) conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dim("hosts", hosts_dfa) + conn_cube["hosts"] = hosts_dfa for (_, paths_dfa, _, peers, ports) in parsed_paths_with_dfa: # every path is converted to allowed connections - conn_cube.set_dims({"dst_ports": ports, "dst_peers": peers, "paths": paths_dfa}) + conn_cube.update({"dst_ports": ports, "dst_peers": peers, "paths": paths_dfa}) conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conns |= conns if not all_paths_dfa: @@ -291,7 +291,7 @@ def parse_policy(self): res_policy.add_rules(self._make_allow_rules(allowed_conns)) protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dims({"protocols": protocols, "src_peers": res_policy.selected_peers}) + conn_cube.update({"protocols": protocols, "src_peers": res_policy.selected_peers}) allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index 389158532..9c64c9ad2 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -196,7 +196,7 @@ def parse_key_values(self, key, values, not_values): elif key == 'destination.port': dst_ports = self.get_rule_ports(values, not_values) # PortSet conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dim("dst_ports", dst_ports) + conn_cube["dst_ports"] = dst_ports return ConnectivityProperties.make_conn_props(conn_cube) # ConnectivityProperties return NotImplemented, False @@ -400,7 +400,7 @@ def parse_operation(self, operation_dict): hosts_dfa = self.parse_regex_dimension_values("hosts", operation.get("hosts"), operation.get("notHosts"), operation) conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dims({"dst_ports": dst_ports, "methods": methods_set, "paths": paths_dfa, 'hosts': hosts_dfa}) + conn_cube.update({"dst_ports": dst_ports, "methods": methods_set, "paths": paths_dfa, 'hosts': hosts_dfa}) return ConnectivityProperties.make_conn_props(conn_cube) def parse_source(self, source_dict): @@ -523,7 +523,7 @@ def parse_ingress_rule(self, rule, selected_peers): condition_props = ConnectivityProperties.make_empty_props() else: conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dims({"src_peers": res_peers, "dst_peers": selected_peers}) + conn_cube.update({"src_peers": res_peers, "dst_peers": selected_peers}) condition_props &= ConnectivityProperties.make_conn_props(conn_cube) connections &= condition_conns conn_props &= condition_props diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index 4d18ea305..85a5e94cb 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -341,9 +341,9 @@ def make_allowed_connections(self, vs, host_dfa): allowed_conns = ConnectivityProperties.make_empty_props() for http_route in vs.http_routes: conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dims({"paths": http_route.uri_dfa, "hosts": host_dfa, "methods": http_route.methods}) + conn_cube.update({"paths": http_route.uri_dfa, "hosts": host_dfa, "methods": http_route.methods}) for dest in http_route.destinations: - conn_cube.set_dims({"dst_ports": dest.port, "dst_peers": dest.service.target_pods}) + conn_cube.update({"dst_ports": dest.port, "dst_peers": dest.service.target_pods}) conns = \ ConnectivityProperties.make_conn_props(conn_cube) allowed_conns |= conns @@ -399,7 +399,7 @@ def create_istio_traffic_policies(self): res_policy.add_rules(self._make_allow_rules(allowed_conns)) protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dims({"protocols": protocols, "src_peers": res_policy.selected_peers}) + conn_cube.update({"protocols": protocols, "src_peers": res_policy.selected_peers}) allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index 059a962d9..006e670b4 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -339,19 +339,19 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): if isinstance(protocol, str): protocol = ProtocolNameResolver.get_protocol_number(protocol) conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dim("dst_ports", dest_port_set) + conn_cube["dst_ports"] = dest_port_set # K8s doesn't reason about src ports res_conns.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) if self.optimized_run != 'false' and src_pods and dst_pods: protocols = ProtocolSet.get_protocol_set_with_single_protocol(protocol) - conn_cube.set_dims({"protocols": protocols, "src_peers": src_pods, "dst_peers": dst_pods}) + conn_cube.update({"protocols": protocols, "src_peers": src_pods, "dst_peers": dst_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) res_opt_props |= conn_props else: res_conns = ConnectionSet(True) if self.optimized_run != 'false': conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.set_dims({"src_peers": src_pods, "dst_peers": dst_pods}) + conn_cube.update({"src_peers": src_pods, "dst_peers": dst_pods}) res_opt_props = ConnectivityProperties.make_conn_props(conn_cube) if not res_pods: self.warning('Rule selects no pods', rule) diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index bf5c1f0ae..b14d38cf7 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -168,11 +168,11 @@ def create_opt_egress_props(self, peer_container): for rule in self.egress_rules: conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) if self.selected_peers and rule.egress_peer_set: - conn_cube.set_dims({"src_peers": self.selected_peers, "dst_peers": rule.egress_peer_set}) + conn_cube.update({"src_peers": self.selected_peers, "dst_peers": rule.egress_peer_set}) self.optimized_egress_props = ConnectivityProperties.make_conn_props(conn_cube) peers_sets_by_ns = self.combine_peer_sets_by_ns(self.selected_peers, rule.special_egress_peer_set, peer_container) for (from_peers, to_peers) in peers_sets_by_ns: if from_peers and to_peers: - conn_cube.set_dims({"src_peers": from_peers, "dst_peers": to_peers}) + conn_cube.update({"src_peers": from_peers, "dst_peers": to_peers}) self.optimized_egress_props |= ConnectivityProperties.make_conn_props(conn_cube) diff --git a/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py b/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py index d50fca841..13c7adfb9 100644 --- a/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py +++ b/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py @@ -14,12 +14,12 @@ def test_k8s_flow(self): dst_res_ports = PortSet() dst_res_ports.add_port("x") conn_cube = ConnectivityCube(PeerSet()) - conn_cube.set_dims({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) - tcp_properties1 = ConnectivityProperties(conn_cube) + conn_cube.update({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) + tcp_properties1 = ConnectivityProperties.create_props_from_cube(conn_cube) dst_res_ports2 = PortSet() dst_res_ports2.add_port("y") - conn_cube.set_dim("dst_ports", dst_res_ports2) - tcp_properties2 = ConnectivityProperties(conn_cube) + conn_cube["dst_ports"] = dst_res_ports2 + tcp_properties2 = ConnectivityProperties.create_props_from_cube(conn_cube) tcp_properties_res = tcp_properties1 | tcp_properties2 named_ports_dict = {"x": (15, 6), "z": (20, 6), "y": (16, 6)} tcp_properties_res.convert_named_ports(named_ports_dict, 6) @@ -40,8 +40,8 @@ def test_calico_flow_1(self): dst_res_ports.add_port("z") dst_res_ports.add_port("w") conn_cube = ConnectivityCube(PeerSet()) - conn_cube.set_dims({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) - tcp_properties = ConnectivityProperties(conn_cube) + conn_cube.update({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) + tcp_properties = ConnectivityProperties.create_props_from_cube(conn_cube) tcp_properties_2 = tcp_properties.copy() self.assertTrue(tcp_properties.has_named_ports()) @@ -73,8 +73,8 @@ def test_calico_flow_2(self): dst_res_ports -= not_ports src_res_ports.add_port_range(1, 100) conn_cube = ConnectivityCube(PeerSet()) - conn_cube.set_dims({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) - tcp_properties = ConnectivityProperties(conn_cube) + conn_cube.update({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) + tcp_properties = ConnectivityProperties.create_props_from_cube(conn_cube) tcp_properties_2 = tcp_properties.copy() self.assertTrue(tcp_properties.has_named_ports()) From f5f579e3becb528fdd2c0e67903a014065403e52 Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Tue, 21 Mar 2023 12:14:18 +0200 Subject: [PATCH 071/187] explain connectivity Signed-off-by: Shmulik Froimovich --- nca/CoreDS/Peer.py | 4 + nca/FileScanners/GenericTreeScanner.py | 3 + nca/NetworkConfig/NetworkConfigQuery.py | 3 + nca/NetworkConfig/NetworkLayer.py | 15 ++- nca/NetworkConfig/ResourcesHandler.py | 8 -- nca/NetworkConfig/TopologyObjectsFinder.py | 3 +- nca/Utils/ExplTracker.py | 136 ++++++++++++++++----- nca/Utils/NcaLogger.py | 4 + 8 files changed, 136 insertions(+), 40 deletions(-) diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index 12d537bde..491370b03 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -599,6 +599,10 @@ def get_all_peers_and_ip_blocks_interval(): res.add_interval(CanonicalIntervalSet.Interval(PeerSet.min_pod_index, PeerSet.max_pod_index)) return res + def get_peer_names_list(self): + names = list(elem.full_name_str for elem in self if not isinstance(elem, IpBlock)) + return names + def update_sorted_peer_list_if_needed(self): """ create self.sorted_peer_list from non IpBlock pods diff --git a/nca/FileScanners/GenericTreeScanner.py b/nca/FileScanners/GenericTreeScanner.py index 2196fe384..60efbe6c6 100644 --- a/nca/FileScanners/GenericTreeScanner.py +++ b/nca/FileScanners/GenericTreeScanner.py @@ -21,6 +21,7 @@ class YamlFile: class ObjectWithLocation: line_number = 0 + path = '' column_number = 0 @@ -36,6 +37,7 @@ def to_yaml_objects(yaml_node): if isinstance(yaml_node, yaml.SequenceNode): res = YamlList() res.line_number = yaml_node.start_mark.line + res.path = yaml_node.start_mark.name res.column_number = yaml_node.start_mark.column for obj in yaml_node.value: res.append(to_yaml_objects(obj)) @@ -43,6 +45,7 @@ def to_yaml_objects(yaml_node): if isinstance(yaml_node, yaml.MappingNode): res = YamlDict() res.line_number = yaml_node.start_mark.line + 1 + res.path = yaml_node.start_mark.name res.column_number = yaml_node.start_mark.column + 1 for obj in yaml_node.value: res[obj[0].value] = to_yaml_objects(obj[1]) diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 26c5deb7a..354d81aac 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -24,6 +24,7 @@ PoliciesAndRulesExplanations, PodsListsExplanations, ConnectionsDiffExplanation, IntersectPodsExplanation, \ PoliciesWithCommonPods, PeersAndConnections, ComputedExplanation from .NetworkLayer import NetworkLayerName +from nca.Utils.ExplTracker import ExplTracker class QueryType(Enum): @@ -755,6 +756,8 @@ def exec(self): subset_conns = TcpLikeProperties.make_tcp_like_properties(self.config.peer_container, src_peers=subset_peers, dst_peers=subset_peers) all_conns_opt &= subset_conns + ExplTracker().set_connections(all_conns_opt) + if self.config.policies_container.layers.does_contain_layer(NetworkLayerName.Istio): output_res, opt_fw_rules_tcp, opt_fw_rules_non_tcp = self.get_props_output_split_by_tcp(all_conns_opt, opt_peers_to_compare) opt_end = time.time() diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index cae55944e..0f6185213 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -247,6 +247,13 @@ def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda poli denied_conns = TcpLikeProperties.make_empty_properties() captured = PeerSet() for policy in self.policies_list: + # Track the peers that were affected by this policy + for peer in policy.selected_peers: + ExplTracker().add_peer_policy(peer, + policy.name, + policy.optimized_egress_props.project_on_one_dimension('dst_peers'), + policy.optimized_ingress_props.project_on_one_dimension('src_peers'), + ) policy_allowed_conns, policy_denied_conns, policy_captured = \ policy.allowed_connections_optimized(is_ingress) if policy_captured: # not empty @@ -261,9 +268,6 @@ def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda poli denied_conns |= policy_denied_conns if captured_func(policy): captured |= policy_captured - # Track the peers that were affected by this policy - for peer in captured: - ExplTracker().add_peer_policy(peer, policy) return allowed_conns, denied_conns, captured @@ -307,6 +311,11 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): TcpLikeProperties.make_tcp_like_properties(peer_container, src_peers=non_captured, dst_peers=base_peer_set_with_ip) + + ExplTracker().add_default_policy(non_captured_conns.project_on_one_dimension('dst_peers'), + non_captured_conns.project_on_one_dimension('src_peers'), + is_ingress + ) allowed_conn |= non_captured_conns return allowed_conn, denied_conns diff --git a/nca/NetworkConfig/ResourcesHandler.py b/nca/NetworkConfig/ResourcesHandler.py index 6bf9ad260..4ab79b796 100644 --- a/nca/NetworkConfig/ResourcesHandler.py +++ b/nca/NetworkConfig/ResourcesHandler.py @@ -408,13 +408,6 @@ def _parse_resources_path(self, resource_list, resource_flags): if ResourceType.Namespaces in resource_flags: self.ns_finder.parse_yaml_code_for_ns(res_code) if ResourceType.Pods in resource_flags: - if res_code: - # Track filenames and content - ExplTracker().add_item(yaml_file.path, - res_code, - res_code.get('metadata').get('name'), - res_code.line_number - ) self.pods_finder.namespaces_finder = self.ns_finder self.pods_finder.add_eps_from_yaml(res_code) self.services_finder.namespaces_finder = self.ns_finder @@ -423,7 +416,6 @@ def _parse_resources_path(self, resource_list, resource_flags): if res_code: # Track filenames and content ExplTracker().add_item(yaml_file.path, - res_code, res_code.get('metadata').get('name'), res_code.line_number ) diff --git a/nca/NetworkConfig/TopologyObjectsFinder.py b/nca/NetworkConfig/TopologyObjectsFinder.py index e5ce12bfc..b3025ba12 100644 --- a/nca/NetworkConfig/TopologyObjectsFinder.py +++ b/nca/NetworkConfig/TopologyObjectsFinder.py @@ -107,6 +107,7 @@ def _add_pod_from_yaml(self, pod_object): for port in container.get('ports') or []: pod.add_named_port(port.get('name'), port.get('containerPort'), port.get('protocol', 'TCP')) self._add_peer(pod) + ExplTracker().add_item(pod_object.path, pod.full_name(), pod_object.line_number) def _add_peer(self, peer): """ @@ -161,7 +162,7 @@ def _add_pod_from_workload_yaml(self, workload_resource): for port in container.get('ports') or []: pod.add_named_port(port.get('name'), port.get('containerPort'), port.get('protocol', 'TCP')) self._add_peer(pod) - ExplTracker().add_item('', workload_resource, pod.name, workload_resource.line_number) + ExplTracker().add_item(workload_resource.path, pod.full_name(), workload_resource.line_number) def _add_networkset_from_yaml(self, networkset_object): """ diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py index 384d2a666..8941ca912 100644 --- a/nca/Utils/ExplTracker.py +++ b/nca/Utils/ExplTracker.py @@ -23,12 +23,29 @@ def get_name(self): return self.name -class ExplPeer(ExplDescriptor): - pass +class ExplPolicies: + def __init__(self): + self.egress_dst = {} + self.ingress_src = {} + self.all_policies = set() + + @staticmethod + def _add_policy(peer_set, peer_list, policy_name): + for peer in peer_set: + peer_name = peer.full_name() + if not peer_list.get(peer_name): + peer_list[peer_name] = set() + peer_list[peer_name].add(policy_name) + + def add_policy(self, policy_name, egress_dst, ingress_src): + + self.all_policies.add(policy_name) + if egress_dst: + self._add_policy(egress_dst, self.egress_dst, policy_name) -class ExplPolicy(ExplDescriptor): - pass + if ingress_src: + self._add_policy(ingress_src, self.ingress_src, policy_name) class ExplTracker(metaclass=Singleton): @@ -42,6 +59,9 @@ def __init__(self): self.ExplDescriptorContainer = {} self.ExplPeerToPolicyContainer = {} self._is_active = False + self.conns = {} + + self.add_item('', 'Default-Policy', 0) def activate(self): self._is_active = True @@ -52,39 +72,99 @@ def is_active(self): def get_path_from_deployment(self, content): return self.ExplDescriptorContainer[content.get('metadata').get('name')].get('path') - def add_item(self, path, content, name, ln): - if path == '': - path = self.get_path_from_deployment(content) + def add_item(self, path, name, ln): self.ExplDescriptorContainer[name] = {'path': path, 'line': ln} - def add_peer_policy(self, peer, policy): - peer_name = peer.name - policy_name = policy.name - if not self.ExplPeerToPolicyContainer.get(peer_name): - self.ExplPeerToPolicyContainer[peer_name] = set() - self.ExplPeerToPolicyContainer[peer_name].add(policy_name) + def add_peer_policy(self, peer, policy_name, egress_dst, ingress_src): + peer_name = peer.full_name() + if self.ExplDescriptorContainer.get(peer_name): + if not self.ExplPeerToPolicyContainer.get(peer_name): + self.ExplPeerToPolicyContainer[peer_name] = ExplPolicies() + self.ExplPeerToPolicyContainer[peer_name].add_policy(policy_name, + egress_dst, + ingress_src, + ) + + def set_connections(self, conns): + self.conns = conns + pass + + def are_peers_connected(self, src, dst): + if not self.conns: + NcaLogger().log_message(f'Explainability error: Connections were not set yet, but peer query was called', + level='E') + for cube in self.conns: + conn = self.conns.get_cube_dict(cube) + src_peers = conn.get('src_peers').get_peer_names_list() + dst_peers = conn.get('dst_peers').get_peer_names_list() + if src in src_peers and dst in dst_peers: + return True + return False + + def add_default_policy(self, currents, peers, is_ingress): + + if is_ingress: + ingress_src = peers + egress_dst = {} + else: + ingress_src = {} + egress_dst = peers + + for peer in currents: + self.add_peer_policy(peer, + 'Default-Policy', + egress_dst, + ingress_src, + ) + + def prepare_node_str(self, direction, node_name, results): + + out = [] + if direction: + out = [f'\n({direction}){node_name}:'] + for name in results: + out.append(f'{name}: line {self.ExplDescriptorContainer.get(name).get("line")} ' + f'in file {self.ExplDescriptorContainer.get(name).get("path")}') + return out def explain(self, nodes): + + out = [] if len(nodes) < 1: - return + return out elif len(nodes) > 2: NcaLogger().log_message(f'Explainability error: only 1 or 2 nodes are allowed for explainability query,' f' found {len(nodes)} ', level='E') - return - results = {} + return out for node in nodes: if not self.ExplDescriptorContainer.get(node): - NcaLogger().log_message(f'Explainability error: {node} was not found in the connectivity results', level='E') - return - results[node] = self.ExplDescriptorContainer.get(node) - for policy in self.ExplPeerToPolicyContainer.get(node): - results[policy] = self.ExplDescriptorContainer.get(policy) - out = [] - if len(nodes) == 1: - out.append(f'Configurations affecting node {nodes[0]}: \n') - else: - out.append(f'Configurations affecting the connectivity between {nodes[0]} and {nodes[1]}:') - for name in results.keys(): - out.append(f'{name}: line {results.get(name).get("line")} in file {results.get(name).get("path")}') + NcaLogger().log_message(f'Explainability error - {node} was not found in the connectivity results', level='E') + return out + if not self.ExplPeerToPolicyContainer.get(node): + NcaLogger().log_message(f'Explainability error - {node} has no explanability results', level='E') + return out + + src_node = nodes[0] + if len(nodes) == 2: + # 2 nodes scenario + dst_node = nodes[1] + if self.are_peers_connected(src_node, dst_node): + # connection valid + out.append(f'\nConfigurations affecting the connectivity between (src){src_node} and (dst){dst_node}:') + src_results = self.ExplPeerToPolicyContainer[src_node].egress_dst.get(dst_node) + dst_results = self.ExplPeerToPolicyContainer[dst_node].ingress_src.get(src_node) + else: + out.append(f'\nConfigurations affecting the LACK of connectivity between (src){src_node} and (dst){dst_node}:') + src_results = self.ExplPeerToPolicyContainer[src_node].all_policies + dst_results = self.ExplPeerToPolicyContainer[dst_node].all_policies + + src_results.add(src_node) + dst_results.add(dst_node) + out.extend(self.prepare_node_str('src', src_node, src_results)) + out.extend(self.prepare_node_str('dst', dst_node, dst_results)) + else: # only one node + results = self.ExplPeerToPolicyContainer[src_node].all_policies + out.append(f'\nConfigurations affecting {src_node}:') + out.extend(self.prepare_node_str(None, src_node, results)) return out diff --git a/nca/Utils/NcaLogger.py b/nca/Utils/NcaLogger.py index 77958579c..98455a4a1 100644 --- a/nca/Utils/NcaLogger.py +++ b/nca/Utils/NcaLogger.py @@ -85,6 +85,10 @@ def log_message(self, msg, file=None, level=None): msg = f'Warning: {msg}' if not file: file = sys.stderr + elif level == 'E': + msg = f'Error: {msg}' + if not file: + file = sys.stderr if self._is_collecting_msgs: if self.is_mute(): From 501365949122d714978a5d508d4d01e6175b875a Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 12:28:36 +0200 Subject: [PATCH 072/187] Made cleaner interface of ConectivityCube class, using __setitem__, __getitem__ and update functions. Made cleaner interface of creating empty/full/by cube ConnectivityProperties. Signed-off-by: Tanya --- nca/CoreDS/ConnectivityProperties.py | 44 ++++++++++++++++------------ 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 9c6919baf..9145613c5 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -58,12 +58,12 @@ def __init__(self, base_peer_set): self.excluded_named_ports = set() # used only in the original solution self.base_peer_set = base_peer_set for dim in self.dimensions_list: - self[dim] = DimensionsManager().get_dimension_domain_by_name(dim) + self.set_dim_directly(dim, DimensionsManager().get_dimension_domain_by_name(dim)) def copy(self): res = ConnectivityCube(self.base_peer_set.copy()) for dim_name, dim_value in self.items(): - if isinstance(self[dim_name], MinDFA): + if isinstance(dim_value, MinDFA): res.set_dim_directly(dim_name, dim_value) else: res.set_dim_directly(dim_name, dim_value.copy()) @@ -89,7 +89,11 @@ def is_active_dim(self, dim_name): def set_dim_directly(self, dim_name, dim_value): assert dim_name in self.dimensions_list - self[dim_name] = dim_value + super().__setitem__(dim_name, dim_value) + + def get_dim_directly(self, dim_name): + assert dim_name in self.dimensions_list + return super().__getitem__(dim_name) def __setitem__(self, dim_name, dim_value): assert dim_name in self.dimensions_list @@ -97,18 +101,18 @@ def __setitem__(self, dim_name, dim_value): return if dim_name == "src_peers" or dim_name == "dst_peers": # translate PeerSet to CanonicalIntervalSet - self[dim_name] = self.base_peer_set.get_peer_interval_of(dim_value) + self.set_dim_directly(dim_name, self.base_peer_set.get_peer_interval_of(dim_value)) elif dim_name == "src_ports" or dim_name == "dst_ports": # extract port_set from PortSet - self[dim_name] = dim_value.port_set + self.set_dim_directly(dim_name, dim_value.port_set) if dim_name == "dst_ports": self.named_ports = dim_value.named_ports self.excluded_named_ports = dim_value.excluded_named_ports elif dim_name == "icmp_type" or dim_name == "icmp_code": # translate int to CanonicalIntervalSet - self[dim_name] = CanonicalIntervalSet.get_interval_set(dim_value, dim_value) + self.set_dim_directly(dim_name, CanonicalIntervalSet.get_interval_set(dim_value, dim_value)) else: # the rest of dimensions do not need a translation - self[dim_name] = dim_value + self.set_dim_directly(dim_name, dim_value) def update(self, dims=None, **f): for dim_name, dim_value in dims.items(): @@ -116,19 +120,20 @@ def update(self, dims=None, **f): def unset_dim(self, dim_name): assert dim_name in self.dimensions_list - self[dim_name] = DimensionsManager().get_dimension_domain_by_name(dim_name) + self.set_dim_directly(dim_name, DimensionsManager().get_dimension_domain_by_name(dim_name)) def __getitem__(self, dim_name): assert dim_name in self.dimensions_list + dim_value = self.get_dim_directly(dim_name) if dim_name == "src_peers" or dim_name == "dst_peers": if self.is_active_dim(dim_name): # translate CanonicalIntervalSet back to PeerSet - return self.base_peer_set.get_peer_set_by_indices(self[dim_name]) + return self.base_peer_set.get_peer_set_by_indices(dim_value) else: return None elif dim_name == "src_ports" or dim_name == "dst_ports": res = PortSet() - res.add_ports(self[dim_name]) + res.add_ports(dim_value) if dim_name == "dst_ports": res.named_ports = self.named_ports res.excluded_named_ports = self.excluded_named_ports @@ -136,18 +141,18 @@ def __getitem__(self, dim_name): elif dim_name == "icmp_type" or dim_name == "icmp_code": if self.is_active_dim(dim_name): # translate CanonicalIntervalSet back to int - return self[dim_name].validate_and_get_single_value() + return dim_value.validate_and_get_single_value() else: return None else: # the rest of dimensions do not need a translation - if isinstance(self[dim_name], MinDFA): - return self[dim_name] + if isinstance(dim_value, MinDFA): + return dim_value else: - return self[dim_name].copy() # TODO - do we need this copy? + return dim_value.copy() # TODO - do we need this copy? def has_active_dim(self): for dim in self.dimensions_list: - if self[dim] != DimensionsManager().get_dimension_domain_by_name(dim): + if self.get_dim_directly(dim) != DimensionsManager().get_dimension_domain_by_name(dim): return True return False @@ -161,11 +166,12 @@ def get_ordered_cube_and_active_dims(self): has_empty_dim_value = False # add values to cube by required order of dimensions for dim in self.dimensions_list: - if self[dim] != DimensionsManager().get_dimension_domain_by_name(dim): - if isinstance(self[dim], MinDFA): - cube.append(self[dim]) + dim_value = self.get_dim_directly(dim) + if dim_value != DimensionsManager().get_dimension_domain_by_name(dim): + if isinstance(dim_value, MinDFA): + cube.append(dim_value) else: - cube.append(self[dim].copy()) # TODO - do we need this copy? + cube.append(dim_value.copy()) # TODO - do we need this copy? active_dims.append(dim) has_empty_dim_value |= self.is_empty_dim(dim) return cube, active_dims, has_empty_dim_value From 90bbceef5b033c502713e94aaea90dbb9b42cc0c Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Tue, 21 Mar 2023 12:29:50 +0200 Subject: [PATCH 073/187] explain connectivity Signed-off-by: Shmulik Froimovich --- nca/Utils/ExplTracker.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py index 8941ca912..1af795822 100644 --- a/nca/Utils/ExplTracker.py +++ b/nca/Utils/ExplTracker.py @@ -123,8 +123,12 @@ def prepare_node_str(self, direction, node_name, results): if direction: out = [f'\n({direction}){node_name}:'] for name in results: - out.append(f'{name}: line {self.ExplDescriptorContainer.get(name).get("line")} ' - f'in file {self.ExplDescriptorContainer.get(name).get("path")}') + path = self.ExplDescriptorContainer.get(name).get("path") + if path == '': # special element (like Default Policy) + out.append(f'{name}') + else: + out.append(f'{name}: line {self.ExplDescriptorContainer.get(name).get("line")} ' + f'in file {path}') return out def explain(self, nodes): From 4119c29aff5661d9d03476ae00b41a23ef2f8d82 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 12:30:32 +0200 Subject: [PATCH 074/187] Fixed lint error. Signed-off-by: Tanya --- nca/NetworkConfig/NetworkLayer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 1765f0d86..7672929fa 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -341,7 +341,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conn |= (non_captured_conns - denied_conns) conn_cube.update({"src_peers": base_peer_set_with_ip, "dst_peers": base_peer_set_with_ip, - "protocols": ProtocolSet.get_non_tcp_protocols()}) + "protocols": ProtocolSet.get_non_tcp_protocols()}) allowed_conn |= ConnectivityProperties.make_conn_props(conn_cube) return allowed_conn, denied_conns From 9fef3cea10b6842b18bffd166cd53ef5b9d16a46 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 12:47:32 +0200 Subject: [PATCH 075/187] Small fix Signed-off-by: Tanya --- nca/CoreDS/ConnectivityProperties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 9145613c5..0ede8b263 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -243,7 +243,7 @@ def create_props_from_cube(conn_cube): cube, active_dims, has_empty_dim_value = conn_cube.get_ordered_cube_and_active_dims() if has_empty_dim_value: - return + return res if not active_dims: res.set_all() From e59169fdcaab8a6909f72fce7f5e18693bc4a655 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 15:26:28 +0200 Subject: [PATCH 076/187] Added documentation and small code beautifications. Signed-off-by: Tanya --- nca/CoreDS/ConnectivityProperties.py | 82 +++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 14 deletions(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 0ede8b263..1407878b4 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -61,6 +61,9 @@ def __init__(self, base_peer_set): self.set_dim_directly(dim, DimensionsManager().get_dimension_domain_by_name(dim)) def copy(self): + """ + Returns a copy of the given ConnectivityCube object + """ res = ConnectivityCube(self.base_peer_set.copy()) for dim_name, dim_value in self.items(): if isinstance(dim_value, MinDFA): @@ -71,31 +74,60 @@ def copy(self): @staticmethod def get_empty_dim(dim_name): + """ + Returns an empty value of a given dimension + :param dim_name: the given dimension name + """ return ConnectivityCube.external_empty_dim_values.get(dim_name) def is_empty_dim(self, dim_name): - assert dim_name in self.dimensions_list - if dim_name == "dst_ports": # can have named ports in original solution - return self.get(dim_name) == self.internal_empty_dim_values.get(dim_name) and \ - not self.named_ports and not self.excluded_named_ports - return self.get(dim_name) == self.internal_empty_dim_values.get(dim_name) + """ + Returns True iff a given dimension is empty + :param str dim_name: the given dimension name + """ + if self.get_dim_directly(dim_name) != self.internal_empty_dim_values.get(dim_name): + return False + + # for "dst_ports" can have named ports in original solution + return not self.named_ports and not self.excluded_named_ports if dim_name == "dst_ports" else True def is_full_dim(self, dim_name): - assert dim_name in self.dimensions_list - return self.get(dim_name) == DimensionsManager().get_dimension_domain_by_name(dim_name) + """ + Returns True iff a given dimension is full + :param str dim_name: the given dimension name + """ + return self.get_dim_directly(dim_name) == DimensionsManager().get_dimension_domain_by_name(dim_name) def is_active_dim(self, dim_name): + """ + Returns True iff a given dimension is active (i.e., not full) + :param str dim_name: the given dimension name + """ return not self.is_full_dim(dim_name) def set_dim_directly(self, dim_name, dim_value): + """ + Sets a given dimension value directly, assuming the value is in the internal format of that dimension. + :param str dim_name: the given dimension name + :param dim_value: the given dimension value + """ assert dim_name in self.dimensions_list super().__setitem__(dim_name, dim_value) def get_dim_directly(self, dim_name): + """ + Returns a given dimension value directly (in its internal format). + :param str dim_name: the given dimension name + """ assert dim_name in self.dimensions_list return super().__getitem__(dim_name) def __setitem__(self, dim_name, dim_value): + """ + Sets a given dimension value after converting it into the internal format of that dimension. + :param str dim_name: the given dimension name + :param dim_value: the given dimension value + """ assert dim_name in self.dimensions_list if dim_value is None: return @@ -115,14 +147,27 @@ def __setitem__(self, dim_name, dim_value): self.set_dim_directly(dim_name, dim_value) def update(self, dims=None, **f): + """ + Sets multiple dimension values at once, after converting them into their internal formats. + :param dict dims: a dictionary from dimension names to dimension values, having all dimensions to be set + :param f: Not used; required by the base class (dict) interface. + """ for dim_name, dim_value in dims.items(): self[dim_name] = dim_value def unset_dim(self, dim_name): + """ + Sets a given dimension to its default (containing all possible values) + :param str dim_name: the given dimension name + """ assert dim_name in self.dimensions_list self.set_dim_directly(dim_name, DimensionsManager().get_dimension_domain_by_name(dim_name)) def __getitem__(self, dim_name): + """ + Returns a given dimension value after converting it from internal to external format. + :param str dim_name: the given dimension name + """ assert dim_name in self.dimensions_list dim_value = self.get_dim_directly(dim_name) if dim_name == "src_peers" or dim_name == "dst_peers": @@ -151,19 +196,30 @@ def __getitem__(self, dim_name): return dim_value.copy() # TODO - do we need this copy? def has_active_dim(self): + """ + Returns True iff the cube has at least one active dimension. Otherwise, returns False. + """ for dim in self.dimensions_list: if self.get_dim_directly(dim) != DimensionsManager().get_dimension_domain_by_name(dim): return True return False + def is_empty(self): + """ + Returns True iff the cube has at least one empty dimension. Otherwise, returns False. + """ + for dim in self.dimensions_list: + if self.is_empty_dim(dim): + return True + return False + def get_ordered_cube_and_active_dims(self): """ Translate the connectivity cube to an ordered cube, and compute its active dimensions - :return: tuple with: (1) cube values (2) active dimensions (3) bool indication if some dimension is empty + :return: tuple with: (1) cube values (2) active dimensions """ cube = [] active_dims = [] - has_empty_dim_value = False # add values to cube by required order of dimensions for dim in self.dimensions_list: dim_value = self.get_dim_directly(dim) @@ -173,8 +229,7 @@ def get_ordered_cube_and_active_dims(self): else: cube.append(dim_value.copy()) # TODO - do we need this copy? active_dims.append(dim) - has_empty_dim_value |= self.is_empty_dim(dim) - return cube, active_dims, has_empty_dim_value + return cube, active_dims class ConnectivityProperties(CanonicalHyperCubeSet): @@ -240,11 +295,10 @@ def create_props_from_cube(conn_cube): """ res = ConnectivityProperties() res.base_peer_set = conn_cube.base_peer_set.copy() - - cube, active_dims, has_empty_dim_value = conn_cube.get_ordered_cube_and_active_dims() - if has_empty_dim_value: + if conn_cube.is_empty(): return res + cube, active_dims = conn_cube.get_ordered_cube_and_active_dims() if not active_dims: res.set_all() else: From 26904f8a82315d046bb9560791e057a8e8f40de0 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 16:09:39 +0200 Subject: [PATCH 077/187] Improved documentation. Simplified interface by adding ConnectivityCube.make_from_dict method. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 7 +++---- nca/CoreDS/ConnectivityProperties.py | 14 +++++++++++--- nca/NetworkConfig/NetworkConfigQuery.py | 12 ++++++------ nca/NetworkConfig/NetworkLayer.py | 4 ++-- nca/Parsers/CalicoPolicyYamlParser.py | 16 ++++++++-------- nca/Parsers/IngressPolicyYamlParser.py | 16 +++++++--------- nca/Parsers/IstioPolicyYamlParser.py | 12 ++++++------ nca/Parsers/IstioTrafficResourcesYamlParser.py | 8 ++++---- nca/Parsers/K8sPolicyYamlParser.py | 8 ++++---- .../testConnectivityPropertiesNamedPorts.py | 9 +++------ 10 files changed, 54 insertions(+), 52 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index d82612a45..cb89b1da0 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -549,8 +549,7 @@ def convert_to_connectivity_properties(self, peer_container): res = ConnectivityProperties.make_empty_props() for protocol, properties in self.allowed_protocols.items(): protocols = ProtocolSet.get_protocol_set_with_single_protocol(protocol) - conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) - conn_cube["protocols"] = protocols + conn_cube = ConnectivityCube.make_from_dict(peer_container.get_all_peers_group(), {"protocols": protocols}) this_prop = ConnectivityProperties.make_conn_props(conn_cube) if isinstance(properties, bool): if properties: @@ -673,8 +672,8 @@ def fw_rules_to_conn_props(fw_rules, peer_container): conn_props = fw_rule.conn.convert_to_connectivity_properties(peer_container) src_peers = PeerSet(fw_rule.src.get_peer_set(fw_rules.cluster_info)) dst_peers = PeerSet(fw_rule.dst.get_peer_set(fw_rules.cluster_info)) - conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) - conn_cube.update({"src_peers": src_peers, "dst_peers": dst_peers}) + conn_cube = ConnectivityCube.make_from_dict(peer_container.get_all_peers_group(), + {"src_peers": src_peers, "dst_peers": dst_peers}) rule_props = ConnectivityProperties.make_conn_props(conn_cube) & conn_props res |= rule_props return res diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 1407878b4..1e7d5b59f 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -50,6 +50,8 @@ class ConnectivityCube(dict): def __init__(self, base_peer_set): """ + By default, each dimension in the cube is initialized with entire domain value, which represents + "don't care" or inactive dimension (i.e., the dimension has no impact). :param PeerSet base_peer_set: the set of all possible peers, which will be referenced by the indices in 'src_peers' and 'dst_peers' """ @@ -146,13 +148,13 @@ def __setitem__(self, dim_name, dim_value): else: # the rest of dimensions do not need a translation self.set_dim_directly(dim_name, dim_value) - def update(self, dims=None, **f): + def update(self, the_dict=None, **f): """ Sets multiple dimension values at once, after converting them into their internal formats. - :param dict dims: a dictionary from dimension names to dimension values, having all dimensions to be set + :param dict the_dict: a dictionary from dimension names to dimension values, having all dimensions to be set :param f: Not used; required by the base class (dict) interface. """ - for dim_name, dim_value in dims.items(): + for dim_name, dim_value in the_dict.items(): self[dim_name] = dim_value def unset_dim(self, dim_name): @@ -231,6 +233,12 @@ def get_ordered_cube_and_active_dims(self): active_dims.append(dim) return cube, active_dims + @staticmethod + def make_from_dict(base_peer_set, the_dict): + ccube = ConnectivityCube(base_peer_set) + ccube.update(the_dict) + return ccube + class ConnectivityProperties(CanonicalHyperCubeSet): """ diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 584779602..94f8fc1c5 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -753,10 +753,10 @@ def exec(self): # noqa: C901 all_conns_opt.project_on_one_dimension('dst_peers') subset_peers = self.compute_subset(opt_peers_to_compare) - src_peers_conn_cube = ConnectivityCube(self.config.peer_container.get_all_peers_group()) - dst_peers_conn_cube = ConnectivityCube(self.config.peer_container.get_all_peers_group()) - src_peers_conn_cube["src_peers"] = subset_peers - dst_peers_conn_cube["dst_peers"] = subset_peers + src_peers_conn_cube = ConnectivityCube.make_from_dict(self.config.peer_container.get_all_peers_group(), + {"src_peers": subset_peers}) + dst_peers_conn_cube = ConnectivityCube.make_from_dict(self.config.peer_container.get_all_peers_group(), + {"dst_peers": subset_peers}) subset_conns = ConnectivityProperties.make_conn_props(src_peers_conn_cube) | \ ConnectivityProperties.make_conn_props(dst_peers_conn_cube) all_conns_opt &= subset_conns @@ -1052,8 +1052,8 @@ def convert_props_to_split_by_tcp(self, props): :rtype: tuple(ConnectivityProperties, ConnectivityProperties) """ tcp_protocol = ProtocolSet.get_protocol_set_with_single_protocol('TCP') - conn_cube = ConnectivityCube(self.config.peer_container.get_all_peers_group()) - conn_cube["protocols"] = tcp_protocol + conn_cube = ConnectivityCube.make_from_dict(self.config.peer_container.get_all_peers_group(), + {"protocols", tcp_protocol}) tcp_props = props & ConnectivityProperties.make_conn_props(conn_cube) non_tcp_props = props - tcp_props return tcp_props, non_tcp_props diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 7672929fa..08e306ff5 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -176,8 +176,8 @@ def allowed_connections_optimized(self, peer_container): """ all_pods = peer_container.get_all_peers_group() all_ips_peer_set = PeerSet({IpBlock.get_all_ips_block()}) - conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) - conn_cube.update({"src_peers": all_pods, "dst_peers": all_ips_peer_set}) + conn_cube = ConnectivityCube.make_from_dict(peer_container.get_all_peers_group(), + {"src_peers": all_pods, "dst_peers": all_ips_peer_set}) allowed_ingress_conns, denied_ingres_conns = self._allowed_xgress_conns_optimized(True, peer_container) allowed_ingress_conns |= ConnectivityProperties.make_conn_props(conn_cube) allowed_egress_conns, denied_egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index 720980a98..c72973f30 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -521,8 +521,8 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): self.warning('notProtocol field has no effect', rule) else: if protocol_supports_ports: - conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.update({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) + conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { + "src_ports": src_res_ports, "dst_ports": dst_res_ports}) connections.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) if self.optimized_run != 'false': conn_cube.update({"protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) @@ -534,8 +534,8 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): else: connections.add_connections(protocol, True) if self.optimized_run != 'false': - conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.update({"protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) + conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { + "protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) elif not_protocol is not None: connections.add_all_connections() @@ -543,14 +543,14 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): if self.optimized_run != 'false' and src_res_pods and dst_res_pods: protocols = ProtocolSet(True) protocols.remove_protocol(not_protocol) - conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.update({"protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) + conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { + "protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) else: connections.allow_all = True if self.optimized_run != 'false': - conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.update({"src_peers": src_res_pods, "dst_peers": dst_res_pods}) + conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), + {"src_peers": src_res_pods, "dst_peers": dst_res_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) self._verify_named_ports(rule, dst_res_pods, connections) diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index dd3ba6cb1..da0175260 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -187,15 +187,13 @@ def _make_default_connections(self, hosts_dfa, paths_dfa=None): """ default_conns = ConnectivityProperties.make_empty_props() if self.default_backend_peers: - conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.update({"dst_ports": self.default_backend_ports, "dst_peers": self.default_backend_peers}) + conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { + "dst_ports": self.default_backend_ports, "dst_peers": self.default_backend_peers}) if hosts_dfa: conn_cube["hosts"] = hosts_dfa if paths_dfa: conn_cube["paths"] = paths_dfa - default_conns = ConnectivityProperties.make_conn_props(conn_cube) - else: - default_conns = ConnectivityProperties.make_conn_props(conn_cube) + default_conns = ConnectivityProperties.make_conn_props(conn_cube) return default_conns def parse_rule(self, rule): @@ -223,8 +221,8 @@ def parse_rule(self, rule): parsed_paths.append(path_resources) if parsed_paths: parsed_paths_with_dfa = self.segregate_longest_paths_and_make_dfa(parsed_paths) - conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube["hosts"] = hosts_dfa + conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), + {"hosts": hosts_dfa}) for (_, paths_dfa, _, peers, ports) in parsed_paths_with_dfa: # every path is converted to allowed connections conn_cube.update({"dst_ports": ports, "dst_peers": peers, "paths": paths_dfa}) @@ -290,8 +288,8 @@ def parse_policy(self): if allowed_conns: res_policy.add_rules(self._make_allow_rules(allowed_conns)) protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') - conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.update({"protocols": protocols, "src_peers": res_policy.selected_peers}) + conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { + "protocols": protocols, "src_peers": res_policy.selected_peers}) allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index 9c64c9ad2..89ff9f0f1 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -195,8 +195,8 @@ def parse_key_values(self, key, values, not_values): return self.parse_principals(values, not_values) # PeerSet elif key == 'destination.port': dst_ports = self.get_rule_ports(values, not_values) # PortSet - conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube["dst_ports"] = dst_ports + conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), + {"dst_ports": dst_ports}) return ConnectivityProperties.make_conn_props(conn_cube) # ConnectivityProperties return NotImplemented, False @@ -399,8 +399,8 @@ def parse_operation(self, operation_dict): operation) hosts_dfa = self.parse_regex_dimension_values("hosts", operation.get("hosts"), operation.get("notHosts"), operation) - conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.update({"dst_ports": dst_ports, "methods": methods_set, "paths": paths_dfa, 'hosts': hosts_dfa}) + conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { + "dst_ports": dst_ports, "methods": methods_set, "paths": paths_dfa, 'hosts': hosts_dfa}) return ConnectivityProperties.make_conn_props(conn_cube) def parse_source(self, source_dict): @@ -522,8 +522,8 @@ def parse_ingress_rule(self, rule, selected_peers): if not res_peers or not selected_peers: condition_props = ConnectivityProperties.make_empty_props() else: - conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.update({"src_peers": res_peers, "dst_peers": selected_peers}) + conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), + {"src_peers": res_peers, "dst_peers": selected_peers}) condition_props &= ConnectivityProperties.make_conn_props(conn_cube) connections &= condition_conns conn_props &= condition_props diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index 85a5e94cb..21d6863c8 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -340,8 +340,8 @@ def make_allowed_connections(self, vs, host_dfa): """ allowed_conns = ConnectivityProperties.make_empty_props() for http_route in vs.http_routes: - conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.update({"paths": http_route.uri_dfa, "hosts": host_dfa, "methods": http_route.methods}) + conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { + "paths": http_route.uri_dfa, "hosts": host_dfa, "methods": http_route.methods}) for dest in http_route.destinations: conn_cube.update({"dst_ports": dest.port, "dst_peers": dest.service.target_pods}) conns = \ @@ -398,8 +398,8 @@ def create_istio_traffic_policies(self): if allowed_conns: res_policy.add_rules(self._make_allow_rules(allowed_conns)) protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') - conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.update({"protocols": protocols, "src_peers": res_policy.selected_peers}) + conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { + "protocols": protocols, "src_peers": res_policy.selected_peers}) allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index 006e670b4..01d5191a4 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -338,8 +338,8 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): protocol, dest_port_set = self.parse_port(port) if isinstance(protocol, str): protocol = ProtocolNameResolver.get_protocol_number(protocol) - conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube["dst_ports"] = dest_port_set + conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), + {"dst_ports", dest_port_set}) # K8s doesn't reason about src ports res_conns.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) if self.optimized_run != 'false' and src_pods and dst_pods: @@ -350,8 +350,8 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): else: res_conns = ConnectionSet(True) if self.optimized_run != 'false': - conn_cube = ConnectivityCube(self.peer_container.get_all_peers_group()) - conn_cube.update({"src_peers": src_pods, "dst_peers": dst_pods}) + conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), + {"src_peers": src_pods, "dst_peers": dst_pods}) res_opt_props = ConnectivityProperties.make_conn_props(conn_cube) if not res_pods: self.warning('Rule selects no pods', rule) diff --git a/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py b/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py index 13c7adfb9..c86f21bcc 100644 --- a/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py +++ b/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py @@ -13,8 +13,7 @@ def test_k8s_flow(self): src_res_ports = PortSet(True) dst_res_ports = PortSet() dst_res_ports.add_port("x") - conn_cube = ConnectivityCube(PeerSet()) - conn_cube.update({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) + conn_cube = ConnectivityCube.make_from_dict(PeerSet(), {"src_ports": src_res_ports, "dst_ports": dst_res_ports}) tcp_properties1 = ConnectivityProperties.create_props_from_cube(conn_cube) dst_res_ports2 = PortSet() dst_res_ports2.add_port("y") @@ -39,8 +38,7 @@ def test_calico_flow_1(self): dst_res_ports.add_port("y") dst_res_ports.add_port("z") dst_res_ports.add_port("w") - conn_cube = ConnectivityCube(PeerSet()) - conn_cube.update({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) + conn_cube = ConnectivityCube.make_from_dict(PeerSet(), {"src_ports": src_res_ports, "dst_ports": dst_res_ports}) tcp_properties = ConnectivityProperties.create_props_from_cube(conn_cube) tcp_properties_2 = tcp_properties.copy() @@ -72,8 +70,7 @@ def test_calico_flow_2(self): dst_res_ports = PortSet(True) dst_res_ports -= not_ports src_res_ports.add_port_range(1, 100) - conn_cube = ConnectivityCube(PeerSet()) - conn_cube.update({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) + conn_cube = ConnectivityCube.make_from_dict(PeerSet(), {"src_ports": src_res_ports, "dst_ports": dst_res_ports}) tcp_properties = ConnectivityProperties.create_props_from_cube(conn_cube) tcp_properties_2 = tcp_properties.copy() From 3aa9f521f3d9e3600db2067d644971f02c1499f4 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 16:22:16 +0200 Subject: [PATCH 078/187] Small fix. Signed-off-by: Tanya --- nca/Parsers/K8sPolicyYamlParser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index 01d5191a4..6bb12c7d3 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -339,7 +339,7 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): if isinstance(protocol, str): protocol = ProtocolNameResolver.get_protocol_number(protocol) conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), - {"dst_ports", dest_port_set}) + {"dst_ports": dest_port_set}) # K8s doesn't reason about src ports res_conns.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) if self.optimized_run != 'false' and src_pods and dst_pods: From 4797f3cd3492d42dca28620d8328a6138117c473 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 17:05:50 +0200 Subject: [PATCH 079/187] Moved empty dimension values to DimensionsManager. Fixed project_on_one_dimension method. Signed-off-by: Tanya --- nca/CoreDS/ConnectivityProperties.py | 50 ++++++---------------------- nca/CoreDS/DimensionsManager.py | 30 +++++++++++------ 2 files changed, 31 insertions(+), 49 deletions(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 1e7d5b59f..cee4e09e2 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -23,30 +23,6 @@ class ConnectivityCube(dict): dimensions_list = ["src_peers", "dst_peers", "protocols", "src_ports", "dst_ports", "methods", "hosts", "paths", "icmp_type", "icmp_code"] - internal_empty_dim_values = { - "src_peers": CanonicalIntervalSet(), - "dst_peers": CanonicalIntervalSet(), - "protocols": ProtocolSet(), - "src_ports": CanonicalIntervalSet(), - "dst_ports": CanonicalIntervalSet(), - "methods": MethodSet(), - "hosts": MinDFA.dfa_from_regex(""), - "paths": MinDFA.dfa_from_regex(""), - "icmp_type": CanonicalIntervalSet(), - "icmp_code": CanonicalIntervalSet() - } - external_empty_dim_values = { - "src_peers": PeerSet(), - "dst_peers": PeerSet(), - "protocols": ProtocolSet(), - "src_ports": PortSet(), - "dst_ports": PortSet(), - "methods": MethodSet(), - "hosts": MinDFA.dfa_from_regex(""), - "paths": MinDFA.dfa_from_regex(""), - "icmp_type": None, - "icmp_code": None - } def __init__(self, base_peer_set): """ @@ -74,20 +50,12 @@ def copy(self): res.set_dim_directly(dim_name, dim_value.copy()) return res - @staticmethod - def get_empty_dim(dim_name): - """ - Returns an empty value of a given dimension - :param dim_name: the given dimension name - """ - return ConnectivityCube.external_empty_dim_values.get(dim_name) - def is_empty_dim(self, dim_name): """ Returns True iff a given dimension is empty :param str dim_name: the given dimension name """ - if self.get_dim_directly(dim_name) != self.internal_empty_dim_values.get(dim_name): + if self.get_dim_directly(dim_name) != DimensionsManager().get_empty_dimension_by_name(dim_name): return False # for "dst_ports" can have named ports in original solution @@ -557,20 +525,24 @@ def _get_first_item_str(self): def project_on_one_dimension(self, dim_name): """ - Build the projection of self to the given dimension + Build the projection of self to the given dimension. + Supports any dimension except of icmp data (icmp_type and icmp_code). :param str dim_name: the given dimension :return: the projection on the given dimension, having that dimension type. - or empty dimension value if the given dimension is not active + or None if the given dimension is not active """ - res = ConnectivityCube.get_empty_dim(dim_name) + if dim_name == "icmp_type" or dim_name == "icmp_code": + return None # not supporting icmp dimensions if dim_name not in self.active_dimensions: - return res + return None + res = None for cube in self: conn_cube = self.get_connectivity_cube(cube) values = conn_cube[dim_name] - if values: + if values and res: res |= values - + elif values: + res = values return res @staticmethod diff --git a/nca/CoreDS/DimensionsManager.py b/nca/CoreDS/DimensionsManager.py index b8645f001..04c35f112 100644 --- a/nca/CoreDS/DimensionsManager.py +++ b/nca/CoreDS/DimensionsManager.py @@ -34,20 +34,22 @@ def __init__(self): all_methods_interval = MethodSet(True) all_protocols_interval = ProtocolSet(True) all_peers_interval = PeerSet.get_all_peers_and_ip_blocks_interval() + # dim_dict is a map from a dimension name to a tuple + # (dimension type, dimension full domain, dimension empty value) self.dim_dict = dict() - self.dim_dict["src_ports"] = (DimensionsManager.DimensionType.IntervalSet, ports_interval) - self.dim_dict["dst_ports"] = (DimensionsManager.DimensionType.IntervalSet, ports_interval) - self.dim_dict["methods"] = (DimensionsManager.DimensionType.IntervalSet, all_methods_interval) - self.dim_dict["protocols"] = (DimensionsManager.DimensionType.IntervalSet, all_protocols_interval) - self.dim_dict["src_peers"] = (DimensionsManager.DimensionType.IntervalSet, all_peers_interval) - self.dim_dict["dst_peers"] = (DimensionsManager.DimensionType.IntervalSet, all_peers_interval) - self.dim_dict["paths"] = (DimensionsManager.DimensionType.DFA, dfa_all_words_path_domain) - self.dim_dict["hosts"] = (DimensionsManager.DimensionType.DFA, dfa_all_words_default) + self.dim_dict["src_ports"] = (DimensionsManager.DimensionType.IntervalSet, ports_interval, CanonicalIntervalSet()) + self.dim_dict["dst_ports"] = (DimensionsManager.DimensionType.IntervalSet, ports_interval, CanonicalIntervalSet()) + self.dim_dict["methods"] = (DimensionsManager.DimensionType.IntervalSet, all_methods_interval, MethodSet()) + self.dim_dict["protocols"] = (DimensionsManager.DimensionType.IntervalSet, all_protocols_interval, ProtocolSet()) + self.dim_dict["src_peers"] = (DimensionsManager.DimensionType.IntervalSet, all_peers_interval, CanonicalIntervalSet()) + self.dim_dict["dst_peers"] = (DimensionsManager.DimensionType.IntervalSet, all_peers_interval, CanonicalIntervalSet()) + self.dim_dict["paths"] = (DimensionsManager.DimensionType.DFA, dfa_all_words_path_domain, MinDFA.dfa_from_regex("")) + self.dim_dict["hosts"] = (DimensionsManager.DimensionType.DFA, dfa_all_words_default, MinDFA.dfa_from_regex("")) icmp_type_interval = CanonicalIntervalSet.get_interval_set(0, 254) icmp_code_interval = CanonicalIntervalSet.get_interval_set(0, 255) - self.dim_dict["icmp_type"] = (DimensionsManager.DimensionType.IntervalSet, icmp_type_interval) - self.dim_dict["icmp_code"] = (DimensionsManager.DimensionType.IntervalSet, icmp_code_interval) + self.dim_dict["icmp_type"] = (DimensionsManager.DimensionType.IntervalSet, icmp_type_interval, CanonicalIntervalSet()) + self.dim_dict["icmp_code"] = (DimensionsManager.DimensionType.IntervalSet, icmp_code_interval, CanonicalIntervalSet()) def _get_dfa_from_alphabet_str(self, alphabet_str): """ @@ -98,6 +100,14 @@ def get_dimension_domain_by_name(self, dim_name): """ return self.dim_dict[dim_name][1] + def get_empty_dimension_by_name(self, dim_name): + """ + get empty dimension value from its name + :param str dim_name: dimension name + :return: CanonicalIntervalSet object or MinDFA object (depends on dimension type) + """ + return self.dim_dict[dim_name][2] + def set_domain(self, dim_name, dim_type, interval_tuple=None, alphabet_str=None): """ set a new dimension, or change an existing dimension From 484bf2af6b34e78fa30cf6bd35dd64dfb581eed9 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 17:29:12 +0200 Subject: [PATCH 080/187] Moved empty dimension values to DimensionsManager. Fixed lint errors. Signed-off-by: Tanya --- nca/CoreDS/ConnectivityProperties.py | 1 - nca/CoreDS/DimensionsManager.py | 34 +++++++++++++++++++--------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index cee4e09e2..9d0fbe153 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -8,7 +8,6 @@ from .DimensionsManager import DimensionsManager from .PortSet import PortSet from .MethodSet import MethodSet -from .ProtocolSet import ProtocolSet from .Peer import PeerSet from .ProtocolNameResolver import ProtocolNameResolver from .MinDFA import MinDFA diff --git a/nca/CoreDS/DimensionsManager.py b/nca/CoreDS/DimensionsManager.py index 04c35f112..c195bd256 100644 --- a/nca/CoreDS/DimensionsManager.py +++ b/nca/CoreDS/DimensionsManager.py @@ -37,19 +37,29 @@ def __init__(self): # dim_dict is a map from a dimension name to a tuple # (dimension type, dimension full domain, dimension empty value) self.dim_dict = dict() - self.dim_dict["src_ports"] = (DimensionsManager.DimensionType.IntervalSet, ports_interval, CanonicalIntervalSet()) - self.dim_dict["dst_ports"] = (DimensionsManager.DimensionType.IntervalSet, ports_interval, CanonicalIntervalSet()) - self.dim_dict["methods"] = (DimensionsManager.DimensionType.IntervalSet, all_methods_interval, MethodSet()) - self.dim_dict["protocols"] = (DimensionsManager.DimensionType.IntervalSet, all_protocols_interval, ProtocolSet()) - self.dim_dict["src_peers"] = (DimensionsManager.DimensionType.IntervalSet, all_peers_interval, CanonicalIntervalSet()) - self.dim_dict["dst_peers"] = (DimensionsManager.DimensionType.IntervalSet, all_peers_interval, CanonicalIntervalSet()) - self.dim_dict["paths"] = (DimensionsManager.DimensionType.DFA, dfa_all_words_path_domain, MinDFA.dfa_from_regex("")) - self.dim_dict["hosts"] = (DimensionsManager.DimensionType.DFA, dfa_all_words_default, MinDFA.dfa_from_regex("")) + self.dim_dict["src_ports"] = \ + (DimensionsManager.DimensionType.IntervalSet, ports_interval, CanonicalIntervalSet()) + self.dim_dict["dst_ports"] = \ + (DimensionsManager.DimensionType.IntervalSet, ports_interval, CanonicalIntervalSet()) + self.dim_dict["methods"] = \ + (DimensionsManager.DimensionType.IntervalSet, all_methods_interval, MethodSet()) + self.dim_dict["protocols"] = \ + (DimensionsManager.DimensionType.IntervalSet, all_protocols_interval, ProtocolSet()) + self.dim_dict["src_peers"] = \ + (DimensionsManager.DimensionType.IntervalSet, all_peers_interval, CanonicalIntervalSet()) + self.dim_dict["dst_peers"] = \ + (DimensionsManager.DimensionType.IntervalSet, all_peers_interval, CanonicalIntervalSet()) + self.dim_dict["paths"] = \ + (DimensionsManager.DimensionType.DFA, dfa_all_words_path_domain, MinDFA.dfa_from_regex("")) + self.dim_dict["hosts"] = \ + (DimensionsManager.DimensionType.DFA, dfa_all_words_default, MinDFA.dfa_from_regex("")) icmp_type_interval = CanonicalIntervalSet.get_interval_set(0, 254) icmp_code_interval = CanonicalIntervalSet.get_interval_set(0, 255) - self.dim_dict["icmp_type"] = (DimensionsManager.DimensionType.IntervalSet, icmp_type_interval, CanonicalIntervalSet()) - self.dim_dict["icmp_code"] = (DimensionsManager.DimensionType.IntervalSet, icmp_code_interval, CanonicalIntervalSet()) + self.dim_dict["icmp_type"] = \ + (DimensionsManager.DimensionType.IntervalSet, icmp_type_interval, CanonicalIntervalSet()) + self.dim_dict["icmp_code"] =\ + (DimensionsManager.DimensionType.IntervalSet, icmp_code_interval, CanonicalIntervalSet()) def _get_dfa_from_alphabet_str(self, alphabet_str): """ @@ -119,10 +129,12 @@ def set_domain(self, dim_name, dim_type, interval_tuple=None, alphabet_str=None) if dim_type == DimensionsManager.DimensionType.IntervalSet: interval = interval_tuple if interval_tuple is not None else self.default_interval_domain_tuple domain = CanonicalIntervalSet.get_interval_set(interval[0], interval[1]) + empty_val = CanonicalIntervalSet() else: alphabet = alphabet_str if alphabet_str is not None else MinDFA.default_alphabet_regex domain = self._get_dfa_from_alphabet_str(alphabet) - self.dim_dict[dim_name] = (dim_type, domain) + empty_val = MinDFA.dfa_from_regex("") + self.dim_dict[dim_name] = (dim_type, domain, empty_val) def validate_value_by_domain(self, value, dim_name, value_name): """ From c221c3b425d08f80ee6ffef76e79583c861552e2 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 17:34:40 +0200 Subject: [PATCH 081/187] Update nca/CoreDS/ConnectivityProperties.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/CoreDS/ConnectivityProperties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 9d0fbe153..22059b626 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -100,7 +100,7 @@ def __setitem__(self, dim_name, dim_value): assert dim_name in self.dimensions_list if dim_value is None: return - if dim_name == "src_peers" or dim_name == "dst_peers": + if dim_name in ["src_peers", "dst_peers"]: # translate PeerSet to CanonicalIntervalSet self.set_dim_directly(dim_name, self.base_peer_set.get_peer_interval_of(dim_value)) elif dim_name == "src_ports" or dim_name == "dst_ports": From 2cdb462ef240cfa8e568dbf87ce1ae65697b49c8 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 17:35:55 +0200 Subject: [PATCH 082/187] Update nca/CoreDS/ConnectivityProperties.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/CoreDS/ConnectivityProperties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 22059b626..80ff0e638 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -103,7 +103,7 @@ def __setitem__(self, dim_name, dim_value): if dim_name in ["src_peers", "dst_peers"]: # translate PeerSet to CanonicalIntervalSet self.set_dim_directly(dim_name, self.base_peer_set.get_peer_interval_of(dim_value)) - elif dim_name == "src_ports" or dim_name == "dst_ports": + elif dim_name in ["src_ports", "dst_ports" ]: # extract port_set from PortSet self.set_dim_directly(dim_name, dim_value.port_set) if dim_name == "dst_ports": From a6d62ffbe2ff68c0d171ba26442491593bac325a Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 17:36:28 +0200 Subject: [PATCH 083/187] Update nca/CoreDS/ConnectivityProperties.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/CoreDS/ConnectivityProperties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 80ff0e638..7a2d4a4d5 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -109,7 +109,7 @@ def __setitem__(self, dim_name, dim_value): if dim_name == "dst_ports": self.named_ports = dim_value.named_ports self.excluded_named_ports = dim_value.excluded_named_ports - elif dim_name == "icmp_type" or dim_name == "icmp_code": + elif dim_name in ["icmp_type", "icmp_code"]: # translate int to CanonicalIntervalSet self.set_dim_directly(dim_name, CanonicalIntervalSet.get_interval_set(dim_value, dim_value)) else: # the rest of dimensions do not need a translation From c6ec4270876d8ba8f5d652310fb7a409322e2d52 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 17:36:57 +0200 Subject: [PATCH 084/187] Update nca/CoreDS/ConnectivityProperties.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/CoreDS/ConnectivityProperties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 7a2d4a4d5..9e147fc62 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -139,7 +139,7 @@ def __getitem__(self, dim_name): """ assert dim_name in self.dimensions_list dim_value = self.get_dim_directly(dim_name) - if dim_name == "src_peers" or dim_name == "dst_peers": + if dim_name in ["src_peers", "dst_peers"]: if self.is_active_dim(dim_name): # translate CanonicalIntervalSet back to PeerSet return self.base_peer_set.get_peer_set_by_indices(dim_value) From 495ac1e3e48463a3dd49db52d4b56e8262c8e612 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 17:37:22 +0200 Subject: [PATCH 085/187] Update nca/CoreDS/ConnectivityProperties.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/CoreDS/ConnectivityProperties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 9e147fc62..628d51aed 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -145,7 +145,7 @@ def __getitem__(self, dim_name): return self.base_peer_set.get_peer_set_by_indices(dim_value) else: return None - elif dim_name == "src_ports" or dim_name == "dst_ports": + elif dim_name in ["src_ports", "dst_ports"]: res = PortSet() res.add_ports(dim_value) if dim_name == "dst_ports": From 1d952db77ffd544e6af3d7291272bb3ff8370991 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 17:38:03 +0200 Subject: [PATCH 086/187] Update nca/CoreDS/ConnectivityProperties.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/CoreDS/ConnectivityProperties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 628d51aed..c6fd7037d 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -152,7 +152,7 @@ def __getitem__(self, dim_name): res.named_ports = self.named_ports res.excluded_named_ports = self.excluded_named_ports return res - elif dim_name == "icmp_type" or dim_name == "icmp_code": + elif dim_name in ["icmp_type", "icmp_code"]: if self.is_active_dim(dim_name): # translate CanonicalIntervalSet back to int return dim_value.validate_and_get_single_value() From 6e956bbb9d4f0a1e4eded52b31caae757546d1b6 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 17:52:52 +0200 Subject: [PATCH 087/187] Fixed small errors. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 2 +- nca/CoreDS/ConnectivityProperties.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index cb89b1da0..76291c510 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -610,7 +610,7 @@ def conn_props_to_fw_rules(conn_props, cluster_info, peer_container, ip_blocks_m dst_peers.filter_ipv6_blocks(ip_blocks_mask) protocols = conn_cube["protocols"] conn_cube.unset_dim("protocols") - if not conn_cube.has_active_dim() and (not protocols or protocols == ignore_protocols): + if not conn_cube.has_active_dim() and protocols == ignore_protocols: conns = ConnectionSet(True) else: conns = ConnectionSet() diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index c6fd7037d..b9cde1008 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -103,7 +103,7 @@ def __setitem__(self, dim_name, dim_value): if dim_name in ["src_peers", "dst_peers"]: # translate PeerSet to CanonicalIntervalSet self.set_dim_directly(dim_name, self.base_peer_set.get_peer_interval_of(dim_value)) - elif dim_name in ["src_ports", "dst_ports" ]: + elif dim_name in ["src_ports", "dst_ports"]: # extract port_set from PortSet self.set_dim_directly(dim_name, dim_value.port_set) if dim_name == "dst_ports": From 80613a8d2fd29e14e12bbfddd52e953e73367416 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 18:10:55 +0200 Subject: [PATCH 088/187] Update nca/CoreDS/ConnectivityProperties.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/CoreDS/ConnectivityProperties.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index b9cde1008..7fcd23d97 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -344,8 +344,9 @@ def get_cube_dict(self, cube, is_txt=False): if dim == 'protocols' or dim == 'methods': values_list = str(dim_values) elif dim == "src_peers" or dim == "dst_peers": - values_list = self.base_peer_set.get_peer_set_by_indices(dim_values) - values_list = ','.join(str(peer.full_name()) for peer in values_list) + peers_set = self.base_peer_set.get_peer_set_by_indices(dim_values) + peers_str_list = [str(peer.full_name()) for peer in peers_set] + values_list = ','.join(peers_str_list) if is_txt else peers_str_list elif dim_type == DimensionsManager.DimensionType.IntervalSet: values_list = dim_values.get_interval_set_list_numbers_and_ranges() if is_txt: From df3afe178323a55ef5040ed668b0c74c3393f326 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 18:11:39 +0200 Subject: [PATCH 089/187] Update nca/CoreDS/ConnectivityProperties.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/CoreDS/ConnectivityProperties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 7fcd23d97..7a6d22687 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -341,7 +341,7 @@ def get_cube_dict(self, cube, is_txt=False): dim_domain = DimensionsManager().get_dimension_domain_by_name(dim) if dim_domain == dim_values: continue # skip dimensions with all values allowed in a cube - if dim == 'protocols' or dim == 'methods': + if dim in ['protocols', 'methods']: values_list = str(dim_values) elif dim == "src_peers" or dim == "dst_peers": peers_set = self.base_peer_set.get_peer_set_by_indices(dim_values) From 380d3795c3e8e28be7c9459ca24e627128863f0e Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 18:12:08 +0200 Subject: [PATCH 090/187] Update nca/CoreDS/ConnectivityProperties.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/CoreDS/ConnectivityProperties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 7a6d22687..d81a96216 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -343,7 +343,7 @@ def get_cube_dict(self, cube, is_txt=False): continue # skip dimensions with all values allowed in a cube if dim in ['protocols', 'methods']: values_list = str(dim_values) - elif dim == "src_peers" or dim == "dst_peers": + elif dim in ["src_peers", "dst_peers"]: peers_set = self.base_peer_set.get_peer_set_by_indices(dim_values) peers_str_list = [str(peer.full_name()) for peer in peers_set] values_list = ','.join(peers_str_list) if is_txt else peers_str_list From 70f128013a2a08e2a53e37f8eeca76cd789b80a6 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 18:12:50 +0200 Subject: [PATCH 091/187] Update nca/CoreDS/ConnectivityProperties.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/CoreDS/ConnectivityProperties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index d81a96216..65391a8a4 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -220,7 +220,7 @@ class ConnectivityProperties(CanonicalHyperCubeSet): the named ports cannot be resolved at parse time, and so are kept in named_ports and excluded_named_ports, as explained below. Second, at the query time, ConnectivityProperties is calculated for every pair of peers, and the named ports - are resolved. The pairs of peers and the protocols are keps in ConnectionSet class, together with + are resolved. The pairs of peers and the protocols are kept in ConnectionSet class, together with the resulting ConnectivityProperties. The usage of this class in the optimized solution: From 62bc9b9d9bec02f7784a8c86d1427cf28f389891 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 18:13:38 +0200 Subject: [PATCH 092/187] Update nca/CoreDS/ConnectivityProperties.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/CoreDS/ConnectivityProperties.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 65391a8a4..c6a28c8f4 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -588,9 +588,8 @@ def make_conn_props(conn_cube): # Initialize conn_properties if dst_ports.port_set: - dst_ports_no_named_ports = dst_ports.copy() - dst_ports_no_named_ports.named_ports = set() - dst_ports_no_named_ports.excluded_named_ports = set() + dst_ports_no_named_ports = PortSet() + dst_ports_no_named_ports.port_set = dst_ports.port_set.copy() conn_cube["dst_ports"] = dst_ports_no_named_ports conn_properties = ConnectivityProperties.create_props_from_cube(conn_cube) else: From ce3cbda7f37ec1a33bfd3f9f6f9509e71e73b17e Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 18:32:50 +0200 Subject: [PATCH 093/187] Update nca/Parsers/CalicoPolicyYamlParser.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/Parsers/CalicoPolicyYamlParser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index c72973f30..ad41613fb 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -429,6 +429,7 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): res = ConnectivityProperties.make_empty_props() self.warning('icmp and notICMP are conflicting - no traffic will be matched', not_icmp_data) elif icmp_type == not_icmp_type and icmp_code is None: + # this is the only case where it makes sense to combine icmp and notICMP tmp = ConnectivityProperties.make_conn_props(not_conn_cube) res -= tmp if self.optimized_run != 'false': From 66fecd8657a3ec16ae36139aa7400c6065acf95c Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 21 Mar 2023 18:44:25 +0200 Subject: [PATCH 094/187] Fixed lint errors. Signed-off-by: Tanya --- nca/CoreDS/ConnectivityProperties.py | 4 ++++ nca/Parsers/CalicoPolicyYamlParser.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index c6a28c8f4..f20c51402 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -40,6 +40,7 @@ def __init__(self, base_peer_set): def copy(self): """ Returns a copy of the given ConnectivityCube object + :rtype: ConnectivityCube """ res = ConnectivityCube(self.base_peer_set.copy()) for dim_name, dim_value in self.items(): @@ -479,6 +480,9 @@ def convert_named_ports(self, named_ports, protocol): self.add_hole(rectangle, active_dims) def copy(self): + """ + :rtype: ConnectivityProperties + """ res = ConnectivityProperties.create_props_from_cube(ConnectivityCube(self.base_peer_set)) for layer in self.layers: res.layers[self._copy_layer_elem(layer)] = self.layers[layer].copy() diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index ad41613fb..c38210f51 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -429,7 +429,7 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): res = ConnectivityProperties.make_empty_props() self.warning('icmp and notICMP are conflicting - no traffic will be matched', not_icmp_data) elif icmp_type == not_icmp_type and icmp_code is None: - # this is the only case where it makes sense to combine icmp and notICMP + # this is the only case where it makes sense to combine icmp and notICMP tmp = ConnectivityProperties.make_conn_props(not_conn_cube) res -= tmp if self.optimized_run != 'false': From 7d6a08c46dba4be5dc91f52e168ad65a1c854b89 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 26 Mar 2023 16:18:56 +0300 Subject: [PATCH 095/187] Removed base_peer_set from ConnectivityProperties and ConnectivityCube. Instead, added a singleton class BasePeerSet that keeps all peers and translates PeerSets to CanonicalIntervalSets and vice versa. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 5 +- nca/CoreDS/ConnectivityProperties.py | 30 +-- nca/CoreDS/DimensionsManager.py | 4 +- nca/CoreDS/Peer.py | 208 ++++++++++-------- nca/NetworkConfig/NetworkConfigQuery.py | 9 +- nca/NetworkConfig/NetworkLayer.py | 9 +- nca/NetworkConfig/TopologyObjectsFinder.py | 3 +- nca/Parsers/CalicoPolicyYamlParser.py | 19 +- nca/Parsers/IngressPolicyYamlParser.py | 10 +- nca/Parsers/IstioPolicyYamlParser.py | 10 +- .../IstioTrafficResourcesYamlParser.py | 8 +- nca/Parsers/K8sPolicyYamlParser.py | 6 +- nca/Resources/IstioSidecar.py | 2 +- .../testConnectivityPropertiesNamedPorts.py | 6 +- 14 files changed, 162 insertions(+), 167 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 76291c510..6d23633ae 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -549,7 +549,7 @@ def convert_to_connectivity_properties(self, peer_container): res = ConnectivityProperties.make_empty_props() for protocol, properties in self.allowed_protocols.items(): protocols = ProtocolSet.get_protocol_set_with_single_protocol(protocol) - conn_cube = ConnectivityCube.make_from_dict(peer_container.get_all_peers_group(), {"protocols": protocols}) + conn_cube = ConnectivityCube.make_from_dict({"protocols": protocols}) this_prop = ConnectivityProperties.make_conn_props(conn_cube) if isinstance(properties, bool): if properties: @@ -672,8 +672,7 @@ def fw_rules_to_conn_props(fw_rules, peer_container): conn_props = fw_rule.conn.convert_to_connectivity_properties(peer_container) src_peers = PeerSet(fw_rule.src.get_peer_set(fw_rules.cluster_info)) dst_peers = PeerSet(fw_rule.dst.get_peer_set(fw_rules.cluster_info)) - conn_cube = ConnectivityCube.make_from_dict(peer_container.get_all_peers_group(), - {"src_peers": src_peers, "dst_peers": dst_peers}) + conn_cube = ConnectivityCube.make_from_dict({"src_peers": src_peers, "dst_peers": dst_peers}) rule_props = ConnectivityProperties.make_conn_props(conn_cube) & conn_props res |= rule_props return res diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index f20c51402..57e02a8d8 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -8,7 +8,7 @@ from .DimensionsManager import DimensionsManager from .PortSet import PortSet from .MethodSet import MethodSet -from .Peer import PeerSet +from .Peer import PeerSet, BasePeerSet from .ProtocolNameResolver import ProtocolNameResolver from .MinDFA import MinDFA @@ -23,17 +23,14 @@ class ConnectivityCube(dict): dimensions_list = ["src_peers", "dst_peers", "protocols", "src_ports", "dst_ports", "methods", "hosts", "paths", "icmp_type", "icmp_code"] - def __init__(self, base_peer_set): + def __init__(self): """ By default, each dimension in the cube is initialized with entire domain value, which represents "don't care" or inactive dimension (i.e., the dimension has no impact). - :param PeerSet base_peer_set: the set of all possible peers, which will be referenced by the indices - in 'src_peers' and 'dst_peers' """ super().__init__() self.named_ports = set() # used only in the original solution self.excluded_named_ports = set() # used only in the original solution - self.base_peer_set = base_peer_set for dim in self.dimensions_list: self.set_dim_directly(dim, DimensionsManager().get_dimension_domain_by_name(dim)) @@ -42,7 +39,7 @@ def copy(self): Returns a copy of the given ConnectivityCube object :rtype: ConnectivityCube """ - res = ConnectivityCube(self.base_peer_set.copy()) + res = ConnectivityCube() for dim_name, dim_value in self.items(): if isinstance(dim_value, MinDFA): res.set_dim_directly(dim_name, dim_value) @@ -103,7 +100,7 @@ def __setitem__(self, dim_name, dim_value): return if dim_name in ["src_peers", "dst_peers"]: # translate PeerSet to CanonicalIntervalSet - self.set_dim_directly(dim_name, self.base_peer_set.get_peer_interval_of(dim_value)) + self.set_dim_directly(dim_name, BasePeerSet().get_peer_interval_of(dim_value)) elif dim_name in ["src_ports", "dst_ports"]: # extract port_set from PortSet self.set_dim_directly(dim_name, dim_value.port_set) @@ -143,7 +140,7 @@ def __getitem__(self, dim_name): if dim_name in ["src_peers", "dst_peers"]: if self.is_active_dim(dim_name): # translate CanonicalIntervalSet back to PeerSet - return self.base_peer_set.get_peer_set_by_indices(dim_value) + return BasePeerSet().get_peer_set_by_indices(dim_value) else: return None elif dim_name in ["src_ports", "dst_ports"]: @@ -202,8 +199,8 @@ def get_ordered_cube_and_active_dims(self): return cube, active_dims @staticmethod - def make_from_dict(base_peer_set, the_dict): - ccube = ConnectivityCube(base_peer_set) + def make_from_dict(the_dict): + ccube = ConnectivityCube() ccube.update(the_dict) return ccube @@ -257,7 +254,6 @@ def __init__(self, create_all=False): super().__init__(ConnectivityCube.dimensions_list) self.named_ports = {} # a mapping from dst named port (String) to src ports interval set self.excluded_named_ports = {} # a mapping from dst named port (String) to src ports interval set - self.base_peer_set = PeerSet() if create_all: self.set_all() @@ -270,7 +266,6 @@ def create_props_from_cube(conn_cube): whereas missing dimensions are represented by their default values (representing all possible values). """ res = ConnectivityProperties() - res.base_peer_set = conn_cube.base_peer_set.copy() if conn_cube.is_empty(): return res @@ -319,7 +314,7 @@ def get_connectivity_cube(self, cube): :return: the cube in ConnectivityCube format :rtype: ConnectivityCube """ - res = ConnectivityCube(self.base_peer_set) + res = ConnectivityCube() for i, dim in enumerate(self.active_dimensions): if isinstance(cube[i], MinDFA): res.set_dim_directly(dim, cube[i]) @@ -345,7 +340,7 @@ def get_cube_dict(self, cube, is_txt=False): if dim in ['protocols', 'methods']: values_list = str(dim_values) elif dim in ["src_peers", "dst_peers"]: - peers_set = self.base_peer_set.get_peer_set_by_indices(dim_values) + peers_set = BasePeerSet().get_peer_set_by_indices(dim_values) peers_str_list = [str(peer.full_name()) for peer in peers_set] values_list = ','.join(peers_str_list) if is_txt else peers_str_list elif dim_type == DimensionsManager.DimensionType.IntervalSet: @@ -399,8 +394,6 @@ def __iand__(self, other): assert not self.has_named_ports() assert not isinstance(other, ConnectivityProperties) or not other.has_named_ports() super().__iand__(other) - if isinstance(other, ConnectivityProperties): - self.base_peer_set |= other.base_peer_set return self def __ior__(self, other): @@ -408,7 +401,6 @@ def __ior__(self, other): assert not isinstance(other, ConnectivityProperties) or not other.excluded_named_ports super().__ior__(other) if isinstance(other, ConnectivityProperties): - self.base_peer_set |= other.base_peer_set res_named_ports = dict({}) for port_name in self.named_ports: res_named_ports[port_name] = self.named_ports[port_name] @@ -424,8 +416,6 @@ def __isub__(self, other): assert not self.has_named_ports() assert not isinstance(other, ConnectivityProperties) or not other.has_named_ports() super().__isub__(other) - if isinstance(other, ConnectivityProperties): - self.base_peer_set |= other.base_peer_set return self def contained_in(self, other): @@ -483,7 +473,7 @@ def copy(self): """ :rtype: ConnectivityProperties """ - res = ConnectivityProperties.create_props_from_cube(ConnectivityCube(self.base_peer_set)) + res = ConnectivityProperties.create_props_from_cube(ConnectivityCube()) for layer in self.layers: res.layers[self._copy_layer_elem(layer)] = self.layers[layer].copy() res.active_dimensions = self.active_dimensions.copy() diff --git a/nca/CoreDS/DimensionsManager.py b/nca/CoreDS/DimensionsManager.py index c195bd256..e7f41e687 100644 --- a/nca/CoreDS/DimensionsManager.py +++ b/nca/CoreDS/DimensionsManager.py @@ -6,7 +6,7 @@ from .CanonicalIntervalSet import CanonicalIntervalSet from .MethodSet import MethodSet from .ProtocolSet import ProtocolSet -from .Peer import PeerSet +from .Peer import BasePeerSet from .MinDFA import MinDFA @@ -33,7 +33,7 @@ def __init__(self): ports_interval = CanonicalIntervalSet.get_interval_set(1, 65535) all_methods_interval = MethodSet(True) all_protocols_interval = ProtocolSet(True) - all_peers_interval = PeerSet.get_all_peers_and_ip_blocks_interval() + all_peers_interval = BasePeerSet.get_all_peers_and_ip_blocks_interval() # dim_dict is a map from a dimension name to a tuple # (dimension type, dimension full domain, dimension empty value) self.dim_dict = dict() diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index bfe661389..0069a9aef 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -478,22 +478,8 @@ class PeerSet(set): Note #2: __contains__ is implemented under the assumption that item arg is from disjoint_ip_blocks() """ - ipv4_highest_number = int(ip_network('0.0.0.0/0').broadcast_address) - ipv6_highest_number = int(ip_network('::/0').broadcast_address) - max_num_of_pods = 10000 - gap_width = 5 # the gap is needed to avoid mixed-type intervals union - min_ipv4_index = 0 - max_ipv4_index = min_ipv4_index + ipv4_highest_number - min_ipv6_index = max_ipv4_index + gap_width - max_ipv6_index = min_ipv6_index + ipv6_highest_number - min_pod_index = max_ipv6_index + gap_width - max_pod_index = min_pod_index + max_num_of_pods - 1 - def __init__(self, peer_set=None): super().__init__(peer_set or set()) - self.sorted_peer_list = [] # for converting PeerSet to CanonicalIntervalSet - self.last_size_when_updated_sorted_peer_list = 0 - assert len(self.get_set_without_ip_block()) <= self.max_num_of_pods def __contains__(self, item): if isinstance(item, IpBlock): # a special check here because an IpBlock may be contained in another IpBlock @@ -519,8 +505,6 @@ def copy(self): # TODO: shallow copy or deep copy? # res = PeerSet(set(elem.copy() for elem in self)) res = PeerSet(super().copy()) - res.sorted_peer_list = self.sorted_peer_list - res.last_size_when_updated_sorted_peer_list = self.last_size_when_updated_sorted_peer_list return res # TODO: what is expected for ipblock name/namespace result on intersection? @@ -543,12 +527,10 @@ def __and__(self, other): def __ior__(self, other): res = PeerSet(super().__ior__(other)) - assert len(res.get_set_without_ip_block()) <= self.max_num_of_pods return res def __or__(self, other): res = PeerSet(super().__or__(other)) - assert len(res.get_set_without_ip_block()) <= self.max_num_of_pods return res def __isub__(self, other): @@ -573,8 +555,7 @@ def __hash__(self): Note: PeerSet is a mutable type. Use with caution! :return: hash value for this object. """ - self.update_sorted_peer_list_if_needed() - return hash(','.join(str(peer.full_name()) for peer in self.sorted_peer_list)) + return hash(','.join(str(peer.full_name()) for peer in sorted(list(self)))) def rep(self): """ @@ -602,81 +583,6 @@ def get_ip_block_canonical_form(self): res |= elem return res - @staticmethod - def get_all_peers_and_ip_blocks_interval(): - res = CanonicalIntervalSet() - res.add_interval(CanonicalIntervalSet.Interval(PeerSet.min_ipv4_index, PeerSet.max_ipv4_index)) - res.add_interval(CanonicalIntervalSet.Interval(PeerSet.min_ipv6_index, PeerSet.max_ipv6_index)) - res.add_interval(CanonicalIntervalSet.Interval(PeerSet.min_pod_index, PeerSet.max_pod_index)) - return res - - def update_sorted_peer_list_if_needed(self): - """ - create self.sorted_peer_list from non IpBlock pods - :return: None - """ - if self.last_size_when_updated_sorted_peer_list != len(self): - self.sorted_peer_list = \ - sorted(list(elem for elem in self if not isinstance(elem, IpBlock)), key=by_full_name) - self.last_size_when_updated_sorted_peer_list = len(self) - - def get_peer_interval_of(self, peer_set): - """ - Calculates interval set of a given peer_set, based on the self peer_set - :param PeerSet peer_set: the peer_set to be converted to the interval set - :return: CanonicalIntervalSet for the peer_set - """ - res = CanonicalIntervalSet() - self.update_sorted_peer_list_if_needed() - for index, peer in enumerate(self.sorted_peer_list): - if peer in peer_set: - assert not isinstance(peer, IpBlock) - res.add_interval(CanonicalIntervalSet.Interval(self.min_pod_index + index, - self.min_pod_index + index)) - # Now pick IpBlocks - for ipb in peer_set: - if isinstance(ipb, IpBlock): - for cidr in ipb: - if isinstance(cidr.start.address, ipaddress.IPv4Address): - res.add_interval(CanonicalIntervalSet.Interval(self.min_ipv4_index + int(cidr.start), - self.min_ipv4_index + int(cidr.end))) - elif isinstance(cidr.start.address, ipaddress.IPv6Address): - res.add_interval(CanonicalIntervalSet.Interval(self.min_ipv6_index + int(cidr.start), - self.min_ipv6_index + int(cidr.end))) - else: - assert False - return res - - def get_peer_set_by_indices(self, peer_interval_set): - """ - Return peer set from interval set of indices - :param peer_interval_set: the interval set of indices into the sorted peer list - :return: the PeerSet of peers referenced by the indices in the interval set - """ - self.update_sorted_peer_list_if_needed() - peer_list = [] - for interval in peer_interval_set: - if interval.end <= self.max_ipv4_index: - # this is IPv4Address - start = ipaddress.IPv4Address(interval.start - self.min_ipv4_index) - end = ipaddress.IPv4Address(interval.end - self.min_ipv4_index) - ipb = IpBlock(interval=CanonicalIntervalSet.Interval(IPNetworkAddress(start), IPNetworkAddress(end))) - peer_list.append(ipb) - elif interval.end <= self.max_ipv6_index: - # this is IPv6Address - start = ipaddress.IPv6Address(interval.start - self.min_ipv6_index) - end = ipaddress.IPv6Address(interval.end - self.min_ipv6_index) - ipb = IpBlock(interval=CanonicalIntervalSet.Interval(IPNetworkAddress(start), IPNetworkAddress(end))) - peer_list.append(ipb) - else: - # this is Pod - assert interval.end <= self.max_pod_index - curr_pods_max_ind = len(self)-1 - for ind in range(min(interval.start-self.min_pod_index, curr_pods_max_ind), - min(interval.end-self.min_pod_index, curr_pods_max_ind) + 1): - peer_list.append(self.sorted_peer_list[ind]) - return PeerSet(set(peer_list)) - def filter_ipv6_blocks(self, ip_blocks_mask): """ Update ip blocks in the peer set by keeping only parts overlapping with the given mask. @@ -705,3 +611,115 @@ def by_full_name(elem): :return: Peer's name (the key being used by the sort) """ return elem.full_name() + + +class BasePeerSet: + """ + A singleton class that keeps the set of all possible pods in the system (from all configs), + and translates PeerSets to CanonicalIntervalSets and vice versa. + """ + + # the inner class is needed to make the outer class a singleton + class __BasePeerSet: + + ipv4_highest_number = int(ip_network('0.0.0.0/0').broadcast_address) + ipv6_highest_number = int(ip_network('::/0').broadcast_address) + max_num_of_pods = 10000 + gap_width = 5 # the gap is needed to avoid mixed-type intervals union + min_ipv4_index = 0 + max_ipv4_index = min_ipv4_index + ipv4_highest_number + min_ipv6_index = max_ipv4_index + gap_width + max_ipv6_index = min_ipv6_index + ipv6_highest_number + min_pod_index = max_ipv6_index + gap_width + max_pod_index = min_pod_index + max_num_of_pods - 1 + + def __init__(self): + self.ordered_peer_list = [] # for converting PeerSet to CanonicalIntervalSet + self.peer_to_index = dict() # a map from peer name to index in self.ordered_peer_list + + def add_peer(self, peer): + """ + Adds a given peer to self peers. + :param peer: the peer to add. + """ + if not isinstance(peer, IpBlock) and not peer in self.peer_to_index: + assert len(self.ordered_peer_list) < self.max_num_of_pods + self.peer_to_index[peer] = len(self.ordered_peer_list) + self.ordered_peer_list.append(peer) + + def get_peer_interval_of(self, peer_set): + """ + Translates the given peer_set to an interval set of indices + :param PeerSet peer_set: the peer_set to be converted to the interval set + :return: CanonicalIntervalSet for the peer_set + """ + res = CanonicalIntervalSet() + for index, peer in enumerate(self.ordered_peer_list): + if peer in peer_set: + assert not isinstance(peer, IpBlock) + res.add_interval(CanonicalIntervalSet.Interval(self.min_pod_index + index, + self.min_pod_index + index)) + # Now pick IpBlocks + for ipb in peer_set: + if isinstance(ipb, IpBlock): + for cidr in ipb: + if isinstance(cidr.start.address, ipaddress.IPv4Address): + res.add_interval(CanonicalIntervalSet.Interval(self.min_ipv4_index + int(cidr.start), + self.min_ipv4_index + int(cidr.end))) + elif isinstance(cidr.start.address, ipaddress.IPv6Address): + res.add_interval(CanonicalIntervalSet.Interval(self.min_ipv6_index + int(cidr.start), + self.min_ipv6_index + int(cidr.end))) + else: + assert False + return res + + def get_peer_set_by_indices(self, peer_interval_set): + """ + Translates the given interval set of indices to a peer set. + :param peer_interval_set: the interval set of indices + :return: the PeerSet of peers referenced by the indices in the interval set + """ + peer_list = [] + for interval in peer_interval_set: + if interval.end <= self.max_ipv4_index: + # this is IPv4Address + start = ipaddress.IPv4Address(interval.start - self.min_ipv4_index) + end = ipaddress.IPv4Address(interval.end - self.min_ipv4_index) + ipb = IpBlock( + interval=CanonicalIntervalSet.Interval(IPNetworkAddress(start), IPNetworkAddress(end))) + peer_list.append(ipb) + elif interval.end <= self.max_ipv6_index: + # this is IPv6Address + start = ipaddress.IPv6Address(interval.start - self.min_ipv6_index) + end = ipaddress.IPv6Address(interval.end - self.min_ipv6_index) + ipb = IpBlock( + interval=CanonicalIntervalSet.Interval(IPNetworkAddress(start), IPNetworkAddress(end))) + peer_list.append(ipb) + else: + # this is Pod + assert interval.end <= self.max_pod_index + curr_pods_max_ind = len(self.ordered_peer_list) - 1 + for ind in range(min(interval.start - self.min_pod_index, curr_pods_max_ind), + min(interval.end - self.min_pod_index, curr_pods_max_ind) + 1): + peer_list.append(self.ordered_peer_list[ind]) + return PeerSet(set(peer_list)) + + instance = None + + def __init__(self): + if not BasePeerSet.instance: + BasePeerSet.instance = BasePeerSet.__BasePeerSet() + + def __getattr__(self, name): + return getattr(self.instance, name) + + @staticmethod + def get_all_peers_and_ip_blocks_interval(): + res = CanonicalIntervalSet() + res.add_interval(CanonicalIntervalSet.Interval(BasePeerSet.__BasePeerSet.min_ipv4_index, + BasePeerSet.__BasePeerSet.max_ipv4_index)) + res.add_interval(CanonicalIntervalSet.Interval(BasePeerSet.__BasePeerSet.min_ipv6_index, + BasePeerSet.__BasePeerSet.max_ipv6_index)) + res.add_interval(CanonicalIntervalSet.Interval(BasePeerSet.__BasePeerSet.min_pod_index, + BasePeerSet.__BasePeerSet.max_pod_index)) + return res diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 94f8fc1c5..b5e13ddd4 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -753,10 +753,8 @@ def exec(self): # noqa: C901 all_conns_opt.project_on_one_dimension('dst_peers') subset_peers = self.compute_subset(opt_peers_to_compare) - src_peers_conn_cube = ConnectivityCube.make_from_dict(self.config.peer_container.get_all_peers_group(), - {"src_peers": subset_peers}) - dst_peers_conn_cube = ConnectivityCube.make_from_dict(self.config.peer_container.get_all_peers_group(), - {"dst_peers": subset_peers}) + src_peers_conn_cube = ConnectivityCube.make_from_dict({"src_peers": subset_peers}) + dst_peers_conn_cube = ConnectivityCube.make_from_dict({"dst_peers": subset_peers}) subset_conns = ConnectivityProperties.make_conn_props(src_peers_conn_cube) | \ ConnectivityProperties.make_conn_props(dst_peers_conn_cube) all_conns_opt &= subset_conns @@ -1052,8 +1050,7 @@ def convert_props_to_split_by_tcp(self, props): :rtype: tuple(ConnectivityProperties, ConnectivityProperties) """ tcp_protocol = ProtocolSet.get_protocol_set_with_single_protocol('TCP') - conn_cube = ConnectivityCube.make_from_dict(self.config.peer_container.get_all_peers_group(), - {"protocols", tcp_protocol}) + conn_cube = ConnectivityCube.make_from_dict({"protocols": tcp_protocol}) tcp_props = props & ConnectivityProperties.make_conn_props(conn_cube) non_tcp_props = props - tcp_props return tcp_props, non_tcp_props diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 08e306ff5..7d1f3d685 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -176,8 +176,7 @@ def allowed_connections_optimized(self, peer_container): """ all_pods = peer_container.get_all_peers_group() all_ips_peer_set = PeerSet({IpBlock.get_all_ips_block()}) - conn_cube = ConnectivityCube.make_from_dict(peer_container.get_all_peers_group(), - {"src_peers": all_pods, "dst_peers": all_ips_peer_set}) + conn_cube = ConnectivityCube.make_from_dict({"src_peers": all_pods, "dst_peers": all_ips_peer_set}) allowed_ingress_conns, denied_ingres_conns = self._allowed_xgress_conns_optimized(True, peer_container) allowed_ingress_conns |= ConnectivityProperties.make_conn_props(conn_cube) allowed_egress_conns, denied_egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) @@ -296,7 +295,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): base_peer_set_no_hep = PeerSet(set([peer for peer in base_peer_set_no_ip if not isinstance(peer, HostEP)])) non_captured = base_peer_set_no_hep - captured if non_captured: - conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) + conn_cube = ConnectivityCube() if is_ingress: conn_cube.update({"src_peers": base_peer_set_with_ip, "dst_peers": non_captured}) else: @@ -332,7 +331,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): base_peer_set_with_ip = peer_container.get_all_peers_group(True) base_peer_set_no_ip = peer_container.get_all_peers_group() non_captured_peers = base_peer_set_no_ip - captured - conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) + conn_cube = ConnectivityCube() if non_captured_peers: if is_ingress: conn_cube.update({"src_peers": base_peer_set_with_ip, "dst_peers": non_captured_peers}) @@ -363,7 +362,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): allowed_conn, denied_conns, captured = self.collect_policies_conns_optimized(is_ingress) base_peer_set_with_ip = peer_container.get_all_peers_group(True) base_peer_set_no_ip = peer_container.get_all_peers_group() - conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) + conn_cube = ConnectivityCube() if is_ingress: conn_cube.update({"src_peers": base_peer_set_with_ip, "dst_peers": base_peer_set_no_ip}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) diff --git a/nca/NetworkConfig/TopologyObjectsFinder.py b/nca/NetworkConfig/TopologyObjectsFinder.py index c346506a3..bc94a0cbb 100644 --- a/nca/NetworkConfig/TopologyObjectsFinder.py +++ b/nca/NetworkConfig/TopologyObjectsFinder.py @@ -6,7 +6,7 @@ from sys import stderr import yaml from nca.Utils.CmdlineRunner import CmdlineRunner -from nca.CoreDS.Peer import PeerSet, Pod, IpBlock, HostEP +from nca.CoreDS.Peer import PeerSet, Pod, IpBlock, HostEP, BasePeerSet from nca.Resources.K8sNamespace import K8sNamespace from nca.Parsers.K8sServiceYamlParser import K8sServiceYamlParser from nca.Utils.NcaLogger import NcaLogger @@ -120,6 +120,7 @@ def _add_peer(self, peer): return self.representative_peers[canonical_form] = peers_with_same_canonical_form + 1 self.peer_set.add(peer) + BasePeerSet().add_peer(peer) def _add_pod_from_workload_yaml(self, workload_resource): """ diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index c38210f51..262410b17 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -402,13 +402,12 @@ def _parse_icmp(self, icmp_data, not_icmp_data, protocol, src_pods, dst_pods): self.syntax_error(err, not_icmp_data) protocols = ProtocolSet.get_protocol_set_with_single_protocol(protocol) - base_peer_set = self.peer_container.get_all_peers_group() - conn_cube = ConnectivityCube(base_peer_set) + conn_cube = ConnectivityCube() if icmp_type: conn_cube["icmp_type"] = icmp_type if icmp_code: conn_cube["icmp_code"] = icmp_code - not_conn_cube = ConnectivityCube(base_peer_set) + not_conn_cube = ConnectivityCube() if not_icmp_type: not_conn_cube["icmp_type"] = not_icmp_type if not_icmp_code: @@ -522,8 +521,7 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): self.warning('notProtocol field has no effect', rule) else: if protocol_supports_ports: - conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { - "src_ports": src_res_ports, "dst_ports": dst_res_ports}) + conn_cube = ConnectivityCube.make_from_dict({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) connections.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) if self.optimized_run != 'false': conn_cube.update({"protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) @@ -535,8 +533,8 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): else: connections.add_connections(protocol, True) if self.optimized_run != 'false': - conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { - "protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) + conn_cube = ConnectivityCube.make_from_dict({"protocols": protocols, "src_peers": src_res_pods, + "dst_peers": dst_res_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) elif not_protocol is not None: connections.add_all_connections() @@ -544,14 +542,13 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): if self.optimized_run != 'false' and src_res_pods and dst_res_pods: protocols = ProtocolSet(True) protocols.remove_protocol(not_protocol) - conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { - "protocols": protocols, "src_peers": src_res_pods, "dst_peers": dst_res_pods}) + conn_cube = ConnectivityCube.make_from_dict({"protocols": protocols, "src_peers": src_res_pods, + "dst_peers": dst_res_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) else: connections.allow_all = True if self.optimized_run != 'false': - conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), - {"src_peers": src_res_pods, "dst_peers": dst_res_pods}) + conn_cube = ConnectivityCube.make_from_dict({"src_peers": src_res_pods, "dst_peers": dst_res_pods}) conn_props = ConnectivityProperties.make_conn_props(conn_cube) self._verify_named_ports(rule, dst_res_pods, connections) diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index da0175260..398cfdd40 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -187,8 +187,8 @@ def _make_default_connections(self, hosts_dfa, paths_dfa=None): """ default_conns = ConnectivityProperties.make_empty_props() if self.default_backend_peers: - conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { - "dst_ports": self.default_backend_ports, "dst_peers": self.default_backend_peers}) + conn_cube = ConnectivityCube.make_from_dict({"dst_ports": self.default_backend_ports, + "dst_peers": self.default_backend_peers}) if hosts_dfa: conn_cube["hosts"] = hosts_dfa if paths_dfa: @@ -221,8 +221,7 @@ def parse_rule(self, rule): parsed_paths.append(path_resources) if parsed_paths: parsed_paths_with_dfa = self.segregate_longest_paths_and_make_dfa(parsed_paths) - conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), - {"hosts": hosts_dfa}) + conn_cube = ConnectivityCube.make_from_dict({"hosts": hosts_dfa}) for (_, paths_dfa, _, peers, ports) in parsed_paths_with_dfa: # every path is converted to allowed connections conn_cube.update({"dst_ports": ports, "dst_peers": peers, "paths": paths_dfa}) @@ -288,8 +287,7 @@ def parse_policy(self): if allowed_conns: res_policy.add_rules(self._make_allow_rules(allowed_conns)) protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') - conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { - "protocols": protocols, "src_peers": res_policy.selected_peers}) + conn_cube = ConnectivityCube.make_from_dict({"protocols": protocols, "src_peers": res_policy.selected_peers}) allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index 89ff9f0f1..135b049d1 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -195,8 +195,7 @@ def parse_key_values(self, key, values, not_values): return self.parse_principals(values, not_values) # PeerSet elif key == 'destination.port': dst_ports = self.get_rule_ports(values, not_values) # PortSet - conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), - {"dst_ports": dst_ports}) + conn_cube = ConnectivityCube.make_from_dict({"dst_ports": dst_ports}) return ConnectivityProperties.make_conn_props(conn_cube) # ConnectivityProperties return NotImplemented, False @@ -399,8 +398,8 @@ def parse_operation(self, operation_dict): operation) hosts_dfa = self.parse_regex_dimension_values("hosts", operation.get("hosts"), operation.get("notHosts"), operation) - conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { - "dst_ports": dst_ports, "methods": methods_set, "paths": paths_dfa, 'hosts': hosts_dfa}) + conn_cube = ConnectivityCube.make_from_dict({"dst_ports": dst_ports, "methods": methods_set, "paths": paths_dfa, + 'hosts': hosts_dfa}) return ConnectivityProperties.make_conn_props(conn_cube) def parse_source(self, source_dict): @@ -522,8 +521,7 @@ def parse_ingress_rule(self, rule, selected_peers): if not res_peers or not selected_peers: condition_props = ConnectivityProperties.make_empty_props() else: - conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), - {"src_peers": res_peers, "dst_peers": selected_peers}) + conn_cube = ConnectivityCube.make_from_dict({"src_peers": res_peers, "dst_peers": selected_peers}) condition_props &= ConnectivityProperties.make_conn_props(conn_cube) connections &= condition_conns conn_props &= condition_props diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index 21d6863c8..08b919cf3 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -340,8 +340,8 @@ def make_allowed_connections(self, vs, host_dfa): """ allowed_conns = ConnectivityProperties.make_empty_props() for http_route in vs.http_routes: - conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { - "paths": http_route.uri_dfa, "hosts": host_dfa, "methods": http_route.methods}) + conn_cube = ConnectivityCube.make_from_dict({"paths": http_route.uri_dfa, "hosts": host_dfa, + "methods": http_route.methods}) for dest in http_route.destinations: conn_cube.update({"dst_ports": dest.port, "dst_peers": dest.service.target_pods}) conns = \ @@ -398,8 +398,8 @@ def create_istio_traffic_policies(self): if allowed_conns: res_policy.add_rules(self._make_allow_rules(allowed_conns)) protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') - conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), { - "protocols": protocols, "src_peers": res_policy.selected_peers}) + conn_cube = ConnectivityCube.make_from_dict({"protocols": protocols, + "src_peers": res_policy.selected_peers}) allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) res_policy.add_optimized_egress_props(allowed_conns) res_policy.findings = self.warning_msgs diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index 6bb12c7d3..64d49b69a 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -338,8 +338,7 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): protocol, dest_port_set = self.parse_port(port) if isinstance(protocol, str): protocol = ProtocolNameResolver.get_protocol_number(protocol) - conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), - {"dst_ports": dest_port_set}) + conn_cube = ConnectivityCube.make_from_dict({"dst_ports": dest_port_set}) # K8s doesn't reason about src ports res_conns.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) if self.optimized_run != 'false' and src_pods and dst_pods: @@ -350,8 +349,7 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): else: res_conns = ConnectionSet(True) if self.optimized_run != 'false': - conn_cube = ConnectivityCube.make_from_dict(self.peer_container.get_all_peers_group(), - {"src_peers": src_pods, "dst_peers": dst_pods}) + conn_cube = ConnectivityCube.make_from_dict({"src_peers": src_pods, "dst_peers": dst_pods}) res_opt_props = ConnectivityProperties.make_conn_props(conn_cube) if not res_pods: self.warning('Rule selects no pods', rule) diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index b14d38cf7..a6617d676 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -166,7 +166,7 @@ def combine_peer_sets_by_ns(from_peer_set, to_peer_set, peer_container): def create_opt_egress_props(self, peer_container): for rule in self.egress_rules: - conn_cube = ConnectivityCube(peer_container.get_all_peers_group()) + conn_cube = ConnectivityCube() if self.selected_peers and rule.egress_peer_set: conn_cube.update({"src_peers": self.selected_peers, "dst_peers": rule.egress_peer_set}) self.optimized_egress_props = ConnectivityProperties.make_conn_props(conn_cube) diff --git a/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py b/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py index c86f21bcc..c63cc04eb 100644 --- a/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py +++ b/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py @@ -13,7 +13,7 @@ def test_k8s_flow(self): src_res_ports = PortSet(True) dst_res_ports = PortSet() dst_res_ports.add_port("x") - conn_cube = ConnectivityCube.make_from_dict(PeerSet(), {"src_ports": src_res_ports, "dst_ports": dst_res_ports}) + conn_cube = ConnectivityCube.make_from_dict({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) tcp_properties1 = ConnectivityProperties.create_props_from_cube(conn_cube) dst_res_ports2 = PortSet() dst_res_ports2.add_port("y") @@ -38,7 +38,7 @@ def test_calico_flow_1(self): dst_res_ports.add_port("y") dst_res_ports.add_port("z") dst_res_ports.add_port("w") - conn_cube = ConnectivityCube.make_from_dict(PeerSet(), {"src_ports": src_res_ports, "dst_ports": dst_res_ports}) + conn_cube = ConnectivityCube.make_from_dict({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) tcp_properties = ConnectivityProperties.create_props_from_cube(conn_cube) tcp_properties_2 = tcp_properties.copy() @@ -70,7 +70,7 @@ def test_calico_flow_2(self): dst_res_ports = PortSet(True) dst_res_ports -= not_ports src_res_ports.add_port_range(1, 100) - conn_cube = ConnectivityCube.make_from_dict(PeerSet(), {"src_ports": src_res_ports, "dst_ports": dst_res_ports}) + conn_cube = ConnectivityCube.make_from_dict({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) tcp_properties = ConnectivityProperties.create_props_from_cube(conn_cube) tcp_properties_2 = tcp_properties.copy() From c2ccbfa5ee2be2bae4011a8451b1c43f79a441b3 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 26 Mar 2023 17:02:12 +0300 Subject: [PATCH 096/187] Removed base_peer_set from ConnectivityProperties and ConnectivityCube. Instead, added a singleton class BasePeerSet that keeps all peers and translates PeerSets to CanonicalIntervalSets and vice versa. Signed-off-by: Tanya --- nca/CoreDS/Peer.py | 2 +- nca/Parsers/GenericIngressLikeYamlParser.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index 0069a9aef..71cabe4fd 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -555,7 +555,7 @@ def __hash__(self): Note: PeerSet is a mutable type. Use with caution! :return: hash value for this object. """ - return hash(','.join(str(peer.full_name()) for peer in sorted(list(self)))) + return hash(','.join(str(peer.full_name()) for peer in sorted(list(elem for elem in self), key=by_full_name))) def rep(self): """ diff --git a/nca/Parsers/GenericIngressLikeYamlParser.py b/nca/Parsers/GenericIngressLikeYamlParser.py index 7565c193d..67dbecacd 100644 --- a/nca/Parsers/GenericIngressLikeYamlParser.py +++ b/nca/Parsers/GenericIngressLikeYamlParser.py @@ -86,7 +86,7 @@ def _make_rules_from_conns(conn_props): new_props = ConnectivityProperties.make_conn_props(conn_cube) new_conns = ConnectionSet() new_conns.add_connections('TCP', new_props) - if peers_to_conns.get(dst_peer_set): + if dst_peer_set in peers_to_conns: peers_to_conns[dst_peer_set] |= new_conns # optimize conns for the same peers else: peers_to_conns[dst_peer_set] = new_conns From 7e2e571dd38dd2778584e7081a6f2f2d9f7a3025 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 26 Mar 2023 20:33:07 +0300 Subject: [PATCH 097/187] Added check to BasePeerSet.get_peer_interval_of that all peers are translated to intervals. Signed-off-by: Tanya --- nca/CoreDS/Peer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index 71cabe4fd..acecdb97c 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -654,14 +654,17 @@ def get_peer_interval_of(self, peer_set): :return: CanonicalIntervalSet for the peer_set """ res = CanonicalIntervalSet() + covered_peers = peer_set.copy() # for check that we covered all peers for index, peer in enumerate(self.ordered_peer_list): if peer in peer_set: + covered_peers.add(peer) assert not isinstance(peer, IpBlock) res.add_interval(CanonicalIntervalSet.Interval(self.min_pod_index + index, self.min_pod_index + index)) # Now pick IpBlocks for ipb in peer_set: if isinstance(ipb, IpBlock): + covered_peers.add(ipb) for cidr in ipb: if isinstance(cidr.start.address, ipaddress.IPv4Address): res.add_interval(CanonicalIntervalSet.Interval(self.min_ipv4_index + int(cidr.start), @@ -671,6 +674,7 @@ def get_peer_interval_of(self, peer_set): self.min_ipv6_index + int(cidr.end))) else: assert False + assert covered_peers == peer_set return res def get_peer_set_by_indices(self, peer_interval_set): From cbcba8e8598e97d71cb1130b3d221d64f1a866b8 Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Mon, 27 Mar 2023 11:32:28 +0300 Subject: [PATCH 098/187] updates after merge with HC branch Signed-off-by: Shmulik Froimovich --- nca/CoreDS/ConnectivityProperties.py | 2 +- nca/NetworkConfig/NetworkLayer.py | 18 ++++++++++++++++-- nca/Utils/ExplTracker.py | 4 ++-- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 9145613c5..0ede8b263 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -243,7 +243,7 @@ def create_props_from_cube(conn_cube): cube, active_dims, has_empty_dim_value = conn_cube.get_ordered_cube_and_active_dims() if has_empty_dim_value: - return + return res if not active_dims: res.set_all() diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index d0c0e8f09..b19f38a9b 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -250,10 +250,24 @@ def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda poli for policy in self.policies_list: # Track the peers that were affected by this policy for peer in policy.selected_peers: + opt_egr_props = policy.optimized_egress_props + dst_opt_egr_props_projected = opt_egr_props.project_on_one_dimension('dst_peers') + dst_peers = PeerSet() + for cube in opt_egr_props: + conn_cube = opt_egr_props.get_connectivity_cube(cube) + dst_peers |= conn_cube["dst_peers"] + + opt_ingr_props = policy.optimized_ingress_props + src_opt_ingr_props_projected = opt_ingr_props.project_on_one_dimension('src_peers') + src_peers = PeerSet() + for cube in opt_ingr_props: + conn_cube = opt_ingr_props.get_connectivity_cube(cube) + src_peers |= conn_cube["src_peers"] + ExplTracker().add_peer_policy(peer, policy.name, - policy.optimized_egress_props.project_on_one_dimension('dst_peers'), - policy.optimized_ingress_props.project_on_one_dimension('src_peers'), + dst_peers, + src_peers, ) policy_allowed_conns, policy_denied_conns, policy_captured = \ policy.allowed_connections_optimized(is_ingress) diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py index 1af795822..491a1ef93 100644 --- a/nca/Utils/ExplTracker.py +++ b/nca/Utils/ExplTracker.py @@ -95,8 +95,8 @@ def are_peers_connected(self, src, dst): level='E') for cube in self.conns: conn = self.conns.get_cube_dict(cube) - src_peers = conn.get('src_peers').get_peer_names_list() - dst_peers = conn.get('dst_peers').get_peer_names_list() + src_peers = conn.get('src_peers').split(',') + dst_peers = conn.get('dst_peers').split(',') if src in src_peers and dst in dst_peers: return True return False From ead46b3d03778ccc3caec04910bf0a724ddd6e38 Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Mon, 27 Mar 2023 13:47:18 +0300 Subject: [PATCH 099/187] updates after merge with HC branch Signed-off-by: Shmulik Froimovich --- nca/NetworkConfig/NetworkLayer.py | 21 +++++---------------- nca/Utils/ExplTracker.py | 12 +++++++++++- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index b19f38a9b..89ce48b51 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -250,20 +250,8 @@ def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda poli for policy in self.policies_list: # Track the peers that were affected by this policy for peer in policy.selected_peers: - opt_egr_props = policy.optimized_egress_props - dst_opt_egr_props_projected = opt_egr_props.project_on_one_dimension('dst_peers') - dst_peers = PeerSet() - for cube in opt_egr_props: - conn_cube = opt_egr_props.get_connectivity_cube(cube) - dst_peers |= conn_cube["dst_peers"] - - opt_ingr_props = policy.optimized_ingress_props - src_opt_ingr_props_projected = opt_ingr_props.project_on_one_dimension('src_peers') - src_peers = PeerSet() - for cube in opt_ingr_props: - conn_cube = opt_ingr_props.get_connectivity_cube(cube) - src_peers |= conn_cube["src_peers"] - + src_peers, _ = ExplTracker().extract_peers(policy.optimized_ingress_props) + _, dst_peers = ExplTracker().extract_peers(policy.optimized_egress_props) ExplTracker().add_peer_policy(peer, policy.name, dst_peers, @@ -323,8 +311,9 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): else: conn_cube.update({"src_peers": non_captured, "dst_peers": base_peer_set_with_ip}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) - ExplTracker().add_default_policy(non_captured_conns.project_on_one_dimension('dst_peers'), - non_captured_conns.project_on_one_dimension('src_peers'), + src_peers, dst_peers = ExplTracker().extract_peers(non_captured_conns) + ExplTracker().add_default_policy(dst_peers, + src_peers, is_ingress ) allowed_conn |= non_captured_conns diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py index 491a1ef93..a2a2e2caf 100644 --- a/nca/Utils/ExplTracker.py +++ b/nca/Utils/ExplTracker.py @@ -5,7 +5,7 @@ from nca.Utils.Utils import Singleton from nca.Utils.NcaLogger import NcaLogger - +from nca.CoreDS.Peer import PeerSet class ExplDescriptor: def __init__(self, name, config_code, config_file): @@ -85,6 +85,16 @@ def add_peer_policy(self, peer, policy_name, egress_dst, ingress_src): ingress_src, ) + @staticmethod + def extract_peers(conns): + src_peers = PeerSet() + dst_peers = PeerSet() + for cube in conns: + conn_cube = conns.get_connectivity_cube(cube) + src_peers |= conn_cube["src_peers"] + dst_peers |= conn_cube["dst_peers"] + return src_peers, dst_peers + def set_connections(self, conns): self.conns = conns pass From 760e1da88f3a3282329cd9882f0f4de77029b8b0 Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Mon, 27 Mar 2023 14:53:23 +0300 Subject: [PATCH 100/187] removed unused function Signed-off-by: Shmulik Froimovich --- nca/CoreDS/Peer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index 2dd01d52c..acecdb97c 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -583,10 +583,6 @@ def get_ip_block_canonical_form(self): res |= elem return res - def get_peer_names_list(self): - names = list(elem.full_name_str for elem in self if not isinstance(elem, IpBlock)) - return names - def filter_ipv6_blocks(self, ip_blocks_mask): """ Update ip blocks in the peer set by keeping only parts overlapping with the given mask. From 4cc8de2251ad3ec7b2213c5ddcfcb35303fbb1ac Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Mon, 27 Mar 2023 16:43:37 +0300 Subject: [PATCH 101/187] added some function descriptions Signed-off-by: Shmulik Froimovich --- nca/Utils/ExplTracker.py | 106 ++++++++++++++++++++++++++++++--------- 1 file changed, 82 insertions(+), 24 deletions(-) diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py index e9f684391..d326a9472 100644 --- a/nca/Utils/ExplTracker.py +++ b/nca/Utils/ExplTracker.py @@ -7,23 +7,13 @@ from nca.Utils.NcaLogger import NcaLogger from nca.CoreDS.Peer import PeerSet -class ExplDescriptor: - def __init__(self, name, config_code, config_file): - self.name = name - self.config_code = config_code - self.config_file = config_file - - def __get__(self, obj, type=None) -> object: - return self.config_code, self.config_file - - def __set__(self, obj, value) -> None: - raise AttributeError("Cannot change values, ExplDescriptor is read only") - - def get_name(self): - return self.name - class ExplPolicies: + """ + ExplPolicies holds the policies affecting peers in relation to other peers. + That is, for each peer it holds all the peers in it's egress and ingress and the policies that has effect on the connection + to that peers. + """ def __init__(self): self.egress_dst = {} self.ingress_src = {} @@ -31,6 +21,12 @@ def __init__(self): @staticmethod def _add_policy(peer_set, peer_list, policy_name): + """ + Adds a policy to the list of affecting policies, for each peer in the peer_set + :param PeerSet peer_set: a set of peers to add the policy to + :param dict peer_list: a list of peers that holds the policies affecting them + :param str policy_name: the policy to add + """ for peer in peer_set: peer_name = peer.full_name() if not peer_list.get(peer_name): @@ -38,7 +34,12 @@ def _add_policy(peer_set, peer_list, policy_name): peer_list[peer_name].add(policy_name) def add_policy(self, policy_name, egress_dst, ingress_src): - + """ + Adds a given policy to the relevant peer lists (egress list, ingress list) + :param str policy_name: name of the policy + :param dict egress_dst: the set of egress destinations peers to add the policy too + :param dict ingress_src: the set of ingress source peers to add the policy too + """ self.all_policies.add(policy_name) if egress_dst: @@ -53,6 +54,8 @@ class ExplTracker(metaclass=Singleton): The Explainability Tracker is used for tracking the elements and their configuration so it will be able to specify which configurations are responsible for each peer and each connection or lack of connection between them. + + The ExplTracker is Singletone """ def __init__(self): @@ -64,18 +67,39 @@ def __init__(self): self.add_item('', 'Default-Policy', 0) def activate(self): + """ + Make the ExplTracker active + """ self._is_active = True def is_active(self): + """ + Return the active state of the ExplTracker + :return: bool + """ return self._is_active - def get_path_from_deployment(self, content): - return self.ExplDescriptorContainer[content.get('metadata').get('name')].get('path') - def add_item(self, path, name, ln): - self.ExplDescriptorContainer[name] = {'path': path, 'line': ln} + """ + Adds an item describing a configuration block + :param str path: the path to the configuration file + :param str name: the name of the configuration block (doc) + :param int ln: the line starting the configuration block in it's file + """ + if name: + self.ExplDescriptorContainer[name] = {'path': path, 'line': ln} + else: + NcaLogger().log_message(f'Explainability error: configuration-block name can not be empty', + level='E') def add_peer_policy(self, peer, policy_name, egress_dst, ingress_src): + """ + Add a new policy to a peer + :param Peer peer: peer object + :param srt policy_name: name of the policy + :param egress_dst: a list of peers that the given policy affect, egress wise. + :param ingress_src: a list of peers that the given policy affect, ingress wise. + """ peer_name = peer.full_name() if self.ExplDescriptorContainer.get(peer_name): if not self.ExplPeerToPolicyContainer.get(peer_name): @@ -87,6 +111,11 @@ def add_peer_policy(self, peer, policy_name, egress_dst, ingress_src): @staticmethod def extract_peers(conns): + """ + Utility function to extract the peer names held in a connectivity element + :param ConnectivityProperties conns: + :return: PeerSet src_peers, PeerSet dst_peers: sets of collected peers + """ src_peers = PeerSet() dst_peers = PeerSet() for cube in conns: @@ -96,10 +125,19 @@ def extract_peers(conns): return src_peers, dst_peers def set_connections(self, conns): + """ + Update the calculated connections into ExplTracker + :param ConnectivityProperties conns: the connectivity mapping calculated by the query + """ self.conns = conns - pass def are_peers_connected(self, src, dst): + """ + Check if a given pair of peers are connected + :param str src: name of the source peer + :param str dst: name of the destination peer + :return: bool: True for connected, False for disconnected + """ if not self.conns: NcaLogger().log_message(f'Explainability error: Connections were not set yet, but peer query was called', level='E') @@ -112,7 +150,12 @@ def are_peers_connected(self, src, dst): return False def add_default_policy(self, currents, peers, is_ingress): - + """ + Add the default policy to the peers which were not affected by a specific policy. + :param PeerSet currents: the peers to add the policy too + :param PeerSet peers: the adjacent peers the default policy nakes a connection with + :param is_ingress: is this an ingress or egress policy + """ if is_ingress: ingress_src = peers egress_dst = {} @@ -128,7 +171,13 @@ def add_default_policy(self, currents, peers, is_ingress): ) def prepare_node_str(self, direction, node_name, results): - + """ + A utility function to help format a node explainability description + :param str direction: src/dst + :param str node_name: the name of the node currently described + :param str results: the names of the configurations affecting this node + :return str: string with the description + """ out = [] if direction: out = [f'\n({direction}){node_name}:'] @@ -142,7 +191,16 @@ def prepare_node_str(self, direction, node_name, results): return out def explain(self, nodes): - + """ + The magic function to explain the connectivity or the LACK of it between the given nodes + It has 2 modes: + single node - if a single node is given, all the configurations on that node are displayed. + two nodes - if 2 nodes are given, either they hava a connection between them and the configurations responsible for + the connection are displayed. or, they lack a connection, in which case, all affecting configurations + on those 2 nodes are displayed. + :param list(str) nodes: nodes to explain + :return: str: the explanation out string + """ out = [] if len(nodes) < 1: return out From 7fbc5cad57475d14f070e98ff890cc960a40dffb Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Tue, 28 Mar 2023 16:25:37 +0300 Subject: [PATCH 102/187] txt_no_fe_rules format added Signed-off-by: Shmulik Froimovich --- nca/FWRules/ConnectivityGraph.py | 2 +- nca/NetworkConfig/NetworkConfigQuery.py | 27 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index a0e3465bd..e41d31c89 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -85,7 +85,7 @@ def add_edges_from_cube_dict(self, conn_cube, ip_blocks_mask): protocols = conn_cube["protocols"] conn_cube.unset_dim("protocols") - if not protocols and not conn_cube.has_active_dim(): + if protocols.is_whole_range() and not conn_cube.has_active_dim(): conns = ConnectionSet(True) else: conns = ConnectionSet() diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 62bf31d63..8ae29f36c 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -839,6 +839,9 @@ def get_props_output_full(self, props, peers_to_compare, ip_blocks_mask): dot_full = self.dot_format_from_props(props, peers_to_compare, ip_blocks_mask) return dot_full, None # TODO - handle 'txt_no_fw_rules' output format + if self.output_config.outputFormat == 'txt_no_fw_rules': + conns_wo_fw_rules = self.txt_no_fw_rules_format_from_props(props, peers_to_compare, ip_blocks_mask) + return conns_wo_fw_rules, None # handle other formats formatted_rules, fw_rules = self.fw_rules_from_props(props, peers_to_compare, ip_blocks_mask) return formatted_rules, fw_rules @@ -908,6 +911,14 @@ def get_props_output_split_by_tcp(self, props, peers_to_compare, ip_blocks_mask) res_str = dot_tcp + dot_non_tcp return res_str, None, None # TODO - handle 'txt_no_fw_rules' output format + if self.output_config.outputFormat in ['txt_no_fw_rules']: + txt_no_fw_rules_tcp = self.txt_no_fw_rules_format_from_props(props_tcp, peers_to_compare, ip_blocks_mask, + connectivity_tcp_str) + txt_no_fw_rules_non_tcp = self.txt_no_fw_rules_format_from_props(props_non_tcp, peers_to_compare, ip_blocks_mask, + connectivity_non_tcp_str) + # concatenate the two graphs into one dot file + res_str = txt_no_fw_rules_tcp + txt_no_fw_rules_non_tcp + return res_str, None, None # handle formats other than dot and txt_no_fw_rules formatted_rules_tcp, fw_rules_tcp = self.fw_rules_from_props(props_tcp, peers_to_compare, ip_blocks_mask, connectivity_tcp_str) @@ -977,6 +988,22 @@ def dot_format_from_props(self, props, peers, ip_blocks_mask, connectivity_restr conn_graph.add_edges_from_cube_dict(props.get_connectivity_cube(cube), ip_blocks_mask) return conn_graph.get_connectivity_dot_format_str(connectivity_restriction) + def txt_no_fw_rules_format_from_props(self, props, peers, ip_blocks_mask, connectivity_restriction=None): + """ + :param ConnectivityProperties props: properties describing allowed connections + :param PeerSet peers: the peers to consider for dot output + :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, + whereas all other values should be filtered out in the output + :param Union[str,None] connectivity_restriction: specify if connectivity is restricted to + TCP / non-TCP , or not + :rtype str + :return the connectivity map in dot-format, considering connectivity_restriction if required + """ + conn_graph = ConnectivityGraph(peers, self.config.get_allowed_labels(), self.output_config) + for cube in props: + conn_graph.add_edges_from_cube_dict(props.get_connectivity_cube(cube), ip_blocks_mask) + return conn_graph.get_connections_without_fw_rules_txt_format(connectivity_restriction) + def fw_rules_from_connections_dict(self, connections, peers_to_compare, connectivity_restriction=None): """ :param dict connections: the connections' dict (map from connection-set to peer pairs) From 75846e39c6caed8fde1a97ab42628f82bdd4cb3c Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 2 Apr 2023 11:16:17 +0300 Subject: [PATCH 103/187] Added OptimizedPolicyConnections class to hold allowed, denied and pass connections and captured peers. Signed-off-by: Tanya --- nca/FWRules/ConnectivityGraph.py | 2 +- nca/NetworkConfig/NetworkConfig.py | 45 ++++-- nca/NetworkConfig/NetworkConfigQuery.py | 3 +- nca/NetworkConfig/NetworkLayer.py | 151 ++++++++++-------- nca/Parsers/IngressPolicyYamlParser.py | 2 +- .../IstioTrafficResourcesYamlParser.py | 3 +- nca/Resources/CalicoNetworkPolicy.py | 17 +- nca/Resources/IngressPolicy.py | 51 ++---- nca/Resources/IstioNetworkPolicy.py | 17 +- nca/Resources/IstioSidecar.py | 17 +- nca/Resources/K8sNetworkPolicy.py | 14 +- nca/Resources/NetworkPolicy.py | 14 ++ 12 files changed, 186 insertions(+), 150 deletions(-) diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index a0e3465bd..e41d31c89 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -85,7 +85,7 @@ def add_edges_from_cube_dict(self, conn_cube, ip_blocks_mask): protocols = conn_cube["protocols"] conn_cube.unset_dim("protocols") - if not protocols and not conn_cube.has_active_dim(): + if protocols.is_whole_range() and not conn_cube.has_active_dim(): conns = ConnectionSet(True) else: conns = ConnectionSet() diff --git a/nca/NetworkConfig/NetworkConfig.py b/nca/NetworkConfig/NetworkConfig.py index 0a9c22e76..d6f483c1e 100644 --- a/nca/NetworkConfig/NetworkConfig.py +++ b/nca/NetworkConfig/NetworkConfig.py @@ -6,8 +6,8 @@ from dataclasses import dataclass, field from nca.CoreDS import Peer from nca.CoreDS.ConnectionSet import ConnectionSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties -from nca.Resources.NetworkPolicy import NetworkPolicy +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube +from nca.Resources.NetworkPolicy import NetworkPolicy, OptimizedPolicyConnections from .NetworkLayer import NetworkLayersContainer, NetworkLayerName @@ -277,17 +277,38 @@ def allowed_connections_optimized(self, layer_name=None): """ if layer_name is not None: if layer_name not in self.policies_container.layers: - conns_res = self.policies_container.layers.empty_layer_allowed_connections_optimized(self.peer_container, - layer_name) - else: - conns_res = self.policies_container.layers[layer_name].allowed_connections_optimized(self.peer_container) - else: - conns_res = ConnectivityProperties.make_all_props() # all connections - for layer, layer_obj in self.policies_container.layers.items(): - conns_per_layer = layer_obj.allowed_connections_optimized(self.peer_container) - # all allowed connections: intersection of all allowed connections from all layers - conns_res &= conns_per_layer + return self.policies_container.layers.empty_layer_allowed_connections_optimized(self.peer_container, + layer_name) + return self.policies_container.layers[layer_name].allowed_connections_optimized(self.peer_container) + + all_peers = self.peer_container.get_all_peers_group() + host_eps = Peer.PeerSet(set([peer for peer in all_peers if isinstance(peer, Peer.HostEP)])) + src_peers_conn_cube = ConnectivityCube.make_from_dict({"src_peers": host_eps}) + dst_peers_conn_cube = ConnectivityCube.make_from_dict({"dst_peers": host_eps}) + # all possible connections involving hostEndpoints + conn_hep = ConnectivityProperties.make_conn_props(src_peers_conn_cube) | \ + ConnectivityProperties.make_conn_props(dst_peers_conn_cube) + conns_res = OptimizedPolicyConnections() + conns_res.all_allowed_conns = ConnectivityProperties.make_all_props() + for layer, layer_obj in self.policies_container.layers.items(): + conns_per_layer = layer_obj.allowed_connections_optimized(self.peer_container) + # only K8s_Calico layer handles host_eps + if layer != NetworkLayerName.K8s_Calico: + # connectivity of hostEndpoints is only determined by calico layer + conns_per_layer.allowed_conns -= conn_hep + conns_per_layer.denied_conns -= conn_hep + conns_per_layer.pass_conns -= conn_hep + + # all allowed connections: intersection of all allowed connections from all layers + conns_res.all_allowed_conns &= conns_per_layer.all_allowed_conns + # all allowed captured connections: should be captured by at least one layer + conns_res.allowed_conns |= conns_per_layer.allowed_conns + conns_res.captured |= conns_per_layer.captured + # denied conns: should be denied by at least one layer + conns_res.denied_conns |= conns_per_layer.denied_conns + # allowed captured conn (by at least one layer) has to be allowed by all layers (either implicitly or explicitly) + conns_res.allowed_conns &= conns_res.all_allowed_conns return conns_res def append_policy_to_config(self, policy): diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index b5e13ddd4..dc79706bd 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -745,7 +745,8 @@ def exec(self): # noqa: C901 all_conns_opt = ConnectivityProperties.make_empty_props() opt_start = time.time() if self.config.optimized_run != 'false': - all_conns_opt = self.config.allowed_connections_optimized() + opt_conns = self.config.allowed_connections_optimized() + all_conns_opt = opt_conns.all_allowed_conns if all_conns_opt: opt_peers_to_compare = self.config.peer_container.get_all_peers_group() # add all relevant IpBlocks, used in connections diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 7d1f3d685..364f9f24f 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -10,7 +10,7 @@ from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy -from nca.Resources.NetworkPolicy import PolicyConnections, NetworkPolicy +from nca.Resources.NetworkPolicy import PolicyConnections, OptimizedPolicyConnections, NetworkPolicy # TODO: add a layer for connectivity based on service type (culsterIP / LB / NodePort)? / containers ports? @@ -171,23 +171,31 @@ def allowed_connections_optimized(self, peer_container): Compute per network layer the allowed connections between any relevant peers, considering all layer's policies (and defaults) :param PeerContainer peer_container: the peer container holding the peers - :return: all allowed connections - :rtype: ConnectivityProperties + :return: all allowed, denied and captured connections + :rtype: OptimizedPolicyConnections """ - all_pods = peer_container.get_all_peers_group() - all_ips_peer_set = PeerSet({IpBlock.get_all_ips_block()}) - conn_cube = ConnectivityCube.make_from_dict({"src_peers": all_pods, "dst_peers": all_ips_peer_set}) - allowed_ingress_conns, denied_ingres_conns = self._allowed_xgress_conns_optimized(True, peer_container) - allowed_ingress_conns |= ConnectivityProperties.make_conn_props(conn_cube) - allowed_egress_conns, denied_egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) - conn_cube.update({"src_peers": all_ips_peer_set, "dst_peers": all_pods}) - allowed_egress_conns |= ConnectivityProperties.make_conn_props(conn_cube) - res = allowed_ingress_conns & allowed_egress_conns + res_conns = OptimizedPolicyConnections() + ingress_conns = self._allowed_xgress_conns_optimized(True, peer_container) + egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) + all_pods_peer_set = peer_container.get_all_peers_group() + all_ips_peer_set = IpBlock.get_all_ips_block_peer_set() + # for ingress, all possible connections to IpBlocks are allowed + conn_cube = ConnectivityCube.make_from_dict({"src_peers": all_pods_peer_set, "dst_peers": all_ips_peer_set}) + ingress_conns.all_allowed_conns |= ConnectivityProperties.make_conn_props(conn_cube) + # for egress, all possible connections from IpBlocks are allowed + conn_cube.update({"src_peers": all_ips_peer_set, "dst_peers": all_pods_peer_set}) + egress_conns.all_allowed_conns |= ConnectivityProperties.make_conn_props(conn_cube) + res_conns.captured = ingress_conns.captured | egress_conns.captured + res_conns.denied_conns = ingress_conns.denied_conns | egress_conns.denied_conns + res_conns.all_allowed_conns = ingress_conns.all_allowed_conns & egress_conns.all_allowed_conns + res_conns.allowed_conns = (ingress_conns.allowed_conns & egress_conns.all_allowed_conns) | \ + (egress_conns.allowed_conns & ingress_conns.all_allowed_conns) # exclude IpBlock->IpBlock connections conn_cube.update({"src_peers": all_ips_peer_set, "dst_peers": all_ips_peer_set}) - excluded_conns = ConnectivityProperties.make_conn_props(conn_cube) - res -= excluded_conns - return res + ip_to_ip_conns = ConnectivityProperties.make_conn_props(conn_cube) + res_conns.allowed_conns -= ip_to_ip_conns + res_conns.all_allowed_conns -= ip_to_ip_conns + return res_conns def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): """ @@ -243,26 +251,23 @@ def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda poli :return: allowed_conns, denied_conns and set of peers to be added to captured peers :rtype: tuple (ConnectivityProperties, ConnectivityProperties, PeerSet) """ - allowed_conns = ConnectivityProperties.make_empty_props() - denied_conns = ConnectivityProperties.make_empty_props() - captured = PeerSet() + res_conns = OptimizedPolicyConnections() for policy in self.policies_list: - policy_allowed_conns, policy_denied_conns, policy_captured = \ - policy.allowed_connections_optimized(is_ingress) - if policy_captured: # not empty - policy_denied_conns -= allowed_conns - # policy_denied_conns -= pass_conns # Preparation for handling of pass - policy_allowed_conns -= denied_conns - # policy_allowed_conns -= pass_conns # Preparation for handling of pass - # policy_pass_conns -= denied_conns - # policy_pass_conns -= allowed_conns - # pass_conns |= policy_pass_conns - allowed_conns |= policy_allowed_conns - denied_conns |= policy_denied_conns + policy_conns = policy.allowed_connections_optimized(is_ingress) + if policy_conns.captured: # not empty if captured_func(policy): - captured |= policy_captured + res_conns.captured |= policy_conns.captured + policy_conns.denied_conns -= res_conns.allowed_conns + policy_conns.denied_conns -= res_conns.pass_conns + policy_conns.allowed_conns -= res_conns.denied_conns + policy_conns.allowed_conns -= res_conns.pass_conns + policy_conns.pass_conns -= res_conns.denied_conns + policy_conns.pass_conns -= res_conns.allowed_conns + res_conns.allowed_conns |= policy_conns.allowed_conns + res_conns.denied_conns |= policy_conns.denied_conns + res_conns.pass_conns |= policy_conns.pass_conns - return allowed_conns, denied_conns, captured + return res_conns class K8sCalicoNetworkLayer(NetworkLayer): @@ -286,23 +291,39 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): all_allowed_conns=allowed_conns | allowed_non_captured_conns) def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): - allowed_conn, denied_conns, captured = self.collect_policies_conns_optimized(is_ingress) + res_conns = self.collect_policies_conns_optimized(is_ingress) # Note: The below computation of non-captured conns cannot be done during the parse stage, # since before computing non-captured conns we should collect all policies conns # compute non-captured connections - base_peer_set_with_ip = peer_container.get_all_peers_group(True) - base_peer_set_no_ip = peer_container.get_all_peers_group() - base_peer_set_no_hep = PeerSet(set([peer for peer in base_peer_set_no_ip if not isinstance(peer, HostEP)])) - non_captured = base_peer_set_no_hep - captured - if non_captured: + all_peers_and_ips = peer_container.get_all_peers_group(True) + all_peers_no_ips = peer_container.get_all_peers_group() + base_peer_set_no_hep = PeerSet(set([peer for peer in all_peers_no_ips if not isinstance(peer, HostEP)])) + not_captured_not_hep = base_peer_set_no_hep - res_conns.captured + if not_captured_not_hep: + # default Allow-all in k8s / calico + # (assuming only calico's default profiles for pods with connectivity rules exist) + # assuming host endpoints have no profiles conn_cube = ConnectivityCube() if is_ingress: - conn_cube.update({"src_peers": base_peer_set_with_ip, "dst_peers": non_captured}) + conn_cube.update({"src_peers": all_peers_and_ips, "dst_peers": not_captured_not_hep}) else: - conn_cube.update({"src_peers": non_captured, "dst_peers": base_peer_set_with_ip}) - non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) - allowed_conn |= non_captured_conns - return allowed_conn, denied_conns + conn_cube.update({"src_peers": not_captured_not_hep, "dst_peers": all_peers_and_ips}) + not_captured_not_hep_conns = ConnectivityProperties.make_conn_props(conn_cube) + res_conns.all_allowed_conns |= not_captured_not_hep_conns + + captured_not_hep = base_peer_set_no_hep & res_conns.captured + if captured_not_hep and res_conns.pass_conns: + # assuming only default profiles generated by calico exist, which allow all for pods + conn_cube = ConnectivityCube() + if is_ingress: + conn_cube.update({"src_peers": all_peers_and_ips, "dst_peers": captured_not_hep}) + else: + conn_cube.update({"src_peers": captured_not_hep, "dst_peers": all_peers_and_ips}) + captured_not_hep_conns = ConnectivityProperties.make_conn_props(conn_cube) + res_conns.allowed_conns |= res_conns.pass_conns & captured_not_hep_conns + + res_conns.all_allowed_conns |= res_conns.allowed_conns + return res_conns class IstioNetworkLayer(NetworkLayer): @@ -326,23 +347,22 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): all_allowed_conns=allowed_conns | allowed_non_captured_conns) def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): - allowed_conn, denied_conns, captured = \ - self.collect_policies_conns_optimized(is_ingress, IstioNetworkLayer.captured_cond_func) - base_peer_set_with_ip = peer_container.get_all_peers_group(True) - base_peer_set_no_ip = peer_container.get_all_peers_group() - non_captured_peers = base_peer_set_no_ip - captured - conn_cube = ConnectivityCube() + res_conns = self.collect_policies_conns_optimized(is_ingress, IstioNetworkLayer.captured_cond_func) + all_peers_and_ips = peer_container.get_all_peers_group(True) + # for istio initialize non-captured conns with non-TCP connections + conn_cube = ConnectivityCube.make_from_dict({"src_peers": all_peers_and_ips, "dst_peers": all_peers_and_ips, + "protocols": ProtocolSet.get_non_tcp_protocols()}) + res_conns.all_allowed_conns |= res_conns.allowed_conns | ConnectivityProperties.make_conn_props(conn_cube) + non_captured_peers = all_peers_and_ips - res_conns.captured if non_captured_peers: + conn_cube = ConnectivityCube() if is_ingress: - conn_cube.update({"src_peers": base_peer_set_with_ip, "dst_peers": non_captured_peers}) + conn_cube.update({"src_peers": all_peers_and_ips, "dst_peers": non_captured_peers}) else: - conn_cube.update({"src_peers": non_captured_peers, "dst_peers": base_peer_set_with_ip}) + conn_cube.update({"src_peers": non_captured_peers, "dst_peers": all_peers_and_ips}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) - allowed_conn |= (non_captured_conns - denied_conns) - conn_cube.update({"src_peers": base_peer_set_with_ip, "dst_peers": base_peer_set_with_ip, - "protocols": ProtocolSet.get_non_tcp_protocols()}) - allowed_conn |= ConnectivityProperties.make_conn_props(conn_cube) - return allowed_conn, denied_conns + res_conns.all_allowed_conns |= (non_captured_conns - res_conns.denied_conns) + return res_conns class IngressNetworkLayer(NetworkLayer): @@ -359,18 +379,19 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): all_allowed_conns=all_allowed_conns) def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): - allowed_conn, denied_conns, captured = self.collect_policies_conns_optimized(is_ingress) - base_peer_set_with_ip = peer_container.get_all_peers_group(True) - base_peer_set_no_ip = peer_container.get_all_peers_group() + res_conns = OptimizedPolicyConnections() + all_peers_and_ips = peer_container.get_all_peers_group(True) conn_cube = ConnectivityCube() if is_ingress: - conn_cube.update({"src_peers": base_peer_set_with_ip, "dst_peers": base_peer_set_no_ip}) + # everything is allowed and non captured + conn_cube.update({"src_peers": all_peers_and_ips, "dst_peers": all_peers_and_ips}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) - allowed_conn |= non_captured_conns + res_conns.all_allowed_conns = non_captured_conns else: - non_captured_peers = base_peer_set_no_ip - captured + res_conns = self.collect_policies_conns_optimized(is_ingress) + non_captured_peers = all_peers_and_ips - res_conns.captured if non_captured_peers: - conn_cube.update({"src_peers": non_captured_peers, "dst_peers": base_peer_set_with_ip}) + conn_cube.update({"src_peers": non_captured_peers, "dst_peers": all_peers_and_ips}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) - allowed_conn |= non_captured_conns - return allowed_conn, denied_conns + res_conns.all_allowed_conns = res_conns.allowed_conns | non_captured_conns + return res_conns diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index 398cfdd40..983d33052 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -250,7 +250,7 @@ def parse_policy(self): return None # Not an Ingress object self.namespace = self.peer_container.get_namespace(policy_ns) - res_policy = IngressPolicy(policy_name + '/allow', self.namespace, IngressPolicy.ActionType.Allow) + res_policy = IngressPolicy(policy_name + '/allow', self.namespace) res_policy.policy_kind = NetworkPolicy.PolicyType.Ingress policy_spec = self.policy['spec'] diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index 08b919cf3..fba1e6d6f 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -390,8 +390,7 @@ def create_istio_traffic_policies(self): peers_to_hosts[peers] = host_dfa for peer_set, host_dfa in peers_to_hosts.items(): - res_policy = IngressPolicy(vs.name + '/' + str(host_dfa) + '/allow', vs.namespace, - IngressPolicy.ActionType.Allow) + res_policy = IngressPolicy(vs.name + '/' + str(host_dfa) + '/allow', vs.namespace) res_policy.policy_kind = NetworkPolicy.PolicyType.Ingress res_policy.selected_peers = peer_set allowed_conns = self.make_allowed_connections(vs, host_dfa) diff --git a/nca/Resources/CalicoNetworkPolicy.py b/nca/Resources/CalicoNetworkPolicy.py index 4b4d2215f..2529e344b 100644 --- a/nca/Resources/CalicoNetworkPolicy.py +++ b/nca/Resources/CalicoNetworkPolicy.py @@ -6,7 +6,7 @@ from enum import Enum from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS import Peer -from .NetworkPolicy import PolicyConnections, NetworkPolicy +from .NetworkPolicy import PolicyConnections, OptimizedPolicyConnections, NetworkPolicy class CalicoPolicyRule: @@ -127,15 +127,16 @@ def allowed_connections_optimized(self, is_ingress): and the peer set of captured peers by this policy. :rtype: tuple (ConnectivityProperties, ConnectivityProperties, PeerSet) """ + res_conns = OptimizedPolicyConnections() if is_ingress: - allowed = self.optimized_ingress_props.copy() - denied = self.optimized_denied_ingress_props.copy() - captured = self.selected_peers if self.affects_ingress else Peer.PeerSet() + res_conns.allowed_conns = self.optimized_ingress_props.copy() + res_conns.denied_conns = self.optimized_denied_ingress_props.copy() + res_conns.captured = self.selected_peers if self.affects_ingress else Peer.PeerSet() else: - allowed = self.optimized_egress_props.copy() - denied = self.optimized_denied_egress_props.copy() - captured = self.selected_peers if self.affects_egress else Peer.PeerSet() - return allowed, denied, captured + res_conns.allowed_conns = self.optimized_egress_props.copy() + res_conns.denied_conns = self.optimized_denied_egress_props.copy() + res_conns.captured = self.selected_peers if self.affects_egress else Peer.PeerSet() + return res_conns def clone_without_rule(self, rule_to_exclude, ingress_rule): """ diff --git a/nca/Resources/IngressPolicy.py b/nca/Resources/IngressPolicy.py index 92229b4ce..612846f75 100644 --- a/nca/Resources/IngressPolicy.py +++ b/nca/Resources/IngressPolicy.py @@ -7,7 +7,7 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.Peer import PeerSet -from .NetworkPolicy import PolicyConnections, NetworkPolicy +from .NetworkPolicy import PolicyConnections, OptimizedPolicyConnections, NetworkPolicy class IngressPolicyRule: @@ -41,23 +41,14 @@ class IngressPolicy(NetworkPolicy): and the rules are egress_rules. """ - class ActionType(IntEnum): - """ - Everything that is not defined in Ingress rule's backend or in the default backend is denied - """ - Deny = 0 - Allow = 1 - - def __init__(self, name, namespace, action): + def __init__(self, name, namespace): """ :param str name: Ingress name :param K8sNamespace namespace: the namespace containing this ingress - :param ActionType action: Allow/Deny """ super().__init__(name, namespace) self.affects_ingress = False self.affects_egress = True - self.action = action def __eq__(self, other): return super().__eq__(other) @@ -88,14 +79,12 @@ def allowed_connections(self, from_peer, to_peer, is_ingress): return PolicyConnections(False) allowed_conns = ConnectionSet() - denied_conns = ConnectionSet() - conns = allowed_conns if self.action == IngressPolicy.ActionType.Allow else denied_conns for rule in self.egress_rules: if to_peer in rule.peer_set: assert not rule.connections.has_named_ports() - conns |= rule.connections + allowed_conns |= rule.connections - return PolicyConnections(True, allowed_conns, denied_conns) + return PolicyConnections(True, allowed_conns) def allowed_connections_optimized(self, is_ingress): """ @@ -107,14 +96,14 @@ def allowed_connections_optimized(self, is_ingress): and the peer set of captured peers by this policy. :rtype: tuple (ConnectivityProperties, ConnectivityProperties, PeerSet) """ - + res_conns = OptimizedPolicyConnections() if is_ingress: - allowed = ConnectivityProperties.make_empty_props() - captured = PeerSet() + res_conns.allowed_conns = ConnectivityProperties.make_empty_props() + res_conns.captured = PeerSet() else: - allowed = self.optimized_egress_props.copy() - captured = self.selected_peers if self.affects_egress else PeerSet() - return allowed, ConnectivityProperties.make_empty_props(), captured + res_conns.allowed_conns = self.optimized_egress_props.copy() + res_conns.captured = self.selected_peers if self.affects_egress else PeerSet() + return res_conns def has_empty_rules(self, _config_name=''): """ @@ -144,7 +133,7 @@ def clone_without_rule(self, rule_to_exclude, ingress_rule): :rtype: IngressPolicy """ assert not ingress_rule - res = IngressPolicy(self.name, self.namespace, self.action) + res = IngressPolicy(self.name, self.namespace) res.selected_peers = self.selected_peers res.affects_egress = self.affects_egress res.affects_ingress = self.affects_ingress @@ -159,19 +148,7 @@ def has_allow_rules(self): :return: Whether the policy has rules that allow connections. :rtype: bool """ - if self.action == IngressPolicy.ActionType.Allow: - for rule in self.egress_rules: - if rule.peer_set: - return True - return False - - def has_deny_rules(self): - """ - :return: Whether the policy has deny rules - :rtype: bool - """ - if self.action == IngressPolicy.ActionType.Deny: - for rule in self.egress_rules: - if rule.peer_set: - return True + for rule in self.egress_rules: + if rule.peer_set: + return True return False diff --git a/nca/Resources/IstioNetworkPolicy.py b/nca/Resources/IstioNetworkPolicy.py index 2f902efbf..f4dfca349 100644 --- a/nca/Resources/IstioNetworkPolicy.py +++ b/nca/Resources/IstioNetworkPolicy.py @@ -6,7 +6,7 @@ from enum import Enum from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import PeerSet, IpBlock -from .NetworkPolicy import PolicyConnections, NetworkPolicy +from .NetworkPolicy import PolicyConnections, OptimizedPolicyConnections, NetworkPolicy class IstioPolicyRule: @@ -101,15 +101,16 @@ def allowed_connections_optimized(self, is_ingress): and the peer set of captured peers by this policy. :rtype: tuple (ConnectivityProperties, ConnectivityProperties, PeerSet) """ + res_conns = OptimizedPolicyConnections() if is_ingress: - allowed = self.optimized_ingress_props.copy() - denied = self.optimized_denied_ingress_props.copy() - captured = self.selected_peers + res_conns.allowed_conns = self.optimized_ingress_props.copy() + res_conns.denied_conns = self.optimized_denied_ingress_props.copy() + res_conns.captured = self.selected_peers else: - allowed = self.optimized_egress_props.copy() - denied = self.optimized_denied_egress_props.copy() - captured = PeerSet() - return allowed, denied, captured + res_conns.allowed_conns = self.optimized_egress_props.copy() + res_conns.denied_conns = self.optimized_denied_egress_props.copy() + res_conns.captured = PeerSet() + return res_conns def referenced_ip_blocks(self, exclude_ipv6=False): """ diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index a6617d676..5b8ddd1b2 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -6,7 +6,7 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock, PeerSet from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube -from .NetworkPolicy import PolicyConnections, NetworkPolicy +from .NetworkPolicy import PolicyConnections, OptimizedPolicyConnections, NetworkPolicy from .IstioTrafficResources import istio_root_namespace @@ -72,15 +72,16 @@ def allowed_connections(self, from_peer, to_peer, is_ingress): return PolicyConnections(True, allowed_conns=ConnectionSet()) def allowed_connections_optimized(self, is_ingress): + res_conns = OptimizedPolicyConnections() if is_ingress: - allowed = self.optimized_ingress_props.copy() - denied = self.optimized_denied_ingress_props.copy() - captured = PeerSet() + res_conns.allowed_conns = self.optimized_ingress_props.copy() + res_conns.denied_conns = self.optimized_denied_ingress_props.copy() + res_conns.captured = PeerSet() else: - allowed = self.optimized_egress_props.copy() - denied = self.optimized_denied_egress_props.copy() - captured = self.selected_peers if self.affects_egress else PeerSet() - return allowed, denied, captured + res_conns.allowed_conns = self.optimized_egress_props.copy() + res_conns.denied_conns = self.optimized_denied_egress_props.copy() + res_conns.captured = self.selected_peers if self.affects_egress else PeerSet() + return res_conns def has_empty_rules(self, config_name=''): """ diff --git a/nca/Resources/K8sNetworkPolicy.py b/nca/Resources/K8sNetworkPolicy.py index ac5cc3303..6c3b93d8a 100644 --- a/nca/Resources/K8sNetworkPolicy.py +++ b/nca/Resources/K8sNetworkPolicy.py @@ -4,8 +4,7 @@ # from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS import Peer -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties -from .NetworkPolicy import PolicyConnections, NetworkPolicy +from .NetworkPolicy import PolicyConnections, OptimizedPolicyConnections, NetworkPolicy class K8sPolicyRule: @@ -73,13 +72,14 @@ def allowed_connections_optimized(self, is_ingress): and the peer set of captured peers by this policy. :rtype: tuple (ConnectivityProperties, ConnectivityProperties, PeerSet) """ + res_conns = OptimizedPolicyConnections() if is_ingress: - allowed = self.optimized_ingress_props.copy() - captured = self.selected_peers if self.affects_ingress else Peer.PeerSet() + res_conns.allowed_conns = self.optimized_ingress_props.copy() + res_conns.captured = self.selected_peers if self.affects_ingress else Peer.PeerSet() else: - allowed = self.optimized_egress_props.copy() - captured = self.selected_peers if self.affects_egress else Peer.PeerSet() - return allowed, ConnectivityProperties.make_empty_props(), captured + res_conns.allowed_conns = self.optimized_egress_props.copy() + res_conns.captured = self.selected_peers if self.affects_egress else Peer.PeerSet() + return res_conns def clone_without_rule(self, rule_to_exclude, ingress_rule): """ diff --git a/nca/Resources/NetworkPolicy.py b/nca/Resources/NetworkPolicy.py index 24d46ac42..e2aa02abc 100644 --- a/nca/Resources/NetworkPolicy.py +++ b/nca/Resources/NetworkPolicy.py @@ -305,3 +305,17 @@ class PolicyConnections: denied_conns: ConnectionSet = ConnectionSet() # Connections denied by the policy(ies) pass_conns: ConnectionSet = ConnectionSet() # Connections specified as PASS by the policy(ies) all_allowed_conns: ConnectionSet = ConnectionSet() # all (captured+ non-captured) Connections allowed by the policy(ies) + + +# TODO - making OptimizedPolicyConnections a dataclass does not work +# (probably because PeerSet and ConnectivityProperties are mutable) +class OptimizedPolicyConnections: + """ + A class to contain the effect of applying policies to all src and dst peers + """ + def __init__(self): + self.captured = PeerSet() + self.allowed_conns = ConnectivityProperties().make_empty_props() + self.denied_conns = ConnectivityProperties().make_empty_props() + self.pass_conns = ConnectivityProperties().make_empty_props() + self.all_allowed_conns = ConnectivityProperties().make_empty_props() From dca13d5f294010850930531f864316ea4214da8f Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 2 Apr 2023 11:18:48 +0300 Subject: [PATCH 104/187] Fixed lint errors. Signed-off-by: Tanya --- nca/CoreDS/Peer.py | 2 +- nca/Resources/IngressPolicy.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index acecdb97c..8676e5c6c 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -642,7 +642,7 @@ def add_peer(self, peer): Adds a given peer to self peers. :param peer: the peer to add. """ - if not isinstance(peer, IpBlock) and not peer in self.peer_to_index: + if not isinstance(peer, IpBlock) and peer not in self.peer_to_index: assert len(self.ordered_peer_list) < self.max_num_of_pods self.peer_to_index[peer] = len(self.ordered_peer_list) self.ordered_peer_list.append(peer) diff --git a/nca/Resources/IngressPolicy.py b/nca/Resources/IngressPolicy.py index 612846f75..5ef956524 100644 --- a/nca/Resources/IngressPolicy.py +++ b/nca/Resources/IngressPolicy.py @@ -3,7 +3,6 @@ # SPDX-License-Identifier: Apache2.0 # -from enum import IntEnum from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.Peer import PeerSet From 51d8927a28f252c80b210c8fda8425c0bee5f060 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 2 Apr 2023 13:10:45 +0300 Subject: [PATCH 105/187] The BasePeerSet singleton should be reset in the main (for the cases when running multiple tests, for example from run_all_tests) Signed-off-by: Tanya --- nca/CoreDS/Peer.py | 4 ++++ nca/nca_cli.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index 8676e5c6c..c03ce47cc 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -714,6 +714,10 @@ def __init__(self): if not BasePeerSet.instance: BasePeerSet.instance = BasePeerSet.__BasePeerSet() + @staticmethod + def reset(): + BasePeerSet.instance = None + def __getattr__(self, name): return getattr(self.instance, name) diff --git a/nca/nca_cli.py b/nca/nca_cli.py index 9915097b5..7c0ec9e55 100644 --- a/nca/nca_cli.py +++ b/nca/nca_cli.py @@ -10,6 +10,7 @@ import traceback from sys import stderr from pathlib import Path +from nca.CoreDS.Peer import BasePeerSet from nca.Utils.OutputConfiguration import OutputConfiguration from nca.NetworkConfig.NetworkConfigQueryRunner import NetworkConfigQueryRunner from nca.NetworkConfig.ResourcesHandler import ResourcesHandler @@ -245,6 +246,7 @@ def nca_main(argv=None): :rtype: int """ os.environ['PATH'] = '.' + os.pathsep + os.environ.get('PATH', '.') # for running kubectl and calicoctl + BasePeerSet.reset() # to reset the singleton parser = argparse.ArgumentParser(description='An analyzer for network connectivity configuration') parser.add_argument('--period', type=int, From 835fa33326a385cab4948a67d39b323e5fab3ada Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 2 Apr 2023 17:00:30 +0300 Subject: [PATCH 106/187] Added support to calico PASS rules in optimized solution. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 2 +- nca/Parsers/CalicoPolicyYamlParser.py | 38 +++++++---- nca/Parsers/IngressPolicyYamlParser.py | 2 +- nca/Parsers/IstioPolicyYamlParser.py | 8 ++- nca/Parsers/IstioSidecarYamlParser.py | 2 +- .../IstioTrafficResourcesYamlParser.py | 2 +- nca/Parsers/K8sPolicyYamlParser.py | 4 +- nca/Resources/CalicoNetworkPolicy.py | 10 +-- nca/Resources/IngressPolicy.py | 2 +- nca/Resources/IstioNetworkPolicy.py | 8 +-- nca/Resources/IstioSidecar.py | 8 +-- nca/Resources/K8sNetworkPolicy.py | 4 +- nca/Resources/NetworkPolicy.py | 63 +++++++++++++------ .../testcase18-pass/testcase18-scheme.yaml | 12 ++++ .../testcase18_connectivity_map.txt | 17 +++++ 15 files changed, 127 insertions(+), 55 deletions(-) create mode 100644 tests/calico_testcases/expected_output/testcase18_connectivity_map.txt diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 6d23633ae..d70cd6ef2 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -610,7 +610,7 @@ def conn_props_to_fw_rules(conn_props, cluster_info, peer_container, ip_blocks_m dst_peers.filter_ipv6_blocks(ip_blocks_mask) protocols = conn_cube["protocols"] conn_cube.unset_dim("protocols") - if not conn_cube.has_active_dim() and protocols == ignore_protocols: + if not conn_cube.has_active_dim() and (protocols.is_whole_range() or protocols == ignore_protocols): conns = ConnectionSet(True) else: conns = ConnectionSet() diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index 262410b17..9692a41e0 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -699,24 +699,38 @@ def parse_policy(self): for ingress_rule in policy_spec.get('ingress', []): rule, optimized_props = self._parse_xgress_rule(ingress_rule, True, res_policy.selected_peers, is_profile) res_policy.add_ingress_rule(rule) - if self.optimized_run != 'false' and rule.action != CalicoPolicyRule.ActionType.Pass: + if self.optimized_run != 'false': # handle the order of rules - if rule.action == CalicoPolicyRule.ActionType.Allow and res_policy.optimized_denied_ingress_props: - optimized_props -= res_policy.optimized_denied_ingress_props - elif rule.action == CalicoPolicyRule.ActionType.Deny and res_policy.optimized_ingress_props: - optimized_props -= res_policy.optimized_ingress_props - res_policy.add_optimized_ingress_props(optimized_props, rule.action == CalicoPolicyRule.ActionType.Allow) + if rule.action == CalicoPolicyRule.ActionType.Allow: + optimized_props -= res_policy.optimized_deny_ingress_props + optimized_props -= res_policy.optimized_pass_ingress_props + res_policy.add_optimized_allow_props(optimized_props, True) + elif rule.action == CalicoPolicyRule.ActionType.Deny: + optimized_props -= res_policy.optimized_allow_ingress_props + optimized_props -= res_policy.optimized_pass_ingress_props + res_policy.add_optimized_deny_props(optimized_props, True) + elif rule.action == CalicoPolicyRule.ActionType.Pass: + optimized_props -= res_policy.optimized_allow_ingress_props + optimized_props -= res_policy.optimized_deny_ingress_props + res_policy.add_optimized_pass_props(optimized_props, True) for egress_rule in policy_spec.get('egress', []): rule, optimized_props = self._parse_xgress_rule(egress_rule, False, res_policy.selected_peers, is_profile) res_policy.add_egress_rule(rule) - if self.optimized_run != 'false' and rule.action != CalicoPolicyRule.ActionType.Pass: + if self.optimized_run != 'false': # handle the order of rules - if rule.action == CalicoPolicyRule.ActionType.Allow and res_policy.optimized_denied_egress_props: - optimized_props -= res_policy.optimized_denied_egress_props - elif rule.action == CalicoPolicyRule.ActionType.Deny and res_policy.optimized_egress_props: - optimized_props -= res_policy.optimized_egress_props - res_policy.add_optimized_egress_props(optimized_props, rule.action == CalicoPolicyRule.ActionType.Allow) + if rule.action == CalicoPolicyRule.ActionType.Allow: + optimized_props -= res_policy.optimized_deny_egress_props + optimized_props -= res_policy.optimized_pass_egress_props + res_policy.add_optimized_allow_props(optimized_props, False) + elif rule.action == CalicoPolicyRule.ActionType.Deny: + optimized_props -= res_policy.optimized_allow_egress_props + optimized_props -= res_policy.optimized_pass_egress_props + res_policy.add_optimized_deny_props(optimized_props, False) + elif rule.action == CalicoPolicyRule.ActionType.Pass: + optimized_props -= res_policy.optimized_allow_egress_props + optimized_props -= res_policy.optimized_deny_egress_props + res_policy.add_optimized_pass_props(optimized_props, False) self._apply_extra_labels(policy_spec, is_profile, res_policy.name) res_policy.findings = self.warning_msgs diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index 983d33052..03282aa19 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -289,6 +289,6 @@ def parse_policy(self): protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') conn_cube = ConnectivityCube.make_from_dict({"protocols": protocols, "src_peers": res_policy.selected_peers}) allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) - res_policy.add_optimized_egress_props(allowed_conns) + res_policy.add_optimized_allow_props(allowed_conns, False) res_policy.findings = self.warning_msgs return res_policy diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index 135b049d1..2d7a15128 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -575,9 +575,11 @@ def parse_policy(self): for ingress_rule in policy_spec.get('rules', []): rule, optimized_props = self.parse_ingress_rule(ingress_rule, res_policy.selected_peers) res_policy.add_ingress_rule(rule) - res_policy.add_optimized_ingress_props(optimized_props, - res_policy.action == IstioNetworkPolicy.ActionType.Allow) - res_policy.add_optimized_egress_props(ConnectivityProperties.make_all_props()) + if res_policy.action == IstioNetworkPolicy.ActionType.Allow: + res_policy.add_optimized_allow_props(optimized_props, True) + else: # Deny + res_policy.add_optimized_deny_props(optimized_props, True) + res_policy.add_optimized_allow_props(ConnectivityProperties.make_all_props(), False) if not res_policy.ingress_rules and res_policy.action == IstioNetworkPolicy.ActionType.Deny: self.syntax_error("DENY action without rules is meaningless as it will never be triggered") diff --git a/nca/Parsers/IstioSidecarYamlParser.py b/nca/Parsers/IstioSidecarYamlParser.py index 1bf0d9238..5ce1567f6 100644 --- a/nca/Parsers/IstioSidecarYamlParser.py +++ b/nca/Parsers/IstioSidecarYamlParser.py @@ -177,7 +177,7 @@ def parse_policy(self): self.namespace = self.peer_container.get_namespace(policy_ns, warn_if_missing) res_policy = IstioSidecar(policy_name, self.namespace) res_policy.policy_kind = NetworkPolicy.PolicyType.IstioSidecar - res_policy.add_optimized_ingress_props(ConnectivityProperties.make_all_props()) + res_policy.add_optimized_allow_props(ConnectivityProperties.make_all_props(), True) sidecar_spec = self.policy['spec'] # currently, supported fields in spec are workloadSelector and egress diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index fba1e6d6f..1a1923cea 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -400,7 +400,7 @@ def create_istio_traffic_policies(self): conn_cube = ConnectivityCube.make_from_dict({"protocols": protocols, "src_peers": res_policy.selected_peers}) allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) - res_policy.add_optimized_egress_props(allowed_conns) + res_policy.add_optimized_allow_props(allowed_conns, False) res_policy.findings = self.warning_msgs vs_policies.append(res_policy) if not vs_policies: diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index 64d49b69a..a0e86bedc 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -459,14 +459,14 @@ def parse_policy(self): for ingress_rule in ingress_rules: rule, optimized_props = self.parse_ingress_rule(ingress_rule, res_policy.selected_peers) res_policy.add_ingress_rule(rule) - res_policy.add_optimized_ingress_props(optimized_props) + res_policy.add_optimized_allow_props(optimized_props, True) egress_rules = policy_spec.get('egress', []) if egress_rules: for egress_rule in egress_rules: rule, optimized_props = self.parse_egress_rule(egress_rule, res_policy.selected_peers) res_policy.add_egress_rule(rule) - res_policy.add_optimized_egress_props(optimized_props) + res_policy.add_optimized_allow_props(optimized_props, False) res_policy.findings = self.warning_msgs res_policy.referenced_labels = self.referenced_labels diff --git a/nca/Resources/CalicoNetworkPolicy.py b/nca/Resources/CalicoNetworkPolicy.py index 2529e344b..462b964f1 100644 --- a/nca/Resources/CalicoNetworkPolicy.py +++ b/nca/Resources/CalicoNetworkPolicy.py @@ -129,12 +129,14 @@ def allowed_connections_optimized(self, is_ingress): """ res_conns = OptimizedPolicyConnections() if is_ingress: - res_conns.allowed_conns = self.optimized_ingress_props.copy() - res_conns.denied_conns = self.optimized_denied_ingress_props.copy() + res_conns.allowed_conns = self.optimized_allow_ingress_props.copy() + res_conns.denied_conns = self.optimized_deny_ingress_props.copy() + res_conns.pass_conns = self.optimized_pass_ingress_props.copy() res_conns.captured = self.selected_peers if self.affects_ingress else Peer.PeerSet() else: - res_conns.allowed_conns = self.optimized_egress_props.copy() - res_conns.denied_conns = self.optimized_denied_egress_props.copy() + res_conns.allowed_conns = self.optimized_allow_egress_props.copy() + res_conns.denied_conns = self.optimized_deny_egress_props.copy() + res_conns.pass_conns = self.optimized_pass_egress_props.copy() res_conns.captured = self.selected_peers if self.affects_egress else Peer.PeerSet() return res_conns diff --git a/nca/Resources/IngressPolicy.py b/nca/Resources/IngressPolicy.py index 5ef956524..4edd128e4 100644 --- a/nca/Resources/IngressPolicy.py +++ b/nca/Resources/IngressPolicy.py @@ -100,7 +100,7 @@ def allowed_connections_optimized(self, is_ingress): res_conns.allowed_conns = ConnectivityProperties.make_empty_props() res_conns.captured = PeerSet() else: - res_conns.allowed_conns = self.optimized_egress_props.copy() + res_conns.allowed_conns = self.optimized_allow_egress_props.copy() res_conns.captured = self.selected_peers if self.affects_egress else PeerSet() return res_conns diff --git a/nca/Resources/IstioNetworkPolicy.py b/nca/Resources/IstioNetworkPolicy.py index f4dfca349..bf045e895 100644 --- a/nca/Resources/IstioNetworkPolicy.py +++ b/nca/Resources/IstioNetworkPolicy.py @@ -103,12 +103,12 @@ def allowed_connections_optimized(self, is_ingress): """ res_conns = OptimizedPolicyConnections() if is_ingress: - res_conns.allowed_conns = self.optimized_ingress_props.copy() - res_conns.denied_conns = self.optimized_denied_ingress_props.copy() + res_conns.allowed_conns = self.optimized_allow_ingress_props.copy() + res_conns.denied_conns = self.optimized_deny_ingress_props.copy() res_conns.captured = self.selected_peers else: - res_conns.allowed_conns = self.optimized_egress_props.copy() - res_conns.denied_conns = self.optimized_denied_egress_props.copy() + res_conns.allowed_conns = self.optimized_allow_egress_props.copy() + res_conns.denied_conns = self.optimized_deny_egress_props.copy() res_conns.captured = PeerSet() return res_conns diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index 5b8ddd1b2..fc1319bd4 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -74,12 +74,12 @@ def allowed_connections(self, from_peer, to_peer, is_ingress): def allowed_connections_optimized(self, is_ingress): res_conns = OptimizedPolicyConnections() if is_ingress: - res_conns.allowed_conns = self.optimized_ingress_props.copy() - res_conns.denied_conns = self.optimized_denied_ingress_props.copy() + res_conns.allowed_conns = self.optimized_allow_ingress_props.copy() + res_conns.denied_conns = self.optimized_deny_ingress_props.copy() res_conns.captured = PeerSet() else: - res_conns.allowed_conns = self.optimized_egress_props.copy() - res_conns.denied_conns = self.optimized_denied_egress_props.copy() + res_conns.allowed_conns = self.optimized_allow_egress_props.copy() + res_conns.denied_conns = self.optimized_deny_egress_props.copy() res_conns.captured = self.selected_peers if self.affects_egress else PeerSet() return res_conns diff --git a/nca/Resources/K8sNetworkPolicy.py b/nca/Resources/K8sNetworkPolicy.py index 6c3b93d8a..d946346a6 100644 --- a/nca/Resources/K8sNetworkPolicy.py +++ b/nca/Resources/K8sNetworkPolicy.py @@ -74,10 +74,10 @@ def allowed_connections_optimized(self, is_ingress): """ res_conns = OptimizedPolicyConnections() if is_ingress: - res_conns.allowed_conns = self.optimized_ingress_props.copy() + res_conns.allowed_conns = self.optimized_allow_ingress_props.copy() res_conns.captured = self.selected_peers if self.affects_ingress else Peer.PeerSet() else: - res_conns.allowed_conns = self.optimized_egress_props.copy() + res_conns.allowed_conns = self.optimized_allow_egress_props.copy() res_conns.captured = self.selected_peers if self.affects_egress else Peer.PeerSet() return res_conns diff --git a/nca/Resources/NetworkPolicy.py b/nca/Resources/NetworkPolicy.py index e2aa02abc..5fadab615 100644 --- a/nca/Resources/NetworkPolicy.py +++ b/nca/Resources/NetworkPolicy.py @@ -53,10 +53,12 @@ def __init__(self, name, namespace): self.selected_peers = PeerSet() # The peers affected by this policy self.ingress_rules = [] self.egress_rules = [] - self.optimized_ingress_props = ConnectivityProperties.make_empty_props() - self.optimized_denied_ingress_props = ConnectivityProperties.make_empty_props() - self.optimized_egress_props = ConnectivityProperties.make_empty_props() - self.optimized_denied_egress_props = ConnectivityProperties.make_empty_props() + self.optimized_allow_ingress_props = ConnectivityProperties.make_empty_props() + self.optimized_deny_ingress_props = ConnectivityProperties.make_empty_props() + self.optimized_pass_ingress_props = ConnectivityProperties.make_empty_props() + self.optimized_allow_egress_props = ConnectivityProperties.make_empty_props() + self.optimized_deny_egress_props = ConnectivityProperties.make_empty_props() + self.optimized_pass_egress_props = ConnectivityProperties.make_empty_props() self.affects_ingress = False # whether the policy affects the ingress of the selected peers self.affects_egress = False # whether the policy affects the egress of the selected peers self.findings = [] # accumulated findings which are relevant only to this policy (emptiness and redundancy) @@ -77,12 +79,21 @@ def __eq__(self, other): self.affects_ingress == other.affects_ingress and \ self.selected_peers == other.selected_peers and \ self.ingress_rules == other.ingress_rules and \ - self.egress_rules == other.egress_rules + self.egress_rules == other.egress_rules and \ + self.optimized_allow_ingress_props == other.optimized_allow_ingress_props and \ + self.optimized_deny_ingress_props == other.optimized_deny_ingress_props and \ + self.optimized_pass_ingress_props == other.optimized_pass_ingress_props and \ + self.optimized_allow_egress_props == other.optimized_allow_egress_props and \ + self.optimized_deny_egress_props == other.optimized_deny_egress_props and \ + self.optimized_pass_egress_props == other.optimized_pass_egress_props return False def __lt__(self, other): # required so we can evaluate the policies according to their order if not isinstance(other, NetworkPolicy): return False + if self.get_order() is None and other.get_order() is None: + # to make the operator consistent + return self.full_name() < other.full_name() # If not specified "order" defaults to infinity, so policies with unspecified "order" will be applied last. if self.get_order() is None: return False @@ -126,33 +137,47 @@ def add_egress_rule(self, rule): """ self.egress_rules.append(rule) - def add_optimized_ingress_props(self, props, is_allow=True): + def add_optimized_allow_props(self, props, is_ingress): """ - Adding properties to the CanonicalHyperCubeSet of optimized ingress properties - :param CanonicalHyperCubeSet props: The properties to add - :param Bool is_allow: whether these are an allow or deny properties + Adding allow properties to the ConnectivityProperties of optimized ingress/egress properties + :param ConnectivityProperties props: The properties to add + :param Bool is_ingress: whether these are ingress or egress properties :return: None """ if not props: return - if is_allow: - self.optimized_ingress_props |= props + if is_ingress: + self.optimized_allow_ingress_props |= props else: - self.optimized_denied_ingress_props |= props + self.optimized_allow_egress_props |= props - def add_optimized_egress_props(self, props, is_allow=True): + def add_optimized_deny_props(self, props, is_ingress): """ - Adding properties to the CanonicalHyperCubeSet of optimized egress properties - :param CanonicalHyperCubeSet props: The properties to add - :param Bool is_allow: whether these are an allow or deny properties + Adding deny properties to the ConnectivityProperties of optimized ingress/egress properties + :param ConnectivityProperties props: The properties to add + :param Bool is_ingress: whether these are ingress or egress properties :return: None """ if not props: return - if is_allow: - self.optimized_egress_props |= props + if is_ingress: + self.optimized_deny_ingress_props |= props else: - self.optimized_denied_egress_props |= props + self.optimized_deny_egress_props |= props + + def add_optimized_pass_props(self, props, is_ingress): + """ + Adding pass properties to the ConnectivityProperties of optimized ingress/egress properties + :param ConnectivityProperties props: The properties to add + :param Bool is_ingress: whether these are ingress or egress properties + :return: None + """ + if not props: + return + if is_ingress: + self.optimized_pass_ingress_props |= props + else: + self.optimized_pass_egress_props |= props @staticmethod def get_policy_type_from_dict(policy): # noqa: C901 diff --git a/tests/calico_testcases/example_policies/testcase18-pass/testcase18-scheme.yaml b/tests/calico_testcases/example_policies/testcase18-pass/testcase18-scheme.yaml index 2197ac39c..862a92b11 100644 --- a/tests/calico_testcases/example_policies/testcase18-pass/testcase18-scheme.yaml +++ b/tests/calico_testcases/example_policies/testcase18-pass/testcase18-scheme.yaml @@ -13,6 +13,18 @@ networkConfigList: expectedWarnings: 0 queries: + - name: connectivity_map + connectivityMap: + - np-pod-based-policies + - np-ports-based + + outputConfiguration: + outputFormat: txt + fwRulesRunInTestMode: false + fwRulesGroupByLabelSinglePod: true + excludeIPv6Range: false + expectedOutput: ../../expected_output/testcase18_connectivity_map.txt + - name: policies_not_vacuous vacuity: - np-pod-based-policies diff --git a/tests/calico_testcases/expected_output/testcase18_connectivity_map.txt b/tests/calico_testcases/expected_output/testcase18_connectivity_map.txt new file mode 100644 index 000000000..379dd8351 --- /dev/null +++ b/tests/calico_testcases/expected_output/testcase18_connectivity_map.txt @@ -0,0 +1,17 @@ +final fw rules for query: connectivity_map, config: np-pod-based-policies: +src: 0.0.0.0/0 dst_ns: [default,vendor-system] dst_pods: [*] conn: All connections +src: ::/0 dst_ns: [default,vendor-system] dst_pods: [*] conn: All connections +src_ns: [default,kube-system,vendor-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections +src_ns: [default,kube-system,vendor-system] src_pods: [*] dst: ::/0 conn: All connections +src_ns: [default,kube-system,vendor-system] src_pods: [*] dst_ns: [default,vendor-system] dst_pods: [*] conn: All connections +src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections + +final fw rules for query: connectivity_map, config: np-ports-based: +src: 0.0.0.0/0 dst_ns: [default,vendor-system] dst_pods: [*] conn: All connections +src: ::/0 dst_ns: [default,vendor-system] dst_pods: [*] conn: All connections +src_ns: [default,vendor-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections +src_ns: [default,vendor-system] src_pods: [*] dst: ::/0 conn: All connections +src_ns: [default,vendor-system] src_pods: [*] dst_ns: [default,vendor-system] dst_pods: [*] conn: All connections +src_ns: [kube-system] src_pods: [*] dst: 0.0.0.0/0 conn: TCP +src_ns: [kube-system] src_pods: [*] dst: ::/0 conn: TCP +src_ns: [kube-system] src_pods: [*] dst_ns: [default,kube-system,vendor-system] dst_pods: [*] conn: TCP From ebe400665bf81e09dc85193e4c5309a43cec67d1 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 2 Apr 2023 17:26:40 +0300 Subject: [PATCH 107/187] Added support to calico PASS rules in optimized solution. Signed-off-by: Tanya --- nca/Resources/NetworkPolicy.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nca/Resources/NetworkPolicy.py b/nca/Resources/NetworkPolicy.py index 5fadab615..0dba8ef85 100644 --- a/nca/Resources/NetworkPolicy.py +++ b/nca/Resources/NetworkPolicy.py @@ -91,9 +91,11 @@ def __eq__(self, other): def __lt__(self, other): # required so we can evaluate the policies according to their order if not isinstance(other, NetworkPolicy): return False - if self.get_order() is None and other.get_order() is None: - # to make the operator consistent - return self.full_name() < other.full_name() + # TODO - should add the condition below to make the comparison stable. + # This makes some tests to fail because the order of output is different + # if self.get_order() is None and other.get_order() is None: + # # to make the operator consistent + # return self.full_name() < other.full_name() # If not specified "order" defaults to infinity, so policies with unspecified "order" will be applied last. if self.get_order() is None: return False From 10bceb8cfb9170a3c4855cb4c685a79b5202c34b Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 2 Apr 2023 17:46:09 +0300 Subject: [PATCH 108/187] Simplified calico parsing method to avoid lint error. Fixed typo in the code. Signed-off-by: Tanya --- nca/Parsers/CalicoPolicyYamlParser.py | 28 ++------------------------- nca/Resources/CalicoNetworkPolicy.py | 23 ++++++++++++++++++++++ nca/Resources/IstioSidecar.py | 4 ++-- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index 9692a41e0..76f0617e7 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -700,37 +700,13 @@ def parse_policy(self): rule, optimized_props = self._parse_xgress_rule(ingress_rule, True, res_policy.selected_peers, is_profile) res_policy.add_ingress_rule(rule) if self.optimized_run != 'false': - # handle the order of rules - if rule.action == CalicoPolicyRule.ActionType.Allow: - optimized_props -= res_policy.optimized_deny_ingress_props - optimized_props -= res_policy.optimized_pass_ingress_props - res_policy.add_optimized_allow_props(optimized_props, True) - elif rule.action == CalicoPolicyRule.ActionType.Deny: - optimized_props -= res_policy.optimized_allow_ingress_props - optimized_props -= res_policy.optimized_pass_ingress_props - res_policy.add_optimized_deny_props(optimized_props, True) - elif rule.action == CalicoPolicyRule.ActionType.Pass: - optimized_props -= res_policy.optimized_allow_ingress_props - optimized_props -= res_policy.optimized_deny_ingress_props - res_policy.add_optimized_pass_props(optimized_props, True) + res_policy.update_and_add_optimized_props(optimized_props, rule.action, True) for egress_rule in policy_spec.get('egress', []): rule, optimized_props = self._parse_xgress_rule(egress_rule, False, res_policy.selected_peers, is_profile) res_policy.add_egress_rule(rule) if self.optimized_run != 'false': - # handle the order of rules - if rule.action == CalicoPolicyRule.ActionType.Allow: - optimized_props -= res_policy.optimized_deny_egress_props - optimized_props -= res_policy.optimized_pass_egress_props - res_policy.add_optimized_allow_props(optimized_props, False) - elif rule.action == CalicoPolicyRule.ActionType.Deny: - optimized_props -= res_policy.optimized_allow_egress_props - optimized_props -= res_policy.optimized_pass_egress_props - res_policy.add_optimized_deny_props(optimized_props, False) - elif rule.action == CalicoPolicyRule.ActionType.Pass: - optimized_props -= res_policy.optimized_allow_egress_props - optimized_props -= res_policy.optimized_deny_egress_props - res_policy.add_optimized_pass_props(optimized_props, False) + res_policy.update_and_add_optimized_props(optimized_props, rule.action, False) self._apply_extra_labels(policy_spec, is_profile, res_policy.name) res_policy.findings = self.warning_msgs diff --git a/nca/Resources/CalicoNetworkPolicy.py b/nca/Resources/CalicoNetworkPolicy.py index 462b964f1..21f6bf612 100644 --- a/nca/Resources/CalicoNetworkPolicy.py +++ b/nca/Resources/CalicoNetworkPolicy.py @@ -78,6 +78,29 @@ def __eq__(self, other): return isinstance(other, CalicoNetworkPolicy) and super().__eq__(other) and \ self.order == other.order + def update_and_add_optimized_props(self, props, action, is_ingress): + """ + Updates properties according to earlier added properties + and adds them to the policy according to action and ingress/egress flag + :param props: the given properties + :param action: the action (Allow/Deny/Pass) + :param is_ingress: True for ingress, False for egress + :return: None + """ + # handle the order of rules + if action == CalicoPolicyRule.ActionType.Allow: + props -= self.optimized_deny_ingress_props if is_ingress else self.optimized_deny_egress_props + props -= self.optimized_pass_ingress_props if is_ingress else self.optimized_pass_egress_props + self.add_optimized_allow_props(props, is_ingress) + elif action == CalicoPolicyRule.ActionType.Deny: + props -= self.optimized_allow_ingress_props if is_ingress else self.optimized_allow_egress_props + props -= self.optimized_pass_ingress_props if is_ingress else self.optimized_pass_egress_props + self.add_optimized_deny_props(props, is_ingress) + elif action == CalicoPolicyRule.ActionType.Pass: + props -= self.optimized_allow_ingress_props if is_ingress else self.optimized_allow_egress_props + props -= self.optimized_deny_ingress_props if is_ingress else self.optimized_deny_egress_props + self.add_optimized_pass_props(props, is_ingress) + def allowed_connections(self, from_peer, to_peer, is_ingress): """ Evaluate the set of connections this policy allows/denies/passes between two peers diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index fc1319bd4..e445c0e21 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -170,10 +170,10 @@ def create_opt_egress_props(self, peer_container): conn_cube = ConnectivityCube() if self.selected_peers and rule.egress_peer_set: conn_cube.update({"src_peers": self.selected_peers, "dst_peers": rule.egress_peer_set}) - self.optimized_egress_props = ConnectivityProperties.make_conn_props(conn_cube) + self.optimized_allow_egress_props = ConnectivityProperties.make_conn_props(conn_cube) peers_sets_by_ns = self.combine_peer_sets_by_ns(self.selected_peers, rule.special_egress_peer_set, peer_container) for (from_peers, to_peers) in peers_sets_by_ns: if from_peers and to_peers: conn_cube.update({"src_peers": from_peers, "dst_peers": to_peers}) - self.optimized_egress_props |= ConnectivityProperties.make_conn_props(conn_cube) + self.optimized_allow_egress_props |= ConnectivityProperties.make_conn_props(conn_cube) From 5dc62e9428e6485a751bd7891dd9d8a26812d76a Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Tue, 4 Apr 2023 11:23:16 +0300 Subject: [PATCH 109/187] handling ipBlocks and base ip range --- nca/NetworkConfig/NetworkConfigQuery.py | 2 +- nca/NetworkConfig/NetworkLayer.py | 17 +++- nca/NetworkConfig/PoliciesFinder.py | 32 +++++-- nca/NetworkConfig/ResourcesHandler.py | 6 -- nca/NetworkConfig/TopologyObjectsFinder.py | 3 + nca/Utils/ExplTracker.py | 100 +++++++++++++++------ 6 files changed, 115 insertions(+), 45 deletions(-) diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 8ae29f36c..c79656a8c 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -759,7 +759,7 @@ def exec(self): # noqa: C901 subset_conns = ConnectivityProperties.make_conn_props(src_peers_conn_cube) | \ ConnectivityProperties.make_conn_props(dst_peers_conn_cube) all_conns_opt &= subset_conns - ExplTracker().set_connections(all_conns_opt) + ExplTracker().set_connections_and_peers(all_conns_opt, subset_peers) ip_blocks_mask = IpBlock.get_all_ips_block() if exclude_ipv6: ip_blocks_mask = IpBlock.get_all_ips_block(exclude_ipv6=True) diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index b61902d14..966afc5fd 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -251,7 +251,7 @@ def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda poli for peer in policy.selected_peers: src_peers, _ = ExplTracker().extract_peers(policy.optimized_ingress_props) _, dst_peers = ExplTracker().extract_peers(policy.optimized_egress_props) - ExplTracker().add_peer_policy(peer, + ExplTracker().add_peer_policy(peer.full_name(), policy.name, dst_peers, src_peers, @@ -311,8 +311,8 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): conn_cube.update({"src_peers": non_captured, "dst_peers": base_peer_set_with_ip}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) src_peers, dst_peers = ExplTracker().extract_peers(non_captured_conns) - ExplTracker().add_default_policy(dst_peers, - src_peers, + ExplTracker().add_default_policy(src_peers, + dst_peers, is_ingress ) allowed_conn |= non_captured_conns @@ -353,6 +353,11 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): conn_cube.update({"src_peers": non_captured_peers, "dst_peers": base_peer_set_with_ip}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conn |= (non_captured_conns - denied_conns) + src_peers, dst_peers = ExplTracker().extract_peers(non_captured_conns) + ExplTracker().add_default_policy(src_peers, + dst_peers, + is_ingress + ) conn_cube.update({"src_peers": base_peer_set_with_ip, "dst_peers": base_peer_set_with_ip, "protocols": ProtocolSet.get_non_tcp_protocols()}) allowed_conn |= ConnectivityProperties.make_conn_props(conn_cube) @@ -387,4 +392,10 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): conn_cube.update({"src_peers": non_captured_peers, "dst_peers": base_peer_set_with_ip}) non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) allowed_conn |= non_captured_conns + if non_captured_conns: + src_peers, dst_peers = ExplTracker().extract_peers(non_captured_conns) + ExplTracker().add_default_policy(src_peers, + dst_peers, + is_ingress + ) return allowed_conn, denied_conns diff --git a/nca/NetworkConfig/PoliciesFinder.py b/nca/NetworkConfig/PoliciesFinder.py index e9aeb48bb..4411609e8 100644 --- a/nca/NetworkConfig/PoliciesFinder.py +++ b/nca/NetworkConfig/PoliciesFinder.py @@ -14,6 +14,7 @@ from nca.Parsers.IngressPolicyYamlParser import IngressPolicyYamlParser from nca.Parsers.IstioTrafficResourcesYamlParser import IstioTrafficResourcesYamlParser from .NetworkConfig import PoliciesContainer +from nca.Utils.ExplTracker import ExplTracker class PoliciesFinder: @@ -73,47 +74,62 @@ def parse_policies_in_parse_queue(self): # noqa: C901 if policy_type == NetworkPolicy.PolicyType.CalicoProfile: parsed_element = CalicoPolicyYamlParser(policy, self.peer_container, file_name, self.optimized_run) # only during parsing adding extra labels from profiles (not supporting profiles with rules) - parsed_element.parse_policy() + parsed_policy = parsed_element.parse_policy() elif policy_type == NetworkPolicy.PolicyType.K8sNetworkPolicy: parsed_element = K8sPolicyYamlParser(policy, self.peer_container, file_name, self.optimized_run) - self._add_policy(parsed_element.parse_policy()) + parsed_policy = parsed_element.parse_policy() + self._add_policy(parsed_policy) # add info about missing resources self.missing_dns_pods_with_labels.update(parsed_element.missing_pods_with_labels) elif policy_type == NetworkPolicy.PolicyType.IstioAuthorizationPolicy: parsed_element = IstioPolicyYamlParser(policy, self.peer_container, file_name) - self._add_policy(parsed_element.parse_policy()) + parsed_policy = parsed_element.parse_policy() + self._add_policy(parsed_policy) elif policy_type == NetworkPolicy.PolicyType.IstioSidecar: if not istio_sidecar_parser: istio_sidecar_parser = IstioSidecarYamlParser(policy, self.peer_container, file_name) else: istio_sidecar_parser.reset(policy, self.peer_container, file_name) - istio_sidecar_parser.parse_policy() + parsed_policy = istio_sidecar_parser.parse_policy() elif policy_type == NetworkPolicy.PolicyType.Ingress: parsed_element = IngressPolicyYamlParser(policy, self.peer_container, file_name) - self._add_policy(parsed_element.parse_policy()) + parsed_policy = parsed_element.parse_policy() + self._add_policy(parsed_policy) # add info about missing resources self.missing_k8s_ingress_peers |= parsed_element.missing_k8s_ingress_peers elif policy_type == NetworkPolicy.PolicyType.Gateway: if not istio_traffic_parser: istio_traffic_parser = IstioTrafficResourcesYamlParser(self.peer_container) - istio_traffic_parser.parse_gateway(policy, file_name) + parsed_policy = istio_traffic_parser.parse_gateway(policy, file_name) # add info about missing resources self.missing_istio_gw_pods_with_labels.update(istio_traffic_parser.missing_istio_gw_pods_with_labels) elif policy_type == NetworkPolicy.PolicyType.VirtualService: if not istio_traffic_parser: istio_traffic_parser = IstioTrafficResourcesYamlParser(self.peer_container) - istio_traffic_parser.parse_virtual_service(policy, file_name) + parsed_policy = istio_traffic_parser.parse_virtual_service(policy, file_name) else: parsed_element = CalicoPolicyYamlParser(policy, self.peer_container, file_name, self.optimized_run) - self._add_policy(parsed_element.parse_policy()) + parsed_policy = parsed_element.parse_policy() + self._add_policy(parsed_policy) + # the name is sometimes modified when parsed, like in the ingress case, when "allowed" is added + if parsed_policy: + policy_name = parsed_policy.name + else: # the istio policy is parsed later + policy_name = policy.get('metadata').get('name') + ExplTracker().add_item(policy.path, + policy_name, + policy.line_number + ) if istio_traffic_parser: istio_traffic_policies = istio_traffic_parser.create_istio_traffic_policies() for istio_traffic_policy in istio_traffic_policies: self._add_policy(istio_traffic_policy) + ExplTracker().derive_item(istio_traffic_policy.name) if istio_sidecar_parser: istio_sidecars = istio_sidecar_parser.get_istio_sidecars() for istio_sidecar in istio_sidecars: self._add_policy(istio_sidecar) + ExplTracker().derive_item(istio_sidecar.name) def parse_yaml_code_for_policy(self, policy_object, file_name): policy_type = NetworkPolicy.get_policy_type_from_dict(policy_object) diff --git a/nca/NetworkConfig/ResourcesHandler.py b/nca/NetworkConfig/ResourcesHandler.py index 4ab79b796..392bb58c4 100644 --- a/nca/NetworkConfig/ResourcesHandler.py +++ b/nca/NetworkConfig/ResourcesHandler.py @@ -413,12 +413,6 @@ def _parse_resources_path(self, resource_list, resource_flags): self.services_finder.namespaces_finder = self.ns_finder self.services_finder.parse_yaml_code_for_service(res_code, yaml_file) if ResourceType.Policies in resource_flags: - if res_code: - # Track filenames and content - ExplTracker().add_item(yaml_file.path, - res_code.get('metadata').get('name'), - res_code.line_number - ) self.policies_finder.parse_yaml_code_for_policy(res_code, yaml_file.path) self.policies_finder.parse_policies_in_parse_queue() diff --git a/nca/NetworkConfig/TopologyObjectsFinder.py b/nca/NetworkConfig/TopologyObjectsFinder.py index a743c583f..1e7791634 100644 --- a/nca/NetworkConfig/TopologyObjectsFinder.py +++ b/nca/NetworkConfig/TopologyObjectsFinder.py @@ -194,6 +194,7 @@ def _add_networkset_from_yaml(self, networkset_object): for cidr in cidrs: ipb.add_cidr(cidr) self._add_peer(ipb) + ExplTracker().add_item(networkset_object.path, ipb.full_name(), networkset_object.line_number) def _add_hep_from_yaml(self, hep_object): """ @@ -217,6 +218,7 @@ def _add_hep_from_yaml(self, hep_object): hep.add_profile(profile) self._add_peer(hep) + ExplTracker().add_item(hep_object.path, hep.full_name(), hep_object.line_number) def _add_wep_from_yaml(self, wep_object): """ @@ -243,6 +245,7 @@ def _add_wep_from_yaml(self, wep_object): wep.add_profile(profile) self._add_peer(wep) + ExplTracker().add_item(wep_object.path, wep.full_name(), wep_object.line_number) class NamespacesFinder: diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py index d326a9472..818693671 100644 --- a/nca/Utils/ExplTracker.py +++ b/nca/Utils/ExplTracker.py @@ -62,7 +62,8 @@ def __init__(self): self.ExplDescriptorContainer = {} self.ExplPeerToPolicyContainer = {} self._is_active = False - self.conns = {} + self.all_conns = {} + self.all_peers = {} self.add_item('', 'Default-Policy', 0) @@ -92,15 +93,30 @@ def add_item(self, path, name, ln): NcaLogger().log_message(f'Explainability error: configuration-block name can not be empty', level='E') - def add_peer_policy(self, peer, policy_name, egress_dst, ingress_src): + def derive_item(self, new_name): + """ + Handles resources that change their name after parsing, like virtual-service + that adds the service name and /allowed + :param str new_name: the name for the new derived element + """ + name_parts = new_name.split('/') + name = name_parts[0] + if self.ExplDescriptorContainer.get(name): + self.ExplDescriptorContainer[new_name] = {'path': self.ExplDescriptorContainer[name].get('path'), + 'line': self.ExplDescriptorContainer[name].get('line') + } + else: + NcaLogger().log_message(f'Explainability error: derived item {new_name} found no base item', + level='E') + + def add_peer_policy(self, peer_name, policy_name, egress_dst, ingress_src): """ Add a new policy to a peer - :param Peer peer: peer object + :param str peer_name: peer name to add the policy to :param srt policy_name: name of the policy :param egress_dst: a list of peers that the given policy affect, egress wise. :param ingress_src: a list of peers that the given policy affect, ingress wise. """ - peer_name = peer.full_name() if self.ExplDescriptorContainer.get(peer_name): if not self.ExplPeerToPolicyContainer.get(peer_name): self.ExplPeerToPolicyContainer[peer_name] = ExplPolicies() @@ -124,12 +140,22 @@ def extract_peers(conns): dst_peers |= conn_cube["dst_peers"] return src_peers, dst_peers - def set_connections(self, conns): + def set_connections_and_peers(self, conns, peers): """ - Update the calculated connections into ExplTracker + Update the calculated connections and topology peers into ExplTracker :param ConnectivityProperties conns: the connectivity mapping calculated by the query + :param PeerSet peers: all the peers in the container """ - self.conns = conns + self.all_conns = conns + self.all_peers = peers + # add all missing 'special' peers with default policy. + for peer in self.all_peers: + peer_name = peer.full_name() + if not self.ExplPeerToPolicyContainer.get(peer_name): + if not self.ExplDescriptorContainer.get(peer_name): + self.add_item('', peer_name, 0) + self.add_default_policy([peer], peers, False) + self.add_default_policy(peers, [peer], True) def are_peers_connected(self, src, dst): """ @@ -138,50 +164,55 @@ def are_peers_connected(self, src, dst): :param str dst: name of the destination peer :return: bool: True for connected, False for disconnected """ - if not self.conns: + if not self.all_conns: NcaLogger().log_message(f'Explainability error: Connections were not set yet, but peer query was called', level='E') - for cube in self.conns: - conn = self.conns.get_cube_dict(cube) + for cube in self.all_conns: + conn = self.all_conns.get_cube_dict(cube) src_peers = conn['src_peers'] dst_peers = conn['dst_peers'] if src in src_peers and dst in dst_peers: return True return False - def add_default_policy(self, currents, peers, is_ingress): + def add_default_policy(self, src, dst, is_ingress): """ Add the default policy to the peers which were not affected by a specific policy. - :param PeerSet currents: the peers to add the policy too - :param PeerSet peers: the adjacent peers the default policy nakes a connection with + :param PeerSet src: the peer list for the source of the policy + :param PeerSet dst: the peer list for the destination of the policy :param is_ingress: is this an ingress or egress policy """ if is_ingress: - ingress_src = peers - egress_dst = {} + nodes = dst + egress_list = {} + ingress_list = src else: - ingress_src = {} - egress_dst = peers + nodes = src + egress_list = dst + ingress_list = {} - for peer in currents: - self.add_peer_policy(peer, + for node in nodes: + self.add_peer_policy(node.full_name(), 'Default-Policy', - egress_dst, - ingress_src, + egress_list, + ingress_list, ) - def prepare_node_str(self, direction, node_name, results): + def prepare_node_str(self, node_name, results, direction=None): """ A utility function to help format a node explainability description - :param str direction: src/dst :param str node_name: the name of the node currently described :param str results: the names of the configurations affecting this node + :param str direction: src/dst :return str: string with the description """ out = [] if direction: out = [f'\n({direction}){node_name}:'] for name in results: + if not self.ExplDescriptorContainer.get(name): + out.append(f'{name} - explainability entry not found') + continue path = self.ExplDescriptorContainer.get(name).get("path") if path == '': # special element (like Default Policy) out.append(f'{name}') @@ -190,6 +221,16 @@ def prepare_node_str(self, direction, node_name, results): f'in file {path}') return out + def explain_all(self): + out = [] + for peer1 in self.all_peers: + for peer2 in self.all_peers: + if peer1 == peer2: + out.extend(self.explain([peer1.full_name()])) + else: + out.extend(self.explain([peer1.full_name(), peer2.full_name()])) + return out + def explain(self, nodes): """ The magic function to explain the connectivity or the LACK of it between the given nodes @@ -208,6 +249,12 @@ def explain(self, nodes): NcaLogger().log_message(f'Explainability error: only 1 or 2 nodes are allowed for explainability query,' f' found {len(nodes)} ', level='E') return out + + src_node = nodes[0] + if src_node == 'ALL': + out = self.explain_all() + return out + for node in nodes: if not self.ExplDescriptorContainer.get(node): NcaLogger().log_message(f'Explainability error - {node} was not found in the connectivity results', level='E') @@ -216,7 +263,6 @@ def explain(self, nodes): NcaLogger().log_message(f'Explainability error - {node} has no explanability results', level='E') return out - src_node = nodes[0] if len(nodes) == 2: # 2 nodes scenario dst_node = nodes[1] @@ -232,11 +278,11 @@ def explain(self, nodes): src_results.add(src_node) dst_results.add(dst_node) - out.extend(self.prepare_node_str('src', src_node, src_results)) - out.extend(self.prepare_node_str('dst', dst_node, dst_results)) + out.extend(self.prepare_node_str(src_node, src_results, 'src')) + out.extend(self.prepare_node_str(dst_node, dst_results, 'dst')) else: # only one node results = self.ExplPeerToPolicyContainer[src_node].all_policies out.append(f'\nConfigurations affecting {src_node}:') - out.extend(self.prepare_node_str(None, src_node, results)) + out.extend(self.prepare_node_str(src_node, results)) return out From 49a404626317ae3f095ed0b78c82ccb1423cb0a9 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 16 Apr 2023 19:37:57 +0300 Subject: [PATCH 110/187] Generalized ServiceEntry implementation for optimized solution. Signed-off-by: Tanya --- nca/CoreDS/Peer.py | 7 ++++++ nca/FWRules/FWRule.py | 7 ++++++ nca/NetworkConfig/NetworkConfigQuery.py | 20 +++++++++++++++ nca/NetworkConfig/NetworkLayer.py | 29 ++++++++++++++-------- nca/NetworkConfig/PeerContainer.py | 7 ++++++ nca/NetworkConfig/TopologyObjectsFinder.py | 3 ++- nca/Resources/IstioSidecar.py | 25 ++++++++++++++----- 7 files changed, 80 insertions(+), 18 deletions(-) diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index fef67829f..8411581b3 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -528,6 +528,13 @@ def __eq__(self, other): return self.name == other.name return False + def canonical_form(self): + if self.namespace is None: + return self.name + else: + return self.namespace.name + '_' + self.name + + @staticmethod def compute_re_pattern_from_host_name(host_name): """ diff --git a/nca/FWRules/FWRule.py b/nca/FWRules/FWRule.py index 4a23f1524..19f1d3bde 100644 --- a/nca/FWRules/FWRule.py +++ b/nca/FWRules/FWRule.py @@ -558,6 +558,13 @@ def get_pods_set(self, cluster_info): # an dns-entry element does not represent any pods return set() + def get_peer_set(self, cluster_info): + """ + :param cluster_info: an object of type ClusterInfo, with relevant cluster topology info + :return: a PeerSet (pods and/or IpBlocks) represented by this element + """ + return PeerSet({self.element}) + class FWRule: """ diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index da40ef04a..6adb48488 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -166,6 +166,25 @@ def determine_whether_to_compute_allowed_conns_for_peer_types(self, peer1, peer2 return False # connectivity to DNSEntry peers is only relevant if istio layer exists return True + def filter_conns_by_peer_types(self, conns, all_peers): + res = conns + # avoid IpBlock -> {IpBlock, DNSEntry} connections + all_ips = IpBlock.get_all_ips_block_peer_set() + all_dns_entries = self.config.peer_container.get_all_dns_entries() + conn_cube = ConnectivityCube.make_from_dict({"src_peers": all_ips, "dst_peers": all_ips | all_dns_entries}) + ip_to_ip_or_dns_conns = ConnectivityProperties.make_conn_props(conn_cube) + res -= ip_to_ip_or_dns_conns + # avoid DNSEntry->anything connections + conn_cube.update({"src_peers": all_dns_entries, "dst_peers": all_peers}) + dns_to_any_conns = ConnectivityProperties.make_conn_props(conn_cube) + res -= dns_to_any_conns + # avoid anything->DNSEntry connections if Istio layer does not exist + if not self.config.policies_container.layers.does_contain_layer(NetworkLayerName.Istio): + conn_cube.update({"src_peers": all_peers, "dst_peers": all_dns_entries}) + any_to_dns_conns = ConnectivityProperties.make_conn_props(conn_cube) + res -= any_to_dns_conns + return res + class DisjointnessQuery(NetworkConfigQuery): """ @@ -778,6 +797,7 @@ def exec(self): # noqa: C901 subset_conns = ConnectivityProperties.make_conn_props(src_peers_conn_cube) | \ ConnectivityProperties.make_conn_props(dst_peers_conn_cube) all_conns_opt &= subset_conns + all_conns_opt = self.filter_conns_by_peer_types(all_conns_opt, opt_peers_to_compare) ip_blocks_mask = IpBlock.get_all_ips_block() if exclude_ipv6: ip_blocks_mask = IpBlock.get_all_ips_block(exclude_ipv6=True) diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 71a40be74..9ca8dc36c 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -190,12 +190,6 @@ def allowed_connections_optimized(self, peer_container): res_conns.all_allowed_conns = ingress_conns.all_allowed_conns & egress_conns.all_allowed_conns res_conns.allowed_conns = (ingress_conns.allowed_conns & egress_conns.all_allowed_conns) | \ (egress_conns.allowed_conns & ingress_conns.all_allowed_conns) - # exclude IpBlock->IpBlock connections - conn_cube.update({"src_peers": all_ips_peer_set, "dst_peers": all_ips_peer_set}) - # TODO - update according to ConnectivityMapQuery.determine_whether_to_compute_allowed_conns_for_peer_types - ip_to_ip_conns = ConnectivityProperties.make_conn_props(conn_cube) - res_conns.allowed_conns -= ip_to_ip_conns - res_conns.all_allowed_conns -= ip_to_ip_conns return res_conns def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): @@ -353,19 +347,32 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): res_conns = self.collect_policies_conns_optimized(is_ingress, IstioNetworkLayer.captured_cond_func) all_peers_and_ips = peer_container.get_all_peers_group(True) + dns_entries = peer_container.get_all_dns_entries() # for istio initialize non-captured conns with non-TCP connections conn_cube = ConnectivityCube.make_from_dict({"src_peers": all_peers_and_ips, "dst_peers": all_peers_and_ips, "protocols": ProtocolSet.get_non_tcp_protocols()}) res_conns.all_allowed_conns |= res_conns.allowed_conns | ConnectivityProperties.make_conn_props(conn_cube) non_captured_peers = all_peers_and_ips - res_conns.captured if non_captured_peers: - conn_cube = ConnectivityCube() + protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') if is_ingress: - conn_cube.update({"src_peers": all_peers_and_ips, "dst_peers": non_captured_peers}) + conn_cube = ConnectivityCube.make_from_dict({"src_peers": all_peers_and_ips, "dst_peers": non_captured_peers}) + res_conns.all_allowed_conns |= ConnectivityProperties.make_conn_props(conn_cube) - res_conns.denied_conns + non_captured_dns_entries = dns_entries - res_conns.captured + if non_captured_dns_entries: + # update allowed non-captured conns to DNSEntry dst with TCP only + conn_cube = ConnectivityCube.make_from_dict({"src_peers": all_peers_and_ips, + "dst_peers": non_captured_dns_entries, "protocols": protocols}) + res_conns.all_allowed_conns |= ConnectivityProperties.make_conn_props(conn_cube) else: - conn_cube.update({"src_peers": non_captured_peers, "dst_peers": all_peers_and_ips}) - non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) - res_conns.all_allowed_conns |= (non_captured_conns - res_conns.denied_conns) + conn_cube = ConnectivityCube.make_from_dict({"src_peers": non_captured_peers, "dst_peers": all_peers_and_ips}) + res_conns.all_allowed_conns |= ConnectivityProperties.make_conn_props(conn_cube) - res_conns.denied_conns + # update allowed non-captured conns to DNSEntry dst with TCP only + conn_cube = ConnectivityCube.make_from_dict({"src_peers": non_captured_peers, "dst_peers": dns_entries, + "protocols": protocols}) + res_conns.all_allowed_conns |= ConnectivityProperties.make_conn_props(conn_cube) + + return res_conns diff --git a/nca/NetworkConfig/PeerContainer.py b/nca/NetworkConfig/PeerContainer.py index b0dc5d440..e7182e4c3 100644 --- a/nca/NetworkConfig/PeerContainer.py +++ b/nca/NetworkConfig/PeerContainer.py @@ -317,6 +317,13 @@ def get_all_peers_group(self, add_external_ips=False, include_globals=True, incl res.add(IpBlock.get_all_ips_block()) return res + def get_all_dns_entries(self): + res = PeerSet() + for peer in self.peer_set: + if isinstance(peer, DNSEntry): + res.add(peer) + return res + def get_all_global_peers(self): """ Return all global peers known in the system diff --git a/nca/NetworkConfig/TopologyObjectsFinder.py b/nca/NetworkConfig/TopologyObjectsFinder.py index 13f747d4d..5c1e90f31 100644 --- a/nca/NetworkConfig/TopologyObjectsFinder.py +++ b/nca/NetworkConfig/TopologyObjectsFinder.py @@ -252,7 +252,8 @@ def _add_dns_entries_from_yaml(self, srv_entry_object): """ parser = IstioServiceEntryYamlParser() dns_entries = parser.parse_serviceentry(srv_entry_object, self.peer_set) - self.peer_set |= dns_entries + for dns_entry in dns_entries: + self._add_peer(dns_entry) class NamespacesFinder: diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index 380fc9ae1..f8322eebd 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -6,11 +6,11 @@ from enum import Enum from nca.CoreDS.ConnectionSet import ConnectionSet -from nca.CoreDS.Peer import IpBlock, PeerSet +from nca.CoreDS.Peer import IpBlock, PeerSet, DNSEntry +from nca.CoreDS.ProtocolSet import ProtocolSet from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube from .NetworkPolicy import PolicyConnections, OptimizedPolicyConnections, NetworkPolicy from .IstioTrafficResources import istio_root_namespace -from ..CoreDS.Peer import DNSEntry, IpBlock @dataclass @@ -182,13 +182,26 @@ def combine_peer_sets_by_ns(from_peer_set, to_peer_set, peer_container): def create_opt_egress_props(self, peer_container): for rule in self.egress_rules: - conn_cube = ConnectivityCube() + # connections to IP-block is enabled only if the outbound mode is allow-any (disabled for registry only) + if self.outbound_mode == IstioSidecar.OutboundMode.ALLOW_ANY: + ip_blocks = IpBlock.get_all_ips_block_peer_set() + conn_cube = ConnectivityCube.make_from_dict({"src_peers": self.selected_peers, "dst_peers": ip_blocks}) + self.optimized_allow_egress_props |= ConnectivityProperties.make_conn_props(conn_cube) + + dns_entries = peer_container.get_all_dns_entries() + dst_dns_entries = dns_entries & (rule.egress_peer_set | rule.special_egress_peer_set) + if self.selected_peers and dst_dns_entries: + protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') + conn_cube = ConnectivityCube.make_from_dict({"src_peers": self.selected_peers, + "dst_peers": dst_dns_entries, "protocols": protocols}) + self.optimized_allow_egress_props |= ConnectivityProperties.make_conn_props(conn_cube) + if self.selected_peers and rule.egress_peer_set: - conn_cube.update({"src_peers": self.selected_peers, "dst_peers": rule.egress_peer_set}) - self.optimized_allow_egress_props = ConnectivityProperties.make_conn_props(conn_cube) + conn_cube = ConnectivityCube.make_from_dict({"src_peers": self.selected_peers, "dst_peers": rule.egress_peer_set}) + self.optimized_allow_egress_props |= ConnectivityProperties.make_conn_props(conn_cube) peers_sets_by_ns = self.combine_peer_sets_by_ns(self.selected_peers, rule.special_egress_peer_set, peer_container) for (from_peers, to_peers) in peers_sets_by_ns: if from_peers and to_peers: - conn_cube.update({"src_peers": from_peers, "dst_peers": to_peers}) + conn_cube = ConnectivityCube.make_from_dict({"src_peers": from_peers, "dst_peers": to_peers}) self.optimized_allow_egress_props |= ConnectivityProperties.make_conn_props(conn_cube) From 736ea07f7dd927e2c58ce174133799f147c21c4c Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 16 Apr 2023 19:42:33 +0300 Subject: [PATCH 111/187] Fixing lint errors. Signed-off-by: Tanya --- nca/CoreDS/Peer.py | 1 - nca/NetworkConfig/NetworkLayer.py | 5 ++--- nca/Resources/IstioSidecar.py | 3 ++- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index 8411581b3..e6370054b 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -534,7 +534,6 @@ def canonical_form(self): else: return self.namespace.name + '_' + self.name - @staticmethod def compute_re_pattern_from_host_name(host_name): """ diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 9ca8dc36c..9186718c4 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -362,7 +362,8 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): if non_captured_dns_entries: # update allowed non-captured conns to DNSEntry dst with TCP only conn_cube = ConnectivityCube.make_from_dict({"src_peers": all_peers_and_ips, - "dst_peers": non_captured_dns_entries, "protocols": protocols}) + "dst_peers": non_captured_dns_entries, + "protocols": protocols}) res_conns.all_allowed_conns |= ConnectivityProperties.make_conn_props(conn_cube) else: conn_cube = ConnectivityCube.make_from_dict({"src_peers": non_captured_peers, "dst_peers": all_peers_and_ips}) @@ -371,8 +372,6 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): conn_cube = ConnectivityCube.make_from_dict({"src_peers": non_captured_peers, "dst_peers": dns_entries, "protocols": protocols}) res_conns.all_allowed_conns |= ConnectivityProperties.make_conn_props(conn_cube) - - return res_conns diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index f8322eebd..5752f2817 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -197,7 +197,8 @@ def create_opt_egress_props(self, peer_container): self.optimized_allow_egress_props |= ConnectivityProperties.make_conn_props(conn_cube) if self.selected_peers and rule.egress_peer_set: - conn_cube = ConnectivityCube.make_from_dict({"src_peers": self.selected_peers, "dst_peers": rule.egress_peer_set}) + conn_cube = ConnectivityCube.make_from_dict({"src_peers": self.selected_peers, + "dst_peers": rule.egress_peer_set}) self.optimized_allow_egress_props |= ConnectivityProperties.make_conn_props(conn_cube) peers_sets_by_ns = self.combine_peer_sets_by_ns(self.selected_peers, rule.special_egress_peer_set, peer_container) From 9db0f9199f233e27c0a90bfe2ca08178a8a2edbb Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Tue, 18 Apr 2023 09:36:44 +0300 Subject: [PATCH 112/187] xml support for explain_all and default-policy fix Signed-off-by: Shmulik Froimovich --- nca/NetworkConfig/NetworkConfigQueryRunner.py | 2 +- nca/Utils/ExplTracker.py | 49 +++++++++++++++---- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/nca/NetworkConfig/NetworkConfigQueryRunner.py b/nca/NetworkConfig/NetworkConfigQueryRunner.py index b3fed139c..1b5fc62c7 100644 --- a/nca/NetworkConfig/NetworkConfigQueryRunner.py +++ b/nca/NetworkConfig/NetworkConfigQueryRunner.py @@ -56,7 +56,7 @@ def compute_final_results(self, output_format, expl_nodes): sort_keys=False) else: output = '\n'.join(self.query_iterations_output) - expl_out = '\n'.join(ExplTracker().explain(expl_nodes)) + expl_out = ExplTracker().explain(expl_nodes) return self.numerical_result, output+expl_out, self.num_not_executed diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py index 818693671..b943a031d 100644 --- a/nca/Utils/ExplTracker.py +++ b/nca/Utils/ExplTracker.py @@ -6,6 +6,8 @@ from nca.Utils.Utils import Singleton from nca.Utils.NcaLogger import NcaLogger from nca.CoreDS.Peer import PeerSet +from bs4 import BeautifulSoup +from bs4.element import Tag class ExplPolicies: @@ -31,6 +33,10 @@ def _add_policy(peer_set, peer_list, policy_name): peer_name = peer.full_name() if not peer_list.get(peer_name): peer_list[peer_name] = set() + # We don't want Default-Policy if we have any other policy, + # so we first remove it and then add the policy (even if we currently add + # the Default-Policy itself). + peer_list[peer_name].discard('Default-Policy') peer_list[peer_name].add(policy_name) def add_policy(self, policy_name, egress_dst, ingress_src): @@ -192,11 +198,23 @@ def add_default_policy(self, src, dst, is_ingress): ingress_list = {} for node in nodes: - self.add_peer_policy(node.full_name(), - 'Default-Policy', - egress_list, - ingress_list, - ) + # we dont add Default-Policy if there is already an explicit + # policy allowing the connectivity + if self.is_policy_list_empty(node.full_name(), is_ingress): + self.add_peer_policy(node.full_name(), + 'Default-Policy', + egress_list, + ingress_list, + ) + + def is_policy_list_empty(self, node_name, check_ingress): + peer = self.ExplPeerToPolicyContainer.get(node_name) + if peer: + if check_ingress and peer.ingress_src: + return False + if not check_ingress and peer.egress_dst: + return False + return True def prepare_node_str(self, node_name, results, direction=None): """ @@ -222,14 +240,25 @@ def prepare_node_str(self, node_name, results, direction=None): return out def explain_all(self): - out = [] + soup = BeautifulSoup(features='xml') + entry_id = 0 for peer1 in self.all_peers: for peer2 in self.all_peers: if peer1 == peer2: - out.extend(self.explain([peer1.full_name()])) + text = self.explain([peer1.full_name()]) else: - out.extend(self.explain([peer1.full_name(), peer2.full_name()])) - return out + text = self.explain([peer1.full_name(), peer2.full_name()]) + # Create the XML entry element + entry = soup.new_tag('entry') + entry_id += 1 + entry['id'] = str(entry_id) + entry['src'] = peer1.full_name() + entry['dst'] = peer2.full_name() + text_elem = Tag(soup, name='text') + text_elem.string = text + entry.append(text_elem) + soup.append(entry) + return soup.prettify() def explain(self, nodes): """ @@ -285,4 +314,6 @@ def explain(self, nodes): out.append(f'\nConfigurations affecting {src_node}:') out.extend(self.prepare_node_str(src_node, results)) + # convert the list of expl' directives into string + out = '\n'.join(out) return out From 31ff80540b000209121b2e549fc7ee9aa5173e2d Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 18 Apr 2023 11:12:53 +0300 Subject: [PATCH 113/187] Removed unused functions. Optimized BasePeerSet.get_peer_interval_of method. Signed-off-by: Tanya --- nca/CoreDS/Peer.py | 22 ++++++++-------------- nca/Resources/IstioSidecar.py | 30 ------------------------------ 2 files changed, 8 insertions(+), 44 deletions(-) diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index e6370054b..1ea3412eb 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -66,7 +66,6 @@ def __init__(self, name, namespace=None): super().__init__(name, namespace) self.named_ports = {} # A map from port name to the port number and its protocol self.profiles = [] # The set of attached profiles (Calico only) - self.prior_sidecar = None # the first injected sidecar with workloadSelector selecting current peer def __eq__(self, other): if isinstance(other, type(self)): @@ -734,18 +733,9 @@ def get_peer_interval_of(self, peer_set): :return: CanonicalIntervalSet for the peer_set """ res = CanonicalIntervalSet() - covered_peers = peer_set.copy() # for check that we covered all peers - for index, peer in enumerate(self.ordered_peer_list): - if peer in peer_set: - covered_peers.add(peer) - assert not isinstance(peer, IpBlock) - res.add_interval(CanonicalIntervalSet.Interval(self.min_pod_index + index, - self.min_pod_index + index)) - # Now pick IpBlocks - for ipb in peer_set: - if isinstance(ipb, IpBlock): - covered_peers.add(ipb) - for cidr in ipb: + for peer in peer_set: + if isinstance(peer, IpBlock): + for cidr in peer: if isinstance(cidr.start.address, ipaddress.IPv4Address): res.add_interval(CanonicalIntervalSet.Interval(self.min_ipv4_index + int(cidr.start), self.min_ipv4_index + int(cidr.end))) @@ -754,7 +744,11 @@ def get_peer_interval_of(self, peer_set): self.min_ipv6_index + int(cidr.end))) else: assert False - assert covered_peers == peer_set + else: + index = self.peer_to_index.get(peer) + assert index is not None + res.add_interval(CanonicalIntervalSet.Interval(self.min_pod_index + index, + self.min_pod_index + index)) return res def get_peer_set_by_indices(self, peer_interval_set): diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index 5752f2817..a335f1445 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -136,36 +136,6 @@ def clone_without_rule(self, rule_to_exclude, _ingress_rule): res.add_egress_rule(rule) return res - def _is_sidecar_prior(self, from_peer): - """ - Check if the current sidecar is in the top priority of the captured from_peer - to be considered in its connections or not - :param Peer.Peer from_peer: the source peer captured by the current sidecar - :return: True if the sidecar is in the peer's top priority to consider it in its connections, otherwise False - computing the return value is according to following: - 1- for from_peer, preference will be given to the first injected sidecar with - a workloadSelector that selected the peer. - 2- if the specific sidecar from (1) does not exist, preference will be given to the - first injected selector-less sidecar in the peer's namespace - 3- if sidecars from (1) and (2) don't exist, the preference will be given to the first default - sidecar of the istio root namespace - :rtype: bool - """ - if not self.default_sidecar: # specific sidecar - if from_peer.prior_sidecar and self == from_peer.prior_sidecar: - return True - else: # selector-less sidecar - if from_peer.prior_sidecar: - return False - if from_peer.namespace.prior_default_sidecar: - if self == from_peer.namespace.prior_default_sidecar: - return True - else: - if str(self.namespace) == istio_root_namespace and \ - self == self.namespace.prior_default_sidecar: - return True - return False - @staticmethod def combine_peer_sets_by_ns(from_peer_set, to_peer_set, peer_container): res = [] From 2604eb6e61060bbadede6533d3da264599d97ed6 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 18 Apr 2023 11:34:52 +0300 Subject: [PATCH 114/187] Separated ConnectivityCube class to its own file. Removed unused methods/params/imports. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 9 +- nca/CoreDS/ConnectivityCube.py | 202 ++++++++++++++++++ nca/CoreDS/ConnectivityProperties.py | 193 +---------------- nca/NetworkConfig/NetworkConfig.py | 3 +- nca/NetworkConfig/NetworkConfigQuery.py | 7 +- nca/NetworkConfig/NetworkLayer.py | 3 +- nca/Parsers/CalicoPolicyYamlParser.py | 3 +- nca/Parsers/IngressPolicyYamlParser.py | 3 +- nca/Parsers/IstioPolicyYamlParser.py | 3 +- .../IstioTrafficResourcesYamlParser.py | 3 +- nca/Parsers/K8sPolicyYamlParser.py | 3 +- nca/Resources/IstioSidecar.py | 4 +- .../testConnectivityPropertiesNamedPorts.py | 4 +- 13 files changed, 230 insertions(+), 210 deletions(-) create mode 100644 nca/CoreDS/ConnectivityCube.py diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index d70cd6ef2..b2bc6bc73 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -5,7 +5,8 @@ from collections import defaultdict from .CanonicalIntervalSet import CanonicalIntervalSet -from .ConnectivityProperties import ConnectivityProperties, ConnectivityCube +from .ConnectivityCube import ConnectivityCube +from .ConnectivityProperties import ConnectivityProperties from .ProtocolNameResolver import ProtocolNameResolver from .ProtocolSet import ProtocolSet from .Peer import PeerSet, IpBlock @@ -542,7 +543,7 @@ def print_diff(self, other, self_name, other_name): return 'No diff.' - def convert_to_connectivity_properties(self, peer_container): + def convert_to_connectivity_properties(self): if self.allow_all: return ConnectivityProperties.make_all_props() @@ -665,11 +666,11 @@ def split_peer_set_to_fw_rule_elements(peer_set, cluster_info): return res @staticmethod - def fw_rules_to_conn_props(fw_rules, peer_container): + def fw_rules_to_conn_props(fw_rules): res = ConnectivityProperties.make_empty_props() for fw_rules_list in fw_rules.fw_rules_map.values(): for fw_rule in fw_rules_list: - conn_props = fw_rule.conn.convert_to_connectivity_properties(peer_container) + conn_props = fw_rule.conn.convert_to_connectivity_properties() src_peers = PeerSet(fw_rule.src.get_peer_set(fw_rules.cluster_info)) dst_peers = PeerSet(fw_rule.dst.get_peer_set(fw_rules.cluster_info)) conn_cube = ConnectivityCube.make_from_dict({"src_peers": src_peers, "dst_peers": dst_peers}) diff --git a/nca/CoreDS/ConnectivityCube.py b/nca/CoreDS/ConnectivityCube.py new file mode 100644 index 000000000..9e4dd5e18 --- /dev/null +++ b/nca/CoreDS/ConnectivityCube.py @@ -0,0 +1,202 @@ +# +# Copyright 2020- IBM Inc. All rights reserved +# SPDX-License-Identifier: Apache2.0 +# + +from .CanonicalIntervalSet import CanonicalIntervalSet +from .DimensionsManager import DimensionsManager +from .PortSet import PortSet +from .Peer import BasePeerSet +from .MinDFA import MinDFA + + +class ConnectivityCube(dict): + """ + This class manages forth and back translations of all dimensions of ConnectivityProperties + (translations between input format and internal format). + It is used as an input interface for ConnectivityProperties methods. + """ + + dimensions_list = ["src_peers", "dst_peers", "protocols", "src_ports", "dst_ports", "methods", "hosts", "paths", + "icmp_type", "icmp_code"] + + def __init__(self): + """ + By default, each dimension in the cube is initialized with entire domain value, which represents + "don't care" or inactive dimension (i.e., the dimension has no impact). + """ + super().__init__() + self.named_ports = set() # used only in the original solution + self.excluded_named_ports = set() # used only in the original solution + for dim in self.dimensions_list: + self.set_dim_directly(dim, DimensionsManager().get_dimension_domain_by_name(dim)) + + def copy(self): + """ + Returns a copy of the given ConnectivityCube object + :rtype: ConnectivityCube + """ + res = ConnectivityCube() + for dim_name, dim_value in self.items(): + if isinstance(dim_value, MinDFA): + res.set_dim_directly(dim_name, dim_value) + else: + res.set_dim_directly(dim_name, dim_value.copy()) + return res + + def is_empty_dim(self, dim_name): + """ + Returns True iff a given dimension is empty + :param str dim_name: the given dimension name + """ + if self.get_dim_directly(dim_name) != DimensionsManager().get_empty_dimension_by_name(dim_name): + return False + + # for "dst_ports" can have named ports in original solution + return not self.named_ports and not self.excluded_named_ports if dim_name == "dst_ports" else True + + def is_full_dim(self, dim_name): + """ + Returns True iff a given dimension is full + :param str dim_name: the given dimension name + """ + return self.get_dim_directly(dim_name) == DimensionsManager().get_dimension_domain_by_name(dim_name) + + def is_active_dim(self, dim_name): + """ + Returns True iff a given dimension is active (i.e., not full) + :param str dim_name: the given dimension name + """ + return not self.is_full_dim(dim_name) + + def set_dim_directly(self, dim_name, dim_value): + """ + Sets a given dimension value directly, assuming the value is in the internal format of that dimension. + :param str dim_name: the given dimension name + :param dim_value: the given dimension value + """ + assert dim_name in self.dimensions_list + super().__setitem__(dim_name, dim_value) + + def get_dim_directly(self, dim_name): + """ + Returns a given dimension value directly (in its internal format). + :param str dim_name: the given dimension name + """ + assert dim_name in self.dimensions_list + return super().__getitem__(dim_name) + + def __setitem__(self, dim_name, dim_value): + """ + Sets a given dimension value after converting it into the internal format of that dimension. + :param str dim_name: the given dimension name + :param dim_value: the given dimension value + """ + assert dim_name in self.dimensions_list + if dim_value is None: + return + if dim_name in ["src_peers", "dst_peers"]: + # translate PeerSet to CanonicalIntervalSet + self.set_dim_directly(dim_name, BasePeerSet().get_peer_interval_of(dim_value)) + elif dim_name in ["src_ports", "dst_ports"]: + # extract port_set from PortSet + self.set_dim_directly(dim_name, dim_value.port_set) + if dim_name == "dst_ports": + self.named_ports = dim_value.named_ports + self.excluded_named_ports = dim_value.excluded_named_ports + elif dim_name in ["icmp_type", "icmp_code"]: + # translate int to CanonicalIntervalSet + self.set_dim_directly(dim_name, CanonicalIntervalSet.get_interval_set(dim_value, dim_value)) + else: # the rest of dimensions do not need a translation + self.set_dim_directly(dim_name, dim_value) + + def update(self, the_dict=None, **f): + """ + Sets multiple dimension values at once, after converting them into their internal formats. + :param dict the_dict: a dictionary from dimension names to dimension values, having all dimensions to be set + :param f: Not used; required by the base class (dict) interface. + """ + for dim_name, dim_value in the_dict.items(): + self[dim_name] = dim_value + + def unset_dim(self, dim_name): + """ + Sets a given dimension to its default (containing all possible values) + :param str dim_name: the given dimension name + """ + assert dim_name in self.dimensions_list + self.set_dim_directly(dim_name, DimensionsManager().get_dimension_domain_by_name(dim_name)) + + def __getitem__(self, dim_name): + """ + Returns a given dimension value after converting it from internal to external format. + :param str dim_name: the given dimension name + """ + assert dim_name in self.dimensions_list + dim_value = self.get_dim_directly(dim_name) + if dim_name in ["src_peers", "dst_peers"]: + if self.is_active_dim(dim_name): + # translate CanonicalIntervalSet back to PeerSet + return BasePeerSet().get_peer_set_by_indices(dim_value) + else: + return None + elif dim_name in ["src_ports", "dst_ports"]: + res = PortSet() + res.add_ports(dim_value) + if dim_name == "dst_ports": + res.named_ports = self.named_ports + res.excluded_named_ports = self.excluded_named_ports + return res + elif dim_name in ["icmp_type", "icmp_code"]: + if self.is_active_dim(dim_name): + # translate CanonicalIntervalSet back to int + return dim_value.validate_and_get_single_value() + else: + return None + else: # the rest of dimensions do not need a translation + if isinstance(dim_value, MinDFA): + return dim_value + else: + return dim_value.copy() # TODO - do we need this copy? + + def has_active_dim(self): + """ + Returns True iff the cube has at least one active dimension. Otherwise, returns False. + """ + for dim in self.dimensions_list: + if self.get_dim_directly(dim) != DimensionsManager().get_dimension_domain_by_name(dim): + return True + return False + + def is_empty(self): + """ + Returns True iff the cube has at least one empty dimension. Otherwise, returns False. + """ + for dim in self.dimensions_list: + if self.is_empty_dim(dim): + return True + return False + + def get_ordered_cube_and_active_dims(self): + """ + Translate the connectivity cube to an ordered cube, and compute its active dimensions + :return: tuple with: (1) cube values (2) active dimensions + """ + cube = [] + active_dims = [] + # add values to cube by required order of dimensions + for dim in self.dimensions_list: + dim_value = self.get_dim_directly(dim) + if dim_value != DimensionsManager().get_dimension_domain_by_name(dim): + if isinstance(dim_value, MinDFA): + cube.append(dim_value) + else: + cube.append(dim_value.copy()) # TODO - do we need this copy? + active_dims.append(dim) + return cube, active_dims + + @staticmethod + def make_from_dict(the_dict): + ccube = ConnectivityCube() + ccube.update(the_dict) + return ccube diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 57e02a8d8..08905b599 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -11,198 +11,7 @@ from .Peer import PeerSet, BasePeerSet from .ProtocolNameResolver import ProtocolNameResolver from .MinDFA import MinDFA - - -class ConnectivityCube(dict): - """ - This class manages forth and back translations of all dimensions of ConnectivityProperties - (translations between input format and internal format). - It is used as an input interface for ConnectivityProperties methods. - """ - - dimensions_list = ["src_peers", "dst_peers", "protocols", "src_ports", "dst_ports", "methods", "hosts", "paths", - "icmp_type", "icmp_code"] - - def __init__(self): - """ - By default, each dimension in the cube is initialized with entire domain value, which represents - "don't care" or inactive dimension (i.e., the dimension has no impact). - """ - super().__init__() - self.named_ports = set() # used only in the original solution - self.excluded_named_ports = set() # used only in the original solution - for dim in self.dimensions_list: - self.set_dim_directly(dim, DimensionsManager().get_dimension_domain_by_name(dim)) - - def copy(self): - """ - Returns a copy of the given ConnectivityCube object - :rtype: ConnectivityCube - """ - res = ConnectivityCube() - for dim_name, dim_value in self.items(): - if isinstance(dim_value, MinDFA): - res.set_dim_directly(dim_name, dim_value) - else: - res.set_dim_directly(dim_name, dim_value.copy()) - return res - - def is_empty_dim(self, dim_name): - """ - Returns True iff a given dimension is empty - :param str dim_name: the given dimension name - """ - if self.get_dim_directly(dim_name) != DimensionsManager().get_empty_dimension_by_name(dim_name): - return False - - # for "dst_ports" can have named ports in original solution - return not self.named_ports and not self.excluded_named_ports if dim_name == "dst_ports" else True - - def is_full_dim(self, dim_name): - """ - Returns True iff a given dimension is full - :param str dim_name: the given dimension name - """ - return self.get_dim_directly(dim_name) == DimensionsManager().get_dimension_domain_by_name(dim_name) - - def is_active_dim(self, dim_name): - """ - Returns True iff a given dimension is active (i.e., not full) - :param str dim_name: the given dimension name - """ - return not self.is_full_dim(dim_name) - - def set_dim_directly(self, dim_name, dim_value): - """ - Sets a given dimension value directly, assuming the value is in the internal format of that dimension. - :param str dim_name: the given dimension name - :param dim_value: the given dimension value - """ - assert dim_name in self.dimensions_list - super().__setitem__(dim_name, dim_value) - - def get_dim_directly(self, dim_name): - """ - Returns a given dimension value directly (in its internal format). - :param str dim_name: the given dimension name - """ - assert dim_name in self.dimensions_list - return super().__getitem__(dim_name) - - def __setitem__(self, dim_name, dim_value): - """ - Sets a given dimension value after converting it into the internal format of that dimension. - :param str dim_name: the given dimension name - :param dim_value: the given dimension value - """ - assert dim_name in self.dimensions_list - if dim_value is None: - return - if dim_name in ["src_peers", "dst_peers"]: - # translate PeerSet to CanonicalIntervalSet - self.set_dim_directly(dim_name, BasePeerSet().get_peer_interval_of(dim_value)) - elif dim_name in ["src_ports", "dst_ports"]: - # extract port_set from PortSet - self.set_dim_directly(dim_name, dim_value.port_set) - if dim_name == "dst_ports": - self.named_ports = dim_value.named_ports - self.excluded_named_ports = dim_value.excluded_named_ports - elif dim_name in ["icmp_type", "icmp_code"]: - # translate int to CanonicalIntervalSet - self.set_dim_directly(dim_name, CanonicalIntervalSet.get_interval_set(dim_value, dim_value)) - else: # the rest of dimensions do not need a translation - self.set_dim_directly(dim_name, dim_value) - - def update(self, the_dict=None, **f): - """ - Sets multiple dimension values at once, after converting them into their internal formats. - :param dict the_dict: a dictionary from dimension names to dimension values, having all dimensions to be set - :param f: Not used; required by the base class (dict) interface. - """ - for dim_name, dim_value in the_dict.items(): - self[dim_name] = dim_value - - def unset_dim(self, dim_name): - """ - Sets a given dimension to its default (containing all possible values) - :param str dim_name: the given dimension name - """ - assert dim_name in self.dimensions_list - self.set_dim_directly(dim_name, DimensionsManager().get_dimension_domain_by_name(dim_name)) - - def __getitem__(self, dim_name): - """ - Returns a given dimension value after converting it from internal to external format. - :param str dim_name: the given dimension name - """ - assert dim_name in self.dimensions_list - dim_value = self.get_dim_directly(dim_name) - if dim_name in ["src_peers", "dst_peers"]: - if self.is_active_dim(dim_name): - # translate CanonicalIntervalSet back to PeerSet - return BasePeerSet().get_peer_set_by_indices(dim_value) - else: - return None - elif dim_name in ["src_ports", "dst_ports"]: - res = PortSet() - res.add_ports(dim_value) - if dim_name == "dst_ports": - res.named_ports = self.named_ports - res.excluded_named_ports = self.excluded_named_ports - return res - elif dim_name in ["icmp_type", "icmp_code"]: - if self.is_active_dim(dim_name): - # translate CanonicalIntervalSet back to int - return dim_value.validate_and_get_single_value() - else: - return None - else: # the rest of dimensions do not need a translation - if isinstance(dim_value, MinDFA): - return dim_value - else: - return dim_value.copy() # TODO - do we need this copy? - - def has_active_dim(self): - """ - Returns True iff the cube has at least one active dimension. Otherwise, returns False. - """ - for dim in self.dimensions_list: - if self.get_dim_directly(dim) != DimensionsManager().get_dimension_domain_by_name(dim): - return True - return False - - def is_empty(self): - """ - Returns True iff the cube has at least one empty dimension. Otherwise, returns False. - """ - for dim in self.dimensions_list: - if self.is_empty_dim(dim): - return True - return False - - def get_ordered_cube_and_active_dims(self): - """ - Translate the connectivity cube to an ordered cube, and compute its active dimensions - :return: tuple with: (1) cube values (2) active dimensions - """ - cube = [] - active_dims = [] - # add values to cube by required order of dimensions - for dim in self.dimensions_list: - dim_value = self.get_dim_directly(dim) - if dim_value != DimensionsManager().get_dimension_domain_by_name(dim): - if isinstance(dim_value, MinDFA): - cube.append(dim_value) - else: - cube.append(dim_value.copy()) # TODO - do we need this copy? - active_dims.append(dim) - return cube, active_dims - - @staticmethod - def make_from_dict(the_dict): - ccube = ConnectivityCube() - ccube.update(the_dict) - return ccube +from .ConnectivityCube import ConnectivityCube class ConnectivityProperties(CanonicalHyperCubeSet): diff --git a/nca/NetworkConfig/NetworkConfig.py b/nca/NetworkConfig/NetworkConfig.py index d6f483c1e..6d68ca95b 100644 --- a/nca/NetworkConfig/NetworkConfig.py +++ b/nca/NetworkConfig/NetworkConfig.py @@ -6,7 +6,8 @@ from dataclasses import dataclass, field from nca.CoreDS import Peer from nca.CoreDS.ConnectionSet import ConnectionSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube +from nca.CoreDS.ConnectivityCube import ConnectivityCube +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.Resources.NetworkPolicy import NetworkPolicy, OptimizedPolicyConnections from .NetworkLayer import NetworkLayersContainer, NetworkLayerName diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 6adb48488..9d461dbb6 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -12,7 +12,8 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import PeerSet, IpBlock, Pod, Peer, DNSEntry from nca.CoreDS.ProtocolSet import ProtocolSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube +from nca.CoreDS.ConnectivityCube import ConnectivityCube +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.FWRules.ConnectivityGraph import ConnectivityGraph from nca.FWRules.MinimizeFWRules import MinimizeFWRules from nca.FWRules.ClusterInfo import ClusterInfo @@ -834,8 +835,8 @@ def exec(self): # noqa: C901 return res def compare_fw_rules(self, fw_rules1, fw_rules2): - conn_props1 = ConnectionSet.fw_rules_to_conn_props(fw_rules1, self.config.peer_container) - conn_props2 = ConnectionSet.fw_rules_to_conn_props(fw_rules2, self.config.peer_container) + conn_props1 = ConnectionSet.fw_rules_to_conn_props(fw_rules1) + conn_props2 = ConnectionSet.fw_rules_to_conn_props(fw_rules2) if conn_props1 == conn_props2: print("Original and optimized fw-rules are semantically equivalent") else: diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 9186718c4..dbb2deb6c 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -7,7 +7,8 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock, HostEP, PeerSet, DNSEntry -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube +from nca.CoreDS.ConnectivityCube import ConnectivityCube +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy from nca.Resources.NetworkPolicy import PolicyConnections, OptimizedPolicyConnections, NetworkPolicy diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index 76f0617e7..c65196913 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -7,7 +7,8 @@ from nca.CoreDS.ProtocolNameResolver import ProtocolNameResolver from nca.CoreDS.Peer import PeerSet, IpBlock from nca.CoreDS.PortSet import PortSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube +from nca.CoreDS.ConnectivityCube import ConnectivityCube +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.ProtocolSet import ProtocolSet from nca.CoreDS.DimensionsManager import DimensionsManager from nca.CoreDS.ConnectionSet import ConnectionSet diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index 03282aa19..f740728c7 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -8,7 +8,8 @@ from nca.CoreDS.DimensionsManager import DimensionsManager from nca.CoreDS.Peer import PeerSet from nca.CoreDS.PortSet import PortSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube +from nca.CoreDS.ConnectivityCube import ConnectivityCube +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.IngressPolicy import IngressPolicy from nca.Resources.NetworkPolicy import NetworkPolicy diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index 2d7a15128..3d2e5b5d3 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -10,7 +10,8 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.PortSet import PortSet from nca.CoreDS.MethodSet import MethodSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube +from nca.CoreDS.ConnectivityCube import ConnectivityCube +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy, IstioPolicyRule from nca.Resources.IstioTrafficResources import istio_root_namespace from nca.Resources.NetworkPolicy import NetworkPolicy diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index 1a1923cea..15114d1b0 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -8,7 +8,8 @@ from nca.CoreDS.Peer import PeerSet from nca.CoreDS.MethodSet import MethodSet from nca.CoreDS.ProtocolSet import ProtocolSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube +from nca.CoreDS.ConnectivityCube import ConnectivityCube +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.Resources.IstioTrafficResources import Gateway, VirtualService from nca.Resources.IngressPolicy import IngressPolicy from nca.Resources.NetworkPolicy import NetworkPolicy diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index a0e86bedc..4046a7d59 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -7,7 +7,8 @@ from nca.CoreDS import Peer from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.PortSet import PortSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube +from nca.CoreDS.ConnectivityCube import ConnectivityCube +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.ProtocolNameResolver import ProtocolNameResolver from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.NetworkPolicy import NetworkPolicy diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index a335f1445..16a5859f2 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -8,9 +8,9 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock, PeerSet, DNSEntry from nca.CoreDS.ProtocolSet import ProtocolSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube +from nca.CoreDS.ConnectivityCube import ConnectivityCube +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from .NetworkPolicy import PolicyConnections, OptimizedPolicyConnections, NetworkPolicy -from .IstioTrafficResources import istio_root_namespace @dataclass diff --git a/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py b/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py index c63cc04eb..0325fb03c 100644 --- a/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py +++ b/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py @@ -1,8 +1,8 @@ import unittest from nca.CoreDS.CanonicalIntervalSet import CanonicalIntervalSet from nca.CoreDS.PortSet import PortSet -from nca.CoreDS.Peer import PeerSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties, ConnectivityCube +from nca.CoreDS.ConnectivityCube import ConnectivityCube +from nca.CoreDS.ConnectivityProperties import ConnectivityProperties class TestNamedPorts(unittest.TestCase): From 368bb78b4c158c07337355d383ea94bc9f128e43 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 18 Apr 2023 12:35:40 +0300 Subject: [PATCH 115/187] Update nca/NetworkConfig/NetworkConfig.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/NetworkConfig/NetworkConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/NetworkConfig/NetworkConfig.py b/nca/NetworkConfig/NetworkConfig.py index 6d68ca95b..77334fb34 100644 --- a/nca/NetworkConfig/NetworkConfig.py +++ b/nca/NetworkConfig/NetworkConfig.py @@ -274,7 +274,7 @@ def allowed_connections_optimized(self, layer_name=None): Computes the set of allowed connections between any relevant peers. :param NetworkLayerName layer_name: The name of the layer to use, if requested to use a specific layer only :return: allowed_conns: all allowed connections for relevant peers. - :rtype: ConnectivityProperties + :rtype: OptimizedPolicyConnections """ if layer_name is not None: if layer_name not in self.policies_container.layers: From c39f53f9983e1547bba256ed0cd95d47c8153655 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 18 Apr 2023 12:35:55 +0300 Subject: [PATCH 116/187] Update nca/Resources/NetworkPolicy.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/Resources/NetworkPolicy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nca/Resources/NetworkPolicy.py b/nca/Resources/NetworkPolicy.py index 0dba8ef85..017c6d6d6 100644 --- a/nca/Resources/NetworkPolicy.py +++ b/nca/Resources/NetworkPolicy.py @@ -342,7 +342,7 @@ class OptimizedPolicyConnections: """ def __init__(self): self.captured = PeerSet() - self.allowed_conns = ConnectivityProperties().make_empty_props() - self.denied_conns = ConnectivityProperties().make_empty_props() - self.pass_conns = ConnectivityProperties().make_empty_props() - self.all_allowed_conns = ConnectivityProperties().make_empty_props() + self.allowed_conns = ConnectivityProperties() + self.denied_conns = ConnectivityProperties() + self.pass_conns = ConnectivityProperties() + self.all_allowed_conns = ConnectivityProperties() From f5d1581ce610e4d841c8f6af0885dc89a3387dd6 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 18 Apr 2023 16:29:47 +0300 Subject: [PATCH 117/187] Added assertions avoiding incorrect comparisons of "src_peers" and "dst_peers" dimensions. Split complex ConnectivityMapQuery.exec method to multiple methods. Signed-off-by: Tanya --- nca/CoreDS/ConnectivityProperties.py | 13 ++ nca/NetworkConfig/NetworkConfigQuery.py | 180 +++++++++++++----------- 2 files changed, 111 insertions(+), 82 deletions(-) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 08905b599..2ca2024b8 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -36,6 +36,15 @@ class ConnectivityProperties(CanonicalHyperCubeSet): The named ports are resolved during the construction, therefore in the optimized solution named_ports and excluded_named_ports fields are not used. + The src_peers and dst_peers dimensions are special dimensions, they do not have constant domain. Their domain + depends on the current set of peers in the system (as appears in BasePeerSet singleton). This set grows when + adding more configurations. Thus, there is no unique 'all values' representation. In particular, those + dimensions are never reduced to inactive. + This might be a problem in comparison and inclusion operators of ConnectivityProperties. The possible solution + may be to keep 'reference full domain value' for these dimensions (as another member in the BasePeerSet), + and to set it to relevant values per query, and to make a special treatment of these dimensions + in the above operators. + Also, including support for (included and excluded) named ports (relevant for dest ports only). The representation with named ports is considered a mid-representation, and is required due to the late binding @@ -179,6 +188,8 @@ def get_properties_obj(self): def __eq__(self, other): if isinstance(other, ConnectivityProperties): + assert ("src_peers" in self.active_dimensions) == ("src_peers" in other.active_dimensions) + assert ("dst_peers" in self.active_dimensions) == ("dst_peers" in other.active_dimensions) res = super().__eq__(other) and self.named_ports == other.named_ports and \ self.excluded_named_ports == other.excluded_named_ports return res @@ -235,6 +246,8 @@ def contained_in(self, other): """ assert not self.has_named_ports() assert not other.has_named_ports() + assert ("src_peers" in self.active_dimensions) == ("src_peers" in other.active_dimensions) + assert ("dst_peers" in self.active_dimensions) == ("dst_peers" in other.active_dimensions) return super().contained_in(other) def has_named_ports(self): diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 9d461dbb6..4ea54489f 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -731,103 +731,119 @@ def are_labels_all_included(target_labels, pool_labels): return False return True - def exec(self): # noqa: C901 + def compute_connectivity_output_original(self): + """ + Compute connectivity output with original implementation (running for every pair of peers). + :return: a tuple of output result (in a required format), FwRules, tcp FWRules and non-tcp FWRules. + """ + fw_rules = None + fw_rules_tcp = None + fw_rules_non_tcp = None + exclude_ipv6 = self.output_config.excludeIPv6Range + connections = defaultdict(list) + peers_to_compare = self.config.peer_container.get_all_peers_group(include_dns_entries=True) + ref_ip_blocks = IpBlock.disjoint_ip_blocks(self.config.get_referenced_ip_blocks(exclude_ipv6), + IpBlock.get_all_ips_block_peer_set(exclude_ipv6), exclude_ipv6) + peers_to_compare |= ref_ip_blocks + peers = PeerSet() + for peer1 in peers_to_compare: + for peer2 in peers_to_compare: + if self.is_in_subset(peer1): + peers.add(peer1) + elif not self.is_in_subset(peer2): + continue # skipping pairs if none of them are in the given subset + if not self.determine_whether_to_compute_allowed_conns_for_peer_types(peer1, peer2): + continue + if peer1 == peer2: + # cannot restrict pod's connection to itself + connections[ConnectionSet(True)].append((peer1, peer2)) + else: + conns, _, _, _ = self.config.allowed_connections(peer1, peer2) + if conns: + connections[conns].append((peer1, peer2)) + # collect both peers, even if one of them is not in the subset + peers.add(peer1) + peers.add(peer2) + # if Istio is a layer in the network config - produce 2 maps, for TCP and for non-TCP + # because Istio policies can only capture TCP connectivity + if self.config.policies_container.layers.does_contain_layer(NetworkLayerName.Istio): + output_res, fw_rules_tcp, fw_rules_non_tcp = \ + self.get_connectivity_output_split_by_tcp(connections, peers, peers_to_compare) + else: + output_res, fw_rules = self.get_connectivity_output_full(connections, peers, peers_to_compare) + return output_res, fw_rules, fw_rules_tcp, fw_rules_non_tcp + + def compute_connectivity_output_optimized(self): + """ + Compute connectivity output with optimized implementation. + :return: a tuple of output result (in a required format), FwRules, tcp FWRules and non-tcp FWRules. + """ + opt_fw_rules = None + opt_fw_rules_tcp = None + opt_fw_rules_non_tcp = None + exclude_ipv6 = self.output_config.excludeIPv6Range + opt_conns = self.config.allowed_connections_optimized() + all_conns_opt = opt_conns.all_allowed_conns + opt_peers_to_compare = self.config.peer_container.get_all_peers_group(include_dns_entries=True) + # add all relevant IpBlocks, used in connections + opt_peers_to_compare |= all_conns_opt.project_on_one_dimension('src_peers') | \ + all_conns_opt.project_on_one_dimension('dst_peers') + subset_peers = self.compute_subset(opt_peers_to_compare) + src_peers_conn_cube = ConnectivityCube.make_from_dict({"src_peers": subset_peers}) + dst_peers_conn_cube = ConnectivityCube.make_from_dict({"dst_peers": subset_peers}) + subset_conns = ConnectivityProperties.make_conn_props(src_peers_conn_cube) | \ + ConnectivityProperties.make_conn_props(dst_peers_conn_cube) + all_conns_opt &= subset_conns + all_conns_opt = self.filter_conns_by_peer_types(all_conns_opt, opt_peers_to_compare) + ip_blocks_mask = IpBlock.get_all_ips_block() + if exclude_ipv6: + ip_blocks_mask = IpBlock.get_all_ips_block(exclude_ipv6=True) + ref_ip_blocks = self.config.get_referenced_ip_blocks(exclude_ipv6) + for ip_block in ref_ip_blocks: + ip_blocks_mask |= ip_block + opt_peers_to_compare.filter_ipv6_blocks(ip_blocks_mask) + if self.config.policies_container.layers.does_contain_layer(NetworkLayerName.Istio): + output_res, opt_fw_rules_tcp, opt_fw_rules_non_tcp = \ + self.get_props_output_split_by_tcp(all_conns_opt, opt_peers_to_compare, ip_blocks_mask) + else: + output_res, opt_fw_rules = self.get_props_output_full(all_conns_opt, opt_peers_to_compare, + ip_blocks_mask) + return output_res, opt_fw_rules, opt_fw_rules_tcp, opt_fw_rules_non_tcp + + def exec(self): self.output_config.fullExplanation = True # assign true for this query - it is always ok to compare its results self.output_config.configName = os.path.basename(self.config.name) if self.config.name.startswith('./') else \ self.config.name - connections = defaultdict(list) res = QueryAnswer(True) fw_rules = None fw_rules_tcp = None fw_rules_non_tcp = None - exclude_ipv6 = self.output_config.excludeIPv6Range if self.config.optimized_run != 'true': - peers_to_compare = self.config.peer_container.get_all_peers_group(include_dns_entries=True) - ref_ip_blocks = IpBlock.disjoint_ip_blocks(self.config.get_referenced_ip_blocks(exclude_ipv6), - IpBlock.get_all_ips_block_peer_set(exclude_ipv6), - exclude_ipv6) - peers_to_compare |= ref_ip_blocks - peers = PeerSet() - peers1_start = time.time() - for peer1_cnt, peer1 in enumerate(peers_to_compare): - for peer2 in peers_to_compare: - if self.is_in_subset(peer1): - peers.add(peer1) - elif not self.is_in_subset(peer2): - continue # skipping pairs if none of them are in the given subset - if not self.determine_whether_to_compute_allowed_conns_for_peer_types(peer1, peer2): - continue - if peer1 == peer2: - # cannot restrict pod's connection to itself - connections[ConnectionSet(True)].append((peer1, peer2)) - else: - conns, _, _, _ = self.config.allowed_connections(peer1, peer2) - if conns: - connections[conns].append((peer1, peer2)) - # collect both peers, even if one of them is not in the subset - peers.add(peer1) - peers.add(peer2) - # if Istio is a layer in the network config - produce 2 maps, for TCP and for non-TCP - # because Istio policies can only capture TCP connectivity - if self.config.policies_container.layers.does_contain_layer(NetworkLayerName.Istio): - output_res, fw_rules_tcp, fw_rules_non_tcp = \ - self.get_connectivity_output_split_by_tcp(connections, peers, peers_to_compare) - else: - output_res, fw_rules = self.get_connectivity_output_full(connections, peers, peers_to_compare) - peers1_end = time.time() - print(f'Original loop: time: {(peers1_end - peers1_start):6.2f} seconds') + orig_start = time.time() + output_res, fw_rules, fw_rules_tcp, fw_rules_non_tcp = self.compute_connectivity_output_original() + orig_end = time.time() + print(f'Original loop: time: {(orig_end - orig_start):6.2f} seconds') if self.output_config.outputFormat in ['json', 'yaml']: res.output_explanation = [ComputedExplanation(dict_explanation=output_res)] else: res.output_explanation = [ComputedExplanation(str_explanation=output_res)] - all_conns_opt = ConnectivityProperties.make_empty_props() - opt_start = time.time() if self.config.optimized_run != 'false': - opt_conns = self.config.allowed_connections_optimized() - all_conns_opt = opt_conns.all_allowed_conns - if all_conns_opt: - opt_peers_to_compare = self.config.peer_container.get_all_peers_group(include_dns_entries=True) - # add all relevant IpBlocks, used in connections - opt_peers_to_compare |= all_conns_opt.project_on_one_dimension('src_peers') | \ - all_conns_opt.project_on_one_dimension('dst_peers') - - subset_peers = self.compute_subset(opt_peers_to_compare) - src_peers_conn_cube = ConnectivityCube.make_from_dict({"src_peers": subset_peers}) - dst_peers_conn_cube = ConnectivityCube.make_from_dict({"dst_peers": subset_peers}) - subset_conns = ConnectivityProperties.make_conn_props(src_peers_conn_cube) | \ - ConnectivityProperties.make_conn_props(dst_peers_conn_cube) - all_conns_opt &= subset_conns - all_conns_opt = self.filter_conns_by_peer_types(all_conns_opt, opt_peers_to_compare) - ip_blocks_mask = IpBlock.get_all_ips_block() - if exclude_ipv6: - ip_blocks_mask = IpBlock.get_all_ips_block(exclude_ipv6=True) - ref_ip_blocks = self.config.get_referenced_ip_blocks(exclude_ipv6) - for ip_block in ref_ip_blocks: - ip_blocks_mask |= ip_block - opt_peers_to_compare.filter_ipv6_blocks(ip_blocks_mask) - - if self.config.policies_container.layers.does_contain_layer(NetworkLayerName.Istio): - output_res, opt_fw_rules_tcp, opt_fw_rules_non_tcp = \ - self.get_props_output_split_by_tcp(all_conns_opt, opt_peers_to_compare, ip_blocks_mask) - opt_end = time.time() - print(f'Opt time: {(opt_end - opt_start):6.2f} seconds') - if self.config.optimized_run == 'debug': - if fw_rules_tcp and fw_rules_tcp.fw_rules_map and \ - opt_fw_rules_tcp and opt_fw_rules_tcp.fw_rules_map: - self.compare_fw_rules(fw_rules_tcp, opt_fw_rules_tcp) - if fw_rules_non_tcp and fw_rules_non_tcp.fw_rules_map and \ - opt_fw_rules_non_tcp and opt_fw_rules_non_tcp.fw_rules_map: - self.compare_fw_rules(fw_rules_non_tcp, opt_fw_rules_non_tcp) - else: - output_res, opt_fw_rules = self.get_props_output_full(all_conns_opt, opt_peers_to_compare, - ip_blocks_mask) - opt_end = time.time() - print(f'Opt time: {(opt_end - opt_start):6.2f} seconds') - if self.config.optimized_run == 'debug' and fw_rules and fw_rules.fw_rules_map and \ - opt_fw_rules and opt_fw_rules.fw_rules_map: + opt_start = time.time() + output_res, opt_fw_rules, opt_fw_rules_tcp, opt_fw_rules_non_tcp = \ + self.compute_connectivity_output_optimized() + opt_end = time.time() + print(f'Opt time: {(opt_end - opt_start):6.2f} seconds') + if self.config.optimized_run == 'debug': + if fw_rules and fw_rules.fw_rules_map and opt_fw_rules and opt_fw_rules.fw_rules_map: self.compare_fw_rules(fw_rules, opt_fw_rules) - if self.config.optimized_run == 'true': + if fw_rules_tcp and fw_rules_tcp.fw_rules_map and \ + opt_fw_rules_tcp and opt_fw_rules_tcp.fw_rules_map: + self.compare_fw_rules(fw_rules_tcp, opt_fw_rules_tcp) + if fw_rules_non_tcp and fw_rules_non_tcp.fw_rules_map and \ + opt_fw_rules_non_tcp and opt_fw_rules_non_tcp.fw_rules_map: + self.compare_fw_rules(fw_rules_non_tcp, opt_fw_rules_non_tcp) + else: # self.config.optimized_run == 'true': if self.output_config.outputFormat in ['json', 'yaml']: res.output_explanation = [ComputedExplanation(dict_explanation=output_res)] else: From ec1235167a7c92149fc9b7ad15eb12e3cf02d9e5 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 18 Apr 2023 17:36:06 +0300 Subject: [PATCH 118/187] Update nca/CoreDS/Peer.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/CoreDS/Peer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index 1ea3412eb..5b81c81d6 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -757,7 +757,7 @@ def get_peer_set_by_indices(self, peer_interval_set): :param peer_interval_set: the interval set of indices :return: the PeerSet of peers referenced by the indices in the interval set """ - peer_list = [] + peer_set = PeerSet() for interval in peer_interval_set: if interval.end <= self.max_ipv4_index: # this is IPv4Address @@ -765,22 +765,22 @@ def get_peer_set_by_indices(self, peer_interval_set): end = ipaddress.IPv4Address(interval.end - self.min_ipv4_index) ipb = IpBlock( interval=CanonicalIntervalSet.Interval(IPNetworkAddress(start), IPNetworkAddress(end))) - peer_list.append(ipb) + peer_set.add(ipb) elif interval.end <= self.max_ipv6_index: # this is IPv6Address start = ipaddress.IPv6Address(interval.start - self.min_ipv6_index) end = ipaddress.IPv6Address(interval.end - self.min_ipv6_index) ipb = IpBlock( interval=CanonicalIntervalSet.Interval(IPNetworkAddress(start), IPNetworkAddress(end))) - peer_list.append(ipb) + peer_set.add(ipb) else: # this is Pod assert interval.end <= self.max_pod_index curr_pods_max_ind = len(self.ordered_peer_list) - 1 for ind in range(min(interval.start - self.min_pod_index, curr_pods_max_ind), - min(interval.end - self.min_pod_index, curr_pods_max_ind) + 1): - peer_list.append(self.ordered_peer_list[ind]) - return PeerSet(set(peer_list)) + min(interval.end - self.min_pod_index, curr_pods_max_ind) + 1): + peer_set.add(self.ordered_peer_list[ind]) + return peer_set instance = None From 9c8ffca70f269899ec695e1370813b990e79d415 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 18 Apr 2023 17:42:29 +0300 Subject: [PATCH 119/187] Update nca/CoreDS/ConnectionSet.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/CoreDS/ConnectionSet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index b2bc6bc73..97b909421 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -161,7 +161,7 @@ def _get_protocol_with_properties_representation(is_str, protocol_text, properti """ :param bool is_str: should get str representation (True) or list representation (False) :param str protocol_text: str description of protocol - :param Union[bool, ConnectivityProperties, ICMPDataSet] properties: properties object of the protocol + :param Union[bool, ConnectivityProperties] properties: properties object of the protocol :return: representation required for a given pair of protocol and its properties :rtype: Union[dict, str] """ From 310a81dab7fdbea313f78d13dab008f842a47bf0 Mon Sep 17 00:00:00 2001 From: Tanya Date: Tue, 18 Apr 2023 17:42:52 +0300 Subject: [PATCH 120/187] Update nca/CoreDS/ConnectionSet.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/CoreDS/ConnectionSet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 97b909421..b7cf4078a 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -404,7 +404,7 @@ def add_connections(self, protocol, properties=True): Add connections to the set of connections :param int,str protocol: protocol number of the connections to add :param properties: an object with protocol properties (e.g., ports), if relevant - :type properties: Union[bool, ConnectivityProperties, ICMPDataSet] + :type properties: Union[bool, ConnectivityProperties] :return: None """ if isinstance(protocol, str): From 93a8ffc3e0432a889665d42d1afec0eafdc53bd6 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 23 Apr 2023 10:59:29 +0300 Subject: [PATCH 121/187] Added shortcut function ConnectivityProperties.make_conn_props_from_dict. Moved BasePeerSet.reset() to run_args. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 13 +++-- nca/CoreDS/ConnectivityProperties.py | 17 +++--- nca/NetworkConfig/NetworkConfig.py | 6 +-- nca/NetworkConfig/NetworkConfigQuery.py | 21 ++++---- nca/NetworkConfig/NetworkLayer.py | 53 +++++++++++-------- nca/Parsers/CalicoPolicyYamlParser.py | 16 +++--- nca/Parsers/IngressPolicyYamlParser.py | 4 +- nca/Parsers/IstioPolicyYamlParser.py | 12 ++--- .../IstioTrafficResourcesYamlParser.py | 6 +-- nca/Parsers/K8sPolicyYamlParser.py | 4 +- nca/Resources/IstioSidecar.py | 23 ++++---- nca/nca_cli.py | 4 +- .../testConnectivityPropertiesNamedPorts.py | 8 +-- 13 files changed, 101 insertions(+), 86 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index b2bc6bc73..e42f1103f 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -544,14 +544,19 @@ def print_diff(self, other, self_name, other_name): return 'No diff.' def convert_to_connectivity_properties(self): + """ + Convert the current ConnectionSet to ConnectivityProperties format. + This function is used for comparing fw-rules output between original and optimized implementation, + when optimized_run == 'debug' + :return: the connection set in ConnectivityProperties format + """ if self.allow_all: return ConnectivityProperties.make_all_props() res = ConnectivityProperties.make_empty_props() for protocol, properties in self.allowed_protocols.items(): protocols = ProtocolSet.get_protocol_set_with_single_protocol(protocol) - conn_cube = ConnectivityCube.make_from_dict({"protocols": protocols}) - this_prop = ConnectivityProperties.make_conn_props(conn_cube) + this_prop = ConnectivityProperties.make_conn_props_from_dict({"protocols": protocols}) if isinstance(properties, bool): if properties: res |= this_prop @@ -673,7 +678,7 @@ def fw_rules_to_conn_props(fw_rules): conn_props = fw_rule.conn.convert_to_connectivity_properties() src_peers = PeerSet(fw_rule.src.get_peer_set(fw_rules.cluster_info)) dst_peers = PeerSet(fw_rule.dst.get_peer_set(fw_rules.cluster_info)) - conn_cube = ConnectivityCube.make_from_dict({"src_peers": src_peers, "dst_peers": dst_peers}) - rule_props = ConnectivityProperties.make_conn_props(conn_cube) & conn_props + rule_props = ConnectivityProperties.make_conn_props_from_dict({"src_peers": src_peers, + "dst_peers": dst_peers}) & conn_props res |= rule_props return res diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 2ca2024b8..5b5c9ff08 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -76,7 +76,7 @@ def __init__(self, create_all=False): self.set_all() @staticmethod - def create_props_from_cube(conn_cube): + def _create_props_from_cube(conn_cube): """ This will create connectivity properties made of the given connectivity cube. This includes tcp properties, non-tcp properties, icmp data properties. @@ -295,7 +295,7 @@ def copy(self): """ :rtype: ConnectivityProperties """ - res = ConnectivityProperties.create_props_from_cube(ConnectivityCube()) + res = ConnectivityProperties._create_props_from_cube(ConnectivityCube()) for layer in self.layers: res.layers[self._copy_layer_elem(layer)] = self.layers[layer].copy() res.active_dimensions = self.active_dimensions.copy() @@ -400,14 +400,14 @@ def make_conn_props(conn_cube): assert not src_ports.named_ports and not src_ports.excluded_named_ports if (not dst_ports.named_ports and not dst_ports.excluded_named_ports) or not dst_peers: # Should not resolve named ports - return ConnectivityProperties.create_props_from_cube(conn_cube) + return ConnectivityProperties._create_props_from_cube(conn_cube) # Initialize conn_properties if dst_ports.port_set: dst_ports_no_named_ports = PortSet() dst_ports_no_named_ports.port_set = dst_ports.port_set.copy() conn_cube["dst_ports"] = dst_ports_no_named_ports - conn_properties = ConnectivityProperties.create_props_from_cube(conn_cube) + conn_properties = ConnectivityProperties._create_props_from_cube(conn_cube) else: conn_properties = ConnectivityProperties.make_empty_props() @@ -418,13 +418,18 @@ def make_conn_props(conn_cube): real_ports = ConnectivityProperties.resolve_named_ports(dst_ports.named_ports, peer, protocols) if real_ports: conn_cube.update({"dst_ports": real_ports, "dst_peers": PeerSet({peer})}) - conn_properties |= ConnectivityProperties.create_props_from_cube(conn_cube) + conn_properties |= ConnectivityProperties._create_props_from_cube(conn_cube) excluded_real_ports = ConnectivityProperties.resolve_named_ports(dst_ports.excluded_named_ports, peer, protocols) if excluded_real_ports: conn_cube.update({"dst_ports": excluded_real_ports, "dst_peers": PeerSet({peer})}) - conn_properties -= ConnectivityProperties.create_props_from_cube(conn_cube) + conn_properties -= ConnectivityProperties._create_props_from_cube(conn_cube) return conn_properties + @staticmethod + def make_conn_props_from_dict(the_dict): + cube = ConnectivityCube.make_from_dict(the_dict) + return ConnectivityProperties.make_conn_props(cube) + @staticmethod def make_empty_props(): """ diff --git a/nca/NetworkConfig/NetworkConfig.py b/nca/NetworkConfig/NetworkConfig.py index 77334fb34..a4d2ca45c 100644 --- a/nca/NetworkConfig/NetworkConfig.py +++ b/nca/NetworkConfig/NetworkConfig.py @@ -284,11 +284,9 @@ def allowed_connections_optimized(self, layer_name=None): all_peers = self.peer_container.get_all_peers_group() host_eps = Peer.PeerSet(set([peer for peer in all_peers if isinstance(peer, Peer.HostEP)])) - src_peers_conn_cube = ConnectivityCube.make_from_dict({"src_peers": host_eps}) - dst_peers_conn_cube = ConnectivityCube.make_from_dict({"dst_peers": host_eps}) # all possible connections involving hostEndpoints - conn_hep = ConnectivityProperties.make_conn_props(src_peers_conn_cube) | \ - ConnectivityProperties.make_conn_props(dst_peers_conn_cube) + conn_hep = ConnectivityProperties.make_conn_props_from_dict({"src_peers": host_eps}) | \ + ConnectivityProperties.make_conn_props_from_dict({"dst_peers": host_eps}) conns_res = OptimizedPolicyConnections() conns_res.all_allowed_conns = ConnectivityProperties.make_all_props() for layer, layer_obj in self.policies_container.layers.items(): diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 4ea54489f..947678e17 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -172,17 +172,17 @@ def filter_conns_by_peer_types(self, conns, all_peers): # avoid IpBlock -> {IpBlock, DNSEntry} connections all_ips = IpBlock.get_all_ips_block_peer_set() all_dns_entries = self.config.peer_container.get_all_dns_entries() - conn_cube = ConnectivityCube.make_from_dict({"src_peers": all_ips, "dst_peers": all_ips | all_dns_entries}) - ip_to_ip_or_dns_conns = ConnectivityProperties.make_conn_props(conn_cube) + ip_to_ip_or_dns_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_ips, + "dst_peers": all_ips | all_dns_entries}) res -= ip_to_ip_or_dns_conns # avoid DNSEntry->anything connections - conn_cube.update({"src_peers": all_dns_entries, "dst_peers": all_peers}) - dns_to_any_conns = ConnectivityProperties.make_conn_props(conn_cube) + dns_to_any_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_dns_entries, + "dst_peers": all_peers}) res -= dns_to_any_conns # avoid anything->DNSEntry connections if Istio layer does not exist if not self.config.policies_container.layers.does_contain_layer(NetworkLayerName.Istio): - conn_cube.update({"src_peers": all_peers, "dst_peers": all_dns_entries}) - any_to_dns_conns = ConnectivityProperties.make_conn_props(conn_cube) + any_to_dns_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers, + "dst_peers": all_dns_entries}) res -= any_to_dns_conns return res @@ -789,10 +789,8 @@ def compute_connectivity_output_optimized(self): opt_peers_to_compare |= all_conns_opt.project_on_one_dimension('src_peers') | \ all_conns_opt.project_on_one_dimension('dst_peers') subset_peers = self.compute_subset(opt_peers_to_compare) - src_peers_conn_cube = ConnectivityCube.make_from_dict({"src_peers": subset_peers}) - dst_peers_conn_cube = ConnectivityCube.make_from_dict({"dst_peers": subset_peers}) - subset_conns = ConnectivityProperties.make_conn_props(src_peers_conn_cube) | \ - ConnectivityProperties.make_conn_props(dst_peers_conn_cube) + subset_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": subset_peers}) | \ + ConnectivityProperties.make_conn_props_from_dict({"dst_peers": subset_peers}) all_conns_opt &= subset_conns all_conns_opt = self.filter_conns_by_peer_types(all_conns_opt, opt_peers_to_compare) ip_blocks_mask = IpBlock.get_all_ips_block() @@ -1108,8 +1106,7 @@ def convert_props_to_split_by_tcp(props): :rtype: tuple(ConnectivityProperties, ConnectivityProperties) """ tcp_protocol = ProtocolSet.get_protocol_set_with_single_protocol('TCP') - conn_cube = ConnectivityCube.make_from_dict({"protocols": tcp_protocol}) - tcp_props = props & ConnectivityProperties.make_conn_props(conn_cube) + tcp_props = props & ConnectivityProperties.make_conn_props_from_dict({"protocols": tcp_protocol}) non_tcp_props = props - tcp_props return tcp_props, non_tcp_props diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index dbb2deb6c..3576d8ff8 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -181,11 +181,13 @@ def allowed_connections_optimized(self, peer_container): all_pods_peer_set = peer_container.get_all_peers_group() all_ips_peer_set = IpBlock.get_all_ips_block_peer_set() # for ingress, all possible connections to IpBlocks are allowed - conn_cube = ConnectivityCube.make_from_dict({"src_peers": all_pods_peer_set, "dst_peers": all_ips_peer_set}) - ingress_conns.all_allowed_conns |= ConnectivityProperties.make_conn_props(conn_cube) + ingress_conns.all_allowed_conns |= \ + ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_pods_peer_set, + "dst_peers": all_ips_peer_set}) # for egress, all possible connections from IpBlocks are allowed - conn_cube.update({"src_peers": all_ips_peer_set, "dst_peers": all_pods_peer_set}) - egress_conns.all_allowed_conns |= ConnectivityProperties.make_conn_props(conn_cube) + egress_conns.all_allowed_conns |= \ + ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_ips_peer_set, + "dst_peers": all_pods_peer_set}) res_conns.captured = ingress_conns.captured | egress_conns.captured res_conns.denied_conns = ingress_conns.denied_conns | egress_conns.denied_conns res_conns.all_allowed_conns = ingress_conns.all_allowed_conns & egress_conns.all_allowed_conns @@ -350,29 +352,35 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): all_peers_and_ips = peer_container.get_all_peers_group(True) dns_entries = peer_container.get_all_dns_entries() # for istio initialize non-captured conns with non-TCP connections - conn_cube = ConnectivityCube.make_from_dict({"src_peers": all_peers_and_ips, "dst_peers": all_peers_and_ips, - "protocols": ProtocolSet.get_non_tcp_protocols()}) - res_conns.all_allowed_conns |= res_conns.allowed_conns | ConnectivityProperties.make_conn_props(conn_cube) + all_all_conns = \ + ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers_and_ips, + "dst_peers": all_peers_and_ips, + "protocols": ProtocolSet.get_non_tcp_protocols()}) + res_conns.all_allowed_conns |= res_conns.allowed_conns | all_all_conns non_captured_peers = all_peers_and_ips - res_conns.captured if non_captured_peers: protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') if is_ingress: - conn_cube = ConnectivityCube.make_from_dict({"src_peers": all_peers_and_ips, "dst_peers": non_captured_peers}) - res_conns.all_allowed_conns |= ConnectivityProperties.make_conn_props(conn_cube) - res_conns.denied_conns + all_nc_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers_and_ips, + "dst_peers": non_captured_peers}) + res_conns.all_allowed_conns |= all_nc_conns - res_conns.denied_conns non_captured_dns_entries = dns_entries - res_conns.captured if non_captured_dns_entries: # update allowed non-captured conns to DNSEntry dst with TCP only - conn_cube = ConnectivityCube.make_from_dict({"src_peers": all_peers_and_ips, - "dst_peers": non_captured_dns_entries, - "protocols": protocols}) - res_conns.all_allowed_conns |= ConnectivityProperties.make_conn_props(conn_cube) + all_nc_dns_conns = \ + ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers_and_ips, + "dst_peers": non_captured_dns_entries, + "protocols": protocols}) + res_conns.all_allowed_conns |= all_nc_dns_conns else: - conn_cube = ConnectivityCube.make_from_dict({"src_peers": non_captured_peers, "dst_peers": all_peers_and_ips}) - res_conns.all_allowed_conns |= ConnectivityProperties.make_conn_props(conn_cube) - res_conns.denied_conns + nc_all_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": non_captured_peers, + "dst_peers": all_peers_and_ips}) + res_conns.all_allowed_conns |= nc_all_conns - res_conns.denied_conns # update allowed non-captured conns to DNSEntry dst with TCP only - conn_cube = ConnectivityCube.make_from_dict({"src_peers": non_captured_peers, "dst_peers": dns_entries, - "protocols": protocols}) - res_conns.all_allowed_conns |= ConnectivityProperties.make_conn_props(conn_cube) + nc_dns_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": non_captured_peers, + "dst_peers": dns_entries, + "protocols": protocols}) + res_conns.all_allowed_conns |= nc_dns_conns return res_conns @@ -392,17 +400,16 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): res_conns = OptimizedPolicyConnections() all_peers_and_ips = peer_container.get_all_peers_group(True) - conn_cube = ConnectivityCube() if is_ingress: # everything is allowed and non captured - conn_cube.update({"src_peers": all_peers_and_ips, "dst_peers": all_peers_and_ips}) - non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) + non_captured_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers_and_ips, + "dst_peers": all_peers_and_ips}) res_conns.all_allowed_conns = non_captured_conns else: res_conns = self.collect_policies_conns_optimized(is_ingress) non_captured_peers = all_peers_and_ips - res_conns.captured if non_captured_peers: - conn_cube.update({"src_peers": non_captured_peers, "dst_peers": all_peers_and_ips}) - non_captured_conns = ConnectivityProperties.make_conn_props(conn_cube) + non_captured_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": non_captured_peers, + "dst_peers": all_peers_and_ips}) res_conns.all_allowed_conns = res_conns.allowed_conns | non_captured_conns return res_conns diff --git a/nca/Parsers/CalicoPolicyYamlParser.py b/nca/Parsers/CalicoPolicyYamlParser.py index c65196913..95a385376 100644 --- a/nca/Parsers/CalicoPolicyYamlParser.py +++ b/nca/Parsers/CalicoPolicyYamlParser.py @@ -534,23 +534,23 @@ def _parse_xgress_rule(self, rule, is_ingress, policy_selected_eps, is_profile): else: connections.add_connections(protocol, True) if self.optimized_run != 'false': - conn_cube = ConnectivityCube.make_from_dict({"protocols": protocols, "src_peers": src_res_pods, - "dst_peers": dst_res_pods}) - conn_props = ConnectivityProperties.make_conn_props(conn_cube) + conn_props = ConnectivityProperties.make_conn_props_from_dict({"protocols": protocols, + "src_peers": src_res_pods, + "dst_peers": dst_res_pods}) elif not_protocol is not None: connections.add_all_connections() connections.remove_protocol(not_protocol) if self.optimized_run != 'false' and src_res_pods and dst_res_pods: protocols = ProtocolSet(True) protocols.remove_protocol(not_protocol) - conn_cube = ConnectivityCube.make_from_dict({"protocols": protocols, "src_peers": src_res_pods, - "dst_peers": dst_res_pods}) - conn_props = ConnectivityProperties.make_conn_props(conn_cube) + conn_props = ConnectivityProperties.make_conn_props_from_dict({"protocols": protocols, + "src_peers": src_res_pods, + "dst_peers": dst_res_pods}) else: connections.allow_all = True if self.optimized_run != 'false': - conn_cube = ConnectivityCube.make_from_dict({"src_peers": src_res_pods, "dst_peers": dst_res_pods}) - conn_props = ConnectivityProperties.make_conn_props(conn_cube) + conn_props = ConnectivityProperties.make_conn_props_from_dict({"src_peers": src_res_pods, + "dst_peers": dst_res_pods}) self._verify_named_ports(rule, dst_res_pods, connections) if not src_res_pods and policy_selected_eps and (is_ingress or not is_profile): diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index f740728c7..eea7a519c 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -288,8 +288,8 @@ def parse_policy(self): if allowed_conns: res_policy.add_rules(self._make_allow_rules(allowed_conns)) protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') - conn_cube = ConnectivityCube.make_from_dict({"protocols": protocols, "src_peers": res_policy.selected_peers}) - allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) + allowed_conns &= ConnectivityProperties.make_conn_props_from_dict({"protocols": protocols, + "src_peers": res_policy.selected_peers}) res_policy.add_optimized_allow_props(allowed_conns, False) res_policy.findings = self.warning_msgs return res_policy diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index 3d2e5b5d3..9aeaa0eed 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -196,8 +196,7 @@ def parse_key_values(self, key, values, not_values): return self.parse_principals(values, not_values) # PeerSet elif key == 'destination.port': dst_ports = self.get_rule_ports(values, not_values) # PortSet - conn_cube = ConnectivityCube.make_from_dict({"dst_ports": dst_ports}) - return ConnectivityProperties.make_conn_props(conn_cube) # ConnectivityProperties + return ConnectivityProperties.make_conn_props_from_dict({"dst_ports": dst_ports}) return NotImplemented, False def parse_condition(self, condition): @@ -399,9 +398,8 @@ def parse_operation(self, operation_dict): operation) hosts_dfa = self.parse_regex_dimension_values("hosts", operation.get("hosts"), operation.get("notHosts"), operation) - conn_cube = ConnectivityCube.make_from_dict({"dst_ports": dst_ports, "methods": methods_set, "paths": paths_dfa, - 'hosts': hosts_dfa}) - return ConnectivityProperties.make_conn_props(conn_cube) + return ConnectivityProperties.make_conn_props_from_dict({"dst_ports": dst_ports, "methods": methods_set, + "paths": paths_dfa, "hosts": hosts_dfa}) def parse_source(self, source_dict): """ @@ -522,8 +520,8 @@ def parse_ingress_rule(self, rule, selected_peers): if not res_peers or not selected_peers: condition_props = ConnectivityProperties.make_empty_props() else: - conn_cube = ConnectivityCube.make_from_dict({"src_peers": res_peers, "dst_peers": selected_peers}) - condition_props &= ConnectivityProperties.make_conn_props(conn_cube) + condition_props &= ConnectivityProperties.make_conn_props_from_dict({"src_peers": res_peers, + "dst_peers": selected_peers}) connections &= condition_conns conn_props &= condition_props return IstioPolicyRule(res_peers, connections), conn_props diff --git a/nca/Parsers/IstioTrafficResourcesYamlParser.py b/nca/Parsers/IstioTrafficResourcesYamlParser.py index 15114d1b0..a6364faca 100644 --- a/nca/Parsers/IstioTrafficResourcesYamlParser.py +++ b/nca/Parsers/IstioTrafficResourcesYamlParser.py @@ -398,9 +398,9 @@ def create_istio_traffic_policies(self): if allowed_conns: res_policy.add_rules(self._make_allow_rules(allowed_conns)) protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') - conn_cube = ConnectivityCube.make_from_dict({"protocols": protocols, - "src_peers": res_policy.selected_peers}) - allowed_conns &= ConnectivityProperties.make_conn_props(conn_cube) + allowed_conns &= \ + ConnectivityProperties.make_conn_props_from_dict({"protocols": protocols, + "src_peers": res_policy.selected_peers}) res_policy.add_optimized_allow_props(allowed_conns, False) res_policy.findings = self.warning_msgs vs_policies.append(res_policy) diff --git a/nca/Parsers/K8sPolicyYamlParser.py b/nca/Parsers/K8sPolicyYamlParser.py index 4046a7d59..92e1fb692 100644 --- a/nca/Parsers/K8sPolicyYamlParser.py +++ b/nca/Parsers/K8sPolicyYamlParser.py @@ -350,8 +350,8 @@ def parse_ingress_egress_rule(self, rule, peer_array_key, policy_selected_pods): else: res_conns = ConnectionSet(True) if self.optimized_run != 'false': - conn_cube = ConnectivityCube.make_from_dict({"src_peers": src_pods, "dst_peers": dst_pods}) - res_opt_props = ConnectivityProperties.make_conn_props(conn_cube) + res_opt_props = ConnectivityProperties.make_conn_props_from_dict({"src_peers": src_pods, + "dst_peers": dst_pods}) if not res_pods: self.warning('Rule selects no pods', rule) diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index 16a5859f2..637dfe053 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -155,24 +155,27 @@ def create_opt_egress_props(self, peer_container): # connections to IP-block is enabled only if the outbound mode is allow-any (disabled for registry only) if self.outbound_mode == IstioSidecar.OutboundMode.ALLOW_ANY: ip_blocks = IpBlock.get_all_ips_block_peer_set() - conn_cube = ConnectivityCube.make_from_dict({"src_peers": self.selected_peers, "dst_peers": ip_blocks}) - self.optimized_allow_egress_props |= ConnectivityProperties.make_conn_props(conn_cube) + self.optimized_allow_egress_props |= \ + ConnectivityProperties.make_conn_props_from_dict({"src_peers": self.selected_peers, + "dst_peers": ip_blocks}) dns_entries = peer_container.get_all_dns_entries() dst_dns_entries = dns_entries & (rule.egress_peer_set | rule.special_egress_peer_set) if self.selected_peers and dst_dns_entries: protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') - conn_cube = ConnectivityCube.make_from_dict({"src_peers": self.selected_peers, - "dst_peers": dst_dns_entries, "protocols": protocols}) - self.optimized_allow_egress_props |= ConnectivityProperties.make_conn_props(conn_cube) + self.optimized_allow_egress_props |= \ + ConnectivityProperties.make_conn_props_from_dict({"src_peers": self.selected_peers, + "dst_peers": dst_dns_entries, + "protocols": protocols}) if self.selected_peers and rule.egress_peer_set: - conn_cube = ConnectivityCube.make_from_dict({"src_peers": self.selected_peers, - "dst_peers": rule.egress_peer_set}) - self.optimized_allow_egress_props |= ConnectivityProperties.make_conn_props(conn_cube) + self.optimized_allow_egress_props |= \ + ConnectivityProperties.make_conn_props_from_dict({"src_peers": self.selected_peers, + "dst_peers": rule.egress_peer_set}) peers_sets_by_ns = self.combine_peer_sets_by_ns(self.selected_peers, rule.special_egress_peer_set, peer_container) for (from_peers, to_peers) in peers_sets_by_ns: if from_peers and to_peers: - conn_cube = ConnectivityCube.make_from_dict({"src_peers": from_peers, "dst_peers": to_peers}) - self.optimized_allow_egress_props |= ConnectivityProperties.make_conn_props(conn_cube) + self.optimized_allow_egress_props |= \ + ConnectivityProperties.make_conn_props_from_dict({"src_peers": from_peers, + "dst_peers": to_peers}) diff --git a/nca/nca_cli.py b/nca/nca_cli.py index 7c0ec9e55..1a40be3b1 100644 --- a/nca/nca_cli.py +++ b/nca/nca_cli.py @@ -141,6 +141,9 @@ def run_args(args): the query result (if command-line queries are used) :rtype: int """ + # reset the singleton before running a new shceme or cli query + # so that configs from certain run do not affect a potential following run. + BasePeerSet.reset() if args.scheme: return SchemeRunner(args.scheme, args.output_format, args.file_out, args.optimized_run).run_scheme() ns_list = args.ns_list @@ -246,7 +249,6 @@ def nca_main(argv=None): :rtype: int """ os.environ['PATH'] = '.' + os.pathsep + os.environ.get('PATH', '.') # for running kubectl and calicoctl - BasePeerSet.reset() # to reset the singleton parser = argparse.ArgumentParser(description='An analyzer for network connectivity configuration') parser.add_argument('--period', type=int, diff --git a/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py b/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py index 0325fb03c..9c780ed5f 100644 --- a/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py +++ b/tests/classes_unit_tests/testConnectivityPropertiesNamedPorts.py @@ -14,11 +14,11 @@ def test_k8s_flow(self): dst_res_ports = PortSet() dst_res_ports.add_port("x") conn_cube = ConnectivityCube.make_from_dict({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) - tcp_properties1 = ConnectivityProperties.create_props_from_cube(conn_cube) + tcp_properties1 = ConnectivityProperties.make_conn_props(conn_cube) dst_res_ports2 = PortSet() dst_res_ports2.add_port("y") conn_cube["dst_ports"] = dst_res_ports2 - tcp_properties2 = ConnectivityProperties.create_props_from_cube(conn_cube) + tcp_properties2 = ConnectivityProperties.make_conn_props(conn_cube) tcp_properties_res = tcp_properties1 | tcp_properties2 named_ports_dict = {"x": (15, 6), "z": (20, 6), "y": (16, 6)} tcp_properties_res.convert_named_ports(named_ports_dict, 6) @@ -39,7 +39,7 @@ def test_calico_flow_1(self): dst_res_ports.add_port("z") dst_res_ports.add_port("w") conn_cube = ConnectivityCube.make_from_dict({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) - tcp_properties = ConnectivityProperties.create_props_from_cube(conn_cube) + tcp_properties = ConnectivityProperties.make_conn_props(conn_cube) tcp_properties_2 = tcp_properties.copy() self.assertTrue(tcp_properties.has_named_ports()) @@ -71,7 +71,7 @@ def test_calico_flow_2(self): dst_res_ports -= not_ports src_res_ports.add_port_range(1, 100) conn_cube = ConnectivityCube.make_from_dict({"src_ports": src_res_ports, "dst_ports": dst_res_ports}) - tcp_properties = ConnectivityProperties.create_props_from_cube(conn_cube) + tcp_properties = ConnectivityProperties.make_conn_props(conn_cube) tcp_properties_2 = tcp_properties.copy() self.assertTrue(tcp_properties.has_named_ports()) From 0cbee6c0aee1bff2baf9c5f155d4a33a37193d9f Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 23 Apr 2023 11:07:28 +0300 Subject: [PATCH 122/187] Fixed lint errors. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 1 - nca/CoreDS/Peer.py | 2 +- nca/NetworkConfig/NetworkConfig.py | 1 - nca/NetworkConfig/NetworkConfigQuery.py | 1 - nca/Parsers/IstioPolicyYamlParser.py | 1 - nca/Resources/IstioSidecar.py | 1 - 6 files changed, 1 insertion(+), 6 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 33c27dc89..7dc0bee2d 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -5,7 +5,6 @@ from collections import defaultdict from .CanonicalIntervalSet import CanonicalIntervalSet -from .ConnectivityCube import ConnectivityCube from .ConnectivityProperties import ConnectivityProperties from .ProtocolNameResolver import ProtocolNameResolver from .ProtocolSet import ProtocolSet diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index 5b81c81d6..64ab06d09 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -778,7 +778,7 @@ def get_peer_set_by_indices(self, peer_interval_set): assert interval.end <= self.max_pod_index curr_pods_max_ind = len(self.ordered_peer_list) - 1 for ind in range(min(interval.start - self.min_pod_index, curr_pods_max_ind), - min(interval.end - self.min_pod_index, curr_pods_max_ind) + 1): + min(interval.end - self.min_pod_index, curr_pods_max_ind) + 1): peer_set.add(self.ordered_peer_list[ind]) return peer_set diff --git a/nca/NetworkConfig/NetworkConfig.py b/nca/NetworkConfig/NetworkConfig.py index a4d2ca45c..beff016f8 100644 --- a/nca/NetworkConfig/NetworkConfig.py +++ b/nca/NetworkConfig/NetworkConfig.py @@ -6,7 +6,6 @@ from dataclasses import dataclass, field from nca.CoreDS import Peer from nca.CoreDS.ConnectionSet import ConnectionSet -from nca.CoreDS.ConnectivityCube import ConnectivityCube from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.Resources.NetworkPolicy import NetworkPolicy, OptimizedPolicyConnections from .NetworkLayer import NetworkLayersContainer, NetworkLayerName diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 947678e17..fe7b26e06 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -12,7 +12,6 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import PeerSet, IpBlock, Pod, Peer, DNSEntry from nca.CoreDS.ProtocolSet import ProtocolSet -from nca.CoreDS.ConnectivityCube import ConnectivityCube from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.FWRules.ConnectivityGraph import ConnectivityGraph from nca.FWRules.MinimizeFWRules import MinimizeFWRules diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index 9aeaa0eed..3e6b659b1 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -10,7 +10,6 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.PortSet import PortSet from nca.CoreDS.MethodSet import MethodSet -from nca.CoreDS.ConnectivityCube import ConnectivityCube from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy, IstioPolicyRule from nca.Resources.IstioTrafficResources import istio_root_namespace diff --git a/nca/Resources/IstioSidecar.py b/nca/Resources/IstioSidecar.py index 637dfe053..cd7fb2aeb 100644 --- a/nca/Resources/IstioSidecar.py +++ b/nca/Resources/IstioSidecar.py @@ -8,7 +8,6 @@ from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.Peer import IpBlock, PeerSet, DNSEntry from nca.CoreDS.ProtocolSet import ProtocolSet -from nca.CoreDS.ConnectivityCube import ConnectivityCube from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from .NetworkPolicy import PolicyConnections, OptimizedPolicyConnections, NetworkPolicy From 2e7010523cf11385777b6ea8b5beb3d7be1cb1c3 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 23 Apr 2023 16:20:12 +0300 Subject: [PATCH 123/187] Code reuse optimization. Added documentation. Removed unused method. Improved ConnectivityProperties.project_on_one_dimension() method. Signed-off-by: Tanya --- nca/CoreDS/ConnectionSet.py | 73 ++++++++++++++----------- nca/CoreDS/ConnectivityProperties.py | 7 ++- nca/FWRules/ConnectivityGraph.py | 43 +-------------- nca/NetworkConfig/NetworkConfigQuery.py | 3 +- 4 files changed, 53 insertions(+), 73 deletions(-) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index 7dc0bee2d..7873f4236 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -578,6 +578,35 @@ def get_non_tcp_connections(): # TODO - after moving to the optimized HC set implementation, # get rid of ConnectionSet and move the code below to ConnectivityProperties.py + + @staticmethod + def get_connection_set_and_peers_from_cube(conn_cube, peer_container, ip_blocks_mask, + relevant_protocols=ProtocolSet(True)): + src_peers = conn_cube["src_peers"] or peer_container.get_all_peers_group(True) + conn_cube.unset_dim("src_peers") + dst_peers = conn_cube["dst_peers"] or peer_container.get_all_peers_group(True) + conn_cube.unset_dim("dst_peers") + if IpBlock.get_all_ips_block() != ip_blocks_mask: + src_peers.filter_ipv6_blocks(ip_blocks_mask) + dst_peers.filter_ipv6_blocks(ip_blocks_mask) + protocols = conn_cube["protocols"] + conn_cube.unset_dim("protocols") + if not conn_cube.has_active_dim() and (protocols.is_whole_range() or protocols == relevant_protocols): + conns = ConnectionSet(True) + else: + conns = ConnectionSet() + assert protocols + protocol_names = ProtocolSet.get_protocol_names_from_interval_set(protocols) + for protocol in protocol_names: + if conn_cube.has_active_dim(): + conns.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) + else: + if ConnectionSet.protocol_supports_ports(protocol) or ConnectionSet.protocol_is_icmp(protocol): + conns.add_connections(protocol, ConnectivityProperties.make_all_props()) + else: + conns.add_connections(protocol, True) + return conns, src_peers, dst_peers + @staticmethod def conn_props_to_fw_rules(conn_props, cluster_info, peer_container, ip_blocks_mask, connectivity_restriction): @@ -592,44 +621,19 @@ def conn_props_to_fw_rules(conn_props, cluster_info, peer_container, ip_blocks_m TCP / non-TCP , or not :return: FWRules map """ - ignore_protocols = ProtocolSet() + relevant_protocols = ProtocolSet() if connectivity_restriction: if connectivity_restriction == 'TCP': - ignore_protocols.add_protocol('TCP') + relevant_protocols.add_protocol('TCP') else: # connectivity_restriction == 'non-TCP' - ignore_protocols = ProtocolSet.get_non_tcp_protocols() + relevant_protocols = ProtocolSet.get_non_tcp_protocols() fw_rules_map = defaultdict(list) for cube in conn_props: conn_cube = conn_props.get_connectivity_cube(cube) - src_peers = conn_cube["src_peers"] - if not src_peers: - src_peers = peer_container.get_all_peers_group(True) - conn_cube.unset_dim("src_peers") - dst_peers = conn_cube["dst_peers"] - if not dst_peers: - dst_peers = peer_container.get_all_peers_group(True) - conn_cube.unset_dim("dst_peers") - if IpBlock.get_all_ips_block() != ip_blocks_mask: - src_peers.filter_ipv6_blocks(ip_blocks_mask) - dst_peers.filter_ipv6_blocks(ip_blocks_mask) - protocols = conn_cube["protocols"] - conn_cube.unset_dim("protocols") - if not conn_cube.has_active_dim() and (protocols.is_whole_range() or protocols == ignore_protocols): - conns = ConnectionSet(True) - else: - conns = ConnectionSet() - protocol_names = ProtocolSet.get_protocol_names_from_interval_set(protocols) if protocols else ['TCP'] - for protocol in protocol_names: - if conn_cube.has_active_dim(): - conns.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) - else: - if ConnectionSet.protocol_supports_ports(protocol): - conns.add_connections(protocol, ConnectivityProperties.make_all_props()) - elif ConnectionSet.protocol_is_icmp(protocol): - conns.add_connections(protocol, ConnectivityProperties.make_all_props()) - else: - conns.add_connections(protocol, True) + conns, src_peers, dst_peers = \ + ConnectionSet.get_connection_set_and_peers_from_cube(conn_cube, peer_container, ip_blocks_mask, + relevant_protocols) # create FWRules for src_peers and dst_peers fw_rules_map[conns] += ConnectionSet.create_fw_rules_list_from_conns(conns, src_peers, dst_peers, cluster_info) @@ -671,6 +675,13 @@ def split_peer_set_to_fw_rule_elements(peer_set, cluster_info): @staticmethod def fw_rules_to_conn_props(fw_rules): + """ + Converting FWRules to ConnectivityProperties format. + This function is used for comparing FWRules output between original and optimized solutions, + when optimized_run == 'debug' + :param list[FWRule] fw_rules: the given FWRules. + :return: the resulting ConnectivityProperties. + """ res = ConnectivityProperties.make_empty_props() for fw_rules_list in fw_rules.fw_rules_map.values(): for fw_rule in fw_rules_list: diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 5b5c9ff08..335043b93 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -351,7 +351,12 @@ def project_on_one_dimension(self, dim_name): return None # not supporting icmp dimensions if dim_name not in self.active_dimensions: return None - res = None + if dim_name == "src_peers" or dim_name == "dst_peers": + res = PeerSet() + elif dim_name == "src_ports" or dim_name == "dst_ports": + res = PortSet() + else: + res = DimensionsManager().get_empty_dimension_by_name(dim_name) for cube in self: conn_cube = self.get_connectivity_cube(cube) values = conn_cube[dim_name] diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index e41d31c89..a3df43b5b 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -36,20 +36,6 @@ def __init__(self, all_peers, allowed_labels, output_config): self.cluster_info = ClusterInfo(all_peers, allowed_labels) self.allowed_labels = allowed_labels - def _get_peer_name(self, peer): - """ - Get the name of a peer object for connectivity graph + flag indicating if it is ip-block - :param Peer peer: the peer object - :return: tuple(str, bool) - str: the peer name - bool: flag to indicate if peer is ip-block (True) or not (False) - """ - if isinstance(peer, IpBlock): - return peer.get_ip_range_or_cidr_str(), True - if self.output_config.outputEndpoints == 'deployments' and isinstance(peer, Pod): - return peer.workload_name, False - return str(peer), False - def add_edge(self, source_peer, dest_peer, connections): """ Adding a labeled edge to the graph @@ -68,38 +54,15 @@ def add_edges(self, connections): """ self.connections_to_peers.update(connections) - def add_edges_from_cube_dict(self, conn_cube, ip_blocks_mask): + def add_edges_from_cube_dict(self, conn_cube, peer_container, ip_blocks_mask): """ Add edges to the graph according to the give cube :param ConnectivityCube conn_cube: the given cube :param IpBlock ip_blocks_mask: IpBlock containing all allowed ip values, whereas all other values should be filtered out in the output """ - src_peers = conn_cube["src_peers"] - conn_cube.unset_dim("src_peers") - dst_peers = conn_cube["dst_peers"] - conn_cube.unset_dim("dst_peers") - if IpBlock.get_all_ips_block() != ip_blocks_mask: - src_peers.filter_ipv6_blocks(ip_blocks_mask) - dst_peers.filter_ipv6_blocks(ip_blocks_mask) - protocols = conn_cube["protocols"] - conn_cube.unset_dim("protocols") - - if protocols.is_whole_range() and not conn_cube.has_active_dim(): - conns = ConnectionSet(True) - else: - conns = ConnectionSet() - protocol_names = ProtocolSet.get_protocol_names_from_interval_set(protocols) if protocols else ['TCP'] - for protocol in protocol_names: - if conn_cube.has_active_dim(): - conns.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube)) - else: - if ConnectionSet.protocol_supports_ports(protocol): - conns.add_connections(protocol, ConnectivityProperties.make_all_props()) - elif ConnectionSet.protocol_is_icmp(protocol): - conns.add_connections(protocol, ConnectivityProperties.make_all_props()) - else: - conns.add_connections(protocol, True) + conns, src_peers, dst_peers = \ + ConnectionSet.get_connection_set_and_peers_from_cube(conn_cube, peer_container, ip_blocks_mask) for src_peer in src_peers: for dst_peer in dst_peers: self.connections_to_peers[conns].append((src_peer, dst_peer)) diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index fe7b26e06..daa01b484 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -1026,7 +1026,8 @@ def dot_format_from_props(self, props, peers, ip_blocks_mask, connectivity_restr """ conn_graph = ConnectivityGraph(peers, self.config.get_allowed_labels(), self.output_config) for cube in props: - conn_graph.add_edges_from_cube_dict(props.get_connectivity_cube(cube), ip_blocks_mask) + conn_graph.add_edges_from_cube_dict(props.get_connectivity_cube(cube), self.config.peer_container, + ip_blocks_mask) return conn_graph.get_connectivity_dot_format_str(connectivity_restriction) def fw_rules_from_connections_dict(self, connections, peers_to_compare, connectivity_restriction=None): From 1a84d7e937a525eb21fff51e49ad01e7325bc99f Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 23 Apr 2023 16:30:48 +0300 Subject: [PATCH 124/187] Update nca/NetworkConfig/NetworkLayer.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/NetworkConfig/NetworkLayer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 3576d8ff8..58b540c2e 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -110,7 +110,7 @@ def empty_layer_allowed_connections_optimized(peer_container, layer_name): Get allowed connections between for all relevant peers for an empty layer (no policies). :param PeerContainer peer_container: holds all the peers :param NetworkLayerName layer_name: The empty layer name - :rtype: ConnectivityProperties + :rtype: OptimizedPolicyConnections """ empty_layer_obj = layer_name.create_network_layer([]) return empty_layer_obj.allowed_connections_optimized(peer_container) From 6d92ca743135078d3abfb2460fac8ac68c67acb3 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 23 Apr 2023 16:31:15 +0300 Subject: [PATCH 125/187] Update nca/NetworkConfig/NetworkLayer.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/NetworkConfig/NetworkLayer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 58b540c2e..2a5d9533e 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -247,7 +247,7 @@ def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda poli :param captured_func: callable that returns True if the policy satisfies additional conditions required for considering captured pods instead of applying the default connections. :return: allowed_conns, denied_conns and set of peers to be added to captured peers - :rtype: tuple (ConnectivityProperties, ConnectivityProperties, PeerSet) + :rtype: OptimizedPolicyConnections """ res_conns = OptimizedPolicyConnections() for policy in self.policies_list: From 21984e1021230550540784e09437be118096b745 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 23 Apr 2023 16:31:54 +0300 Subject: [PATCH 126/187] Update nca/NetworkConfig/NetworkLayer.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/NetworkConfig/NetworkLayer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 2a5d9533e..0dd00639a 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -204,6 +204,7 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): """ Implemented by derived classes to get ingress/egress connections between any relevant peers + :rtype: OptimizedPolicyConnections """ return NotImplemented From fd7bc60714113559317a14e090a7bd9af046c4cc Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 23 Apr 2023 16:33:24 +0300 Subject: [PATCH 127/187] Update nca/NetworkConfig/NetworkLayer.py Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- nca/NetworkConfig/NetworkLayer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 0dd00639a..fff4aa732 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -293,6 +293,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): res_conns = self.collect_policies_conns_optimized(is_ingress) # Note: The below computation of non-captured conns cannot be done during the parse stage, # since before computing non-captured conns we should collect all policies conns + # compute non-captured connections all_peers_and_ips = peer_container.get_all_peers_group(True) all_peers_no_ips = peer_container.get_all_peers_group() From 64e568418b38b7020966137af1ce162cf99229db Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 23 Apr 2023 19:21:18 +0300 Subject: [PATCH 128/187] Removed unused functions and imports. Avoid including IpBlocks in non-captured peers. Signed-off-by: Tanya --- nca/FWRules/ClusterInfo.py | 6 ------ nca/FWRules/ConnectivityGraph.py | 2 -- nca/FWRules/MinimizeFWRules.py | 4 ---- nca/NetworkConfig/NetworkLayer.py | 14 ++++++++------ 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/nca/FWRules/ClusterInfo.py b/nca/FWRules/ClusterInfo.py index 6e752954c..dcc02f46a 100644 --- a/nca/FWRules/ClusterInfo.py +++ b/nca/FWRules/ClusterInfo.py @@ -52,12 +52,6 @@ def __init__(self, all_peers, allowed_labels): # labels values for a combination of multiple labels, in a fw-rule self.add_update_pods_labels_map_with_required_conjunction_labels() - def __eq__(self, other): - - return self.all_peers == other.all_peers and self.allowed_labels == other.allowed_labels \ - and self.ns_dict == other.ns_dict and self.pods_labels_map == other.pods_labels_map \ - and self.all_label_values_per_ns == other.all_label_values_per_ns - def add_update_pods_labels_map_with_invalid_val(self, all_pods): """ Updating the pods_labels_map with (key,"NO_LABEL_VALUE") for the set of pods without this label diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index a3df43b5b..626d0fa10 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -8,8 +8,6 @@ import networkx from nca.CoreDS.Peer import IpBlock, ClusterEP, Pod from nca.CoreDS.ConnectionSet import ConnectionSet -from nca.CoreDS.ProtocolSet import ProtocolSet -from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from .DotGraph import DotGraph from .MinimizeFWRules import MinimizeCsFwRules, MinimizeFWRules from .ClusterInfo import ClusterInfo diff --git a/nca/FWRules/MinimizeFWRules.py b/nca/FWRules/MinimizeFWRules.py index 68ac491c8..d42cd301c 100644 --- a/nca/FWRules/MinimizeFWRules.py +++ b/nca/FWRules/MinimizeFWRules.py @@ -667,10 +667,6 @@ def __init__(self, fw_rules_map, cluster_info, output_config, results_map): self.output_config = output_config self.results_map = results_map - def __eq__(self, other): - return self.fw_rules_map == other.fw_rules_map and self.cluster_info == other.cluster_info \ - and self.output_config == other.output_config and self.results_map == other.results_map - def get_fw_rules_in_required_format(self, add_txt_header=True, add_csv_header=True, connectivity_restriction=None): """ :param add_txt_header: bool flag to indicate if header of fw-rules query should be added in txt format diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 3576d8ff8..768b53273 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -350,6 +350,7 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): res_conns = self.collect_policies_conns_optimized(is_ingress, IstioNetworkLayer.captured_cond_func) all_peers_and_ips = peer_container.get_all_peers_group(True) + all_peers_no_ips = peer_container.get_all_peers_group() dns_entries = peer_container.get_all_dns_entries() # for istio initialize non-captured conns with non-TCP connections all_all_conns = \ @@ -357,9 +358,9 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): "dst_peers": all_peers_and_ips, "protocols": ProtocolSet.get_non_tcp_protocols()}) res_conns.all_allowed_conns |= res_conns.allowed_conns | all_all_conns - non_captured_peers = all_peers_and_ips - res_conns.captured + non_captured_peers = all_peers_no_ips - res_conns.captured if non_captured_peers: - protocols = ProtocolSet.get_protocol_set_with_single_protocol('TCP') + tcp_protocol = ProtocolSet.get_protocol_set_with_single_protocol('TCP') if is_ingress: all_nc_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers_and_ips, "dst_peers": non_captured_peers}) @@ -370,7 +371,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): all_nc_dns_conns = \ ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers_and_ips, "dst_peers": non_captured_dns_entries, - "protocols": protocols}) + "protocols": tcp_protocol}) res_conns.all_allowed_conns |= all_nc_dns_conns else: nc_all_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": non_captured_peers, @@ -379,7 +380,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): # update allowed non-captured conns to DNSEntry dst with TCP only nc_dns_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": non_captured_peers, "dst_peers": dns_entries, - "protocols": protocols}) + "protocols": tcp_protocol}) res_conns.all_allowed_conns |= nc_dns_conns return res_conns @@ -400,14 +401,15 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): res_conns = OptimizedPolicyConnections() all_peers_and_ips = peer_container.get_all_peers_group(True) + all_peers_no_ips = peer_container.get_all_peers_group() if is_ingress: # everything is allowed and non captured non_captured_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers_and_ips, - "dst_peers": all_peers_and_ips}) + "dst_peers": all_peers_no_ips}) res_conns.all_allowed_conns = non_captured_conns else: res_conns = self.collect_policies_conns_optimized(is_ingress) - non_captured_peers = all_peers_and_ips - res_conns.captured + non_captured_peers = all_peers_no_ips - res_conns.captured if non_captured_peers: non_captured_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": non_captured_peers, "dst_peers": all_peers_and_ips}) From f3eeeba445c8623f8f39951294ac50c5884f4880 Mon Sep 17 00:00:00 2001 From: Tanya Date: Sun, 23 Apr 2023 19:49:24 +0300 Subject: [PATCH 129/187] Making more accurate default all properties, according to all peers in the current config. Signed-off-by: Tanya --- nca/NetworkConfig/NetworkConfig.py | 5 ++++- nca/Parsers/IstioPolicyYamlParser.py | 5 ++++- nca/Parsers/IstioSidecarYamlParser.py | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/nca/NetworkConfig/NetworkConfig.py b/nca/NetworkConfig/NetworkConfig.py index beff016f8..be1918838 100644 --- a/nca/NetworkConfig/NetworkConfig.py +++ b/nca/NetworkConfig/NetworkConfig.py @@ -287,7 +287,10 @@ def allowed_connections_optimized(self, layer_name=None): conn_hep = ConnectivityProperties.make_conn_props_from_dict({"src_peers": host_eps}) | \ ConnectivityProperties.make_conn_props_from_dict({"dst_peers": host_eps}) conns_res = OptimizedPolicyConnections() - conns_res.all_allowed_conns = ConnectivityProperties.make_all_props() + all_peers_and_ips_and_dns = self.peer_container.get_all_peers_group(True, True, True) + conns_res.all_allowed_conns = \ + ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers_and_ips_and_dns, + "dst_peers": all_peers_and_ips_and_dns}) for layer, layer_obj in self.policies_container.layers.items(): conns_per_layer = layer_obj.allowed_connections_optimized(self.peer_container) # only K8s_Calico layer handles host_eps diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index 3e6b659b1..4e640e1c6 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -577,7 +577,10 @@ def parse_policy(self): res_policy.add_optimized_allow_props(optimized_props, True) else: # Deny res_policy.add_optimized_deny_props(optimized_props, True) - res_policy.add_optimized_allow_props(ConnectivityProperties.make_all_props(), False) + all_peers_and_ips = self.peer_container.get_all_peers_group(True) + all_props = ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers_and_ips, + "dst_peers": all_peers_and_ips}) + res_policy.add_optimized_allow_props(all_props, False) if not res_policy.ingress_rules and res_policy.action == IstioNetworkPolicy.ActionType.Deny: self.syntax_error("DENY action without rules is meaningless as it will never be triggered") diff --git a/nca/Parsers/IstioSidecarYamlParser.py b/nca/Parsers/IstioSidecarYamlParser.py index 2d35354a9..048e2c862 100644 --- a/nca/Parsers/IstioSidecarYamlParser.py +++ b/nca/Parsers/IstioSidecarYamlParser.py @@ -203,7 +203,10 @@ def parse_policy(self): self.namespace = self.peer_container.get_namespace(policy_ns, warn_if_missing) res_policy = IstioSidecar(policy_name, self.namespace) res_policy.policy_kind = NetworkPolicy.PolicyType.IstioSidecar - res_policy.add_optimized_allow_props(ConnectivityProperties.make_all_props(), True) + all_peers_and_ips_and_dns = self.peer_container.get_all_peers_group(True, True, True) + all_props = ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers_and_ips_and_dns, + "dst_peers": all_peers_and_ips_and_dns}) + res_policy.add_optimized_allow_props(all_props, True) sidecar_spec = self.policy['spec'] # currently, supported fields in spec are workloadSelector and egress From 2a2606bbbcc2b630a4b0803d9c140c66be2de3f3 Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Mon, 24 Apr 2023 15:06:47 +0300 Subject: [PATCH 130/187] output_endpoints support. Signed-off-by: Shmulik Froimovich --- nca/NetworkConfig/NetworkLayer.py | 5 +- nca/NetworkConfig/TopologyObjectsFinder.py | 8 +- nca/Utils/ExplTracker.py | 124 ++++++++++++--------- nca/nca_cli.py | 2 +- 4 files changed, 77 insertions(+), 62 deletions(-) diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 926a2b005..0a913d1a5 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -6,7 +6,7 @@ from enum import Enum from nca.CoreDS.ConnectionSet import ConnectionSet -from nca.CoreDS.Peer import IpBlock, HostEP, PeerSet, DNSEntry +from nca.CoreDS.Peer import IpBlock, HostEP, PeerSet, DNSEntry, Pod from nca.CoreDS.ConnectivityCube import ConnectivityCube from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.ProtocolSet import ProtocolSet @@ -254,7 +254,8 @@ def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda poli for peer in policy.selected_peers: src_peers, _ = ExplTracker().extract_peers(policy.optimized_allow_ingress_props) _, dst_peers = ExplTracker().extract_peers(policy.optimized_allow_egress_props) - ExplTracker().add_peer_policy(peer.full_name(), + peer_name = ExplTracker().get_peer_ep_name(peer) + ExplTracker().add_peer_policy(peer_name, policy.name, dst_peers, src_peers, diff --git a/nca/NetworkConfig/TopologyObjectsFinder.py b/nca/NetworkConfig/TopologyObjectsFinder.py index 48c78346a..4ad3ef2ab 100644 --- a/nca/NetworkConfig/TopologyObjectsFinder.py +++ b/nca/NetworkConfig/TopologyObjectsFinder.py @@ -110,7 +110,7 @@ def _add_pod_from_yaml(self, pod_object): for port in container.get('ports') or []: pod.add_named_port(port.get('name'), port.get('containerPort'), port.get('protocol', 'TCP')) self._add_peer(pod) - ExplTracker().add_item(pod_object.path, pod.full_name(), pod_object.line_number) + ExplTracker().add_item(pod_object.path, ExplTracker().get_peer_ep_name(pod), pod_object.line_number) def _add_peer(self, peer): """ @@ -166,7 +166,7 @@ def _add_pod_from_workload_yaml(self, workload_resource): for port in container.get('ports') or []: pod.add_named_port(port.get('name'), port.get('containerPort'), port.get('protocol', 'TCP')) self._add_peer(pod) - ExplTracker().add_item(workload_resource.path, pod.full_name(), workload_resource.line_number) + ExplTracker().add_item(workload_resource.path, ExplTracker().get_peer_ep_name(pod), workload_resource.line_number) def _add_networkset_from_yaml(self, networkset_object): """ @@ -221,7 +221,7 @@ def _add_hep_from_yaml(self, hep_object): hep.add_profile(profile) self._add_peer(hep) - ExplTracker().add_item(hep_object.path, hep.full_name(), hep_object.line_number) + ExplTracker().add_item(hep_object.path, ExplTracker().get_peer_ep_name(hep), hep_object.line_number) def _add_wep_from_yaml(self, wep_object): """ @@ -248,7 +248,7 @@ def _add_wep_from_yaml(self, wep_object): wep.add_profile(profile) self._add_peer(wep) - ExplTracker().add_item(wep_object.path, wep.full_name(), wep_object.line_number) + ExplTracker().add_item(wep_object.path, ExplTracker().get_peer_ep_name(wep), wep_object.line_number) def _add_dns_entries_from_yaml(self, srv_entry_object): """ diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py index b943a031d..90b8ff135 100644 --- a/nca/Utils/ExplTracker.py +++ b/nca/Utils/ExplTracker.py @@ -5,56 +5,11 @@ from nca.Utils.Utils import Singleton from nca.Utils.NcaLogger import NcaLogger -from nca.CoreDS.Peer import PeerSet +from nca.CoreDS.Peer import PeerSet, Pod from bs4 import BeautifulSoup from bs4.element import Tag -class ExplPolicies: - """ - ExplPolicies holds the policies affecting peers in relation to other peers. - That is, for each peer it holds all the peers in it's egress and ingress and the policies that has effect on the connection - to that peers. - """ - def __init__(self): - self.egress_dst = {} - self.ingress_src = {} - self.all_policies = set() - - @staticmethod - def _add_policy(peer_set, peer_list, policy_name): - """ - Adds a policy to the list of affecting policies, for each peer in the peer_set - :param PeerSet peer_set: a set of peers to add the policy to - :param dict peer_list: a list of peers that holds the policies affecting them - :param str policy_name: the policy to add - """ - for peer in peer_set: - peer_name = peer.full_name() - if not peer_list.get(peer_name): - peer_list[peer_name] = set() - # We don't want Default-Policy if we have any other policy, - # so we first remove it and then add the policy (even if we currently add - # the Default-Policy itself). - peer_list[peer_name].discard('Default-Policy') - peer_list[peer_name].add(policy_name) - - def add_policy(self, policy_name, egress_dst, ingress_src): - """ - Adds a given policy to the relevant peer lists (egress list, ingress list) - :param str policy_name: name of the policy - :param dict egress_dst: the set of egress destinations peers to add the policy too - :param dict ingress_src: the set of ingress source peers to add the policy too - """ - self.all_policies.add(policy_name) - - if egress_dst: - self._add_policy(egress_dst, self.egress_dst, policy_name) - - if ingress_src: - self._add_policy(ingress_src, self.ingress_src, policy_name) - - class ExplTracker(metaclass=Singleton): """ The Explainability Tracker is used for tracking the elements and their configuration @@ -64,15 +19,61 @@ class ExplTracker(metaclass=Singleton): The ExplTracker is Singletone """ - def __init__(self): + def __init__(self, ep=''): self.ExplDescriptorContainer = {} self.ExplPeerToPolicyContainer = {} self._is_active = False self.all_conns = {} self.all_peers = {} + self.ep = ep self.add_item('', 'Default-Policy', 0) + class ExplPolicies: + """ + ExplPolicies holds the policies affecting peers in relation to other peers. + That is, for each peer it holds all the peers in it's egress and ingress and the policies + that has effect on the connection to that peers. + """ + + def __init__(self): + self.egress_dst = {} + self.ingress_src = {} + self.all_policies = set() + + @staticmethod + def _add_policy(peer_set, peer_list, policy_name): + """ + Adds a policy to the list of affecting policies, for each peer in the peer_set + :param PeerSet peer_set: a set of peers to add the policy to + :param dict peer_list: a list of peers that holds the policies affecting them + :param str policy_name: the policy to add + """ + for peer in peer_set: + peer_name = ExplTracker().get_peer_ep_name(peer) + if not peer_list.get(peer_name): + peer_list[peer_name] = set() + # We don't want Default-Policy if we have any other policy, + # so we first remove it and then add the policy (even if we currently add + # the Default-Policy itself). + peer_list[peer_name].discard('Default-Policy') + peer_list[peer_name].add(policy_name) + + def add_policy(self, policy_name, egress_dst, ingress_src): + """ + Adds a given policy to the relevant peer lists (egress list, ingress list) + :param str policy_name: name of the policy + :param PeerSet egress_dst: the set of egress destinations peers to add the policy too + :param PeerSet ingress_src: the set of ingress source peers to add the policy too + """ + self.all_policies.add(policy_name) + + if egress_dst: + self._add_policy(egress_dst, self.egress_dst, policy_name) + + if ingress_src: + self._add_policy(ingress_src, self.ingress_src, policy_name) + def activate(self): """ Make the ExplTracker active @@ -125,7 +126,7 @@ def add_peer_policy(self, peer_name, policy_name, egress_dst, ingress_src): """ if self.ExplDescriptorContainer.get(peer_name): if not self.ExplPeerToPolicyContainer.get(peer_name): - self.ExplPeerToPolicyContainer[peer_name] = ExplPolicies() + self.ExplPeerToPolicyContainer[peer_name] = self.ExplPolicies() self.ExplPeerToPolicyContainer[peer_name].add_policy(policy_name, egress_dst, ingress_src, @@ -201,7 +202,8 @@ def add_default_policy(self, src, dst, is_ingress): # we dont add Default-Policy if there is already an explicit # policy allowing the connectivity if self.is_policy_list_empty(node.full_name(), is_ingress): - self.add_peer_policy(node.full_name(), + node_name = self.get_peer_ep_name(node) + self.add_peer_policy(node_name, 'Default-Policy', egress_list, ingress_list, @@ -239,21 +241,33 @@ def prepare_node_str(self, node_name, results, direction=None): f'in file {path}') return out + def get_peer_ep_name(self, peer): + if self.ep == 'deployments' and isinstance(peer, Pod): + return peer.workload_name + else: + return peer.full_name() + def explain_all(self): soup = BeautifulSoup(features='xml') entry_id = 0 - for peer1 in self.all_peers: - for peer2 in self.all_peers: + # use the peer names as defined in the endpoints configuration, + # also use one peer for each deployment + peer_names = set() + for peer in self.all_peers: + peer_names.add(self.get_peer_ep_name(peer)) + + for peer1 in peer_names: + for peer2 in peer_names: if peer1 == peer2: - text = self.explain([peer1.full_name()]) + text = self.explain([peer1]) else: - text = self.explain([peer1.full_name(), peer2.full_name()]) + text = self.explain([peer1, peer2]) # Create the XML entry element entry = soup.new_tag('entry') entry_id += 1 entry['id'] = str(entry_id) - entry['src'] = peer1.full_name() - entry['dst'] = peer2.full_name() + entry['src'] = peer1 + entry['dst'] = peer2 text_elem = Tag(soup, name='text') text_elem.string = text entry.append(text_elem) diff --git a/nca/nca_cli.py b/nca/nca_cli.py index 505fa8ffd..cdf10dcd6 100644 --- a/nca/nca_cli.py +++ b/nca/nca_cli.py @@ -183,7 +183,7 @@ def run_args(args): if args.explain is not None: output_config['expl'] = args.explain.split(',') - ExplTracker().activate() + ExplTracker(output_config.outputEndpoints).activate() if args.equiv is not None: np_list = args.equiv if args.equiv != [''] else None From 99a6b73ff199348e694cec5da7d6d6c6b1f8eda5 Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Tue, 25 Apr 2023 10:28:03 +0300 Subject: [PATCH 131/187] support ep modes Signed-off-by: Shmulik Froimovich --- nca/Utils/ExplTracker.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py index 90b8ff135..05d2be663 100644 --- a/nca/Utils/ExplTracker.py +++ b/nca/Utils/ExplTracker.py @@ -157,7 +157,7 @@ def set_connections_and_peers(self, conns, peers): self.all_peers = peers # add all missing 'special' peers with default policy. for peer in self.all_peers: - peer_name = peer.full_name() + peer_name = self.get_peer_ep_name(peer) if not self.ExplPeerToPolicyContainer.get(peer_name): if not self.ExplDescriptorContainer.get(peer_name): self.add_item('', peer_name, 0) @@ -175,10 +175,16 @@ def are_peers_connected(self, src, dst): NcaLogger().log_message(f'Explainability error: Connections were not set yet, but peer query was called', level='E') for cube in self.all_conns: - conn = self.all_conns.get_cube_dict(cube) - src_peers = conn['src_peers'] - dst_peers = conn['dst_peers'] - if src in src_peers and dst in dst_peers: + src_peers_names = [] + dst_peers_names = [] + conn_cube = self.all_conns.get_connectivity_cube(cube) + src_peers = conn_cube['src_peers'] + dst_peers = conn_cube['dst_peers'] + for peer in src_peers: + src_peers_names.append(self.get_peer_ep_name(peer)) + for peer in dst_peers: + dst_peers_names.append(self.get_peer_ep_name(peer)) + if src in src_peers_names and dst in dst_peers_names: return True return False @@ -201,7 +207,7 @@ def add_default_policy(self, src, dst, is_ingress): for node in nodes: # we dont add Default-Policy if there is already an explicit # policy allowing the connectivity - if self.is_policy_list_empty(node.full_name(), is_ingress): + if self.is_policy_list_empty(self.get_peer_ep_name(node), is_ingress): node_name = self.get_peer_ep_name(node) self.add_peer_policy(node_name, 'Default-Policy', From 4f9c0c29bbeac4aff2c05bccf45a71fef42ebf9d Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Tue, 25 Apr 2023 13:26:57 +0300 Subject: [PATCH 132/187] Documentation added Signed-off-by: Shmulik Froimovich --- nca/Utils/ExplTracker.py | 47 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py index 05d2be663..069394237 100644 --- a/nca/Utils/ExplTracker.py +++ b/nca/Utils/ExplTracker.py @@ -17,6 +17,28 @@ class ExplTracker(metaclass=Singleton): or lack of connection between them. The ExplTracker is Singletone + Members: + ExplDescriptorContainer - A container for all expl' items' in the system. + Each entry has a peer or a policy with their name, file and line number of the configurations. + + ExplPeerToPolicyContainer - A container for finding the affecting policies from peers in egress and ingress + For each peer name it has a ExplPolicies class object with 3 items: + all_policies - a set of all the policies affecting the current peer + egress_dst - a dict of destination peers allowed in the current peer's egress and for each of them, + the policies that allows it + ingress_src - a dict of source peers allowed in the current peer's ingress and for each of them, + the policies that allows it + That way, when given a src and dst peers we can extract which policies allow the connection in each side. + When there is no connection, we list all the policies that affect the peers, so the user may have all the info + to find problems. + + _is_active - flag for checking if expl' was activated + + all_conns - all the calculated connection. This is used to shortcut the check if 2 peers are connected or not + + all_peers - all the peers. This is used for the explain-all feature. + + ep - endpoints configurations (use either full_name for pod mode, or workload_name for deployments mode) """ def __init__(self, ep=''): @@ -32,7 +54,7 @@ def __init__(self, ep=''): class ExplPolicies: """ ExplPolicies holds the policies affecting peers in relation to other peers. - That is, for each peer it holds all the peers in it's egress and ingress and the policies + That is, for each peer it holds all the peers in its egress and ingress and the policies that has effect on the connection to that peers. """ @@ -155,14 +177,14 @@ def set_connections_and_peers(self, conns, peers): """ self.all_conns = conns self.all_peers = peers - # add all missing 'special' peers with default policy. + # add all missing 'special' peers (like 0.0.0.0/0) with default policy. for peer in self.all_peers: peer_name = self.get_peer_ep_name(peer) if not self.ExplPeerToPolicyContainer.get(peer_name): if not self.ExplDescriptorContainer.get(peer_name): self.add_item('', peer_name, 0) - self.add_default_policy([peer], peers, False) - self.add_default_policy(peers, [peer], True) + self.add_default_policy(PeerSet([peer]), peers, False) + self.add_default_policy(peers, PeerSet([peer]), True) def are_peers_connected(self, src, dst): """ @@ -216,6 +238,12 @@ def add_default_policy(self, src, dst, is_ingress): ) def is_policy_list_empty(self, node_name, check_ingress): + """ + A service function to check if the expl' list of ingress or egress is empty. + :param node_name: the node to check + :param check_ingress: list to check (ingress or egress) + :return: + """ peer = self.ExplPeerToPolicyContainer.get(node_name) if peer: if check_ingress and peer.ingress_src: @@ -248,12 +276,23 @@ def prepare_node_str(self, node_name, results, direction=None): return out def get_peer_ep_name(self, peer): + """ + Get the name of the peer based on the endpoints configurations: + full_name for Pods mode + workload_name for Deployments mode + :param peer: the peer to query + :return: string: name of peer + """ if self.ep == 'deployments' and isinstance(peer, Pod): return peer.workload_name else: return peer.full_name() def explain_all(self): + """ + Get a full expl' description of all the peers in the connectivity map + :return: string: xml format of all the expl' entries for every 2 nodes. + """ soup = BeautifulSoup(features='xml') entry_id = 0 # use the peer names as defined in the endpoints configuration, From 8b0e3ace4b8a0d78fe4953c9971001053782361c Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Tue, 25 Apr 2023 14:00:05 +0300 Subject: [PATCH 133/187] minor fix Signed-off-by: Shmulik Froimovich --- nca/Utils/ExplTracker.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py index 069394237..872be5992 100644 --- a/nca/Utils/ExplTracker.py +++ b/nca/Utils/ExplTracker.py @@ -330,13 +330,12 @@ def explain(self, nodes): :param list(str) nodes: nodes to explain :return: str: the explanation out string """ - out = [] if len(nodes) < 1: - return out + return '' elif len(nodes) > 2: NcaLogger().log_message(f'Explainability error: only 1 or 2 nodes are allowed for explainability query,' f' found {len(nodes)} ', level='E') - return out + return '' src_node = nodes[0] if src_node == 'ALL': @@ -346,21 +345,22 @@ def explain(self, nodes): for node in nodes: if not self.ExplDescriptorContainer.get(node): NcaLogger().log_message(f'Explainability error - {node} was not found in the connectivity results', level='E') - return out + return '' if not self.ExplPeerToPolicyContainer.get(node): NcaLogger().log_message(f'Explainability error - {node} has no explanability results', level='E') - return out + return '' + out = [] if len(nodes) == 2: # 2 nodes scenario dst_node = nodes[1] if self.are_peers_connected(src_node, dst_node): # connection valid - out.append(f'\nConfigurations affecting the connectivity between (src){src_node} and (dst){dst_node}:') + out.append(f'Configurations affecting the connectivity between (src){src_node} and (dst){dst_node}:') src_results = self.ExplPeerToPolicyContainer[src_node].egress_dst.get(dst_node) dst_results = self.ExplPeerToPolicyContainer[dst_node].ingress_src.get(src_node) else: - out.append(f'\nConfigurations affecting the LACK of connectivity between (src){src_node} and (dst){dst_node}:') + out.append(f'Configurations affecting the LACK of connectivity between (src){src_node} and (dst){dst_node}:') src_results = self.ExplPeerToPolicyContainer[src_node].all_policies dst_results = self.ExplPeerToPolicyContainer[dst_node].all_policies @@ -370,7 +370,7 @@ def explain(self, nodes): out.extend(self.prepare_node_str(dst_node, dst_results, 'dst')) else: # only one node results = self.ExplPeerToPolicyContainer[src_node].all_policies - out.append(f'\nConfigurations affecting {src_node}:') + out.append(f'Configurations affecting {src_node}:') out.extend(self.prepare_node_str(src_node, results)) # convert the list of expl' directives into string From 802b3fc15b9ca5d420e19246daef0a9c20ad61b3 Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Thu, 27 Apr 2023 12:26:00 +0300 Subject: [PATCH 134/187] use Expl' functions only when activated by user Signed-off-by: Shmulik Froimovich --- nca/NetworkConfig/NetworkConfigQuery.py | 3 +- nca/NetworkConfig/NetworkConfigQueryRunner.py | 4 ++- nca/NetworkConfig/NetworkLayer.py | 35 ++++++++++--------- nca/NetworkConfig/PoliciesFinder.py | 23 ++++++------ nca/NetworkConfig/TopologyObjectsFinder.py | 15 +++++--- 5 files changed, 46 insertions(+), 34 deletions(-) diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index b82843dd5..2a2b1288f 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -793,7 +793,8 @@ def compute_connectivity_output_optimized(self): ConnectivityProperties.make_conn_props_from_dict({"dst_peers": subset_peers}) all_conns_opt &= subset_conns all_conns_opt = self.filter_conns_by_peer_types(all_conns_opt, opt_peers_to_compare) - ExplTracker().set_connections_and_peers(all_conns_opt, subset_peers) + if ExplTracker().is_active(): + ExplTracker().set_connections_and_peers(all_conns_opt, subset_peers) ip_blocks_mask = IpBlock.get_all_ips_block() if exclude_ipv6: ip_blocks_mask = IpBlock.get_all_ips_block(exclude_ipv6=True) diff --git a/nca/NetworkConfig/NetworkConfigQueryRunner.py b/nca/NetworkConfig/NetworkConfigQueryRunner.py index 1b5fc62c7..ca102f562 100644 --- a/nca/NetworkConfig/NetworkConfigQueryRunner.py +++ b/nca/NetworkConfig/NetworkConfigQueryRunner.py @@ -56,7 +56,9 @@ def compute_final_results(self, output_format, expl_nodes): sort_keys=False) else: output = '\n'.join(self.query_iterations_output) - expl_out = ExplTracker().explain(expl_nodes) + expl_out = '' + if ExplTracker().is_active(): + expl_out = ExplTracker().explain(expl_nodes) return self.numerical_result, output+expl_out, self.num_not_executed diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 739f279ca..1bb4a2061 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -254,15 +254,16 @@ def collect_policies_conns_optimized(self, is_ingress, captured_func=lambda poli res_conns = OptimizedPolicyConnections() for policy in self.policies_list: # Track the peers that were affected by this policy - for peer in policy.selected_peers: - src_peers, _ = ExplTracker().extract_peers(policy.optimized_allow_ingress_props) - _, dst_peers = ExplTracker().extract_peers(policy.optimized_allow_egress_props) - peer_name = ExplTracker().get_peer_ep_name(peer) - ExplTracker().add_peer_policy(peer_name, - policy.name, - dst_peers, - src_peers, - ) + if ExplTracker().is_active(): + for peer in policy.selected_peers: + src_peers, _ = ExplTracker().extract_peers(policy.optimized_allow_ingress_props) + _, dst_peers = ExplTracker().extract_peers(policy.optimized_allow_egress_props) + peer_name = ExplTracker().get_peer_ep_name(peer) + ExplTracker().add_peer_policy(peer_name, + policy.name, + dst_peers, + src_peers, + ) policy_conns = policy.allowed_connections_optimized(is_ingress) if policy_conns.captured: # not empty if captured_func(policy): @@ -320,11 +321,12 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): else: conn_cube.update({"src_peers": not_captured_not_hep, "dst_peers": all_peers_and_ips}) not_captured_not_hep_conns = ConnectivityProperties.make_conn_props(conn_cube) - src_peers, dst_peers = ExplTracker().extract_peers(not_captured_not_hep_conns) - ExplTracker().add_default_policy(src_peers, - dst_peers, - is_ingress - ) + if ExplTracker().is_active(): + src_peers, dst_peers = ExplTracker().extract_peers(not_captured_not_hep_conns) + ExplTracker().add_default_policy(src_peers, + dst_peers, + is_ingress + ) res_conns.all_allowed_conns |= not_captured_not_hep_conns captured_not_hep = base_peer_set_no_hep & res_conns.captured @@ -403,8 +405,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): "protocols": tcp_protocol}) non_captured_conns = nc_dns_conns res_conns.all_allowed_conns |= nc_dns_conns - if non_captured_conns: - # non_captured_conns = ConnectivityProperties.make_conn_props(nc_dns_conns) + if non_captured_conns and ExplTracker().is_active(): src_peers, dst_peers = ExplTracker().extract_peers(non_captured_conns) ExplTracker().add_default_policy(src_peers, dst_peers, @@ -443,7 +444,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): non_captured_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": non_captured_peers, "dst_peers": all_peers_and_ips}) res_conns.all_allowed_conns = res_conns.allowed_conns | non_captured_conns - if non_captured_conns: + if non_captured_conns and ExplTracker().is_active(): src_peers, dst_peers = ExplTracker().extract_peers(non_captured_conns) ExplTracker().add_default_policy(src_peers, dst_peers, diff --git a/nca/NetworkConfig/PoliciesFinder.py b/nca/NetworkConfig/PoliciesFinder.py index 4411609e8..06ffd7099 100644 --- a/nca/NetworkConfig/PoliciesFinder.py +++ b/nca/NetworkConfig/PoliciesFinder.py @@ -112,24 +112,27 @@ def parse_policies_in_parse_queue(self): # noqa: C901 parsed_policy = parsed_element.parse_policy() self._add_policy(parsed_policy) # the name is sometimes modified when parsed, like in the ingress case, when "allowed" is added - if parsed_policy: - policy_name = parsed_policy.name - else: # the istio policy is parsed later - policy_name = policy.get('metadata').get('name') - ExplTracker().add_item(policy.path, - policy_name, - policy.line_number - ) + if ExplTracker().is_active(): + if parsed_policy: + policy_name = parsed_policy.name + else: # the istio policy is parsed later + policy_name = policy.get('metadata').get('name') + ExplTracker().add_item(policy.path, + policy_name, + policy.line_number + ) if istio_traffic_parser: istio_traffic_policies = istio_traffic_parser.create_istio_traffic_policies() for istio_traffic_policy in istio_traffic_policies: self._add_policy(istio_traffic_policy) - ExplTracker().derive_item(istio_traffic_policy.name) + if ExplTracker().is_active(): + ExplTracker().derive_item(istio_traffic_policy.name) if istio_sidecar_parser: istio_sidecars = istio_sidecar_parser.get_istio_sidecars() for istio_sidecar in istio_sidecars: self._add_policy(istio_sidecar) - ExplTracker().derive_item(istio_sidecar.name) + if ExplTracker().is_active(): + ExplTracker().derive_item(istio_sidecar.name) def parse_yaml_code_for_policy(self, policy_object, file_name): policy_type = NetworkPolicy.get_policy_type_from_dict(policy_object) diff --git a/nca/NetworkConfig/TopologyObjectsFinder.py b/nca/NetworkConfig/TopologyObjectsFinder.py index 4ad3ef2ab..3f594b5e6 100644 --- a/nca/NetworkConfig/TopologyObjectsFinder.py +++ b/nca/NetworkConfig/TopologyObjectsFinder.py @@ -110,7 +110,8 @@ def _add_pod_from_yaml(self, pod_object): for port in container.get('ports') or []: pod.add_named_port(port.get('name'), port.get('containerPort'), port.get('protocol', 'TCP')) self._add_peer(pod) - ExplTracker().add_item(pod_object.path, ExplTracker().get_peer_ep_name(pod), pod_object.line_number) + if ExplTracker().is_active(): + ExplTracker().add_item(pod_object.path, ExplTracker().get_peer_ep_name(pod), pod_object.line_number) def _add_peer(self, peer): """ @@ -166,7 +167,8 @@ def _add_pod_from_workload_yaml(self, workload_resource): for port in container.get('ports') or []: pod.add_named_port(port.get('name'), port.get('containerPort'), port.get('protocol', 'TCP')) self._add_peer(pod) - ExplTracker().add_item(workload_resource.path, ExplTracker().get_peer_ep_name(pod), workload_resource.line_number) + if ExplTracker().is_active(): + ExplTracker().add_item(workload_resource.path, ExplTracker().get_peer_ep_name(pod), workload_resource.line_number) def _add_networkset_from_yaml(self, networkset_object): """ @@ -197,7 +199,8 @@ def _add_networkset_from_yaml(self, networkset_object): for cidr in cidrs: ipb.add_cidr(cidr) self._add_peer(ipb) - ExplTracker().add_item(networkset_object.path, ipb.full_name(), networkset_object.line_number) + if ExplTracker().is_active(): + ExplTracker().add_item(networkset_object.path, ipb.full_name(), networkset_object.line_number) def _add_hep_from_yaml(self, hep_object): """ @@ -221,7 +224,8 @@ def _add_hep_from_yaml(self, hep_object): hep.add_profile(profile) self._add_peer(hep) - ExplTracker().add_item(hep_object.path, ExplTracker().get_peer_ep_name(hep), hep_object.line_number) + if ExplTracker().is_active(): + ExplTracker().add_item(hep_object.path, ExplTracker().get_peer_ep_name(hep), hep_object.line_number) def _add_wep_from_yaml(self, wep_object): """ @@ -248,7 +252,8 @@ def _add_wep_from_yaml(self, wep_object): wep.add_profile(profile) self._add_peer(wep) - ExplTracker().add_item(wep_object.path, ExplTracker().get_peer_ep_name(wep), wep_object.line_number) + if ExplTracker().is_active(): + ExplTracker().add_item(wep_object.path, ExplTracker().get_peer_ep_name(wep), wep_object.line_number) def _add_dns_entries_from_yaml(self, srv_entry_object): """ From 0587e83c4dc335d8d5f072a81d74152d52d13052 Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Thu, 27 Apr 2023 12:46:43 +0300 Subject: [PATCH 135/187] some lintings Signed-off-by: Shmulik Froimovich --- nca/NetworkConfig/NetworkLayer.py | 2 +- nca/NetworkConfig/ResourcesHandler.py | 1 + nca/NetworkConfig/TopologyObjectsFinder.py | 5 ++++- nca/Utils/ExplTracker.py | 6 ++---- nca/Utils/Utils.py | 1 - 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index 1bb4a2061..69961d3d1 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -6,7 +6,7 @@ from enum import Enum from nca.CoreDS.ConnectionSet import ConnectionSet -from nca.CoreDS.Peer import IpBlock, HostEP, PeerSet, DNSEntry, Pod +from nca.CoreDS.Peer import IpBlock, HostEP, PeerSet, DNSEntry from nca.CoreDS.ConnectivityCube import ConnectivityCube from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.ProtocolSet import ProtocolSet diff --git a/nca/NetworkConfig/ResourcesHandler.py b/nca/NetworkConfig/ResourcesHandler.py index 392bb58c4..3457f5ed8 100644 --- a/nca/NetworkConfig/ResourcesHandler.py +++ b/nca/NetworkConfig/ResourcesHandler.py @@ -14,6 +14,7 @@ from .PeerContainer import PeerContainer from nca.Utils.ExplTracker import ExplTracker + class ResourceType(Enum): Unknown = 0 Pods = 1 diff --git a/nca/NetworkConfig/TopologyObjectsFinder.py b/nca/NetworkConfig/TopologyObjectsFinder.py index 3f594b5e6..177f3e984 100644 --- a/nca/NetworkConfig/TopologyObjectsFinder.py +++ b/nca/NetworkConfig/TopologyObjectsFinder.py @@ -168,7 +168,10 @@ def _add_pod_from_workload_yaml(self, workload_resource): pod.add_named_port(port.get('name'), port.get('containerPort'), port.get('protocol', 'TCP')) self._add_peer(pod) if ExplTracker().is_active(): - ExplTracker().add_item(workload_resource.path, ExplTracker().get_peer_ep_name(pod), workload_resource.line_number) + ExplTracker().add_item(workload_resource.path, + ExplTracker().get_peer_ep_name(pod), + workload_resource.line_number + ) def _add_networkset_from_yaml(self, networkset_object): """ diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py index 872be5992..804c33d5e 100644 --- a/nca/Utils/ExplTracker.py +++ b/nca/Utils/ExplTracker.py @@ -119,8 +119,7 @@ def add_item(self, path, name, ln): if name: self.ExplDescriptorContainer[name] = {'path': path, 'line': ln} else: - NcaLogger().log_message(f'Explainability error: configuration-block name can not be empty', - level='E') + NcaLogger().log_message('Explainability error: configuration-block name can not be empty', level='E') def derive_item(self, new_name): """ @@ -194,8 +193,7 @@ def are_peers_connected(self, src, dst): :return: bool: True for connected, False for disconnected """ if not self.all_conns: - NcaLogger().log_message(f'Explainability error: Connections were not set yet, but peer query was called', - level='E') + NcaLogger().log_message('Explainability error: Connections were not set yet, but peer query was called', level='E') for cube in self.all_conns: src_peers_names = [] dst_peers_names = [] diff --git a/nca/Utils/Utils.py b/nca/Utils/Utils.py index 5e8e6408b..adc22afa6 100644 --- a/nca/Utils/Utils.py +++ b/nca/Utils/Utils.py @@ -14,4 +14,3 @@ def __call__(cls, *args, **kwargs): if cls not in cls._instances: cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) return cls._instances[cls] - From e2d059febc9ea94cc48ea544c8dc1b50eb9a1b08 Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Sun, 30 Apr 2023 20:27:09 +0300 Subject: [PATCH 136/187] adding html config params Signed-off-by: Shmulik Froimovich --- nca/NetworkConfig/NetworkConfigQuery.py | 10 +++++----- nca/Utils/ExplTracker.py | 5 ++++- nca/Utils/OutputConfiguration.py | 15 +++++++++++++++ nca/nca_cli.py | 6 +++++- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index f1e44aefa..75aef72e4 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -630,7 +630,7 @@ class ConnectivityMapQuery(NetworkConfigQuery): @staticmethod def get_supported_output_formats(): - return {'txt', 'yaml', 'csv', 'md', 'dot', 'json', 'jpg', 'txt_no_fw_rules'} + return {'txt', 'yaml', 'csv', 'md', 'dot', 'json', 'jpg', 'html', 'txt_no_fw_rules'} def is_in_subset(self, peer): """ @@ -817,7 +817,7 @@ def get_connectivity_output_full(self, connections, peers, peers_to_compare): :param PeerSet peers_to_compare: the peers to consider for fw-rules output :rtype Union[str,dict] """ - if self.output_config.outputFormat in ['dot', 'jpg']: + if self.output_config.outputFormat in ['dot', 'jpg', 'html']: dot_full = self.dot_format_from_connections_dict(connections, peers) return dot_full, None if self.output_config.outputFormat == 'txt_no_fw_rules': @@ -836,7 +836,7 @@ def get_props_output_full(self, props, peers_to_compare, ip_blocks_mask): whereas all other values should be filtered out in the output :rtype Union[str,dict] """ - if self.output_config.outputFormat in ['dot', 'jpg']: + if self.output_config.outputFormat in ['dot', 'jpg', 'html']: dot_full = self.dot_format_from_props(props, peers_to_compare, ip_blocks_mask) return dot_full, None # TODO - handle 'txt_no_fw_rules' output format @@ -858,7 +858,7 @@ def get_connectivity_output_split_by_tcp(self, connections, peers, peers_to_comp connectivity_tcp_str = 'TCP' connectivity_non_tcp_str = 'non-TCP' connections_tcp, connections_non_tcp = self.convert_connections_to_split_by_tcp(connections) - if self.output_config.outputFormat in ['dot', 'jpg']: + if self.output_config.outputFormat in ['dot', 'jpg', 'html']: dot_tcp = self.dot_format_from_connections_dict(connections_tcp, peers, connectivity_tcp_str) dot_non_tcp = self.dot_format_from_connections_dict(connections_non_tcp, peers, connectivity_non_tcp_str) # concatenate the two graphs into one dot file @@ -903,7 +903,7 @@ def get_props_output_split_by_tcp(self, props, peers_to_compare, ip_blocks_mask) connectivity_tcp_str = 'TCP' connectivity_non_tcp_str = 'non-TCP' props_tcp, props_non_tcp = self.convert_props_to_split_by_tcp(props) - if self.output_config.outputFormat in ['dot', 'jpg']: + if self.output_config.outputFormat in ['dot', 'jpg', 'html']: dot_tcp = self.dot_format_from_props(props_tcp, peers_to_compare, ip_blocks_mask, connectivity_tcp_str) dot_non_tcp = self.dot_format_from_props(props_non_tcp, peers_to_compare, ip_blocks_mask, diff --git a/nca/Utils/ExplTracker.py b/nca/Utils/ExplTracker.py index b943a031d..4821b01e5 100644 --- a/nca/Utils/ExplTracker.py +++ b/nca/Utils/ExplTracker.py @@ -70,6 +70,7 @@ def __init__(self): self._is_active = False self.all_conns = {} self.all_peers = {} + self.explain_all_results = '' self.add_item('', 'Default-Policy', 0) @@ -258,7 +259,9 @@ def explain_all(self): text_elem.string = text entry.append(text_elem) soup.append(entry) - return soup.prettify() + + self.explain_all_results = soup.prettify() + return self.explain_all_results def explain(self, nodes): """ diff --git a/nca/Utils/OutputConfiguration.py b/nca/Utils/OutputConfiguration.py index bef6fe4f4..d73ad0ce6 100644 --- a/nca/Utils/OutputConfiguration.py +++ b/nca/Utils/OutputConfiguration.py @@ -62,6 +62,21 @@ def print_query_output(self, output, supported_output_formats=None): print(f'Command {dot_cmd_string}\n did not create {path}\n', file=sys.stderr) if os.path.isfile(tmp_dot_file): os.remove(tmp_dot_file) + elif self.outputFormat == 'html': + tmp_dot_file = f'{path}.nca_tmp.dot' + dot_cmd = ['dot', tmp_dot_file, '-Tjpg', f'-o{path}'] + try: + with open(tmp_dot_file, "w") as f: + f.write(output) + CmdlineRunner.run_and_get_output(dot_cmd) + except Exception as e: + print(f'Failed to create a jpg file: {path}\n{e}', file=sys.stderr) + if not os.path.isfile(path): + dot_cmd_string = ' '.join(dot_cmd) + print(f'Command {dot_cmd_string}\n did not create {path}\n', file=sys.stderr) + if os.path.isfile(tmp_dot_file): + os.remove(tmp_dot_file) + else: try: with open(path, "a") as f: diff --git a/nca/nca_cli.py b/nca/nca_cli.py index 505fa8ffd..3d995f605 100644 --- a/nca/nca_cli.py +++ b/nca/nca_cli.py @@ -185,6 +185,10 @@ def run_args(args): output_config['expl'] = args.explain.split(',') ExplTracker().activate() + if args.output_format == 'html': + output_config['expl'] = ['ALL'] + ExplTracker().activate() + if args.equiv is not None: np_list = args.equiv if args.equiv != [''] else None query_name = 'twoWayContainment' @@ -312,7 +316,7 @@ def nca_main(argv=None): help='A list of labels to subset the query by') parser.add_argument('--ghe_token', '--gh_token', type=str, help='A valid token to access a GitHub repository') parser.add_argument('--output_format', '-o', type=str, - help='Output format specification (txt, txt_no_fw_rules, csv, md, dot, jpg or yaml). ' + help='Output format specification (txt, txt_no_fw_rules, csv, md, dot, jpg, html or yaml). ' 'The default is txt.') parser.add_argument('--file_out', '-f', type=str, help='A file path to which output is redirected') parser.add_argument('--expected_output', type=str, help='A file path of the expected query output,' From 8159fb00cbd0ce89c9f106cdb83e7756e45b1303 Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Tue, 2 May 2023 17:19:43 +0300 Subject: [PATCH 137/187] integrating SVG, expl and JS code Signed-off-by: Shmulik Froimovich --- nca/FWRules/InteractiveConnectivityGraph.py | 162 +++++++++++++++++- nca/NetworkConfig/NetworkConfigQueryRunner.py | 2 +- nca/Utils/OutputConfiguration.py | 11 +- nca/nca_cli.py | 2 +- 4 files changed, 168 insertions(+), 9 deletions(-) diff --git a/nca/FWRules/InteractiveConnectivityGraph.py b/nca/FWRules/InteractiveConnectivityGraph.py index 8d6a38a70..f8c46f51d 100644 --- a/nca/FWRules/InteractiveConnectivityGraph.py +++ b/nca/FWRules/InteractiveConnectivityGraph.py @@ -59,13 +59,13 @@ class ElementRelations: highlights: set = field(default_factory=set) explanation: list = field(default_factory=set) - def __init__(self, svg_file_name, output_directory): + def __init__(self, svg_file_name, output_directory, expl_xml=None): """ Creates the InteractiveConnectivityGraph param: svg_file_name: str param: output_directory: str """ - self.svg_graph = self.SvgGraph(svg_file_name, output_directory) + self.svg_graph = self.SvgGraph(svg_file_name, output_directory, expl_xml) self.abstract_graph = self.AbstractGraph() def create_interactive_graph(self): @@ -86,7 +86,7 @@ def create_interactive_graph(self): # (5b) from the abstract graph, for each element, set the explanation of its connectivity graph: self.abstract_graph.set_tags_explanation(elements_relations) # (6) for each element, create an svg file containing these related elements: - self.svg_graph.create_output(elements_relations) + self.svg_graph.create_html() class SvgGraph: """ @@ -111,7 +111,7 @@ class SvgGraph: ELEMENTS_DIRECTORY = 'elements' - def __init__(self, input_svg_file, output_directory): + def __init__(self, input_svg_file, output_directory, expl_xml=None): """ Creates the InteractiveConnectivityGraph param: svg_file_name: str @@ -120,6 +120,7 @@ def __init__(self, input_svg_file, output_directory): self.input_svg_file = input_svg_file self.output_directory = output_directory self.soup = None + self.expl_xml = expl_xml def read_input_file(self): """ @@ -289,6 +290,56 @@ def _set_explanation(self, tag_soup, explanation): for holder, line in zip(place_holders, explanation + ['']*(len(place_holders) - len(explanation))): holder.string = line + def create_html(self): + # make a node element for each table entry + node_elements = self.soup.find_all(class_='node') + for node in node_elements: + # Find all text elements within the current node + text_elements = node.find_all('text') + # Check if the current node has more than one text element + if len(text_elements) > 1: + namespace = node.find('title').string + namespace = namespace.split('/')[0] + # group each text element with the polygon before it + for text in text_elements: + # Find the previous polygon element + polygon = text.find_previous('polygon') + text['fill'] = 'blue' + # Create a new 'g' element + full_name = namespace + '/' + text.string + group = self.soup.new_tag('g', attrs={'class': 'node', 'title': full_name}) + # Move the polygon and text elements inside the new 'g' element + polygon.insert_before(group) + group.append(polygon.extract()) + group.append(text.extract()) + del node['class'] + + xml_soup_str = str(self.soup) + lxml_soup = BeautifulSoup(xml_soup_str, 'lxml') + + # add the expl' xml block to the svg graph + svg_root = lxml_soup.find('svg') + script = lxml_soup.new_tag('script') + script['type'] = "text/xml" # You can use a custom MIME type if needed + # prepare the expl' buffer for js: + self.expl_xml = '\n'.join(self.expl_xml.splitlines()[1:]) + + # cdata_section = etree.CDATA(self.expl_xml) + cdata_section = f'{self.expl_xml}]]>' + script.append(cdata_section) + svg_root.append(script) + + # add js code + html_soup = BeautifulSoup(self.HTML_TEMPLATE, 'html.parser') + graph_container = html_soup.find(id='graph-container') + graph_container.insert(0, BeautifulSoup(str(lxml_soup), 'html.parser')) + + # write to file + tag_file_name = self.output_directory + with open(tag_file_name, 'wb') as tag_svg_file: + tag_svg_file.write(html_soup.prettify(encoding='utf-8')) + return + def create_output(self, elements_relations): """ Creates the set of svg files as an interactive graph @@ -327,6 +378,109 @@ def create_output(self, elements_relations): self._highlight_tag(related_tag, related_tag_info.t_class) self._save_tag_file(tag_soup, tag_info) + HTML_TEMPLATE = ''' + + + + + + NCA Graph + + + +
+ 
Please select the SOURCE node
+ + + + ''' + class AbstractGraph: """ AbstractGraph is responsible of diff --git a/nca/NetworkConfig/NetworkConfigQueryRunner.py b/nca/NetworkConfig/NetworkConfigQueryRunner.py index ca102f562..1bfd35a63 100644 --- a/nca/NetworkConfig/NetworkConfigQueryRunner.py +++ b/nca/NetworkConfig/NetworkConfigQueryRunner.py @@ -57,7 +57,7 @@ def compute_final_results(self, output_format, expl_nodes): else: output = '\n'.join(self.query_iterations_output) expl_out = '' - if ExplTracker().is_active(): + if ExplTracker().is_active() and not expl_nodes[0] == 'ALL': expl_out = ExplTracker().explain(expl_nodes) return self.numerical_result, output+expl_out, self.num_not_executed diff --git a/nca/Utils/OutputConfiguration.py b/nca/Utils/OutputConfiguration.py index d73ad0ce6..9119e4af4 100644 --- a/nca/Utils/OutputConfiguration.py +++ b/nca/Utils/OutputConfiguration.py @@ -8,6 +8,8 @@ import sys from urllib import request from nca.Utils.CmdlineRunner import CmdlineRunner +from nca.FWRules.InteractiveConnectivityGraph import InteractiveConnectivityGraph +from nca.Utils.ExplTracker import ExplTracker class OutputConfiguration(dict): @@ -46,6 +48,7 @@ def print_query_output(self, output, supported_output_formats=None): print(f'{self.outputFormat} output format is not supported for this query') return path = self.outputPath + results = '' if path is not None: # print output to a file if self.outputFormat == 'jpg': @@ -64,19 +67,21 @@ def print_query_output(self, output, supported_output_formats=None): os.remove(tmp_dot_file) elif self.outputFormat == 'html': tmp_dot_file = f'{path}.nca_tmp.dot' - dot_cmd = ['dot', tmp_dot_file, '-Tjpg', f'-o{path}'] + tmp_svg_file = f'{path}.nca_tmp.svg' + dot_cmd = ['dot', tmp_dot_file, '-Tsvg', f'-o{tmp_svg_file}'] try: with open(tmp_dot_file, "w") as f: f.write(output) CmdlineRunner.run_and_get_output(dot_cmd) + InteractiveConnectivityGraph(tmp_svg_file, path, ExplTracker().explain_all())\ + .create_interactive_graph() except Exception as e: - print(f'Failed to create a jpg file: {path}\n{e}', file=sys.stderr) + print(f'Failed to create a svg file: {path}\n{e}', file=sys.stderr) if not os.path.isfile(path): dot_cmd_string = ' '.join(dot_cmd) print(f'Command {dot_cmd_string}\n did not create {path}\n', file=sys.stderr) if os.path.isfile(tmp_dot_file): os.remove(tmp_dot_file) - else: try: with open(path, "a") as f: diff --git a/nca/nca_cli.py b/nca/nca_cli.py index 1a3a39ebf..b658db733 100644 --- a/nca/nca_cli.py +++ b/nca/nca_cli.py @@ -190,7 +190,7 @@ def run_args(args): if args.output_format == 'html': output_config['expl'] = ['ALL'] - ExplTracker().activate() + ExplTracker(output_config.outputEndpoints).activate() if args.equiv is not None: np_list = args.equiv if args.equiv != [''] else None From 4e291e54b4934aec95604d3d1edee778490eda5c Mon Sep 17 00:00:00 2001 From: Shmulik Froimovich Date: Thu, 4 May 2023 13:14:35 +0300 Subject: [PATCH 138/187] implemented subgraph with expl into the html generation Signed-off-by: Shmulik Froimovich --- nca/FWRules/InteractiveConnectivityGraph.py | 210 +++++++++++++------- 1 file changed, 133 insertions(+), 77 deletions(-) diff --git a/nca/FWRules/InteractiveConnectivityGraph.py b/nca/FWRules/InteractiveConnectivityGraph.py index f8c46f51d..2f4aeef3e 100644 --- a/nca/FWRules/InteractiveConnectivityGraph.py +++ b/nca/FWRules/InteractiveConnectivityGraph.py @@ -13,6 +13,7 @@ from collections import defaultdict import posixpath import networkx +import json from bs4 import BeautifulSoup @@ -86,7 +87,7 @@ def create_interactive_graph(self): # (5b) from the abstract graph, for each element, set the explanation of its connectivity graph: self.abstract_graph.set_tags_explanation(elements_relations) # (6) for each element, create an svg file containing these related elements: - self.svg_graph.create_html() + self.svg_graph.create_html(elements_relations) class SvgGraph: """ @@ -290,7 +291,7 @@ def _set_explanation(self, tag_soup, explanation): for holder, line in zip(place_holders, explanation + ['']*(len(place_holders) - len(explanation))): holder.string = line - def create_html(self): + def create_html(self, elements_relations): # make a node element for each table entry node_elements = self.soup.find_all(class_='node') for node in node_elements: @@ -333,6 +334,19 @@ def create_html(self): html_soup = BeautifulSoup(self.HTML_TEMPLATE, 'html.parser') graph_container = html_soup.find(id='graph-container') graph_container.insert(0, BeautifulSoup(str(lxml_soup), 'html.parser')) + # add elements_relations to the js as a json parameter + # convert elements_relations to Json serializable + er_dict = {} + for key, value in elements_relations.items(): + er_dict[key] = {'relations': list(elements_relations[key].relations), + 'highlights': list(elements_relations[key].highlights), + 'explanation': list(elements_relations[key].explanation) + } + json_string = json.dumps(er_dict) + head = html_soup.head + script_tag = html_soup.new_tag('script') + script_tag.string = f'const jsObject = {json_string};' + head.append(script_tag) # write to file tag_file_name = self.output_directory @@ -386,24 +400,6 @@ def create_output(self, elements_relations): NCA Graph