Finalizing optimized HC set (#690)

* Initial implementation of building and minimizing fw-rules directly from connectivity properties.

Signed-off-by: Tanya <[email protected]>

* Fixed lint errors.

Signed-off-by: Tanya <[email protected]>

* Updating (some of) expected results for explainability queries, according to more condensed optimized output.

Signed-off-by: Tanya <[email protected]>

* Fixed converting fw-rules to connectivity properties, while taking into account TCP/non-TCP protocol restriction.

Signed-off-by: Tanya <[email protected]>

* Optimized handling IpBlocks in optimized fw-rules minimization

Signed-off-by: Tanya <[email protected]>

* Optimized initial namespace grouping (by grouping few namespaces together, according to grouping in cubes). Also, added grouping by labels to initial grouping.

Signed-off-by: Tanya <[email protected]>

* Optimized initial namespace grouping (by grouping few namespaces together, according to grouping in cubes). Also, added grouping by labels to initial grouping.

Signed-off-by: Tanya <[email protected]>

* More optimization in calculation partial ns grouping.

Signed-off-by: Tanya <[email protected]>

* Fixed lint error

Signed-off-by: Tanya <[email protected]>

* Refining basic namespace grouping by finding more opportunities to use properties in containing connections.

Signed-off-by: Tanya <[email protected]>

* One more refinemenet of basic namespace grouping

Signed-off-by: Tanya <[email protected]>

* One more refinemenet of basic namespace grouping

Signed-off-by: Tanya <[email protected]>

* More refinemenets of peer grouping from properties

Signed-off-by: Tanya <[email protected]>

* More refinemenets of peer grouping from properties

Signed-off-by: Tanya <[email protected]>

* More refinements of peer grouping from properties

Signed-off-by: Tanya <[email protected]>

* Added outputEndpoints option handling to PeerSetElement.
Refined ns-set pairs grouping computation -trying starting from src_peers and from dst_peers and choosing a more compact grouping.
Added grouping by full IpBlock.

Signed-off-by: Tanya <[email protected]>

* Fixing lint errors.

Signed-off-by: Tanya <[email protected]>

* Fixing handling txt-no_fw_rules format in the optimized solution

Signed-off-by: Tanya <[email protected]>

* Fixing lint error

Signed-off-by: Tanya <[email protected]>

* Fix: taking into account connectivity restriction (TCP/non-TCP) in generation of dot output in optimized solution

Signed-off-by: Tanya <[email protected]>

* Small fixes in txt_no_fw_rules_format

Signed-off-by: Tanya <[email protected]>

* Small fixes in txt_no_fw_rules_format

Signed-off-by: Tanya <[email protected]>

* Added grouping by dns entries to the optimized algorithm.

Signed-off-by: Tanya <[email protected]>

* Cleaning up unused code and refactoring accordingly.

Signed-off-by: Tanya <[email protected]>

* Fixed lint error.

Signed-off-by: Tanya <[email protected]>

---------

Signed-off-by: Tanya <[email protected]>

nca/CoreDS/ConnectionSet.py

@@ -3,13 +3,10 @@
 # SPDX-License-Identifier: Apache2.0
 #
 
-from collections import defaultdict
 from .CanonicalIntervalSet import CanonicalIntervalSet
 from .ConnectivityProperties import ConnectivityProperties
 from .ProtocolNameResolver import ProtocolNameResolver
 from .ProtocolSet import ProtocolSet
-from .Peer import PeerSet, IpBlock
-from nca.FWRules import FWRule
 
 
 class ConnectionSet:
@@ -546,16 +543,21 @@ def print_diff(self, other, self_name, other_name):
 
  return 'No diff.'
 
- def convert_to_connectivity_properties(self, peer_container):
+ def convert_to_connectivity_properties(self, peer_container, relevant_protocols=ProtocolSet()):
  """
  Convert the current ConnectionSet to ConnectivityProperties format.
  This function is used for comparing fw-rules output between original and optimized implementation,
  when optimized_run == 'debug'
  :param PeerContainer peer_container: the peer container
+ :param ProtocolSet relevant_protocols: specify if all protocols refer to TCP / non-TCP protocols
  :return: the connection set in ConnectivityProperties format
  """
  if self.allow_all:
- return ConnectivityProperties.get_all_conns_props_per_config_peers(peer_container)
+ if relevant_protocols:
+ protocols_conn = ConnectivityProperties.make_conn_props_from_dict({"protocols": relevant_protocols})
+ else:
+ protocols_conn = ConnectivityProperties(create_all=True)
+ return ConnectivityProperties.get_all_conns_props_per_config_peers(peer_container) & protocols_conn
 
  res = ConnectivityProperties.make_empty_props()
  for protocol, properties in self.allowed_protocols.items():
@@ -580,123 +582,3 @@ def get_non_tcp_connections():
  res.add_all_connections([ProtocolNameResolver.get_protocol_number('TCP')])
  return res
  # return ConnectionSet(True) - ConnectionSet.get_all_TCP_connections()
-
- # TODO - after moving to the optimized HC set implementation,
- # get rid of ConnectionSet and move the code below to ConnectivityProperties.py
-
- @staticmethod
- def get_connection_set_and_peers_from_cube(the_cube, peer_container,
- relevant_protocols=ProtocolSet(True)):
- conn_cube = the_cube.copy()
- src_peers = conn_cube["src_peers"] or peer_container.get_all_peers_group(True)
- conn_cube.unset_dim("src_peers")
- dst_peers = conn_cube["dst_peers"] or peer_container.get_all_peers_group(True)
- conn_cube.unset_dim("dst_peers")
- protocols = conn_cube["protocols"]
- conn_cube.unset_dim("protocols")
- if not conn_cube.has_active_dim() and (protocols.is_whole_range() or protocols == relevant_protocols):
- conns = ConnectionSet(True)
- else:
- conns = ConnectionSet()
- protocol_names = ProtocolSet.get_protocol_names_from_interval_set(protocols)
- for protocol in protocol_names:
- if conn_cube.has_active_dim():
- conns.add_connections(protocol, ConnectivityProperties.make_conn_props(conn_cube))
- else:
- if ConnectionSet.protocol_supports_ports(protocol) or ConnectionSet.protocol_is_icmp(protocol):
- conns.add_connections(protocol,
- ConnectivityProperties.get_all_conns_props_per_config_peers(peer_container))
- else:
- conns.add_connections(protocol, True)
- return conns, src_peers, dst_peers
-
- @staticmethod
- def conn_props_to_fw_rules(conn_props, cluster_info, peer_container,
- connectivity_restriction):
- """
- Build FWRules from the given ConnectivityProperties
- :param ConnectivityProperties conn_props: properties describing allowed connections
- :param ClusterInfo cluster_info: the cluster info
- :param PeerContainer peer_container: the peer container
- whereas all other values should be filtered out in the output
- :param Union[str,None] connectivity_restriction: specify if connectivity is restricted to
- TCP / non-TCP , or not
- :return: FWRules map
- """
- relevant_protocols = ProtocolSet()
- if connectivity_restriction:
- if connectivity_restriction == 'TCP':
- relevant_protocols.add_protocol('TCP')
- else: # connectivity_restriction == 'non-TCP'
- relevant_protocols = ProtocolSet.get_non_tcp_protocols()
-
- fw_rules_map = defaultdict(list)
- for cube in conn_props:
- conn_cube = conn_props.get_connectivity_cube(cube)
- conns, src_peers, dst_peers = \
- ConnectionSet.get_connection_set_and_peers_from_cube(conn_cube, peer_container, relevant_protocols)
- # create FWRules for src_peers and dst_peers
- fw_rules_map[conns] += ConnectionSet.create_fw_rules_list_from_conns(conns, src_peers, dst_peers,
- cluster_info)
- return fw_rules_map
-
- @staticmethod
- def create_fw_rules_list_from_conns(conns, src_peers, dst_peers, cluster_info):
- src_fw_elements = ConnectionSet.split_peer_set_to_fw_rule_elements(src_peers, cluster_info)
- dst_fw_elements = ConnectionSet.split_peer_set_to_fw_rule_elements(dst_peers, cluster_info)
- fw_rules_list = []
- for src_elem in src_fw_elements:
- for dst_elem in dst_fw_elements:
- fw_rules_list.append(FWRule.FWRule(src_elem, dst_elem, conns))
- return fw_rules_list
-
- @staticmethod
- def split_peer_set_to_fw_rule_elements(peer_set, cluster_info):
- res = []
- peer_set_copy = peer_set.copy()
- ns_set = set()
- # first, split by namespaces
- while peer_set_copy:
- peer = list(peer_set_copy)[0]
- if isinstance(peer, IpBlock):
- res.append(FWRule.IPBlockElement(peer))
- peer_set_copy.remove(peer)
- continue
- elif isinstance(peer, FWRule.DNSEntry):
- res.append(FWRule.DNSElement(peer))
- peer_set_copy.remove(peer)
- continue
- ns_peers = PeerSet(cluster_info.ns_dict[peer.namespace])
- if ns_peers.issubset(peer_set_copy):
- ns_set.add(peer.namespace)
- else:
- # TODO try to split the element below by labels
- res.append(FWRule.PeerSetElement(ns_peers & peer_set_copy))
- peer_set_copy -= ns_peers
- if ns_set:
- res.append(FWRule.FWRuleElement(ns_set))
-
- return res
-
- @staticmethod
- def fw_rules_to_conn_props(fw_rules, peer_container):
- """
- Converting FWRules to ConnectivityProperties format.
- This function is used for comparing FWRules output between original and optimized solutions,
- when optimized_run == 'debug'
- :param MinimizeFWRules fw_rules: the given FWRules.
- :param PeerContainer peer_container: the peer container
- :return: the resulting ConnectivityProperties.
- """
- res = ConnectivityProperties.make_empty_props()
- if fw_rules.fw_rules_map is None:
- return res
- for fw_rules_list in fw_rules.fw_rules_map.values():
- for fw_rule in fw_rules_list:
- conn_props = fw_rule.conn.convert_to_connectivity_properties(peer_container)
- src_peers = PeerSet(fw_rule.src.get_peer_set(fw_rules.cluster_info))
- dst_peers = PeerSet(fw_rule.dst.get_peer_set(fw_rules.cluster_info))
- rule_props = ConnectivityProperties.make_conn_props_from_dict({"src_peers": src_peers,
- "dst_peers": dst_peers}) & conn_props
- res |= rule_props
- return res

nca/CoreDS/ConnectivityCube.py

@@ -17,15 +17,16 @@ class ConnectivityCube(dict):
  It is used as an input interface for ConnectivityProperties methods.
  """
 
- dimensions_list = ["src_peers", "dst_peers", "protocols", "src_ports", "dst_ports", "methods", "hosts", "paths",
- "icmp_type", "icmp_code"]
+ all_dimensions_list = ["src_peers", "dst_peers", "protocols", "src_ports", "dst_ports", "methods", "hosts", "paths",
+  "icmp_type", "icmp_code"]
 
- def __init__(self):
+ def __init__(self, dimensions_list=None):
  """
  By default, each dimension in the cube is initialized with entire domain value, which represents
  "don't care" or inactive dimension (i.e., the dimension has no impact).
  """
  super().__init__()
+ self.dimensions_list = dimensions_list if dimensions_list else self.all_dimensions_list
  self.named_ports = set() # used only in the original solution
  self.excluded_named_ports = set() # used only in the original solution
  for dim in self.dimensions_list:
@@ -37,7 +38,7 @@ def copy(self):
  Returns a copy of the given ConnectivityCube object
  :rtype: ConnectivityCube
  """
- res = ConnectivityCube()
+ res = ConnectivityCube(self.dimensions_list)
  for dim_name, dim_value in self.items():
  if isinstance(dim_value, MinDFA):
  res.set_dim_directly(dim_name, dim_value)
@@ -129,6 +130,11 @@ def unset_dim(self, dim_name):
  dim_value = DimensionsManager().get_dimension_domain_by_name(dim_name, True)
  self.set_dim_directly(dim_name, dim_value)
 
+ def unset_all_but_peers(self):
+ for dim_name in self.dimensions_list:
+ if dim_name not in ["src_peers", "dst_peers"]:
+ self.unset_dim(dim_name)
+
  def __getitem__(self, dim_name):
  """
  Returns a given dimension value after converting it from internal to external format.

nca/CoreDS/ConnectivityProperties.py

@@ -64,12 +64,12 @@ class ConnectivityProperties(CanonicalHyperCubeSet):
  (2) calico: +ve and -ve named ports, no src named ports, and no use of operators between these objects.
  """
 
- def __init__(self, create_all=False):
+ def __init__(self, dimensions_list=None, create_all=False):
  """
  This will create empty or full connectivity properties, depending on create_all flag.
  :param create_all: whether to create full connectivity properties.
  """
- super().__init__(ConnectivityCube.dimensions_list)
+ super().__init__(dimensions_list if dimensions_list else ConnectivityCube.all_dimensions_list)
  self.named_ports = {} # a mapping from dst named port (String) to src ports interval set
  self.excluded_named_ports = {} # a mapping from dst named port (String) to src ports interval set
  if create_all:
@@ -132,7 +132,7 @@ def get_connectivity_cube(self, cube):
  :return: the cube in ConnectivityCube format
  :rtype: ConnectivityCube
  """
- res = ConnectivityCube()
+ res = ConnectivityCube(self.all_dimensions_list)
  for i, dim in enumerate(self.active_dimensions):
  if isinstance(cube[i], MinDFA):
  res.set_dim_directly(dim, cube[i])
@@ -291,7 +291,7 @@ def copy(self):
  """
  :rtype: ConnectivityProperties
  """
- res = ConnectivityProperties()
+ res = ConnectivityProperties(self.all_dimensions_list)
  for layer in self.layers:
  res.layers[self._copy_layer_elem(layer)] = self.layers[layer].copy()
  res.active_dimensions = self.active_dimensions.copy()
@@ -470,7 +470,14 @@ def make_all_props():
  Returns all connectivity properties, representing logical True
  :return: ConnectivityProperties
  """
- return ConnectivityProperties(True)
+ return ConnectivityProperties(create_all=True)
+
+ def get_all_peers(self):
+ """
+ Return all peers appearing in self.
+ :return: PeerSet
+ """
+ return self.project_on_one_dimension("src_peers") | self.project_on_one_dimension("dst_peers")
 
  def are_auto_conns(self):
  """
@@ -496,9 +503,70 @@ def props_without_auto_conns(self):
  """
  Return the properties after removing all connections from peer to itself
  """
- peers = self.project_on_one_dimension("src_peers") | self.project_on_one_dimension("dst_peers")
+ return self - self.get_auto_conns_from_peers()
+
+ def get_auto_conns_from_peers(self):
+ """
+ Build properties containing all connections from peer to itself, for all peers in the current properties
+ :return: the resulting auto connections properties
+ """
+ peers = self.get_all_peers()
  auto_conns = ConnectivityProperties()
  for peer in peers:
  auto_conns |= ConnectivityProperties.make_conn_props_from_dict({"src_peers": PeerSet({peer}),
  "dst_peers": PeerSet({peer})})
- return self - auto_conns
+ return auto_conns
+
+ def minimize(self):
+ """
+ Try to minimize the current properties by changing the order between "src_peers" and "dst_peers" dimensions
+ """
+ new_props = self.reorder_by_switching_src_dst_peers()
+ return self if len(self) <= len(new_props) else new_props
+
+ def reorder_by_switching_src_dst_peers(self):
+ """
+ Reorder self by switching the order between "src_peers" and "dst_peers" dimensions
+ """
+ new_all_dims_map = [i for i in range(len(self.all_dimensions_list))]
+ src_peers_index = self.all_dimensions_list.index("src_peers")
+ dst_peers_index = self.all_dimensions_list.index("dst_peers")
+ # switch between "src_peers" and "dst_peers" dimensions
+ new_all_dims_map[src_peers_index] = dst_peers_index
+ new_all_dims_map[dst_peers_index] = src_peers_index
+ return self._reorder_by_dim_list(new_all_dims_map)
+
+ def _reorder_by_dim_list(self, new_all_dims_map):
+ """
+ Reorder the current properties by the given dimensions order
+ :param list[int] new_all_dims_map: the given dimensions order
+ :return: the reordered connectivity properties
+ """
+ # Build reordered all dimensions list
+ new_all_dimensions_list = self._reorder_list_by_map(self.all_dimensions_list, new_all_dims_map)
+ new_active_dimensions = []
+ new_active_dims_map = [i for i in range(len(self.active_dimensions))]
+ # Build reordered active dimensions list
+ for dim in new_all_dimensions_list:
+ if dim in self.active_dimensions:
+ new_active_dims_map[len(new_active_dimensions)] = self.active_dimensions.index(dim)
+ new_active_dimensions.append(dim)
+ # Build reordered properties by cubes
+ res = ConnectivityProperties(new_all_dimensions_list)
+ for cube in self:
+ new_cube = self._reorder_list_by_map(cube, new_active_dims_map)
+ res.add_cube(new_cube, new_active_dimensions)
+ return res
+
+ @staticmethod
+ def _reorder_list_by_map(orig_list, new_to_old_map):
+ """
+ Reorder a given list by map from new to old indices.
+ :param list orig_list: the original list
+ :param list[int] new_to_old_map: the list mapping new to old indices
+ :return: the resulting list
+ """
+ res = []
+ for i in range(len(orig_list)):
+ res.append(orig_list[new_to_old_map[i]])
+ return res