diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7f579f4ab..8d103218f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -31,11 +31,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@fdcae64e1484d349b3366718cdfef3d404390e85 + uses: github/codeql-action/init@6c089f53dd51dc3fc7e599c3cb5356453a52ca9e with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -44,4 +44,4 @@ jobs: # queries: ./path/to/local/query, your-org/your-repo/queries@main - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@fdcae64e1484d349b3366718cdfef3d404390e85 + uses: github/codeql-action/analyze@6c089f53dd51dc3fc7e599c3cb5356453a52ca9e diff --git a/.github/workflows/make-github-and-docker-release.yml b/.github/workflows/make-github-and-docker-release.yml index 29a3fd28b..93ff7a898 100644 --- a/.github/workflows/make-github-and-docker-release.yml +++ b/.github/workflows/make-github-and-docker-release.yml @@ -20,7 +20,7 @@ jobs: packages: write steps: - name: Check out the repo - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab - name: Setup NCA environment uses: ./.github/actions/setup-nca-env @@ -31,21 +31,21 @@ jobs: echo "version=$VERSION" >> $GITHUB_ENV - name: Log in to Docker Hub - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push Docker image - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 + uses: docker/build-push-action@44ea916f6c540f9302d50c2b1e5a8dc071f15cdf with: context: . push: true tags: ghcr.io/ibm/nca:${{ env.version }} - name: Build and push ubi-based Docker image - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 + uses: docker/build-push-action@44ea916f6c540f9302d50c2b1e5a8dc071f15cdf with: context: . file: Dockerfile.ubi diff --git a/.github/workflows/reset-tests-expected-runtime.yml b/.github/workflows/reset-tests-expected-runtime.yml index 7790a4099..9b7b72a95 100644 --- a/.github/workflows/reset-tests-expected-runtime.yml +++ b/.github/workflows/reset-tests-expected-runtime.yml @@ -13,21 +13,21 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab - uses: ./.github/actions/setup-nca-env - - uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e + - uses: dawidd6/action-download-artifact@7132ab516fba5f602fafae6fdd4822afa10db76f with: workflow: test-push.yml workflow_conclusion: completed name: k8s-log path: tests/ - - uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e + - uses: dawidd6/action-download-artifact@7132ab516fba5f602fafae6fdd4822afa10db76f with: workflow: test-push.yml workflow_conclusion: completed name: calico-log path: tests/ - - uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e + - uses: dawidd6/action-download-artifact@7132ab516fba5f602fafae6fdd4822afa10db76f with: workflow: test-push.yml workflow_conclusion: completed diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index aad432b53..09d1849cd 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -24,12 +24,12 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab with: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@483ef80eb98fb506c348f7d62e28055e49fe2398 # v2.3.0 + uses: ossf/scorecard-action@80e868c13c90f172d68d1f4501dee99e2479f7af # v2.1.3 with: results_file: results.sarif results_format: sarif @@ -47,6 +47,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@fdcae64e1484d349b3366718cdfef3d404390e85 + uses: github/codeql-action/upload-sarif@6c089f53dd51dc3fc7e599c3cb5356453a52ca9e # v2.1.27 with: sarif_file: results.sarif diff --git a/.github/workflows/update-tests-expected-output.yml b/.github/workflows/update-tests-expected-output.yml index efa6a4a47..c46c70986 100644 --- a/.github/workflows/update-tests-expected-output.yml +++ b/.github/workflows/update-tests-expected-output.yml @@ -13,7 +13,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab - uses: ./.github/actions/setup-nca-env - name: update or add expected output files run: | diff --git a/.github/workflows/update-tests-expected-runtime.yml b/.github/workflows/update-tests-expected-runtime.yml index 265a83458..d98b6bf50 100644 --- a/.github/workflows/update-tests-expected-runtime.yml +++ b/.github/workflows/update-tests-expected-runtime.yml @@ -13,7 +13,7 @@ jobs: outputs: changed_tests: ${{ steps.changes.outputs.changed_tests}} steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab with: fetch-depth: 0 - uses: ./.github/actions/setup-nca-env @@ -28,21 +28,21 @@ jobs: needs: changed-tests if: ${{needs.changed-tests.outputs.changed_tests}} steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab - uses: ./.github/actions/setup-nca-env - - uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e + - uses: dawidd6/action-download-artifact@7132ab516fba5f602fafae6fdd4822afa10db76f with: workflow: test-push.yml workflow_conclusion: completed name: k8s-log path: tests/ - - uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e + - uses: dawidd6/action-download-artifact@7132ab516fba5f602fafae6fdd4822afa10db76f with: workflow: test-push.yml workflow_conclusion: completed name: calico-log path: tests/ - - uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e + - uses: dawidd6/action-download-artifact@7132ab516fba5f602fafae6fdd4822afa10db76f with: workflow: test-push.yml workflow_conclusion: completed diff --git a/Dockerfile b/Dockerfile index 3ff24b8c6..191d8bc16 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ # # Using python:3.9-slim -FROM python@sha256:7476637ee33fae24822294643449e7fe9158708c976ea379037960d3007590a4 +FROM python@sha256:1fc44d17b4ca49a8715af80786f21fa5ed8cfd257a1e14e24f4a79b4ec329388 COPY requirements.txt /nca/ RUN python -m pip install -U pip wheel setuptools && pip install -r /nca/requirements.txt diff --git a/Dockerfile.ubi b/Dockerfile.ubi index abb4b6350..fa98a918b 100644 --- a/Dockerfile.ubi +++ b/Dockerfile.ubi @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache2.0 # -FROM registry.access.redhat.com/ubi8/ubi-minimal@sha256:8d43664c250c72d35af8498c7ff76a9f0d42f16b9b3b29f0caa747121778de0e +FROM registry.access.redhat.com/ubi8/ubi-minimal@sha256:6910799b75ad41f00891978575a0d955be2f800c51b955af73926e7ab59a41c3 USER 0 diff --git a/README.md b/README.md index cc1b48482..9081d100c 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ It takes such resources as input, in addition to a list of relevant endpoints, a - What are the endpoints that are not covered by any policy? - Are my policies implemented efficiently? -## Installation (requires Python 3.9 or above) +## Installation (requires Python 3.8 or above) For command-line use, NCA is installed with: ```shell pip install network-config-analyzer @@ -95,8 +95,6 @@ The arguments to `--resource_list` and to `--base_resource_list` should be one o *shorthand* `-f` - `--expected_output `\ A file path to the expected query output (for connectivity or semantic_diff queries).\ -- `--simplify_graph`\ - simplify the connectivity graph, (relevant only when output_format is dot or jpg) - `--pr_url `\ Write output as GitHub PR comment. URL points to the relevant `comments` resource in the GitHub API.\ e.g., https://api.github.com/repos/shift-left-netconfig/online-boutique/issues/1/comments diff --git a/docs/SchemeFileFormat.md b/docs/SchemeFileFormat.md index 18fc5f225..141f65329 100644 --- a/docs/SchemeFileFormat.md +++ b/docs/SchemeFileFormat.md @@ -82,7 +82,6 @@ The supported entries in the outputConfiguration object are as follows: |------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------| | outputFormat | Output format specification. | string [ txt / yaml / csv / md / dot / jpg/ txt_no_fw_rules] | | outputPath | A file path to redirect output into. | string | -| simplifyGraph | Choose if to simplify the connectivity graph. | bool [default: False] | | outputEndpoints | Choose endpoints type in output. | string [ pods / deployments ] | | subset | A dict object with the defined subset elements to display in the output | [subset](#subset) object | | fullExplanation | Choose if to print all counterexamples causing the query result in the output | bool | diff --git a/nca/CoreDS/CanonicalHyperCubeSet.py b/nca/CoreDS/CanonicalHyperCubeSet.py index a64c8b812..ad5568174 100644 --- a/nca/CoreDS/CanonicalHyperCubeSet.py +++ b/nca/CoreDS/CanonicalHyperCubeSet.py @@ -541,23 +541,20 @@ def _contained_in_aux(self, other, all_active_dims): # noqa: C901 common_part = current_layer_0 & other_layer has_common_part = bool(common_part) if has_common_part: - # if it's not last dim for both self and other, determine containment recursively if not self._is_last_dimension() and not other._is_last_dimension() and \ not (self.layers[layer])._contained_in_aux(other_sub_elem, all_active_dims[1:]): return False - # if it's last dim for self but not for other: the remaining of other should be entire cube - if self._is_last_dimension() and not other._is_last_dimension() and \ - not other_sub_elem._is_sub_elem_entire_sub_space(): - return False - # if it's the last dim for other but not for self -> containment is satisfied on this part - # at this point, sub-object from common_part is contained remaining = current_layer_0 - common_part if remaining: # continue exploring other's cubes for containment of the remaining part from self current_layer_0 = remaining else: - # count current cube (from current_layer_0) as contained in other - is_subset_count += 1 + if self._is_last_dimension() and not other._is_last_dimension(): + # if it's last dim for self but not for other: the remaining of other should be entire cube + if other_sub_elem._is_sub_elem_entire_sub_space(): + is_subset_count += 1 + else: + is_subset_count += 1 break return is_subset_count == len(self.layers) diff --git a/nca/CoreDS/ConnectionSet.py b/nca/CoreDS/ConnectionSet.py index f93142444..398c9f823 100644 --- a/nca/CoreDS/ConnectionSet.py +++ b/nca/CoreDS/ConnectionSet.py @@ -528,12 +528,8 @@ def print_diff(self, other, self_name, other_name): return other_name + ' allows all connections while ' + self_name + ' does not.' for protocol, properties in self.allowed_protocols.items(): if protocol not in other.allowed_protocols: - res = self_name + ' allows communication using protocol ' + \ - ProtocolNameResolver.get_protocol_name(protocol) - if not isinstance(properties, bool) and not properties.is_all(): - res += ' on ' + properties._get_first_item_str() - res += ' while ' + other_name + ' does not.' - return res + return self_name + ' allows communication using protocol ' + ProtocolNameResolver.get_protocol_name(protocol) \ + + ' while ' + other_name + ' does not.' other_properties = other.allowed_protocols[protocol] if properties != other_properties: return ProtocolNameResolver.get_protocol_name(protocol) + ' protocol - ' + \ @@ -585,9 +581,8 @@ def get_non_tcp_connections(): # get rid of ConnectionSet and move the code below to ConnectivityProperties.py @staticmethod - def get_connection_set_and_peers_from_cube(the_cube, peer_container, + def get_connection_set_and_peers_from_cube(conn_cube, peer_container, relevant_protocols=ProtocolSet(True)): - conn_cube = the_cube.copy() src_peers = conn_cube["src_peers"] or peer_container.get_all_peers_group(True) conn_cube.unset_dim("src_peers") dst_peers = conn_cube["dst_peers"] or peer_container.get_all_peers_group(True) @@ -662,10 +657,6 @@ def split_peer_set_to_fw_rule_elements(peer_set, cluster_info): res.append(FWRule.IPBlockElement(peer)) peer_set_copy.remove(peer) continue - elif isinstance(peer, FWRule.DNSEntry): - res.append(FWRule.DNSElement(peer)) - peer_set_copy.remove(peer) - continue ns_peers = PeerSet(cluster_info.ns_dict[peer.namespace]) if ns_peers.issubset(peer_set_copy): ns_set.add(peer.namespace) @@ -689,8 +680,6 @@ def fw_rules_to_conn_props(fw_rules, peer_container): :return: the resulting ConnectivityProperties. """ res = ConnectivityProperties.make_empty_props() - if fw_rules.fw_rules_map is None: - return res for fw_rules_list in fw_rules.fw_rules_map.values(): for fw_rule in fw_rules_list: conn_props = fw_rule.conn.convert_to_connectivity_properties(peer_container) diff --git a/nca/CoreDS/ConnectivityProperties.py b/nca/CoreDS/ConnectivityProperties.py index 84d4db0c4..1d51eabfd 100644 --- a/nca/CoreDS/ConnectivityProperties.py +++ b/nca/CoreDS/ConnectivityProperties.py @@ -159,7 +159,7 @@ def get_cube_dict(self, cube, is_txt=False): values_list = str(dim_values) elif dim in ["src_peers", "dst_peers"]: peers_set = BasePeerSet().get_peer_set_by_indices(dim_values) - peers_str_list = sorted([str(peer.full_name()) for peer in peers_set]) + peers_str_list = [str(peer.full_name()) for peer in peers_set] values_list = ','.join(peers_str_list) if is_txt else peers_str_list elif dim_type == DimensionsManager.DimensionType.IntervalSet: values_list = dim_values.get_interval_set_list_numbers_and_ranges() @@ -491,14 +491,3 @@ def are_auto_conns(self): if cube[src_peers_index] != cube[dst_peers_index] or not cube[src_peers_index].is_single_value(): return False return True - - def props_without_auto_conns(self): - """ - Return the properties after removing all connections from peer to itself - """ - peers = self.project_on_one_dimension("src_peers") | self.project_on_one_dimension("dst_peers") - auto_conns = ConnectivityProperties() - for peer in peers: - auto_conns |= ConnectivityProperties.make_conn_props_from_dict({"src_peers": PeerSet({peer}), - "dst_peers": PeerSet({peer})}) - return self - auto_conns diff --git a/nca/CoreDS/Peer.py b/nca/CoreDS/Peer.py index 02829449f..64ab06d09 100644 --- a/nca/CoreDS/Peer.py +++ b/nca/CoreDS/Peer.py @@ -662,17 +662,15 @@ def get_ip_block_canonical_form(self): res |= elem return res - def filter_ip_blocks_by_mask(self, ip_blocks_mask): + def filter_ipv6_blocks(self, ip_blocks_mask): """ Update ip blocks in the peer set by keeping only parts overlapping with the given mask. - :param IpBlock ip_blocks_mask: the mask according to which ip blocks should be updated + :param ip_blocks_mask: the mask according to which ip blocks should be updated """ peers_to_remove = [] peers_to_add = [] for peer in self: if isinstance(peer, IpBlock): - if peer.contained_in(ip_blocks_mask): - continue # optimization - avoid removing and adding the same peer peers_to_remove.append(peer) if peer.overlaps(ip_blocks_mask): new_peer = peer.copy() diff --git a/nca/FWRules/ConnectivityGraph.py b/nca/FWRules/ConnectivityGraph.py index 3631a6fd1..52ac27a1d 100644 --- a/nca/FWRules/ConnectivityGraph.py +++ b/nca/FWRules/ConnectivityGraph.py @@ -57,7 +57,6 @@ def add_edges_from_cube_dict(self, conn_cube, peer_container): Add edges to the graph according to the give cube :param ConnectivityCube conn_cube: the given cube whereas all other values should be filtered out in the output - :param PeerContainer peer_container: the peer container """ conns, src_peers, dst_peers = \ ConnectionSet.get_connection_set_and_peers_from_cube(conn_cube, peer_container) @@ -65,15 +64,6 @@ def add_edges_from_cube_dict(self, conn_cube, peer_container): for dst_peer in dst_peers: self.connections_to_peers[conns].append((src_peer, dst_peer)) - def add_props_to_graph(self, props, peer_container): - """ - Add edges to the graph according to the given connectivity properties - :param ConnectivityProperties props: the given connectivity properties - :param PeerContainer peer_container: the peer container - """ - for cube in props: - self.add_edges_from_cube_dict(props.get_connectivity_cube(cube), peer_container) - def _get_peer_details(self, peer, format_requirement=False): """ Get the name of a peer object for connectivity graph, the type and the namespace @@ -340,11 +330,10 @@ def get_connections_without_fw_rules_txt_format(self, connectivity_msg=None, exc lines_list.extend(sorted(list(lines))) return '\n'.join(lines_list) - def get_connectivity_dot_format_str(self, connectivity_restriction=None, simplify_graph=False): + def get_connectivity_dot_format_str(self, connectivity_restriction=None): """ :param Union[str,None] connectivity_restriction: specify if connectivity is restricted to TCP / non-TCP , or not - :param simplify_graph[bool, False] whether to simplify the dot output graph :rtype str :return: a string with content of dot format for connectivity graph """ @@ -352,7 +341,7 @@ def get_connectivity_dot_format_str(self, connectivity_restriction=None, simplif query_title = f'{self.output_config.queryName}/' if self.output_config.queryName else '' name = f'{query_title}{self.output_config.configName}{restriction_title}' - dot_graph = DotGraph(name, do_not_subgraph=simplify_graph) + dot_graph = DotGraph(name) peers_groups = self._get_equals_groups() # we are going to treat a peers_group as one peer. # the first peer in the peers_group is representing the group @@ -398,7 +387,7 @@ def get_connectivity_dot_format_str(self, connectivity_restriction=None, simplif dot_graph.add_edge(src_name=edge[0][0], dst_name=edge[1][0], label=conn_str, is_dir=True) for edge in undirected_edges | cliques_edges: dot_graph.add_edge(src_name=edge[0][0], dst_name=edge[1][0], label=conn_str, is_dir=False) - return dot_graph.to_str() + return dot_graph.to_str(self.output_config.outputFormat == 'dot') def get_minimized_firewall_rules(self): """ diff --git a/nca/FWRules/DotGraph.py b/nca/FWRules/DotGraph.py index 34fd2438d..2f3d4a989 100644 --- a/nca/FWRules/DotGraph.py +++ b/nca/FWRules/DotGraph.py @@ -30,7 +30,6 @@ class Node: name: str node_type: int label: str - title: str @dataclass class Edge: @@ -39,14 +38,13 @@ class Edge: label: str is_dir: bool - def __init__(self, name, do_not_subgraph): + def __init__(self, name): self.subgraphs = {} self.name = name self.edges = [] self.all_nodes = {} self.labels = set() self.labels_dict = {} - self.do_not_subgraph = do_not_subgraph self.node_styles = \ {self.NodeType.IPBlock: 'shape=box fontcolor=red2', self.NodeType.Pod: 'shape=box fontcolor=blue', @@ -76,11 +74,9 @@ def add_node(self, subgraph, name, node_type, label): param label: node label """ label = [tok.strip() for tok in label if tok != ''] - title = subgraph if self.do_not_subgraph else '' - subgraph = '' if self.do_not_subgraph else subgraph if subgraph not in self.subgraphs: self.subgraphs[subgraph] = self.Subgraph(subgraph) - node = self.Node(name, node_type, label, title) + node = self.Node(name, node_type, label) self.subgraphs[subgraph].nodes.append(node) self.all_nodes[name] = node if node_type in {self.NodeType.Clique, self.NodeType.BiClique}: @@ -100,19 +96,22 @@ def add_edge(self, src_name, dst_name, label, is_dir): self.edges.append(self.Edge(src_node, dst_node, label, is_dir)) self.labels.add(label) - def to_str(self): + def to_str(self, with_header=True): """ creates a string in a dot file format + :param bool with_header: whether to add a header to the graph (dot has one, but html does not) return str: the string """ output_result = f'// The Connectivity Graph of {self.name}\n' output_result += 'digraph ' + '{\n' - output_result += f'\tlabel=\"Connectivity Graph of {self.name}\"' + if with_header: + output_result += f'\tlabel=\"Connectivity Graph of {self.name}\"' output_result += '\tlabelloc = "t"\n' - output_result += '\tfontsize=30\n' - output_result += '\tfontcolor=maroon\n' - output_result += '\tsubgraph cluster_map_explanation {\n' + if with_header: + output_result += '\tfontsize=30\n' + output_result += '\tfontcolor=maroon\n' + output_result += '\tsubgraph cluster_map_explanation {\n' if self._set_labels_dict(): output_result += self._labels_dict_to_str() self.subgraphs = dict(sorted(self.subgraphs.items())) @@ -121,9 +120,11 @@ def to_str(self): output_result += '\tcolor=white\n' output_result += self._explanation_to_str() output_result += '\tlabelloc = "b"\n' - output_result += '\tfontsize=15\n' + if with_header: + output_result += '\tfontsize=15\n' output_result += '\tfontcolor=maroon\n' - output_result += '\t}\n' + if with_header: + output_result += '\t}\n' output_result += '}\n' return output_result @@ -195,11 +196,7 @@ def _node_to_str(self, node): table = f'<' for line in node.label: if line: - if node.title: - table += f'' - else: - table += f'' - + table += f'' table += '
{node.title}/{line}
{line}
{line}
>' label = f'label={table}' node_desc = f'{label} {self.node_styles[node.node_type]} tooltip=\"{self.node_tooltip[node.node_type]}\"' diff --git a/nca/FWRules/InteractiveConnectivityGraph.py b/nca/FWRules/InteractiveConnectivityGraph.py index 166c157e0..3493d2f69 100644 --- a/nca/FWRules/InteractiveConnectivityGraph.py +++ b/nca/FWRules/InteractiveConnectivityGraph.py @@ -13,6 +13,7 @@ from collections import defaultdict import posixpath import networkx +import json from bs4 import BeautifulSoup @@ -59,13 +60,13 @@ class ElementRelations: highlights: set = field(default_factory=set) explanation: list = field(default_factory=set) - def __init__(self, svg_file_name, output_directory): + def __init__(self, svg_file_name, output_directory, expl_xml=None): """ Creates the InteractiveConnectivityGraph param: svg_file_name: str param: output_directory: str """ - self.svg_graph = self.SvgGraph(svg_file_name, output_directory) + self.svg_graph = self.SvgGraph(svg_file_name, output_directory, expl_xml) self.abstract_graph = self.AbstractGraph() def create_interactive_graph(self): @@ -86,7 +87,7 @@ def create_interactive_graph(self): # (5b) from the abstract graph, for each element, set the explanation of its connectivity graph: self.abstract_graph.set_tags_explanation(elements_relations) # (6) for each element, create an svg file containing these related elements: - self.svg_graph.create_output(elements_relations) + self.svg_graph.create_html(elements_relations) class SvgGraph: """ @@ -111,7 +112,7 @@ class SvgGraph: ELEMENTS_DIRECTORY = 'elements' - def __init__(self, input_svg_file, output_directory): + def __init__(self, input_svg_file, output_directory, expl_xml=None): """ Creates the InteractiveConnectivityGraph param: svg_file_name: str @@ -120,6 +121,7 @@ def __init__(self, input_svg_file, output_directory): self.input_svg_file = input_svg_file self.output_directory = output_directory self.soup = None + self.expl_xml = expl_xml def read_input_file(self): """ @@ -127,7 +129,7 @@ def read_input_file(self): """ try: with open(self.input_svg_file) as svg_file: - self.soup = BeautifulSoup(svg_file.read(), 'html') + self.soup = BeautifulSoup(svg_file.read(), 'xml') except Exception as e: print(f'Failed to open file: {self.input_svg_file}\n{e} for reading', file=sys.stderr) @@ -155,9 +157,6 @@ def set_soup_tags_info(self): for conn in conn_legend.find_all('g'): conn[self.CLASS_TA] = self.LEGEND_MISC_CT - # setting class to explanation tag: - explanation_cluster = self.soup.svg.find('title', string='cluster_map_explanation').find_parent('g') - explanation_cluster[self.CLASS_TA] = self.EXPLANATION_CT # for element that we want to add a link, we replace with , and mark as clickable: for tag in self.soup.svg.find_all(True): if tag.get(self.CLASS_TA): @@ -289,6 +288,73 @@ def _set_explanation(self, tag_soup, explanation): for holder, line in zip(place_holders, explanation + ['']*(len(place_holders) - len(explanation))): holder.string = line + def create_html(self, elements_relations): + # make a node element for each table entry + node_elements = self.soup.find_all(class_='node') + for node in node_elements: + # Find all text elements within the current node + text_elements = node.find_all('text') + # Check if the current node has more than one text element + if len(text_elements) > 1: + namespace = node.find('title').string + namespace = namespace.split('/')[0] + # group each text element with the polygon before it + for text in text_elements: + # Find the previous polygon element + polygon = text.find_previous('polygon') + text['fill'] = 'blue' + # Create a new 'g' element + full_name = namespace + '/' + text.string + group = self.soup.new_tag('g', attrs={'class': 'node', 'title': full_name}) + # Move the polygon and text elements inside the new 'g' element + polygon.insert_before(group) + group.append(polygon.extract()) + group.append(text.extract()) + del node['class'] + + # remove all links + elements_with_href = self.soup.find_all(attrs={'xlink:href': True}) + for element in elements_with_href: + del element['xlink:href'] + + # add the expl' xml block to the svg graph + xml_soup_str = str(self.soup) + lxml_soup = BeautifulSoup(xml_soup_str, 'xml') + svg_root = lxml_soup.find('svg') + script = lxml_soup.new_tag('script') + script['type'] = "text/xml" # You can use a custom MIME type if needed + # prepare the expl' buffer for js: + self.expl_xml = '\n'.join(self.expl_xml.splitlines()[1:]) + + # cdata_section = etree.CDATA(self.expl_xml) + cdata_section = f'{self.expl_xml}]]>' + script.append(cdata_section) + svg_root.append(script) + + # add js code + html_soup = BeautifulSoup(self.HTML_TEMPLATE, 'html.parser') + graph_container = html_soup.find(id='graph-container') + graph_container.insert(0, BeautifulSoup(str(lxml_soup), 'xml')) + # add elements_relations to the js as a json parameter + # convert elements_relations to Json serializable + er_dict = {} + for key, value in elements_relations.items(): + er_dict[key] = {'relations': list(elements_relations[key].relations), + 'highlights': list(elements_relations[key].highlights), + 'explanation': list(elements_relations[key].explanation) + } + json_string = json.dumps(er_dict) + head = html_soup.head + script_tag = html_soup.new_tag('script') + script_tag.string = f'const jsObject = {json_string};' + head.append(script_tag) + + # write to file + tag_file_name = self.output_directory + with open(tag_file_name, 'wb') as tag_svg_file: + tag_svg_file.write(html_soup.prettify(encoding='utf-8')) + return + def create_output(self, elements_relations): """ Creates the set of svg files as an interactive graph @@ -327,6 +393,218 @@ def create_output(self, elements_relations): self._highlight_tag(related_tag, related_tag_info.t_class) self._save_tag_file(tag_soup, tag_info) + HTML_TEMPLATE = ''' + + + + + + NCA Graph + + + +
+ 
Please select the SOURCE node
+ + + + ''' # noqa + class AbstractGraph: """ AbstractGraph is responsible of diff --git a/nca/FileScanners/GenericTreeScanner.py b/nca/FileScanners/GenericTreeScanner.py index e8cd929e4..60efbe6c6 100644 --- a/nca/FileScanners/GenericTreeScanner.py +++ b/nca/FileScanners/GenericTreeScanner.py @@ -75,13 +75,6 @@ def convert_documents(documents): return [to_yaml_objects(document) for document in documents] -def leave_documents_as_is(documents): - """ - Forces the parser to yield all documents and throw parse errors (if any) at this point of time - """ - return [document for document in documents] - - class GenericTreeScanner(abc.ABC): """ A base class for reading yaml files @@ -114,7 +107,7 @@ def _yield_yaml_file(self, path, stream): """ try: if self.fast_load: - documents = leave_documents_as_is(yaml.load_all(stream, Loader=yaml.CSafeLoader)) + documents = yaml.load_all(stream, Loader=yaml.CSafeLoader) else: documents = convert_documents(yaml.compose_all(stream, Loader=yaml.CSafeLoader)) yield YamlFile(documents, path) diff --git a/nca/NetworkConfig/LiveSim/istio_gateway/istio_egress_gateway.yaml b/nca/NetworkConfig/LiveSim/istio_gateway/istio_egress_gateway.yaml deleted file mode 100644 index ce653f6d9..000000000 --- a/nca/NetworkConfig/LiveSim/istio_gateway/istio_egress_gateway.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: istio-egressgateway-livesim - namespace: istio-system - labels: - app: istio-egressgateway - istio: egressgateway -spec: - serviceAccountName: istio-egressgateway - containers: - - name: istio-proxy - image: auto ---- - -apiVersion: v1 -kind: Service -metadata: - name: istio-egressgateway - namespace: istio-system -spec: - ports: - - port: 443 - selector: - app: istio-egressgateway - istio: egressgateway ---- \ No newline at end of file diff --git a/nca/NetworkConfig/LiveSim/istio_gateway/istio_ingress_gateway.yaml b/nca/NetworkConfig/LiveSim/istio_gateway/istio_gateway.yaml similarity index 86% rename from nca/NetworkConfig/LiveSim/istio_gateway/istio_ingress_gateway.yaml rename to nca/NetworkConfig/LiveSim/istio_gateway/istio_gateway.yaml index d71cb7d3e..971dad10a 100644 --- a/nca/NetworkConfig/LiveSim/istio_gateway/istio_ingress_gateway.yaml +++ b/nca/NetworkConfig/LiveSim/istio_gateway/istio_gateway.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Pod metadata: name: istio-ingressgateway-livesim - namespace: istio-system + namespace: istio-ingressgateway-ns labels: app: istio-ingressgateway istio: ingressgateway diff --git a/nca/NetworkConfig/NetworkConfig.py b/nca/NetworkConfig/NetworkConfig.py index 2d647f93e..54274dc71 100644 --- a/nca/NetworkConfig/NetworkConfig.py +++ b/nca/NetworkConfig/NetworkConfig.py @@ -3,11 +3,11 @@ # SPDX-License-Identifier: Apache2.0 # -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field from nca.CoreDS import Peer from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.ConnectivityProperties import ConnectivityProperties -from nca.Resources.NetworkPolicy import NetworkPolicy, OptimizedPolicyConnections, PolicyConnectionsFilter +from nca.Resources.NetworkPolicy import NetworkPolicy, OptimizedPolicyConnections from .NetworkLayer import NetworkLayersContainer, NetworkLayerName from nca.Utils.ExplTracker import ExplTracker @@ -176,7 +176,7 @@ def get_affected_pods(self, is_ingress, layer_name): return affected_pods - def check_for_excluding_ipv6_addresses(self, exclude_ipv6): + def _check_for_excluding_ipv6_addresses(self, exclude_ipv6): """ checks and returns if to exclude non-referenced IPv6 addresses from the config Excluding the IPv6 addresses will be enabled if the exclude_ipv6 param is True and @@ -202,7 +202,7 @@ def get_referenced_ip_blocks(self, exclude_non_ref_ipv6=False): if self.referenced_ip_blocks is not None: return self.referenced_ip_blocks - exclude_non_ref_ipv6_from_policies = self.check_for_excluding_ipv6_addresses(exclude_non_ref_ipv6) + exclude_non_ref_ipv6_from_policies = self._check_for_excluding_ipv6_addresses(exclude_non_ref_ipv6) self.referenced_ip_blocks = Peer.PeerSet() for policy in self.policies_container.policies.values(): self.referenced_ip_blocks |= policy.referenced_ip_blocks(exclude_non_ref_ipv6_from_policies) @@ -269,12 +269,10 @@ def allowed_connections(self, from_peer, to_peer, layer_name=None): return allowed_conns_res, captured_flag_res, allowed_captured_conns_res, denied_conns_res - def allowed_connections_optimized(self, layer_name=None, res_conns_filter=PolicyConnectionsFilter()): + def allowed_connections_optimized(self, layer_name=None): """ Computes the set of allowed connections between any relevant peers. :param NetworkLayerName layer_name: The name of the layer to use, if requested to use a specific layer only - :param PolicyConnectionsFilter res_conns_filter: filter of the required resulting connections - (connections with False value will not be calculated) :return: allowed_conns: all allowed connections for relevant peers. :rtype: OptimizedPolicyConnections """ @@ -283,10 +281,8 @@ def allowed_connections_optimized(self, layer_name=None, res_conns_filter=Policy if layer_name is not None: if layer_name not in self.policies_container.layers: return self.policies_container.layers.empty_layer_allowed_connections_optimized(self.peer_container, - layer_name, - res_conns_filter) - return self.policies_container.layers[layer_name].allowed_connections_optimized(self.peer_container, - res_conns_filter) + layer_name) + return self.policies_container.layers[layer_name].allowed_connections_optimized(self.peer_container) all_peers = self.peer_container.get_all_peers_group() host_eps = Peer.PeerSet(set([peer for peer in all_peers if isinstance(peer, Peer.HostEP)])) @@ -297,33 +293,32 @@ def allowed_connections_optimized(self, layer_name=None, res_conns_filter=Policy # maintain K8s_Calico layer as active if peer container has hostEndpoint conns_res = \ self.policies_container.layers.empty_layer_allowed_connections_optimized(self.peer_container, - NetworkLayerName.K8s_Calico, - res_conns_filter) - conns_res.and_by_filter(conn_hep, replace(res_conns_filter, calc_all_allowed=False)) + NetworkLayerName.K8s_Calico) + conns_res.allowed_conns &= conn_hep + conns_res.denied_conns &= conn_hep + conns_res.pass_conns &= conn_hep else: conns_res = OptimizedPolicyConnections() - if res_conns_filter.calc_all_allowed: - conns_res.all_allowed_conns = ConnectivityProperties.get_all_conns_props_per_config_peers(self.peer_container) + conns_res.all_allowed_conns = ConnectivityProperties.get_all_conns_props_per_config_peers(self.peer_container) for layer, layer_obj in self.policies_container.layers.items(): - conns_per_layer = layer_obj.allowed_connections_optimized(self.peer_container, res_conns_filter) + conns_per_layer = layer_obj.allowed_connections_optimized(self.peer_container) # only K8s_Calico layer handles host_eps if layer != NetworkLayerName.K8s_Calico: # connectivity of hostEndpoints is only determined by calico layer - conns_per_layer.sub_by_filter(conn_hep, replace(res_conns_filter, calc_all_allowed=False)) + conns_per_layer.allowed_conns -= conn_hep + conns_per_layer.denied_conns -= conn_hep + conns_per_layer.pass_conns -= conn_hep + + # all allowed connections: intersection of all allowed connections from all layers + conns_res.all_allowed_conns &= conns_per_layer.all_allowed_conns + # all allowed captured connections: should be captured by at least one layer + conns_res.allowed_conns |= conns_per_layer.allowed_conns conns_res.captured |= conns_per_layer.captured - if res_conns_filter.calc_all_allowed: - # all allowed connections: intersection of all allowed connections from all layers - conns_res.all_allowed_conns &= conns_per_layer.all_allowed_conns - if res_conns_filter.calc_allowed: - # all allowed captured connections: should be captured by at least one layer - conns_res.allowed_conns |= conns_per_layer.allowed_conns - if res_conns_filter.calc_denied: - # denied conns: should be denied by at least one layer - conns_res.denied_conns |= conns_per_layer.denied_conns - - if res_conns_filter.calc_allowed: - # allowed captured conn (by at least one layer) has to be allowed by all layers (either implicitly or explicitly) - conns_res.allowed_conns &= conns_res.all_allowed_conns + # denied conns: should be denied by at least one layer + conns_res.denied_conns |= conns_per_layer.denied_conns + + # allowed captured conn (by at least one layer) has to be allowed by all layers (either implicitly or explicitly) + conns_res.allowed_conns &= conns_res.all_allowed_conns return conns_res def append_policy_to_config(self, policy): @@ -334,7 +329,7 @@ def append_policy_to_config(self, policy): """ self.policies_container.append_policy(policy) - def filter_conns_by_peer_types(self, conns): + def filter_conns_by_peer_types(self, conns, all_peers): """ Filter the given connections by removing several connection kinds that are never allowed (such as IpBlock to IpBlock connections, connections from DNSEntries, and more). @@ -351,6 +346,7 @@ def filter_conns_by_peer_types(self, conns): "dst_peers": all_ips | all_dns_entries}) res -= ip_to_ip_or_dns_conns # avoid DNSEntry->anything connections - dns_to_any_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_dns_entries}) + dns_to_any_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_dns_entries, + "dst_peers": all_peers}) res -= dns_to_any_conns return res diff --git a/nca/NetworkConfig/NetworkConfigQuery.py b/nca/NetworkConfig/NetworkConfigQuery.py index 13b27c880..254ee2469 100644 --- a/nca/NetworkConfig/NetworkConfigQuery.py +++ b/nca/NetworkConfig/NetworkConfigQuery.py @@ -17,9 +17,8 @@ from nca.FWRules.ConnectivityGraph import ConnectivityGraph from nca.FWRules.MinimizeFWRules import MinimizeFWRules from nca.FWRules.ClusterInfo import ClusterInfo -from nca.Resources.NetworkPolicy import PolicyConnectionsFilter from nca.Resources.CalicoNetworkPolicy import CalicoNetworkPolicy -from nca.Resources.GatewayPolicy import GatewayPolicy +from nca.Resources.IngressPolicy import IngressPolicy from nca.Utils.OutputConfiguration import OutputConfiguration from .QueryOutputHandler import QueryAnswer, DictOutputHandler, StringOutputHandler, \ PoliciesAndRulesExplanations, PodsListsExplanations, ConnectionsDiffExplanation, IntersectPodsExplanation, \ @@ -86,12 +85,11 @@ def execute_and_compute_output_in_required_format(self, cmd_line_flag=False): BasePeerSet().get_peer_interval_of(peer_set)) DimensionsManager().set_domain("dst_peers", DimensionsManager.DimensionType.IntervalSet, BasePeerSet().get_peer_interval_of(peer_set)) - if self.get_configs()[0].optimized_run != 'false': - # update all optimized connectivity properties by reducing full src_peers/dst_peers dimensions - # according to their updated domains (above) - for config in self.get_configs(): - for policy in config.policies_container.policies.values(): - policy.reorganize_opt_props_by_new_domains() + # update all optimized connectivity properties by reducing full src_peers/dst_peers dimensions + # according to their updated domains (above) + for config in self.get_configs(): + for policy in config.policies_container.policies.values(): + policy.reorganize_opt_props_by_new_domains() # run the query query_answer = self.execute(cmd_line_flag) # restore peers domains and optimized connectivity properties original values @@ -149,26 +147,6 @@ def determine_whether_to_compute_allowed_conns_for_peer_types(peer1, peer2): return False # connectivity between external peers is not relevant either return True - @staticmethod - def compare_fw_rules(fw_rules1, fw_rules2, peer_container, rules_descr=""): - text_prefix = "Original and optimized fw-rules" - if rules_descr: - text_prefix += " for " + rules_descr - if fw_rules1.fw_rules_map == fw_rules2.fw_rules_map: - print(f"{text_prefix} are semantically equivalent") - return - conn_props1 = ConnectionSet.fw_rules_to_conn_props(fw_rules1, peer_container) - conn_props2 = ConnectionSet.fw_rules_to_conn_props(fw_rules2, peer_container) - if conn_props1 == conn_props2: - print(f"{text_prefix} are semantically equivalent") - else: - diff_prop = (conn_props1 - conn_props2) | (conn_props2 - conn_props1) - if diff_prop.are_auto_conns(): - print(f"{text_prefix} differ only in auto-connections") - else: - print(f"Error: {text_prefix} are different") - assert False - class NetworkConfigQuery(BaseNetworkQuery): """ @@ -217,7 +195,7 @@ def exec(self): # collecting non-disjoint policies per network layer non_disjoint_explanation_list = [] for layer_name, layer in self.config.policies_container.layers.items(): - if layer_name == NetworkLayerName.Gateway: # skip gateway layer + if layer_name == NetworkLayerName.Ingress: # skip ingress layer continue policies_list = layer.policies_list for policy1 in policies_list: @@ -294,12 +272,12 @@ class VacuityQuery(NetworkConfigQuery): """ def exec(self): - # TODO: should handle 'gateway' layer or not? (ingress controller pod is not expected to have egress + # TODO: should handle 'ingress' layer or not? (ingress controller pod is not expected to have egress # traffic without any Ingress resource) - # currently ignoring gateway layer, removing it from configs on this query + # currently ignoring ingres layer, removing it from configs on this query self.output_config.fullExplanation = True # assign true for this query - it is ok to compare its results vacuous_config = self.config.clone_without_policies('vacuousConfig') - self_config = TwoNetworkConfigsQuery.clone_without_gateway_layer(self.config) + self_config = TwoNetworkConfigsQuery.clone_without_ingress(self.config) vacuous_res = EquivalenceQuery(self_config, vacuous_config).exec() if not vacuous_res.bool_result: return QueryAnswer(vacuous_res.bool_result, @@ -381,7 +359,7 @@ def exec(self): redundant_egress_rules = {} self.output_config.fullExplanation = True # assign true for this query - it is ok to compare its results for layer_name, layer in self.config.policies_container.layers.items(): - if layer_name == NetworkLayerName.Gateway: + if layer_name == NetworkLayerName.Ingress: continue policies_list = layer.policies_list redundant_policies = sorted(list(self.redundant_policies(policies_list, layer_name))) @@ -480,7 +458,7 @@ def other_policy_containing_deny(self, self_policy, config_with_self_policy, lay :param NetworkPolicy self_policy: The policy to check :param NetworkConfig config_with_self_policy: A network config with self_policy as its single policy :param NetworkLayerName layer_name: The layer name of the policy - :return: A policy containing self_policy's denied connections if exists, None otherwise + :return: A policy containing self_policy's denied connections if exist, None otherwise :rtype: NetworkPolicy """ policies_list = self.config.policies_container.layers[layer_name].policies_list @@ -493,40 +471,26 @@ def other_policy_containing_deny(self, self_policy, config_with_self_policy, lay if not other_policy.has_deny_rules(): continue config_with_other_policy = self.config.clone_with_just_one_policy(other_policy.full_name()) - if self.config.optimized_run == 'false': - res = self.check_deny_containment_original(config_with_self_policy, config_with_other_policy, layer_name) - else: - res = self.check_deny_containment_optimized(config_with_self_policy, config_with_other_policy, layer_name) - if res: - return other_policy + # calling get_all_peers_group does not require getting dnsEntry peers, since they are not relevant when computing + # deny connections + pods_to_compare = self.config.peer_container.get_all_peers_group() + pods_to_compare |= TwoNetworkConfigsQuery(self.config, + config_with_other_policy).disjoint_referenced_ip_blocks() + for pod1 in pods_to_compare: + for pod2 in pods_to_compare: + if isinstance(pod1, IpBlock) and isinstance(pod2, IpBlock): + continue + if pod1 == pod2: + continue # no way to prevent a pod from communicating with itself + _, _, _, self_deny_conns = config_with_self_policy.allowed_connections(pod1, pod2, layer_name) + _, _, _, other_deny_conns = config_with_other_policy.allowed_connections(pod1, pod2, layer_name) + if not self_deny_conns: + continue + if not self_deny_conns.contained_in(other_deny_conns): + return None + return other_policy return None - def check_deny_containment_original(self, config_with_self_policy, config_with_other_policy, layer_name): - # calling get_all_peers_group does not require getting dnsEntry peers, since they are not relevant when computing - # deny connections - pods_to_compare = self.config.peer_container.get_all_peers_group() - pods_to_compare |= TwoNetworkConfigsQuery(self.config, config_with_other_policy).disjoint_referenced_ip_blocks() - for pod1 in pods_to_compare: - for pod2 in pods_to_compare: - if isinstance(pod1, IpBlock) and isinstance(pod2, IpBlock): - continue - if pod1 == pod2: - continue # no way to prevent a pod from communicating with itself - _, _, _, self_deny_conns = config_with_self_policy.allowed_connections(pod1, pod2, layer_name) - _, _, _, other_deny_conns = config_with_other_policy.allowed_connections(pod1, pod2, layer_name) - if not self_deny_conns: - continue - if not self_deny_conns.contained_in(other_deny_conns): - return False - return True - - @staticmethod - def check_deny_containment_optimized(config_with_self_policy, config_with_other_policy, layer_name): - res_conns_filter = PolicyConnectionsFilter.only_denied_connections() - self_props = config_with_self_policy.allowed_connections_optimized(layer_name, res_conns_filter) - other_props = config_with_other_policy.allowed_connections_optimized(layer_name, res_conns_filter) - return self_props.denied_conns.contained_in(other_props.denied_conns) - def other_rule_containing(self, self_policy, self_rule_index, is_ingress, layer_name): """ Search whether a given policy rule is contained in another policy rule @@ -637,7 +601,7 @@ def exec(self): # noqa: C901 policies_issue += '\tNote that it contains a single policy.\n' for layer_name, layer in self.config.policies_container.layers.items(): - if layer_name == NetworkLayerName.Gateway: + if layer_name == NetworkLayerName.Ingress: continue policies_list = layer.policies_list # check for redundant policies in this layer @@ -705,7 +669,7 @@ class ConnectivityMapQuery(NetworkConfigQuery): @staticmethod def get_supported_output_formats(): - return {'txt', 'yaml', 'csv', 'md', 'dot', 'json', 'jpg', 'txt_no_fw_rules'} + return {'txt', 'yaml', 'csv', 'md', 'dot', 'json', 'jpg', 'html', 'txt_no_fw_rules'} def is_in_subset(self, peer): """ @@ -777,7 +741,7 @@ def compute_connectivity_output_original(self): fw_rules = None fw_rules_tcp = None fw_rules_non_tcp = None - exclude_ipv6 = self.config.check_for_excluding_ipv6_addresses(self.output_config.excludeIPv6Range) + exclude_ipv6 = self.output_config.excludeIPv6Range connections = defaultdict(list) # if dns entry peers exist but no istio policies are configured, # then actually istio layer exists implicitly, connections to these peers will be considered with the @@ -823,18 +787,21 @@ def compute_connectivity_output_optimized(self): opt_fw_rules = None opt_fw_rules_tcp = None opt_fw_rules_non_tcp = None - exclude_ipv6 = self.config.check_for_excluding_ipv6_addresses(self.output_config.excludeIPv6Range) - res_conns_filter = PolicyConnectionsFilter.only_all_allowed_connections() - opt_conns = self.config.allowed_connections_optimized(res_conns_filter=res_conns_filter) + exclude_ipv6 = self.output_config.excludeIPv6Range + opt_conns = self.config.allowed_connections_optimized() all_conns_opt = opt_conns.all_allowed_conns opt_peers_to_compare = self.config.peer_container.get_all_peers_group(include_dns_entries=True) # add all relevant IpBlocks, used in connections opt_peers_to_compare |= all_conns_opt.project_on_one_dimension('src_peers') | \ all_conns_opt.project_on_one_dimension('dst_peers') if exclude_ipv6: - # remove connections where any of src_peers or dst_peers contain automatically-added IPv6 blocks, + ip_blocks_mask = IpBlock.get_all_ips_block(exclude_ipv6=True) + ref_ip_blocks = self.config.get_referenced_ip_blocks(exclude_ipv6) + for ip_block in ref_ip_blocks: + ip_blocks_mask |= ip_block + opt_peers_to_compare.filter_ipv6_blocks(ip_blocks_mask) + # remove connections where any of src_peers or dst_peers contains automatically-added IPv6 blocks, # while keeping connections with IPv6 blocks directly referenced in policies - opt_peers_to_compare.filter_ip_blocks_by_mask(IpBlock.get_all_ips_block(exclude_ipv6=True)) all_conns_opt &= ConnectivityProperties.make_conn_props_from_dict({"src_peers": opt_peers_to_compare, "dst_peers": opt_peers_to_compare}) base_peers_num = len(opt_peers_to_compare) @@ -847,7 +814,7 @@ def compute_connectivity_output_optimized(self): all_conns_opt &= subset_conns src_peers, dst_peers = ExplTracker().extract_peers(all_conns_opt) all_peers = src_peers | dst_peers - all_conns_opt = self.config.filter_conns_by_peer_types(all_conns_opt) + all_conns_opt = self.config.filter_conns_by_peer_types(all_conns_opt, opt_peers_to_compare) expl_conns = all_conns_opt if self.config.policies_container.layers.does_contain_layer(NetworkLayerName.Istio): output_res, opt_fw_rules_tcp, opt_fw_rules_non_tcp = \ @@ -884,15 +851,14 @@ def exec(self): opt_end = time.time() print(f'Opt time: {(opt_end - opt_start):6.2f} seconds') if self.config.optimized_run == 'debug': - if fw_rules and opt_fw_rules: - self.compare_fw_rules(fw_rules, opt_fw_rules, self.config.peer_container, - f"connectivity of {self.config.name}") - if fw_rules_tcp and opt_fw_rules_tcp: - self.compare_fw_rules(fw_rules_tcp, opt_fw_rules_tcp, self.config.peer_container, - f"connectivity - tcp only of {self.config.name}") - if fw_rules_non_tcp and opt_fw_rules_non_tcp: - self.compare_fw_rules(fw_rules_non_tcp, opt_fw_rules_non_tcp, self.config.peer_container, - f"connectivity - non-tcp only of {self.config.name}") + if fw_rules and fw_rules.fw_rules_map and opt_fw_rules and opt_fw_rules.fw_rules_map: + self.compare_fw_rules(fw_rules, opt_fw_rules) + if fw_rules_tcp and fw_rules_tcp.fw_rules_map and \ + opt_fw_rules_tcp and opt_fw_rules_tcp.fw_rules_map: + self.compare_fw_rules(fw_rules_tcp, opt_fw_rules_tcp) + if fw_rules_non_tcp and fw_rules_non_tcp.fw_rules_map and \ + opt_fw_rules_non_tcp and opt_fw_rules_non_tcp.fw_rules_map: + self.compare_fw_rules(fw_rules_non_tcp, opt_fw_rules_non_tcp) else: # self.config.optimized_run == 'true': if self.output_config.outputFormat in ['json', 'yaml']: res.output_explanation = [ComputedExplanation(dict_explanation=output_res)] @@ -900,6 +866,19 @@ def exec(self): res.output_explanation = [ComputedExplanation(str_explanation=output_res)] return res + def compare_fw_rules(self, fw_rules1, fw_rules2): + conn_props1 = ConnectionSet.fw_rules_to_conn_props(fw_rules1, self.config.peer_container) + conn_props2 = ConnectionSet.fw_rules_to_conn_props(fw_rules2, self.config.peer_container) + if conn_props1 == conn_props2: + print("Original and optimized fw-rules are semantically equivalent") + else: + diff_prop = (conn_props1 - conn_props2) | (conn_props2 - conn_props1) + if diff_prop.are_auto_conns(): + print("Original and optimized fw-rules differ only in auto-connections") + else: + print("Error: original and optimized fw-rules are different") + assert False + def get_connectivity_output_full(self, connections, peers, peers_to_compare): """ get the connectivity map output considering all connections in the output @@ -908,7 +887,7 @@ def get_connectivity_output_full(self, connections, peers, peers_to_compare): :param PeerSet peers_to_compare: the peers to consider for fw-rules output :rtype (Union[str,dict], MinimizeFWRules) """ - if self.output_config.outputFormat in ['dot', 'jpg']: + if self.output_config.outputFormat in ['dot', 'jpg', 'html']: dot_full = self.dot_format_from_connections_dict(connections, peers) return dot_full, None if self.output_config.outputFormat == 'txt_no_fw_rules': @@ -926,7 +905,7 @@ def get_props_output_full(self, props, peers_to_compare): whereas all other values should be filtered out in the output :rtype ([Union[str, dict], MinimizeFWRules]) """ - if self.output_config.outputFormat in ['dot', 'jpg']: + if self.output_config.outputFormat in ['dot', 'jpg', 'html']: dot_full = self.dot_format_from_props(props, peers_to_compare) return dot_full, None if self.output_config.outputFormat == 'txt_no_fw_rules': @@ -947,7 +926,7 @@ def get_connectivity_output_split_by_tcp(self, connections, peers, peers_to_comp connectivity_tcp_str = 'TCP' connectivity_non_tcp_str = 'non-TCP' connections_tcp, connections_non_tcp = self.convert_connections_to_split_by_tcp(connections) - if self.output_config.outputFormat in ['dot', 'jpg']: + if self.output_config.outputFormat in ['dot', 'jpg', 'html']: dot_tcp = self.dot_format_from_connections_dict(connections_tcp, peers, connectivity_tcp_str) dot_non_tcp = self.dot_format_from_connections_dict(connections_non_tcp, peers, connectivity_non_tcp_str) # concatenate the two graphs into one dot file @@ -991,7 +970,7 @@ def get_props_output_split_by_tcp(self, props, peers_to_compare): connectivity_tcp_str = 'TCP' connectivity_non_tcp_str = 'non-TCP' props_tcp, props_non_tcp = self.convert_props_to_split_by_tcp(props) - if self.output_config.outputFormat in ['dot', 'jpg']: + if self.output_config.outputFormat in ['dot', 'jpg', 'html']: dot_tcp = self.dot_format_from_props(props_tcp, peers_to_compare, connectivity_tcp_str) dot_non_tcp = self.dot_format_from_props(props_non_tcp, peers_to_compare, connectivity_non_tcp_str) # concatenate the two graphs into one dot file @@ -1036,8 +1015,8 @@ def _txt_no_fw_rules_format_from_connections_dict(self, connections, peers, conn :param PeerSet peers: the peers to consider for dot output :param Union[str,None] connectivity_restriction: specify if connectivity is restricted to TCP / non-TCP , or not :rtype: str - :return the connectivity map in txt_no_fw_rules format: the connections between peers excluding connections - between workload to itself (without grouping as fw-rules). + :return the connectivity map in txt_no_fw_rules format: the connections between peers excluding connections between + workload to itself (without grouping as fw-rules). """ conn_graph = self._get_conn_graph(connections, peers) return conn_graph.get_connections_without_fw_rules_txt_format(connectivity_restriction) @@ -1052,7 +1031,7 @@ def dot_format_from_connections_dict(self, connections, peers, connectivity_rest :return the connectivity map in dot-format, considering connectivity_restriction if required """ conn_graph = self._get_conn_graph(connections, peers) - return conn_graph.get_connectivity_dot_format_str(connectivity_restriction, self.output_config.simplifyGraph) + return conn_graph.get_connectivity_dot_format_str(connectivity_restriction) def dot_format_from_props(self, props, peers, connectivity_restriction=None): """ @@ -1065,7 +1044,8 @@ def dot_format_from_props(self, props, peers, connectivity_restriction=None): :return the connectivity map in dot-format, considering connectivity_restriction if required """ conn_graph = ConnectivityGraph(peers, self.config.get_allowed_labels(), self.output_config) - conn_graph.add_props_to_graph(props, self.config.peer_container) + for cube in props: + conn_graph.add_edges_from_cube_dict(props.get_connectivity_cube(cube), self.config.peer_container) return conn_graph.get_connectivity_dot_format_str(connectivity_restriction) def txt_no_fw_rules_format_from_props(self, props, peers, connectivity_restriction=None): @@ -1078,7 +1058,8 @@ def txt_no_fw_rules_format_from_props(self, props, peers, connectivity_restricti :return the connectivity map in txt_no_fw_rules format, considering connectivity_restriction if required """ conn_graph = ConnectivityGraph(peers, self.config.get_allowed_labels(), self.output_config) - conn_graph.add_props_to_graph(props, self.config.peer_container) + for cube in props: + conn_graph.add_edges_from_cube_dict(props.get_connectivity_cube(cube), self.config.peer_container) return conn_graph.get_connections_without_fw_rules_txt_format(connectivity_restriction) def fw_rules_from_connections_dict(self, connections, peers_to_compare, connectivity_restriction=None): @@ -1149,8 +1130,8 @@ def split_to_tcp_and_non_tcp_conns(conns): @staticmethod def convert_props_to_split_by_tcp(props): """ - given the ConnectivityProperties properties set, convert it to two properties sets, one for TCP only, - and the other for non-TCP only. + given the ConnectivityProperties properties set, convert it to two properties sets, one for TCP only, and the other + for non-TCP only. :param ConnectivityProperties props: properties describing allowed connections :return: a tuple of the two properties sets: first for TCP, second for non-TCP :rtype: tuple(ConnectivityProperties, ConnectivityProperties) @@ -1210,8 +1191,7 @@ def disjoint_referenced_ip_blocks(self): :return: A set of disjoint ip-blocks :rtype: PeerSet """ - exclude_ipv6 = self.config1.check_for_excluding_ipv6_addresses(self.output_config.excludeIPv6Range) and \ - self.config2.check_for_excluding_ipv6_addresses(self.output_config.excludeIPv6Range) + exclude_ipv6 = self.output_config.excludeIPv6Range # TODO - consider including also non referenced IPBlocks, as in ConnectivityMapQuery # (see issue https://github.com/IBM/network-config-analyzer/issues/522) return IpBlock.disjoint_ip_blocks(self.config1.get_referenced_ip_blocks(exclude_ipv6), @@ -1227,64 +1207,37 @@ def filter_conns_by_input_or_internal_constraints(self, conns1, conns2): :rtype: [ConnectivityProperties, ConnectivityProperties] :return: two resulting allowed connections """ - all_peers = conns1.project_on_one_dimension('src_peers') | conns1.project_on_one_dimension('dst_peers') | \ + peers_to_compare = conns1.project_on_one_dimension('src_peers') | conns1.project_on_one_dimension('dst_peers') | \ conns2.project_on_one_dimension('src_peers') | conns2.project_on_one_dimension('dst_peers') - exclude_ipv6 = self.config1.check_for_excluding_ipv6_addresses(self.output_config.excludeIPv6Range) and \ - self.config2.check_for_excluding_ipv6_addresses(self.output_config.excludeIPv6Range) - conns_filter = ConnectivityProperties.make_all_props() - if exclude_ipv6: - all_peers.filter_ip_blocks_by_mask(IpBlock.get_all_ips_block(exclude_ipv6=True)) - conns_filter = ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers, - "dst_peers": all_peers}) - res_conns1 = self.config1.filter_conns_by_peer_types(conns1) & conns_filter - res_conns2 = self.config2.filter_conns_by_peer_types(conns2) & conns_filter + exclude_ipv6 = self.output_config.excludeIPv6Range + ref_ip_blocks = self.config1.get_referenced_ip_blocks(exclude_ipv6) | \ + self.config2.get_referenced_ip_blocks(exclude_ipv6) + ip_blocks_mask = IpBlock() if ref_ip_blocks else IpBlock.get_all_ips_block(exclude_ipv6) + for ip_block in ref_ip_blocks: + ip_blocks_mask |= ip_block + peers_to_compare.filter_ipv6_blocks(ip_blocks_mask) + conns_filter = ConnectivityProperties.make_conn_props_from_dict({"src_peers": peers_to_compare, + "dst_peers": peers_to_compare}) + res_conns1 = self.config1.filter_conns_by_peer_types(conns1, peers_to_compare) & conns_filter + res_conns2 = self.config2.filter_conns_by_peer_types(conns2, peers_to_compare) & conns_filter return res_conns1, res_conns2 - def _append_different_conns_to_list(self, conn_diff_props, different_conns_list, props_based_on_config1=True): - """ - Adds difference between config1 and config2 connectivities into the list of differences - :param ConnectivityProperties conn_diff_props: connectivity properties representing a difference - between config1 and config2 connections (or between config2 and config1 connections) - :param list different_conns_list: the list to add differences to - :param bool props_based_on_config1: whether conn_diff_props represent connections present in config1 but not in config2 - (the value True) or connections present in config2 but not in config1 (the value False) - """ - no_conns = ConnectionSet() - for cube in conn_diff_props: - conn_cube = conn_diff_props.get_connectivity_cube(cube) - conns, src_peers, dst_peers = \ - ConnectionSet.get_connection_set_and_peers_from_cube(conn_cube, self.config1.peer_container) - conns1 = conns if props_based_on_config1 else no_conns - conns2 = no_conns if props_based_on_config1 else conns - if self.output_config.fullExplanation: - if self.config1.optimized_run == 'true': - different_conns_list.append(PeersAndConnections(str(src_peers), str(dst_peers), conns1, conns2)) - else: # 'debug': produce the same output format as in the original implementation (per peer pairs) - for src_peer in src_peers: - for dst_peer in dst_peers: - if src_peer != dst_peer: - different_conns_list.append(PeersAndConnections(str(src_peer), str(dst_peer), - conns1, conns2)) - else: - different_conns_list.append(PeersAndConnections(src_peers.rep(), dst_peers.rep(), conns1, conns2)) - return - @staticmethod - def clone_without_gateway_layer(config): + def clone_without_ingress(config): """ - Clone config without gateway policies + Clone config without ingress policies :param NetworkConfig config: the config to clone - :return: resulting config without gateway policies + :return: resulting config without ingress policies :rtype: NetworkConfig """ - if NetworkLayerName.Gateway not in config.policies_container.layers or not config.policies_container.layers[ - NetworkLayerName.Gateway].policies_list: - return config # no gateway policies in this config - config_without_gateway = config.clone_without_policies(config.name) + if NetworkLayerName.Ingress not in config.policies_container.layers or not config.policies_container.layers[ + NetworkLayerName.Ingress].policies_list: + return config # no ingress policies in this config + config_without_ingress = config.clone_without_policies(config.name) for policy in config.policies_container.policies.values(): - if not isinstance(policy, GatewayPolicy): # ignoring gateway policies - config_without_gateway.append_policy_to_config(policy) - return config_without_gateway + if not isinstance(policy, IngressPolicy): # ignoring ingress policies + config_without_ingress.append_policy_to_config(policy) + return config_without_ingress def execute(self, cmd_line_flag): return self.exec(cmd_line_flag) @@ -1338,10 +1291,36 @@ def check_equivalence_original(self, layer_name=None): return QueryAnswer(True, self.name1 + ' and ' + self.name2 + ' are semantically equivalent.', numerical_result=0) + def _append_different_conns_to_list(self, conn_props, different_conns_list, props_based_on_config1): + """ + Adds difference between config1 and config2 connectivities into the list of differences + :param ConnectivityProperties conn_props: connectivity properties representing a difference between config1 and config2 + :param list different_conns_list: the list to add differences to + :param bool props_based_on_config1: whether conn_props represent connections present in config1 but not in config2 + (the value True) or connections present in config2 but not in config1 (the value False) + """ + no_conns = ConnectionSet() + for cube in conn_props: + conn_cube = conn_props.get_connectivity_cube(cube) + conns, src_peers, dst_peers = \ + ConnectionSet.get_connection_set_and_peers_from_cube(conn_cube, self.config1.peer_container) + conns1 = conns if props_based_on_config1 else no_conns + conns2 = no_conns if props_based_on_config1 else conns + if self.output_config.fullExplanation: + if self.config1.optimized_run == 'true': + different_conns_list.append(PeersAndConnections(str(src_peers), str(dst_peers), conns1, conns2)) + else: # 'debug': produce the same output format as in the original implementation (per peer pairs) + for src_peer in src_peers: + for dst_peer in dst_peers: + if src_peer != dst_peer: + different_conns_list.append(PeersAndConnections(str(src_peer), str(dst_peer), + conns1, conns2)) + else: + different_conns_list.append(PeersAndConnections(src_peers.rep(), dst_peers.rep(), conns1, conns2)) + def check_equivalence_optimized(self, layer_name=None): - res_conns_filter = PolicyConnectionsFilter.only_all_allowed_connections() - conn_props1 = self.config1.allowed_connections_optimized(layer_name, res_conns_filter) - conn_props2 = self.config2.allowed_connections_optimized(layer_name, res_conns_filter) + conn_props1 = self.config1.allowed_connections_optimized(layer_name) + conn_props2 = self.config2.allowed_connections_optimized(layer_name) all_conns1, all_conns2 = self.filter_conns_by_input_or_internal_constraints(conn_props1.all_allowed_conns, conn_props2.all_allowed_conns) if all_conns1 == all_conns2: @@ -1394,13 +1373,13 @@ def get_explanation_from_conn_graph(conn_graph, is_first_connectivity_result): :param conn_graph: a ConnectivityGraph with added/removed connections :param is_first_connectivity_result: bool flag indicating if this is the first connectivity fw-rules computation for the current semantic-diff query - :return: fw-rules summarizing added/removed connections (in required format and as MinimizeFWRules) - :rtype: Union[str, dict], MinimizeFWRules (dict if required format is yaml/json , str otherwise) + :return: fw-rules summarizing added/removed connections + :rtype: Union[str, dict] - dict if required format is yaml/json , str otherwise """ fw_rules = conn_graph.get_minimized_firewall_rules() # for csv format, adding the csv header only for the first connectivity fw-rules computation fw_rules_output = fw_rules.get_fw_rules_in_required_format(False, is_first_connectivity_result) - return fw_rules_output, fw_rules + return fw_rules_output def compute_explanation_for_key(self, key, is_added, conn_graph, is_first_connectivity_result): """ @@ -1413,19 +1392,17 @@ def compute_explanation_for_key(self, key, is_added, conn_graph, is_first_connec :param ConnectivityGraph conn_graph: a ConnectivityGraph with added/removed connections :param bool is_first_connectivity_result: flag indicating if this is the first connectivity fw-rules computation for the current semantic-diff query - :return the computedExplanation of the current key and conn_graph considering the outputFormat, - and fw_rules from which the explanation was computed - :rtype: ComputedExplanation, Union[None, MinimizeFWRules] + :return the computedExplanation of the current key and conn_graph considering the outputFormat + :rtype: ComputedExplanation """ updated_key = self._get_updated_key(key, is_added) topology_config_name = self.name2 if is_added else self.name1 connectivity_changes_header = f'{updated_key} (based on topology from config: {topology_config_name}) :' - fw_rules = None if self.output_config.outputFormat == 'txt_no_fw_rules': conn_graph_explanation = conn_graph.get_connections_without_fw_rules_txt_format( connectivity_changes_header, exclude_self_loop_conns=False) + '\n' else: - conn_graph_explanation, fw_rules = self.get_explanation_from_conn_graph(conn_graph, is_first_connectivity_result) + conn_graph_explanation = self.get_explanation_from_conn_graph(conn_graph, is_first_connectivity_result) if self.output_config.outputFormat in ['json', 'yaml']: explanation_dict = {'description': updated_key} @@ -1436,7 +1413,7 @@ def compute_explanation_for_key(self, key, is_added, conn_graph, is_first_connec str_explanation += conn_graph_explanation key_explanation = ComputedExplanation(str_explanation=str_explanation) - return key_explanation, fw_rules + return key_explanation def get_results_for_computed_fw_rules(self, keys_list, conn_graph_removed_per_key, conn_graph_added_per_key): """ @@ -1459,71 +1436,12 @@ def get_results_for_computed_fw_rules(self, keys_list, conn_graph_removed_per_ke is_removed = conn_graph_removed_conns is not None and conn_graph_removed_conns.conn_graph_has_fw_rules() if is_added: if add_explanation: - key_explanation, _ = self.compute_explanation_for_key(key, True, conn_graph_added_conns, res == 0) - explanation.append(key_explanation) + explanation.append(self.compute_explanation_for_key(key, True, conn_graph_added_conns, res == 0)) res += 1 if is_removed: if add_explanation: - key_explanation, _ = self.compute_explanation_for_key(key, False, conn_graph_removed_conns, res == 0) - explanation.append(key_explanation) - res += 1 - - return res, explanation - - def get_results_for_computed_fw_rules_and_compare_orig_to_opt(self, keys_list, orig_conn_graph_removed_per_key, - orig_conn_graph_added_per_key, - opt_conn_graph_removed_per_key, - opt_conn_graph_added_per_key): - """ - Compute accumulated explanation and res for all keys of changed connections categories. - Also, compare original and optimized results. - :param keys_list: the list of keys - :param orig_conn_graph_removed_per_key: map from key to ConnectivityGraph of original removed connections - :param orig_conn_graph_added_per_key: map from key to ConnectivityGraph of original added connections - :param opt_conn_graph_removed_per_key: map from key to ConnectivityGraph of optimized removed connections - :param opt_conn_graph_added_per_key: map from key to ConnectivityGraph of optimized added connections - :return: - res (int): number of categories with diffs - explanation (list): list of ComputedExplanation, the diffs' explanations, one for each category - :rtype: int, list[ComputedExplanation] - """ - explanation = [] - add_explanation = self.output_config.outputFormat in SemanticDiffQuery.get_supported_output_formats() - res = 0 - for key in keys_list: - orig_conn_graph_added_conns = orig_conn_graph_added_per_key[key] - orig_conn_graph_removed_conns = orig_conn_graph_removed_per_key[key] - is_added = orig_conn_graph_added_conns is not None and orig_conn_graph_added_conns.conn_graph_has_fw_rules() - is_removed = orig_conn_graph_removed_conns is not None and orig_conn_graph_removed_conns.conn_graph_has_fw_rules() - if is_added: - if add_explanation: - key_explanation, orig_fw_rules = self.compute_explanation_for_key( - key, True, orig_conn_graph_added_conns, res == 0) - if not orig_fw_rules: - orig_fw_rules = orig_conn_graph_added_conns.get_minimized_firewall_rules() - opt_conn_graph_added_conns = opt_conn_graph_added_per_key[key] - assert opt_conn_graph_added_conns and opt_conn_graph_added_conns.conn_graph_has_fw_rules() - opt_fw_rules = opt_conn_graph_added_conns.get_minimized_firewall_rules() - self.compare_fw_rules(orig_fw_rules, opt_fw_rules, self.config2.peer_container, - self._get_updated_key(key, True) + - f'between {self.config1.name} and {self.config2.name}') - explanation.append(key_explanation) - res += 1 - - if is_removed: - if add_explanation: - key_explanation, orig_fw_rules = self.compute_explanation_for_key( - key, False, orig_conn_graph_removed_conns, res == 0) - if not orig_fw_rules: - orig_fw_rules = orig_conn_graph_removed_conns.get_minimized_firewall_rules() - opt_conn_graph_removed_conns = opt_conn_graph_removed_per_key[key] - assert opt_conn_graph_removed_conns and opt_conn_graph_removed_conns.conn_graph_has_fw_rules() - opt_fw_rules = opt_conn_graph_removed_conns.get_minimized_firewall_rules() - self.compare_fw_rules(orig_fw_rules, opt_fw_rules, self.config1.peer_container, - self._get_updated_key(key, False) + - f'between {self.config1.name} and {self.config2.name}') - explanation.append(key_explanation) + explanation.append(self.compute_explanation_for_key(key, False, conn_graph_removed_conns, res == 0)) res += 1 return res, explanation @@ -1550,7 +1468,7 @@ def get_conn_graph_changed_conns(self, key, ip_blocks, is_added): output_config = OutputConfiguration(self.output_config, query_name) return ConnectivityGraph(topology_peers, allowed_labels, output_config) - def compute_diff_original(self): # noqa: C901 + def compute_diff(self): # noqa: C901 """ Compute changed connections as following: @@ -1570,11 +1488,9 @@ def compute_diff_original(self): # noqa: C901 Some sections might be empty and can be dropped. :return: - keys_list (list[str]): list of names of connection categories, - being the keys in conn_graph_removed_per_key/conn_graph_added_per_key - conn_graph_removed_per_key (dict): a dictionary of removed connections connectivity graphs per category - conn_graph_added_per_key (dict): a dictionary of added connections connectivity graphs per category - :rtype: list[str], dict, dict + res (int): number of categories with diffs + explanation (list): list of diff explanations - one for each category + :rtype: int, list[ComputedExplanation] """ old_peers = self.config1.peer_container.get_all_peers_group(include_dns_entries=True) new_peers = self.config2.peer_container.get_all_peers_group(include_dns_entries=True) @@ -1582,8 +1498,7 @@ def compute_diff_original(self): # noqa: C901 removed_peers = old_peers - intersected_peers added_peers = new_peers - intersected_peers captured_pods = (self.config1.get_captured_pods() | self.config2.get_captured_pods()) & intersected_peers - exclude_ipv6 = self.config1.check_for_excluding_ipv6_addresses(self.output_config.excludeIPv6Range) and \ - self.config2.check_for_excluding_ipv6_addresses(self.output_config.excludeIPv6Range) + exclude_ipv6 = self.output_config.excludeIPv6Range old_ip_blocks = IpBlock.disjoint_ip_blocks(self.config1.get_referenced_ip_blocks(exclude_ipv6), IpBlock.get_all_ips_block_peer_set(exclude_ipv6), exclude_ipv6) @@ -1613,15 +1528,17 @@ def compute_diff_original(self): # noqa: C901 conn_graph_removed_per_key[key] = self.get_conn_graph_changed_conns(key, old_ip_blocks, False) conn_graph_added_per_key[key] = None for pair in itertools.product(removed_peers, old_ip_blocks): - if self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[0], pair[1]): - lost_conns, _, _, _ = self.config1.allowed_connections(pair[0], pair[1]) - if lost_conns: - conn_graph_removed_per_key[key].add_edge(pair[0], pair[1], lost_conns) + if not self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[0], pair[1]): + continue + lost_conns, _, _, _ = self.config1.allowed_connections(pair[0], pair[1]) + if lost_conns: + conn_graph_removed_per_key[key].add_edge(pair[0], pair[1], lost_conns) - if self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[1], pair[0]): - lost_conns, _, _, _ = self.config1.allowed_connections(pair[1], pair[0]) - if lost_conns: - conn_graph_removed_per_key[key].add_edge(pair[1], pair[0], lost_conns) + if not self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[1], pair[0]): + continue + lost_conns, _, _, _ = self.config1.allowed_connections(pair[1], pair[0]) + if lost_conns: + conn_graph_removed_per_key[key].add_edge(pair[1], pair[0], lost_conns) # 2.1. lost connections between removed peers and intersected peers key = 'Lost connections between removed peers and persistent peers' @@ -1629,15 +1546,17 @@ def compute_diff_original(self): # noqa: C901 conn_graph_removed_per_key[key] = self.get_conn_graph_changed_conns(key, PeerSet(), False) conn_graph_added_per_key[key] = None for pair in itertools.product(removed_peers, intersected_peers): - if self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[0], pair[1]): - lost_conns, _, _, _ = self.config1.allowed_connections(pair[0], pair[1]) - if lost_conns: - conn_graph_removed_per_key[key].add_edge(pair[0], pair[1], lost_conns) + if not self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[0], pair[1]): + continue + lost_conns, _, _, _ = self.config1.allowed_connections(pair[0], pair[1]) + if lost_conns: + conn_graph_removed_per_key[key].add_edge(pair[0], pair[1], lost_conns) - if self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[1], pair[0]): - lost_conns, _, _, _ = self.config1.allowed_connections(pair[1], pair[0]) - if lost_conns: - conn_graph_removed_per_key[key].add_edge(pair[1], pair[0], lost_conns) + if not self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[1], pair[0]): + continue + lost_conns, _, _, _ = self.config1.allowed_connections(pair[1], pair[0]) + if lost_conns: + conn_graph_removed_per_key[key].add_edge(pair[1], pair[0], lost_conns) # 3.1. lost/new connections between intersected peers due to changes in policies and labels of pods/namespaces key = 'Changed connections between persistent peers' @@ -1679,15 +1598,17 @@ def compute_diff_original(self): # noqa: C901 conn_graph_removed_per_key[key] = None conn_graph_added_per_key[key] = self.get_conn_graph_changed_conns(key, PeerSet(), True) for pair in itertools.product(intersected_peers, added_peers): - if self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[0], pair[1]): - new_conns, _, _, _ = self.config2.allowed_connections(pair[0], pair[1]) - if new_conns: - conn_graph_added_per_key[key].add_edge(pair[0], pair[1], new_conns) + if not self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[0], pair[1]): + continue + new_conns, _, _, _ = self.config2.allowed_connections(pair[0], pair[1]) + if new_conns: + conn_graph_added_per_key[key].add_edge(pair[0], pair[1], new_conns) - if self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[1], pair[0]): - new_conns, _, _, _ = self.config2.allowed_connections(pair[1], pair[0]) - if new_conns: - conn_graph_added_per_key[key].add_edge(pair[1], pair[0], new_conns) + if not self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[1], pair[0]): + continue + new_conns, _, _, _ = self.config2.allowed_connections(pair[1], pair[0]) + if new_conns: + conn_graph_added_per_key[key].add_edge(pair[1], pair[0], new_conns) # 5.1. new connections between added peers key = 'New connections between added peers' @@ -1708,198 +1629,27 @@ def compute_diff_original(self): # noqa: C901 conn_graph_added_per_key[key] = self.get_conn_graph_changed_conns(key, new_ip_blocks, True) for pair in itertools.product(added_peers, new_ip_blocks): - if self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[0], pair[1]): - new_conns, _, _, _ = self.config2.allowed_connections(pair[0], pair[1]) - if new_conns: - conn_graph_added_per_key[key].add_edge(pair[0], pair[1], new_conns) - - if self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[1], pair[0]): - new_conns, _, _, _ = self.config2.allowed_connections(pair[1], pair[0]) - if new_conns: - conn_graph_added_per_key[key].add_edge(pair[1], pair[0], new_conns) - - return keys_list, conn_graph_removed_per_key, conn_graph_added_per_key - - def compute_diff_optimized(self): # noqa: C901 - """ - Compute changed connections (by optimized implementation) as following: - - 1.1. lost connections between removed peers - 1.2. lost connections between removed peers and ipBlocks - - 2.1. lost connections between removed peers and intersected peers - - 3.1. lost/new connections between intersected peers due to changes in policies and labels of pods/namespaces - 3.2. lost/new connections between intersected peers and ipBlocks due to changes in policies and labels - - 4.1. new connections between intersected peers and added peers - - 5.1. new connections between added peers - 5.2. new connections between added peers and ipBlocks - - Some sections might be empty and can be dropped. - - :return: - keys_list (list[str]): list of names of connection categories, - being the keys in conn_graph_removed_per_key/conn_graph_added_per_key - conn_graph_removed_per_key (dict): a dictionary of removed connections connectivity graphs per category - conn_graph_added_per_key (dict): a dictionary of added connections connectivity graphs per category - :rtype: list[str], dict, dict - """ - - old_peers = self.config1.peer_container.get_all_peers_group(include_dns_entries=True) - new_peers = self.config2.peer_container.get_all_peers_group(include_dns_entries=True) - intersected_peers = old_peers & new_peers - removed_peers = old_peers - intersected_peers - added_peers = new_peers - intersected_peers - captured_pods = (self.config1.get_captured_pods() | self.config2.get_captured_pods()) & intersected_peers - exclude_ipv6 = self.config1.check_for_excluding_ipv6_addresses(self.output_config.excludeIPv6Range) and \ - self.config2.check_for_excluding_ipv6_addresses(self.output_config.excludeIPv6Range) - old_ip_blocks = IpBlock.disjoint_ip_blocks(self.config1.get_referenced_ip_blocks(exclude_ipv6), - IpBlock.get_all_ips_block_peer_set(exclude_ipv6), - exclude_ipv6) - new_ip_blocks = IpBlock.disjoint_ip_blocks(self.config2.get_referenced_ip_blocks(exclude_ipv6), - IpBlock.get_all_ips_block_peer_set(exclude_ipv6), - exclude_ipv6) - - conn_graph_removed_per_key = dict() - conn_graph_added_per_key = dict() - keys_list = [] - res_conns_filter = PolicyConnectionsFilter.only_all_allowed_connections() - old_conns = self.config1.allowed_connections_optimized(res_conns_filter=res_conns_filter) - new_conns = self.config2.allowed_connections_optimized(res_conns_filter=res_conns_filter) - old_props, new_props = self.filter_conns_by_input_or_internal_constraints(old_conns.all_allowed_conns, - new_conns.all_allowed_conns) - - # 1.1. lost connections between removed peers - key = 'Lost connections between removed peers' - keys_list.append(key) - conn_graph_removed_per_key[key] = self.get_conn_graph_changed_conns(key, PeerSet(), False) - conn_graph_added_per_key[key] = None - props = ConnectivityProperties.make_conn_props_from_dict({"src_peers": removed_peers, - "dst_peers": removed_peers}) - props &= old_props - props = props.props_without_auto_conns() - conn_graph_removed_per_key[key].add_props_to_graph(props, self.config1.peer_container) - - # 1.2. lost connections between removed peers and ipBlocks - key = 'Lost connections between removed peers and ipBlocks' - keys_list.append(key) - conn_graph_removed_per_key[key] = self.get_conn_graph_changed_conns(key, old_ip_blocks, False) - conn_graph_added_per_key[key] = None - props = ConnectivityProperties.make_conn_props_from_dict({"src_peers": removed_peers, - "dst_peers": old_ip_blocks}) | \ - ConnectivityProperties.make_conn_props_from_dict({"src_peers": old_ip_blocks, - "dst_peers": removed_peers}) - props &= old_props - conn_graph_removed_per_key[key].add_props_to_graph(props, self.config1.peer_container) - - # 2.1. lost connections between removed peers and intersected peers - key = 'Lost connections between removed peers and persistent peers' - keys_list.append(key) - conn_graph_removed_per_key[key] = self.get_conn_graph_changed_conns(key, PeerSet(), False) - conn_graph_added_per_key[key] = None - props = ConnectivityProperties.make_conn_props_from_dict({"src_peers": removed_peers, - "dst_peers": intersected_peers}) | \ - ConnectivityProperties.make_conn_props_from_dict({"src_peers": intersected_peers, - "dst_peers": removed_peers}) - props &= old_props - props = props.props_without_auto_conns() - conn_graph_removed_per_key[key].add_props_to_graph(props, self.config1.peer_container) - - # 3.1. lost/new connections between intersected peers due to changes in policies and labels of pods/namespaces - key = 'Changed connections between persistent peers' - keys_list.append(key) - conn_graph_removed_per_key[key] = self.get_conn_graph_changed_conns(key, PeerSet(), False) - conn_graph_added_per_key[key] = self.get_conn_graph_changed_conns(key, PeerSet(), True) - props = ConnectivityProperties.make_conn_props_from_dict({"src_peers": captured_pods, - "dst_peers": intersected_peers}) | \ - ConnectivityProperties.make_conn_props_from_dict({"src_peers": intersected_peers, - "dst_peers": captured_pods}) - props1 = old_props & props - props1 = props1.props_without_auto_conns() - props2 = new_props & props - props2 = props2.props_without_auto_conns() - conn_graph_removed_per_key[key].add_props_to_graph(props1 - props2, self.config1.peer_container) - conn_graph_added_per_key[key].add_props_to_graph(props2 - props1, self.config2.peer_container) - - # 3.2. lost/new connections between intersected peers and ipBlocks due to changes in policies and labels - key = 'Changed connections between persistent peers and ipBlocks' - disjoint_ip_blocks = IpBlock.disjoint_ip_blocks(old_ip_blocks, new_ip_blocks, exclude_ipv6) - keys_list.append(key) - conn_graph_removed_per_key[key] = self.get_conn_graph_changed_conns(key, disjoint_ip_blocks, False) - conn_graph_added_per_key[key] = self.get_conn_graph_changed_conns(key, disjoint_ip_blocks, True) - props = ConnectivityProperties.make_conn_props_from_dict({"src_peers": captured_pods, - "dst_peers": disjoint_ip_blocks}) | \ - ConnectivityProperties.make_conn_props_from_dict({"src_peers": disjoint_ip_blocks, - "dst_peers": captured_pods}) - props1 = old_props & props - props2 = new_props & props - conn_graph_removed_per_key[key].add_props_to_graph(props1 - props2, self.config1.peer_container) - conn_graph_added_per_key[key].add_props_to_graph(props2 - props1, self.config2.peer_container) - - # 4.1. new connections between intersected peers and added peers - key = 'New connections between persistent peers and added peers' - keys_list.append(key) - conn_graph_removed_per_key[key] = None - conn_graph_added_per_key[key] = self.get_conn_graph_changed_conns(key, PeerSet(), True) - props = ConnectivityProperties.make_conn_props_from_dict({"src_peers": intersected_peers, - "dst_peers": added_peers}) | \ - ConnectivityProperties.make_conn_props_from_dict({"src_peers": added_peers, - "dst_peers": intersected_peers}) - props &= new_props - props = props.props_without_auto_conns() - conn_graph_added_per_key[key].add_props_to_graph(props, self.config2.peer_container) - - # 5.1. new connections between added peers - key = 'New connections between added peers' - keys_list.append(key) - conn_graph_removed_per_key[key] = None - conn_graph_added_per_key[key] = self.get_conn_graph_changed_conns(key, PeerSet(), True) - props = ConnectivityProperties.make_conn_props_from_dict({"src_peers": added_peers, - "dst_peers": added_peers}) - props &= new_props - props = props.props_without_auto_conns() - conn_graph_added_per_key[key].add_props_to_graph(props, self.config2.peer_container) + if not self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[0], pair[1]): + continue + new_conns, _, _, _ = self.config2.allowed_connections(pair[0], pair[1]) + if new_conns: + conn_graph_added_per_key[key].add_edge(pair[0], pair[1], new_conns) - # 5.2. new connections between added peers and ipBlocks - key = 'New connections between added peers and ipBlocks' - keys_list.append(key) - conn_graph_removed_per_key[key] = None - conn_graph_added_per_key[key] = self.get_conn_graph_changed_conns(key, new_ip_blocks, True) - props = ConnectivityProperties.make_conn_props_from_dict({"src_peers": added_peers, - "dst_peers": new_ip_blocks}) | \ - ConnectivityProperties.make_conn_props_from_dict({"src_peers": new_ip_blocks, - "dst_peers": added_peers}) - props &= new_props - conn_graph_added_per_key[key].add_props_to_graph(props, self.config2.peer_container) + if not self.determine_whether_to_compute_allowed_conns_for_peer_types(pair[1], pair[0]): + continue + new_conns, _, _, _ = self.config2.allowed_connections(pair[1], pair[0]) + if new_conns: + conn_graph_added_per_key[key].add_edge(pair[1], pair[0], new_conns) - return keys_list, conn_graph_removed_per_key, conn_graph_added_per_key + return self.get_results_for_computed_fw_rules(keys_list, conn_graph_removed_per_key, + conn_graph_added_per_key) def exec(self, cmd_line_flag): self.output_config.fullExplanation = True # assign true for this query - it is always ok to compare its results query_answer = self.is_identical_topologies(True) if query_answer.bool_result and query_answer.output_result: return query_answer - orig_conn_graph_removed_per_key = dict() - orig_conn_graph_added_per_key = dict() - res = 0 - explanation = "" - if self.config1.optimized_run != 'true': - keys_list, orig_conn_graph_removed_per_key, orig_conn_graph_added_per_key = self.compute_diff_original() - if self.config1.optimized_run == 'false': - res, explanation = self.get_results_for_computed_fw_rules(keys_list, orig_conn_graph_removed_per_key, - orig_conn_graph_added_per_key) - if self.config1.optimized_run != 'false': - keys_list, opt_conn_graph_removed_per_key, opt_conn_graph_added_per_key = self.compute_diff_optimized() - if self.config1.optimized_run == 'true': - res, explanation = self.get_results_for_computed_fw_rules(keys_list, opt_conn_graph_removed_per_key, - opt_conn_graph_added_per_key) - else: - res, explanation = self.get_results_for_computed_fw_rules_and_compare_orig_to_opt( - keys_list, orig_conn_graph_removed_per_key, orig_conn_graph_added_per_key, - opt_conn_graph_removed_per_key, opt_conn_graph_added_per_key) - + res, explanation = self.compute_diff() if res > 0: return QueryAnswer(bool_result=False, output_result=f'{self.name1} and {self.name2} are not semantically equivalent.', @@ -1972,13 +1722,6 @@ def exec(self, cmd_line_flag=False, only_captured=False): return QueryAnswer(False, f'{self.name1} is not contained in {self.name2} ', output_explanation=[final_explanation], numerical_result=0 if not cmd_line_flag else 1) - if self.config1.optimized_run == 'false': - return self.check_containment_original(cmd_line_flag, only_captured) - else: - return self.check_containment_optimized(cmd_line_flag, only_captured) - - def check_containment_original(self, cmd_line_flag=False, only_captured=False): - config1_peers = self.config1.peer_container.get_all_peers_group(include_dns_entries=True) peers_to_compare = config1_peers | self.disjoint_referenced_ip_blocks() captured_pods = self.config1.get_captured_pods() | self.config2.get_captured_pods() not_contained_list = [] @@ -2002,26 +1745,6 @@ def check_containment_original(self, cmd_line_flag=False, only_captured=False): return QueryAnswer(True, self.name1 + ' is contained in ' + self.name2, numerical_result=1 if not cmd_line_flag else 0) - def check_containment_optimized(self, cmd_line_flag=False, only_captured=False): - if only_captured: - res_conns_filter1 = PolicyConnectionsFilter.only_allowed_connections() - else: - res_conns_filter1 = PolicyConnectionsFilter.only_all_allowed_connections() - res_conns_filter2 = PolicyConnectionsFilter.only_all_allowed_connections() - conn_props1 = self.config1.allowed_connections_optimized(res_conns_filter=res_conns_filter1) - conn_props2 = self.config2.allowed_connections_optimized(res_conns_filter=res_conns_filter2) - conns1, conns2 = self.filter_conns_by_input_or_internal_constraints( - conn_props1.allowed_conns if only_captured else conn_props1.all_allowed_conns, - conn_props2.all_allowed_conns) - if conns1.contained_in(conns2): - return QueryAnswer(True, self.name1 + ' is contained in ' + self.name2, - numerical_result=1 if not cmd_line_flag else 0) - - conns1_not_in_conns2 = conns1 - conns2 - different_conns_list = [] - self._append_different_conns_to_list(conns1_not_in_conns2, different_conns_list) - return self._query_answer_with_relevant_explanation(sorted(different_conns_list), cmd_line_flag) - def _query_answer_with_relevant_explanation(self, explanation_list, cmd_line_flag): output_result = f'{self.name1} is not contained in {self.name2}' explanation_description = f'Connections allowed in {self.name1} which are not a subset of those in {self.name2}' @@ -2094,13 +1817,13 @@ def exec(self, cmd_line_flag): query_answer.output_result = output_result_on_permit return query_answer - if self.config1.policies_container.layers.does_contain_single_layer(NetworkLayerName.Gateway): + if self.config1.policies_container.layers.does_contain_single_layer(NetworkLayerName.Ingress): return QueryAnswer(bool_result=False, - output_result='Permitted traffic cannot be specified using Ingress/Gateway resources only', + output_result='Permitted traffic cannot be specified using Ingress resources only', query_not_executed=True) - config1_without_gateway = self.clone_without_gateway_layer(self.config1) - query_answer = ContainmentQuery(config1_without_gateway, self.config2, + config1_without_ingress = self.clone_without_ingress(self.config1) + query_answer = ContainmentQuery(config1_without_ingress, self.config2, self.output_config).exec(cmd_line_flag=cmd_line_flag, only_captured=True) if not cmd_line_flag: query_answer.numerical_result = 1 if query_answer.output_explanation else 0 @@ -2123,12 +1846,6 @@ def exec(self, cmd_line_flag): else not query_answer.bool_result return query_answer - if self.config1.optimized_run == 'false': - return self.check_interferes_original(cmd_line_flag) - else: - return self.check_interferes_optimized(cmd_line_flag) - - def check_interferes_original(self, cmd_line_flag): peers_to_compare = \ self.config2.peer_container.get_all_peers_group(include_dns_entries=True) peers_to_compare |= self.disjoint_referenced_ip_blocks() @@ -2154,22 +1871,6 @@ def check_interferes_original(self, cmd_line_flag): return QueryAnswer(False, self.name1 + ' does not interfere with ' + self.name2, numerical_result=0 if not cmd_line_flag else 1) - def check_interferes_optimized(self, cmd_line_flag=False): - res_conns_filter = PolicyConnectionsFilter.only_allowed_connections() - - conn_props1 = self.config1.allowed_connections_optimized(res_conns_filter=res_conns_filter) - conn_props2 = self.config2.allowed_connections_optimized(res_conns_filter=res_conns_filter) - conns1, conns2 = self.filter_conns_by_input_or_internal_constraints(conn_props1.allowed_conns, - conn_props2.allowed_conns) - if conns1.contained_in(conns2): - return QueryAnswer(False, self.name1 + ' does not interfere with ' + self.name2, - numerical_result=0 if not cmd_line_flag else 1) - - conns1_not_in_conns2 = conns1 - conns2 - extended_conns_list = [] - self._append_different_conns_to_list(conns1_not_in_conns2, extended_conns_list, True) - return self._query_answer_with_relevant_explanation(sorted(extended_conns_list), cmd_line_flag) - def _query_answer_with_relevant_explanation(self, explanation_list, cmd_line_flag): interfere_result_msg = self.name1 + ' interferes with ' + self.name2 explanation_description = f'Allowed connections from {self.name2} which are extended in {self.name1}' @@ -2195,10 +1896,6 @@ def exec(self, cmd_line_flag): class IntersectsQuery(TwoNetworkConfigsQuery): """ Checking whether both configs allow the same connection between any pair of peers - Note: this query is only used by ForbidsQuery. - It's not symmetrical: config1 is a "specification config", that explicitly defines things to be checked - in the "implementation" config (config2), i.e., its captured connections are considered, - while config2 is the "implementation" config to be checked, and all its connections are considered. """ def exec(self, cmd_line_flag=False, only_captured=True): @@ -2206,12 +1903,6 @@ def exec(self, cmd_line_flag=False, only_captured=True): if query_answer.output_result: return query_answer - if self.config1.optimized_run == 'false': - return self.check_intersects_original() - else: - return self.check_intersects_optimized() - - def check_intersects_original(self, only_captured=True): peers_to_compare = \ self.config1.peer_container.get_all_peers_group(include_dns_entries=True) peers_to_compare |= self.disjoint_referenced_ip_blocks() @@ -2240,26 +1931,6 @@ def check_intersects_original(self, only_captured=True): return QueryAnswer(False, f'The connections allowed by {self.name1}' f' do not intersect the connections allowed by {self.name2}', numerical_result=1) - def check_intersects_optimized(self, only_captured=True): - if only_captured: - res_conns_filter1 = PolicyConnectionsFilter.only_allowed_connections() - else: - res_conns_filter1 = PolicyConnectionsFilter.only_all_allowed_connections() - res_conns_filter2 = PolicyConnectionsFilter.only_all_allowed_connections() - conn_props1 = self.config1.allowed_connections_optimized(res_conns_filter=res_conns_filter1) - conn_props2 = self.config2.allowed_connections_optimized(res_conns_filter=res_conns_filter2) - conns1, conns2 = self.filter_conns_by_input_or_internal_constraints( - conn_props1.allowed_conns if only_captured else conn_props1.all_allowed_conns, - conn_props2.all_allowed_conns) - conns_in_both = conns1 & conns2 - if conns_in_both: - intersect_connections_list = [] - self._append_different_conns_to_list(conns_in_both, intersect_connections_list) - return self._query_answer_with_relevant_explanation(sorted(intersect_connections_list)) - - return QueryAnswer(False, f'The connections allowed by {self.name1}' - f' do not intersect the connections allowed by {self.name2}', numerical_result=1) - def _query_answer_with_relevant_explanation(self, explanation_list): intersect_result_msg = self.name2 + ' intersects with ' + self.name1 final_explanation = ConnectionsDiffExplanation(peers_diff_connections_list=explanation_list) @@ -2276,15 +1947,15 @@ def exec(self, cmd_line_flag): if not self.config1: return QueryAnswer(False, 'There are no NetworkPolicies in the given forbids config. ' 'No traffic is specified as forbidden.', query_not_executed=True) - if self.config1.policies_container.layers.does_contain_single_layer(NetworkLayerName.Gateway): + if self.config1.policies_container.layers.does_contain_single_layer(NetworkLayerName.Ingress): return QueryAnswer(bool_result=False, - output_result='Forbidden traffic cannot be specified using Ingress/Gateway resources only', + output_result='Forbidden traffic cannot be specified using Ingress resources only', query_not_executed=True) - config1_without_gateway = self.clone_without_gateway_layer(self.config1) + config1_without_ingress = self.clone_without_ingress(self.config1) query_answer = \ - IntersectsQuery(config1_without_gateway, self.config2, self.output_config).exec(only_captured=True) + IntersectsQuery(config1_without_ingress, self.config2, self.output_config).exec(only_captured=True) if query_answer.numerical_result == 1: query_answer.output_result += f'\n{self.name2} forbids connections specified in ' \ f'{self.name1}' @@ -2362,12 +2033,17 @@ def exec(self): self.output_config.fullExplanation = True # assign true for this query - it is always ok to compare its results # get_all_peers_group() does not require getting dnsEntry peers, since they are not ClusterEP (pods) existing_pods = self.config.peer_container.get_all_peers_group() - if not self.config or self.config.policies_container.layers.does_contain_single_layer(NetworkLayerName.Gateway): + if not self.config: return QueryAnswer(bool_result=False, output_result=f'There are no network policies in {self.config.name}. ' f'All workload resources are non captured', numerical_result=len(existing_pods)) + if self.config.policies_container.layers.does_contain_single_layer(NetworkLayerName.Ingress): + return QueryAnswer(bool_result=False, + output_result='AllCapturedQuery cannot be applied using Ingress resources only', + query_not_executed=True) + k8s_calico_pods_list_explanation, k8s_calico_res = self._compute_uncaptured_pods_by_layer(NetworkLayerName.K8s_Calico) istio_pods_list_explanation, istio_res = self._compute_uncaptured_pods_by_layer(NetworkLayerName.Istio, True) diff --git a/nca/NetworkConfig/NetworkLayer.py b/nca/NetworkConfig/NetworkLayer.py index fd5741040..94086d686 100644 --- a/nca/NetworkConfig/NetworkLayer.py +++ b/nca/NetworkConfig/NetworkLayer.py @@ -11,7 +11,7 @@ from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.ProtocolSet import ProtocolSet from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy -from nca.Resources.NetworkPolicy import PolicyConnections, OptimizedPolicyConnections, NetworkPolicy, PolicyConnectionsFilter +from nca.Resources.NetworkPolicy import PolicyConnections, OptimizedPolicyConnections, NetworkPolicy from nca.Utils.ExplTracker import ExplTracker @@ -21,14 +21,14 @@ class NetworkLayerName(Enum): K8s_Calico = 0 Istio = 1 - Gateway = 2 + Ingress = 2 def create_network_layer(self, policies): if self == NetworkLayerName.K8s_Calico: return K8sCalicoNetworkLayer(policies) if self == NetworkLayerName.Istio: return IstioNetworkLayer(policies) - if self == NetworkLayerName.Gateway: + if self == NetworkLayerName.Ingress: return IngressNetworkLayer(policies) return None @@ -40,7 +40,7 @@ def policy_type_to_layer(policy_type): elif policy_type in {NetworkPolicy.PolicyType.IstioAuthorizationPolicy, NetworkPolicy.PolicyType.IstioSidecar}: return NetworkLayerName.Istio elif policy_type == NetworkPolicy.PolicyType.Ingress: - return NetworkLayerName.Gateway + return NetworkLayerName.Ingress return None @@ -106,18 +106,15 @@ def empty_layer_allowed_connections(layer_name, from_peer, to_peer): return empty_layer_obj.allowed_connections(from_peer, to_peer) @staticmethod - def empty_layer_allowed_connections_optimized(peer_container, layer_name, - res_conns_filter=PolicyConnectionsFilter()): + def empty_layer_allowed_connections_optimized(peer_container, layer_name): """ Get allowed connections between for all relevant peers for an empty layer (no policies). :param PeerContainer peer_container: holds all the peers :param NetworkLayerName layer_name: The empty layer name - :param PolicyConnectionsFilter res_conns_filter: filter of the required resulting connections - (connections with None value will not be calculated) :rtype: OptimizedPolicyConnections """ empty_layer_obj = layer_name.create_network_layer([]) - return empty_layer_obj.allowed_connections_optimized(peer_container, res_conns_filter) + return empty_layer_obj.allowed_connections_optimized(peer_container) class NetworkLayer: @@ -171,37 +168,32 @@ def allowed_connections(self, from_peer, to_peer): return allowed_conns, captured_flag, allowed_captured_conns, denied_conns - def allowed_connections_optimized(self, peer_container, res_conns_filter=PolicyConnectionsFilter()): + def allowed_connections_optimized(self, peer_container): """ Compute per network layer the allowed connections between any relevant peers, considering all layer's policies (and defaults) :param PeerContainer peer_container: the peer container holding the peers - :param PolicyConnectionsFilter res_conns_filter: filter of the required resulting connections - (connections with None value will not be calculated) :return: all allowed, denied and captured connections :rtype: OptimizedPolicyConnections """ res_conns = OptimizedPolicyConnections() - ingress_conns = self._allowed_xgress_conns_optimized(True, peer_container, res_conns_filter) - egress_conns = self._allowed_xgress_conns_optimized(False, peer_container, res_conns_filter) + ingress_conns = self._allowed_xgress_conns_optimized(True, peer_container) + egress_conns = self._allowed_xgress_conns_optimized(False, peer_container) all_pods_peer_set = peer_container.get_all_peers_group() all_ips_peer_set = IpBlock.get_all_ips_block_peer_set() - if res_conns_filter.calc_all_allowed: - # for ingress, all possible connections to IpBlocks are allowed - ingress_conns.all_allowed_conns |= \ - ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_pods_peer_set, - "dst_peers": all_ips_peer_set}) - # for egress, all possible connections from IpBlocks are allowed - egress_conns.all_allowed_conns |= \ - ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_ips_peer_set, - "dst_peers": all_pods_peer_set}) - res_conns.all_allowed_conns = ingress_conns.all_allowed_conns & egress_conns.all_allowed_conns + # for ingress, all possible connections to IpBlocks are allowed + ingress_conns.all_allowed_conns |= \ + ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_pods_peer_set, + "dst_peers": all_ips_peer_set}) + # for egress, all possible connections from IpBlocks are allowed + egress_conns.all_allowed_conns |= \ + ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_ips_peer_set, + "dst_peers": all_pods_peer_set}) res_conns.captured = ingress_conns.captured | egress_conns.captured - if res_conns_filter.calc_denied: - res_conns.denied_conns = ingress_conns.denied_conns | egress_conns.denied_conns - if res_conns_filter.calc_allowed: - res_conns.allowed_conns = (ingress_conns.allowed_conns & egress_conns.all_allowed_conns) | \ - (egress_conns.allowed_conns & ingress_conns.all_allowed_conns) + res_conns.denied_conns = ingress_conns.denied_conns | egress_conns.denied_conns + res_conns.all_allowed_conns = ingress_conns.all_allowed_conns & egress_conns.all_allowed_conns + res_conns.allowed_conns = (ingress_conns.allowed_conns & egress_conns.all_allowed_conns) | \ + (egress_conns.allowed_conns & ingress_conns.all_allowed_conns) return res_conns def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): @@ -210,7 +202,7 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): """ return NotImplemented - def _allowed_xgress_conns_optimized(self, is_ingress, peer_container, res_conns_filter=PolicyConnectionsFilter()): + def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): """ Implemented by derived classes to get ingress/egress connections between any relevant peers :rtype: OptimizedPolicyConnections @@ -298,7 +290,7 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): return PolicyConnections(captured_res, allowed_conns, denied_conns, all_allowed_conns=allowed_conns | allowed_non_captured_conns) - def _allowed_xgress_conns_optimized(self, is_ingress, peer_container, res_conns_filter=PolicyConnectionsFilter()): + def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): res_conns = self.collect_policies_conns_optimized(is_ingress) # Note: The below computation of non-captured conns cannot be done during the parse stage, # since before computing non-captured conns we should collect all policies conns @@ -308,7 +300,7 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container, res_conns_ all_peers_no_ips = peer_container.get_all_peers_group(add_external_ips=False, include_dns_entries=True) base_peer_set_no_hep = PeerSet(set([peer for peer in all_peers_no_ips if not isinstance(peer, HostEP)])) not_captured_not_hep = base_peer_set_no_hep - res_conns.captured - if not_captured_not_hep and res_conns_filter.calc_all_allowed: + if not_captured_not_hep: # default Allow-all in k8s / calico # (assuming only calico's default profiles for pods with connectivity rules exist) # assuming host endpoints have no profiles @@ -336,9 +328,8 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container, res_conns_ conn_cube.update({"src_peers": captured_not_hep, "dst_peers": all_peers_and_ips}) captured_not_hep_conns = ConnectivityProperties.make_conn_props(conn_cube) res_conns.allowed_conns |= res_conns.pass_conns & captured_not_hep_conns - if res_conns_filter.calc_all_allowed: - res_conns.all_allowed_conns |= res_conns.allowed_conns + res_conns.all_allowed_conns |= res_conns.allowed_conns return res_conns @@ -365,13 +356,10 @@ def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): return PolicyConnections(captured_res, allowed_conns, denied_conns, all_allowed_conns=allowed_conns | allowed_non_captured_conns) - def _allowed_xgress_conns_optimized(self, is_ingress, peer_container, res_conns_filter=PolicyConnectionsFilter()): + def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): res_conns = self.collect_policies_conns_optimized(is_ingress, IstioNetworkLayer.captured_cond_func) - if not res_conns_filter.calc_all_allowed: - return res_conns - # all the calculations below update res_conns.all_allowed_conns - all_peers_and_ips = peer_container.get_all_peers_group(add_external_ips=True) - all_peers_no_ips = peer_container.get_all_peers_group(add_external_ips=False) + all_peers_and_ips = peer_container.get_all_peers_group(True) + all_peers_no_ips = peer_container.get_all_peers_group() dns_entries = peer_container.get_all_dns_entries() # for istio initialize non-captured conns with all possible non-TCP connections # This is a compact way to represent all peers connections, but it is an over-approximation also containing @@ -421,29 +409,33 @@ def _allowed_xgress_conns_optimized(self, is_ingress, peer_container, res_conns_ class IngressNetworkLayer(NetworkLayer): def _allowed_xgress_conns(self, from_peer, to_peer, is_ingress): + allowed_conns = ConnectionSet() all_allowed_conns = ConnectionSet(True) - allowed_conns, _, _, captured_res = self.collect_policies_conns(from_peer, to_peer, is_ingress) - if captured_res: - all_allowed_conns = allowed_conns + captured_res = False + if not is_ingress: + allowed_conns, _, _, captured_res = self.collect_policies_conns(from_peer, to_peer, is_ingress) + if captured_res: + all_allowed_conns = allowed_conns return PolicyConnections(captured=captured_res, allowed_conns=allowed_conns, denied_conns=ConnectionSet(), all_allowed_conns=all_allowed_conns) - def _allowed_xgress_conns_optimized(self, is_ingress, peer_container, res_conns_filter=PolicyConnectionsFilter()): - all_peers_and_ips = peer_container.get_all_peers_group(add_external_ips=True, include_dns_entries=True) - all_peers_no_ips = peer_container.get_all_peers_group(add_external_ips=False, include_dns_entries=True) + def _allowed_xgress_conns_optimized(self, is_ingress, peer_container): + res_conns = OptimizedPolicyConnections() + all_peers_and_ips = peer_container.get_all_peers_group(True) + all_peers_no_ips = peer_container.get_all_peers_group() non_captured_conns = None - res_conns = self.collect_policies_conns_optimized(is_ingress) - if res_conns_filter.calc_all_allowed: - res_conns.all_allowed_conns = res_conns.allowed_conns + if is_ingress: + # everything is allowed and non captured + non_captured_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers_and_ips, + "dst_peers": all_peers_no_ips}) + res_conns.all_allowed_conns = non_captured_conns + else: + res_conns = self.collect_policies_conns_optimized(is_ingress) non_captured_peers = all_peers_no_ips - res_conns.captured if non_captured_peers: - if is_ingress: - non_captured_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": all_peers_and_ips, - "dst_peers": non_captured_peers}) - else: - non_captured_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": non_captured_peers, - "dst_peers": all_peers_and_ips}) - res_conns.all_allowed_conns |= non_captured_conns + non_captured_conns = ConnectivityProperties.make_conn_props_from_dict({"src_peers": non_captured_peers, + "dst_peers": all_peers_and_ips}) + res_conns.all_allowed_conns = res_conns.allowed_conns | non_captured_conns if non_captured_conns and ExplTracker().is_active(): src_peers, dst_peers = ExplTracker().extract_peers(non_captured_conns) ExplTracker().add_default_policy(src_peers, diff --git a/nca/NetworkConfig/PoliciesFinder.py b/nca/NetworkConfig/PoliciesFinder.py index 1c37e892e..4a6d910d1 100644 --- a/nca/NetworkConfig/PoliciesFinder.py +++ b/nca/NetworkConfig/PoliciesFinder.py @@ -29,9 +29,9 @@ def __init__(self, optimized_run='false'): self.optimized_run = optimized_run # following missing resources fields are relevant for "livesim" mode, # where certain resources are added to enable the analysis - self.missing_istio_gw_pods_with_labels = set() + self.missing_istio_gw_pods_with_labels = {} self.missing_k8s_ingress_peers = False - self.missing_dns_pods_with_labels = set() + self.missing_dns_pods_with_labels = {} def set_peer_container(self, peer_container): """ diff --git a/nca/NetworkConfig/ResourcesHandler.py b/nca/NetworkConfig/ResourcesHandler.py index 44e91196b..3457f5ed8 100644 --- a/nca/NetworkConfig/ResourcesHandler.py +++ b/nca/NetworkConfig/ResourcesHandler.py @@ -69,24 +69,24 @@ def get_full_livesim_resource_path(livesim_resource_path): return os.path.join(current_path, livesim_resource_path) @staticmethod - def get_relevant_livesim_resources_paths_by_labels_matching(livesim_resource_path, missing_resource_labels): + def get_relevant_livesim_resources_paths_by_labels_matching(livesim_resource_path, missing_resource_labels_dict): """ check by labels matching if one of the livesim resources has matching labels for a resource referenced by one of the parsed policies. If yes, return its path to be added to the configuration, to enable the analysis. :param str livesim_resource_path: a path to the relevant livesim dir to check for resources - :param set((key, value)) missing_resource_labels: the labels from parsed policy in the config for + :param dict missing_resource_labels_dict: the labels from parsed policy in the config for which a matching peer was missing :return: list of paths for relevant livesim resources to add :rtype list[str] """ - res = set() + res = [] resource_full_path = ResourcesHandler.get_full_livesim_resource_path(livesim_resource_path) livesim_resource_labels = ResourcesParser.parse_livesim_yamls(resource_full_path) - for (key, value) in missing_resource_labels: + for key in missing_resource_labels_dict.keys(): for yaml_path, labels in livesim_resource_labels.items(): - if (key, value) in labels: - res.add(yaml_path) - return list(res) + if missing_resource_labels_dict.get(key) == labels.get(key): + res.append(yaml_path) + return res @staticmethod def analyze_livesim(policy_finder): @@ -113,12 +113,12 @@ def analyze_livesim(policy_finder): livesim_configuration_addons.append(resource_full_path) ResourcesHandler.livesim_information_message('ingress-controller') - # find Istio ingress/egress gateway + # find Istio ingress gateway istio_gateway_added_resources = ResourcesHandler.get_relevant_livesim_resources_paths_by_labels_matching( LiveSimPaths.IstioGwCfgPath, policy_finder.missing_istio_gw_pods_with_labels) if istio_gateway_added_resources: livesim_configuration_addons += istio_gateway_added_resources - ResourcesHandler.livesim_information_message('Istio-ingress/egress-gateway') + ResourcesHandler.livesim_information_message('Istio-ingress-gateway') return livesim_configuration_addons @@ -367,15 +367,15 @@ def parse_livesim_yamls(path): for yaml_file in yaml_files: pods_finder = PodsFinder() ns_finder = NamespacesFinder() - labels_found = set() + labels_found = {} for res_code in yaml_file.data: ns_finder.parse_yaml_code_for_ns(res_code) pods_finder.namespaces_finder = ns_finder pods_finder.add_eps_from_yaml(res_code) for item in ns_finder.namespaces.values(): - labels_found.update(set(item.labels.items())) + labels_found.update(item.labels) for item in pods_finder.peer_set: - labels_found.update(set(item.labels.items())) + labels_found.update(item.labels) results.update({yaml_file.path: labels_found}) NcaLogger().collect_msgs() diff --git a/nca/NetworkConfig/TopologyObjectsFinder.py b/nca/NetworkConfig/TopologyObjectsFinder.py index ef59f5bf2..85983d86b 100644 --- a/nca/NetworkConfig/TopologyObjectsFinder.py +++ b/nca/NetworkConfig/TopologyObjectsFinder.py @@ -47,12 +47,14 @@ def add_eps_from_yaml(self, yaml_obj, kind_override=None): :param kind_override: if set, ignoring the object kind and using this param instead :return: None """ - if isinstance(yaml_obj, list): - for ep_sub_list in yaml_obj: # e.g. when we have a list of lists - call recursively for each list - self.add_eps_from_yaml(ep_sub_list) - return if not isinstance(yaml_obj, dict): + try: + for ep_sub_list in yaml_obj: # e.g. when we have a list of lists - call recursively for each list + self.add_eps_from_yaml(ep_sub_list) + except TypeError: + pass return + kind = yaml_obj.get('kind') if not kind_override else kind_override if kind in ['List', 'PodList', 'WorkloadEndpointList', 'HostEndpointList', 'NetworkSetList', 'GlobalNetworkSetList']: diff --git a/nca/Parsers/GenericIngressLikeYamlParser.py b/nca/Parsers/GenericIngressLikeYamlParser.py index 34a8ef72e..5abc3e724 100644 --- a/nca/Parsers/GenericIngressLikeYamlParser.py +++ b/nca/Parsers/GenericIngressLikeYamlParser.py @@ -11,7 +11,7 @@ from nca.CoreDS.ProtocolSet import ProtocolSet from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.CoreDS.ConnectionSet import ConnectionSet -from nca.Resources.GatewayPolicy import GatewayPolicyRule +from nca.Resources.IngressPolicy import IngressPolicyRule from .GenericYamlParser import GenericYamlParser @@ -81,7 +81,7 @@ def _make_allow_rules(conn_props, src_peers): new_props = ConnectivityProperties.make_conn_props(new_conn_cube) new_conns = ConnectionSet() new_conns.add_connections('TCP', new_props) - res.append(GatewayPolicyRule(dst_peer_set, new_conns, rule_opt_props)) + res.append(IngressPolicyRule(dst_peer_set, new_conns, rule_opt_props)) return res @staticmethod diff --git a/nca/Parsers/IngressPolicyYamlParser.py b/nca/Parsers/IngressPolicyYamlParser.py index dd1fb20e5..825b95e1b 100644 --- a/nca/Parsers/IngressPolicyYamlParser.py +++ b/nca/Parsers/IngressPolicyYamlParser.py @@ -10,7 +10,7 @@ from nca.CoreDS.PortSet import PortSet from nca.CoreDS.ConnectivityCube import ConnectivityCube from nca.CoreDS.ConnectivityProperties import ConnectivityProperties -from nca.Resources.GatewayPolicy import GatewayPolicy +from nca.Resources.IngressPolicy import IngressPolicy from nca.Resources.NetworkPolicy import NetworkPolicy from .GenericIngressLikeYamlParser import GenericIngressLikeYamlParser @@ -94,11 +94,7 @@ def parse_backend(self, backend, is_default=False): service_port = srv.get_port_by_name(port_name) if port_name else srv.get_port_by_number(port_number) if not service_port: - port_str = f'{port_name if port_name else port_number}' - warning_msg = f'Ingress rule redirects traffic to {service_name}:{port_str}, ' - warning_msg += f' but port {port_str} is not exposed by Service {service_name}' - self.warning(warning_msg, service) - return None, None, False + self.syntax_error(f'Missing port {port_name if port_name else port_number} in the service', service) rule_ports = PortSet() rule_ports.add_port(service_port.target_port) # may be either a number or a named port @@ -245,8 +241,8 @@ def parse_rule(self, rule): def parse_policy(self): """ - Parses the input object to create IstioGatewayPolicy object (with deny rules only) - :return: IstioGatewayPolicy object with proper deny egress_rules, or None for wrong input object + Parses the input object to create IngressPolicy object (with deny rules only) + :return: IngressPolicy object with proper deny egress_rules, or None for wrong input object """ policy_name, policy_ns = self.parse_generic_yaml_objects_fields(self.policy, ['Ingress'], ['networking.k8s.io/v1'], 'k8s', True) @@ -254,9 +250,9 @@ def parse_policy(self): return None # Not an Ingress object self.namespace = self.peer_container.get_namespace(policy_ns) - res_policy = GatewayPolicy(policy_name + '/allow', self.namespace) + res_policy = IngressPolicy(policy_name + '/allow', self.namespace) res_policy.policy_kind = NetworkPolicy.PolicyType.Ingress - res_policy.affects_egress = True + policy_spec = self.policy['spec'] allowed_spec_keys = {'defaultBackend': [0, dict], 'ingressClassName': [0, str], 'rules': [0, list], 'tls': [0, list]} @@ -290,6 +286,6 @@ def parse_policy(self): # allowed_conns = none means that services referenced by this Ingress policy are not found, # then no connections rules to add (Ingress policy has no effect) if allowed_conns: - res_policy.add_egress_rules(self._make_allow_rules(allowed_conns, res_policy.selected_peers)) + res_policy.add_rules(self._make_allow_rules(allowed_conns, res_policy.selected_peers)) res_policy.findings = self.warning_msgs return res_policy diff --git a/nca/Parsers/IstioPolicyYamlParser.py b/nca/Parsers/IstioPolicyYamlParser.py index 4a7db974e..3dd89b064 100644 --- a/nca/Parsers/IstioPolicyYamlParser.py +++ b/nca/Parsers/IstioPolicyYamlParser.py @@ -9,7 +9,6 @@ from nca.CoreDS.Peer import IpBlock, PeerSet from nca.CoreDS.ConnectionSet import ConnectionSet from nca.CoreDS.PortSet import PortSet -from nca.CoreDS.ProtocolSet import ProtocolSet from nca.CoreDS.MethodSet import MethodSet from nca.CoreDS.ConnectivityProperties import ConnectivityProperties from nca.Resources.IstioNetworkPolicy import IstioNetworkPolicy, IstioPolicyRule @@ -490,14 +489,11 @@ def parse_ingress_rule(self, rule, selected_peers): # currently parsing only ports # TODO: extend operations parsing to include other attributes conn_props = ConnectivityProperties.make_empty_props() - tcp_props = ConnectivityProperties.make_conn_props_from_dict( - {"protocols": ProtocolSet.get_protocol_set_with_single_protocol('TCP')}) if to_array is not None: for operation_dict in to_array: conn_props |= self.parse_operation(operation_dict) connections = ConnectionSet() connections.add_connections('TCP', conn_props) - conn_props &= tcp_props else: # no 'to' in the rule => all connections allowed connections = ConnectionSet(True) conn_props = ConnectivityProperties.get_all_conns_props_per_config_peers(self.peer_container) @@ -518,7 +514,6 @@ def parse_ingress_rule(self, rule, selected_peers): condition_props &= condition_res condition_conns = ConnectionSet() condition_conns.add_connections('TCP', condition_props) - condition_props &= tcp_props if not res_peers: self.warning('Rule selects no pods', rule) if not res_peers or not selected_peers: diff --git a/nca/Parsers/IstioSidecarYamlParser.py b/nca/Parsers/IstioSidecarYamlParser.py index 37ef8b7b2..518b293b5 100644 --- a/nca/Parsers/IstioSidecarYamlParser.py +++ b/nca/Parsers/IstioSidecarYamlParser.py @@ -93,12 +93,11 @@ def _validate_dns_name_pattern(self, dns_name): # the dnsName in the internal case form looks like ..svc.cluster.local # also FQDN for external hosts is of the format [hostname].[domain].[tld] - # The entire FQDN has a max length of 255 characters. if alphabet_str: - fqdn_regex = r"(?=.{1,254}$)[A-Za-z0-9]([-A-Za-z0-9]*[A-Za-z0-9])?(\.[A-Za-z0-9]([-A-Za-z0-9]*[A-Za-z0-9])?)*[.]?" + fqdn_regex = "^((?!-)[A-Za-z0-9-]+(? 0: + print(*self._collected_messages, sep="\n") self._collected_messages.clear() diff --git a/nca/Utils/OutputConfiguration.py b/nca/Utils/OutputConfiguration.py index a04e0ee6f..a8363f14e 100644 --- a/nca/Utils/OutputConfiguration.py +++ b/nca/Utils/OutputConfiguration.py @@ -8,6 +8,8 @@ import sys from urllib import request from nca.Utils.CmdlineRunner import CmdlineRunner +from nca.FWRules.InteractiveConnectivityGraph import InteractiveConnectivityGraph +from nca.Utils.ExplTracker import ExplTracker class OutputConfiguration(dict): @@ -19,7 +21,7 @@ def __init__(self, output_config_dict=None, query_name=''): default_output_config = {'fwRulesRunInTestMode': False, 'fwRulesDebug': False, 'fwRulesGroupByLabelSinglePod': False, 'fwRulesFilterSystemNs': False, 'fwRulesMaxIter': 10, 'outputFormat': 'txt', 'outputPath': None, - 'simplifyGraph': False, 'fwRulesOverrideAllowedLabels': None, 'prURL': None, + 'fwRulesOverrideAllowedLabels': None, 'prURL': None, 'connectivityFilterIstioEdges': True, 'outputEndpoints': 'deployments', 'subset': {}, 'explain': None, 'fullExplanation': False, 'excludeIPv6Range': True} @@ -33,7 +35,7 @@ def __init__(self, output_config_dict=None, query_name=''): def __getattr__(self, name): return super().__getitem__(name) - def print_query_output(self, output, supported_output_formats=None): + def print_query_output(self, output, supported_output_formats=None): # noqa: C901 """ print accumulated query's output according to query's output config (in required format, to file or stdout) :param output: string @@ -62,6 +64,23 @@ def print_query_output(self, output, supported_output_formats=None): print(f'Command {dot_cmd_string}\n did not create {path}\n', file=sys.stderr) if os.path.isfile(tmp_dot_file): os.remove(tmp_dot_file) + elif self.outputFormat == 'html': + tmp_dot_file = f'{path}.nca_tmp.dot' + tmp_svg_file = f'{path}.nca_tmp.svg' + dot_cmd = ['dot', tmp_dot_file, '-Tsvg', f'-o{tmp_svg_file}'] + try: + with open(tmp_dot_file, "w") as f: + f.write(output) + CmdlineRunner.run_and_get_output(dot_cmd) + InteractiveConnectivityGraph(tmp_svg_file, path, ExplTracker().explain_all())\ + .create_interactive_graph() + except Exception as e: + print(f'Failed to create a svg file: {path}\n{e}', file=sys.stderr) + if not os.path.isfile(path): + dot_cmd_string = ' '.join(dot_cmd) + print(f'Command {dot_cmd_string}\n did not create {path}\n', file=sys.stderr) + if os.path.isfile(tmp_dot_file): + os.remove(tmp_dot_file) else: try: with open(path, "a") as f: diff --git a/nca/VERSION.txt b/nca/VERSION.txt index 7bc1c4047..8fdcf3869 100644 --- a/nca/VERSION.txt +++ b/nca/VERSION.txt @@ -1 +1 @@ -1.9.6 +1.9.2 diff --git a/nca/nca_cli.py b/nca/nca_cli.py index 5dab590f8..9a90c8a3b 100644 --- a/nca/nca_cli.py +++ b/nca/nca_cli.py @@ -153,7 +153,6 @@ def run_args(args): # noqa: C901 output_config = OutputConfiguration({'outputFormat': args.output_format or 'txt', 'outputPath': args.file_out or None, - 'simplifyGraph': args.simplify_graph or False, 'prURL': args.pr_url or None, 'outputEndpoints': args.output_endpoints, 'subset': {}, @@ -190,6 +189,10 @@ def run_args(args): # noqa: C901 ExplTracker().activate() ExplTracker().set_endpoints(output_config.outputEndpoints) + if args.output_format == 'html': + output_config['expl'] = ['ALL'] + ExplTracker(output_config.outputEndpoints).activate() + if args.equiv is not None: np_list = args.equiv if args.equiv != [''] else None query_name = 'twoWayContainment' @@ -322,14 +325,11 @@ def nca_main(argv=None): help='A list of labels to subset the query by') parser.add_argument('--ghe_token', '--gh_token', type=str, help='A valid token to access a GitHub repository') parser.add_argument('--output_format', '-o', type=str, - help='Output format specification (txt, txt_no_fw_rules, csv, md, dot, jpg or yaml). ' + help='Output format specification (txt, txt_no_fw_rules, csv, md, dot, jpg, html or yaml). ' 'The default is txt.') parser.add_argument('--file_out', '-f', type=str, help='A file path to which output is redirected') parser.add_argument('--expected_output', type=str, help='A file path of the expected query output,' 'relevant only with --connectivity and --semantic_diff') - parser.add_argument('--simplify_graph', action='store_true', - help='simplify the connectivity graph,' - 'relevant only when output_format is dot or jpg') parser.add_argument('--pr_url', type=str, help='The full api url for adding a PR comment') parser.add_argument('--return_0', action='store_true', help='Force a return value 0') parser.add_argument('--version', '-v', action='store_true', help='Print version and exit') diff --git a/requirements.txt b/requirements.txt index 3336cc6a7..db2303398 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ ghapi==1.0.4 PyYAML==6.0 greenery==4.0.0 -networkx==3.2.1 -beautifulsoup4==4.12.0 \ No newline at end of file +networkx==3.1 +beautifulsoup4==4.12.0 +lxml==4.9.2 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index a29d10f98..8940de08c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,10 +22,11 @@ install_requires = ghapi==1.0.4 PyYAML==6.0 greenery==4.0.0 - networkx==3.2.1 + networkx==3.1 beautifulsoup4==4.12.0 + lxml==4.9.2 -python_requires = >=3.9 +python_requires = >=3.8 [options.packages.find] include = nca* diff --git a/tests/bad_yamls/list_of_strings.json b/tests/bad_yamls/list_of_strings.json deleted file mode 100644 index 7ce01ed93..000000000 --- a/tests/bad_yamls/list_of_strings.json +++ /dev/null @@ -1,5 +0,0 @@ -[ - "esnext.global-this", - "esnext.promise.all-settled", - "esnext.string.match-all" -] diff --git a/tests/bad_yamls/single_bracket.json b/tests/bad_yamls/single_bracket.json deleted file mode 100644 index 98232c64f..000000000 --- a/tests/bad_yamls/single_bracket.json +++ /dev/null @@ -1 +0,0 @@ -{ diff --git a/tests/bad_yamls/tab_in_json.json b/tests/bad_yamls/tab_in_json.json deleted file mode 100644 index ead884617..000000000 --- a/tests/bad_yamls/tab_in_json.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "key": "value" -} diff --git a/tests/calico_testcases/example_policies/testcase18-pass/testcase18-scheme.yaml b/tests/calico_testcases/example_policies/testcase18-pass/testcase18-scheme.yaml index 738126145..862a92b11 100644 --- a/tests/calico_testcases/example_policies/testcase18-pass/testcase18-scheme.yaml +++ b/tests/calico_testcases/example_policies/testcase18-pass/testcase18-scheme.yaml @@ -94,9 +94,9 @@ queries: pairwiseInterferes: - np-ports-based/testcase18-different-ranges-writing1 - np-ports-based/testcase18-different-ranges-writing-slightly-bigger -# outputConfiguration: # TODO - uncomment after updating expected results according to optimized solution -# fullExplanation: true -# expectedOutput: ../../expected_output/testcase18-scheme-pair-wise-interferes-different-ranges-writing-additional-port.txt + outputConfiguration: + fullExplanation: true + expectedOutput: ../../expected_output/testcase18-scheme-pair-wise-interferes-different-ranges-writing-additional-port.txt expected: 2 - name: containment_different_ranges_writing_additional_port diff --git a/tests/classes_unit_tests/testCanonicalHyperCubeSet.py b/tests/classes_unit_tests/testCanonicalHyperCubeSet.py index d270ca41b..4ec26a6ad 100644 --- a/tests/classes_unit_tests/testCanonicalHyperCubeSet.py +++ b/tests/classes_unit_tests/testCanonicalHyperCubeSet.py @@ -1434,50 +1434,6 @@ def tearDown(self): # undo changes this test did to DimensionsManager singleton DimensionsManager.reset() - def test_contained_in_issue(self): - my_dimensions1 = ["x", "y"] - my_dimensions2 = ["x", "y", "z"] - cube_values_1 = [CanonicalIntervalSet.get_interval_set(2, 2), - CanonicalIntervalSet.get_interval_set(1, 3)] - conns1 = CanonicalHyperCubeSet.create_from_cube(my_dimensions2, cube_values_1, my_dimensions1) - cube_values_2 = [ - CanonicalIntervalSet.get_interval_set(3, 3), - CanonicalIntervalSet.get_interval_set(1, 1) - ] - conns1.add_cube(cube_values_2, my_dimensions1) - - cube_values_3 = [ - CanonicalIntervalSet.get_interval_set(2, 2), - CanonicalIntervalSet.get_interval_set(2, 3), - CanonicalIntervalSet.get_interval_set(1, 100), - ] - conns2 = CanonicalHyperCubeSet.create_from_cube(my_dimensions2, cube_values_3, my_dimensions2) - conns3 = CanonicalHyperCubeSet.create_from_cube(my_dimensions2, cube_values_3, my_dimensions2) - - cube_values_4 = [ - CanonicalIntervalSet.get_interval_set(2, 2), - CanonicalIntervalSet.get_interval_set(1, 1) - ] - cube_values_5 = [ - CanonicalIntervalSet.get_interval_set(3, 3), - CanonicalIntervalSet.get_interval_set(1, 1) - ] - cube_values_6 = [ - CanonicalIntervalSet.get_interval_set(2, 3), - CanonicalIntervalSet.get_interval_set(1, 1) - ] - conns2.add_cube(cube_values_4, my_dimensions1) - conns2.add_cube(cube_values_5, my_dimensions1) - conns3.add_cube(cube_values_6, my_dimensions1) - - # conns2 should be contained in conns1 - self.assertFalse(conns1.contained_in(conns2)) - self.assertTrue(conns2.contained_in(conns1)) - - self.assertEqual(conns2, conns3) - - - def test_basic(self): a = CanonicalHyperCubeSet(dimensions4) a.add_cube([CanonicalIntervalSet.get_interval_set(1, 2)], ["x"]) diff --git a/tests/classes_unit_tests/testCanonicalHyperCubeSetNew.py b/tests/classes_unit_tests/testCanonicalHyperCubeSetNew.py index 29140bd94..c9f6ce38f 100644 --- a/tests/classes_unit_tests/testCanonicalHyperCubeSetNew.py +++ b/tests/classes_unit_tests/testCanonicalHyperCubeSetNew.py @@ -4,8 +4,6 @@ from nca.CoreDS.MinDFA import MinDFA from nca.CoreDS.CanonicalHyperCubeSet import CanonicalHyperCubeSet from nca.CoreDS.DimensionsManager import DimensionsManager -from nca.CoreDS.Peer import BasePeerSet, IpBlock -from nca.CoreDS.ProtocolSet import ProtocolSet dimensions = ["src_ports", "ports", "methods", "paths"] dimensions2 = ["ports", "src_ports", "methods", "paths"] @@ -973,32 +971,6 @@ def test_contained_in_new(self): d = CanonicalHyperCubeSet.create_from_cube(dimensions, [get_str_dfa("x|y|z")], ["paths"]) self.assertTrue(c.contained_in(d)) - def test_bug_in_contained(self): - BasePeerSet.reset() - BasePeerSet().add_peer("A") - BasePeerSet().add_peer("B") - BasePeerSet().add_peer("C") - my_dimensions1 = ["src_peers", "dst_peers"] - my_dimensions2 = ["src_peers", "dst_peers", "protocols"] - conns1 = CanonicalHyperCubeSet.create_from_cube(my_dimensions2, - [BasePeerSet().get_peer_interval_of({"B"}), - BasePeerSet().get_peer_interval_of({"A", "B", "C"})], - my_dimensions1) - conns1.add_cube([BasePeerSet().get_peer_interval_of({"C"}), - BasePeerSet().get_peer_interval_of({"A"})], my_dimensions1) - - conns2 = CanonicalHyperCubeSet.create_from_cube(my_dimensions2, - [BasePeerSet().get_peer_interval_of({"B"}), - BasePeerSet().get_peer_interval_of({"B", "C"}), - ProtocolSet.get_non_tcp_protocols()], - my_dimensions2) - conns2.add_cube([BasePeerSet().get_peer_interval_of({"B"}), - BasePeerSet().get_peer_interval_of({"A"})], my_dimensions1) - conns2.add_cube([BasePeerSet().get_peer_interval_of({"C"}), - BasePeerSet().get_peer_interval_of({"A"})], my_dimensions1) - self.assertFalse(conns1.contained_in(conns2)) - self.assertTrue(conns2.contained_in(conns1)) - def test_subtract_basic(self): x = CanonicalHyperCubeSet(dimensions) y = CanonicalHyperCubeSet(dimensions) diff --git a/tests/expected_cmdline_output_files/basic_connectivity_expl_output.txt b/tests/expected_cmdline_output_files/basic_connectivity_expl_output.txt index 52b17a60d..6c0f7d1e2 100644 --- a/tests/expected_cmdline_output_files/basic_connectivity_expl_output.txt +++ b/tests/expected_cmdline_output_files/basic_connectivity_expl_output.txt @@ -11,6 +11,7 @@ src_ns: [ns3] src_pods: [*] dst_ns: [default] dst_pods: [Pod4] conn: All connect Explainability results: + Configurations affecting default/Pod1(Pod): diff --git a/tests/expected_cmdline_output_files/istio-ingress_expl_output.txt b/tests/expected_cmdline_output_files/istio-ingress_expl_output.txt index 9e2a86fba..413889bfb 100644 --- a/tests/expected_cmdline_output_files/istio-ingress_expl_output.txt +++ b/tests/expected_cmdline_output_files/istio-ingress_expl_output.txt @@ -9,6 +9,7 @@ src_ns: [prod,qa] src_pods: [*] dst_ns: [default,prod,qa] dst_pods: [*] conn: Al Explainability results: + Configurations affecting 0.0.0.0/0: diff --git a/tests/expected_cmdline_output_files/livesim_test_all_dot.dot b/tests/expected_cmdline_output_files/livesim_test_all_dot.dot index fb358bb98..33ad16810 100644 --- a/tests/expected_cmdline_output_files/livesim_test_all_dot.dot +++ b/tests/expected_cmdline_output_files/livesim_test_all_dot.dot @@ -23,12 +23,12 @@ subgraph cluster_ingress_controller_ns_namespace{ tooltip="Namespace" "ingress-controller-ns/ingress-controller-livesim(Pod)" [label=<
ingress-controller-livesim(Pod)
> shape=box fontcolor=magenta tooltip="Automatically added workload"] } -subgraph cluster_istio_system_namespace{ - label="istio-system" +subgraph cluster_istio_ingressgateway_ns_namespace{ + label="istio-ingressgateway-ns" fontsize=20 fontcolor=blue tooltip="Namespace" - "istio-system/istio-ingressgateway-livesim(Pod)" [label=<
istio-ingressgateway-livesim(Pod)
> shape=box fontcolor=magenta tooltip="Automatically added workload"] + "istio-ingressgateway-ns/istio-ingressgateway-livesim(Pod)" [label=<
istio-ingressgateway-livesim(Pod)
> shape=box fontcolor=magenta tooltip="Automatically added workload"] } subgraph cluster_kube_system_namespace{ label="kube-system" @@ -40,19 +40,19 @@ subgraph cluster_kube_system_namespace{ "0.0.0.0/0" -> "default/foo-app(Pod)"[label="All" labeltooltip="All" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] "0.0.0.0/0" -> "default/httpbin(Deployment)"[label="All" labeltooltip="All" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] "0.0.0.0/0" -> "ingress-controller-ns/ingress-controller-livesim(Pod)"[label="All" labeltooltip="All" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] - "0.0.0.0/0" -> "istio-system/istio-ingressgateway-livesim(Pod)"[label="All" labeltooltip="All" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] + "0.0.0.0/0" -> "istio-ingressgateway-ns/istio-ingressgateway-livesim(Pod)"[label="All" labeltooltip="All" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] "default/deployment-A(Deployment)" -> "kube-system/kube-dns-livesim(Pod)"[label="udp53" labeltooltip="UDP 53" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] "default/deployment-B(Deployment)" -> "default/deployment-A(Deployment)"[label="All" labeltooltip="All" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=normal] "default/deployment-B(Deployment)" -> "kube-system/kube-dns-livesim(Pod)"[label="udp53" labeltooltip="UDP 53" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] "default/foo-app(Pod)" -> "kube-system/kube-dns-livesim(Pod)"[label="udp53" labeltooltip="UDP 53" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] "default/httpbin(Deployment)" -> "kube-system/kube-dns-livesim(Pod)"[label="udp53" labeltooltip="UDP 53" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] "ingress-controller-ns/ingress-controller-livesim(Pod)" -> "default/foo-app(Pod)"[label="tcp5678" labeltooltip="TCP {'dst_ports': '5678', 'paths': '/foo(/*)?'}" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] - "istio-system/istio-ingressgateway-livesim(Pod)" -> "default/httpbin(Deployment)"[label="tcp80" labeltooltip="TCP {'dst_ports': '80', 'hosts': 'httpbin.example.com', 'paths': '(/status(/*)?)|(/delay(/*)?)'}" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] + "istio-ingressgateway-ns/istio-ingressgateway-livesim(Pod)" -> "default/httpbin(Deployment)"[label="tcp80" labeltooltip="TCP {'dst_ports': '80', 'hosts': 'httpbin.example.com', 'paths': '(/status(/*)?)|(/delay(/*)?)'}" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] "kube-system/kube-dns-livesim(Pod)" -> "0.0.0.0/0"[label="All" labeltooltip="All" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=normal] "kube-system/kube-dns-livesim(Pod)" -> "default/foo-app(Pod)"[label="All" labeltooltip="All" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] "kube-system/kube-dns-livesim(Pod)" -> "default/httpbin(Deployment)"[label="All" labeltooltip="All" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] "kube-system/kube-dns-livesim(Pod)" -> "ingress-controller-ns/ingress-controller-livesim(Pod)"[label="All" labeltooltip="All" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] - "kube-system/kube-dns-livesim(Pod)" -> "istio-system/istio-ingressgateway-livesim(Pod)"[label="All" labeltooltip="All" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] + "kube-system/kube-dns-livesim(Pod)" -> "istio-ingressgateway-ns/istio-ingressgateway-livesim(Pod)"[label="All" labeltooltip="All" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] color=white label=<
Application connectivity graph


> labelloc = "b" diff --git a/tests/expected_cmdline_output_files/livesim_test_all_txt.txt b/tests/expected_cmdline_output_files/livesim_test_all_txt.txt index 1d1ec7c39..ddca31875 100644 --- a/tests/expected_cmdline_output_files/livesim_test_all_txt.txt +++ b/tests/expected_cmdline_output_files/livesim_test_all_txt.txt @@ -1,13 +1,13 @@ final fw rules for query: , config: **: src: 0.0.0.0/0 dst_ns: [default] dst_pods: [!has(dep)] conn: All connections -src: 0.0.0.0/0 dst_ns: [ingress-controller-ns,istio-system,kube-system] dst_pods: [*] conn: All connections +src: 0.0.0.0/0 dst_ns: [ingress-controller-ns,istio-ingressgateway-ns,kube-system] dst_pods: [*] conn: All connections src_ns: [default] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: UDP 53 src_ns: [default] src_pods: [dep=A] dst_ns: [default] dst_pods: [dep=B] conn: All connections src_ns: [default] src_pods: [dep=B] dst_ns: [default] dst_pods: [dep=A] conn: All connections src_ns: [ingress-controller-ns,kube-system] src_pods: [*] dst_ns: [ingress-controller-ns] dst_pods: [*] conn: All connections src_ns: [ingress-controller-ns] src_pods: [*] dst_ns: [default] dst_pods: [foo-app] conn: TCP {'dst_ports': '5678', 'paths': '/foo(/*)?'} -src_ns: [istio-system,kube-system] src_pods: [*] dst_ns: [istio-system] dst_pods: [*] conn: All connections -src_ns: [istio-system] src_pods: [*] dst_ns: [default] dst_pods: [httpbin] conn: TCP {'dst_ports': '80', 'hosts': 'httpbin.example.com', 'paths': '(/status(/*)?)|(/delay(/*)?)'} +src_ns: [istio-ingressgateway-ns,kube-system] src_pods: [*] dst_ns: [istio-ingressgateway-ns] dst_pods: [*] conn: All connections +src_ns: [istio-ingressgateway-ns] src_pods: [*] dst_ns: [default] dst_pods: [httpbin] conn: TCP {'dst_ports': '80', 'hosts': 'httpbin.example.com', 'paths': '(/status(/*)?)|(/delay(/*)?)'} src_ns: [kube-system] src_pods: [*] dst: 0.0.0.0/0 conn: All connections src_ns: [kube-system] src_pods: [*] dst_ns: [default] dst_pods: [!has(dep)] conn: All connections -src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections \ No newline at end of file +src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: All connections diff --git a/tests/expected_cmdline_output_files/poc1_expl_output.txt b/tests/expected_cmdline_output_files/poc1_expl_output.txt index 4628937b9..5feff2217 100644 --- a/tests/expected_cmdline_output_files/poc1_expl_output.txt +++ b/tests/expected_cmdline_output_files/poc1_expl_output.txt @@ -27,6 +27,7 @@ src_ns: [kube-system] src_pods: [*] dst_ns: [kube-system] dst_pods: [*] conn: Al Explainability results: + Configurations affecting 0.0.0.0/0: diff --git a/tests/expected_cmdline_output_files/subset_deployment_expl_output.txt b/tests/expected_cmdline_output_files/subset_deployment_expl_output.txt index ba6472b6c..3811c81e4 100644 --- a/tests/expected_cmdline_output_files/subset_deployment_expl_output.txt +++ b/tests/expected_cmdline_output_files/subset_deployment_expl_output.txt @@ -6,6 +6,7 @@ src_ns: [ns2] src_pods: [deployment-C] dst_ns: [ns1] dst_pods: [deployment-A] co Explainability results: + Configurations affecting default/Pod4(Pod): diff --git a/tests/expected_cmdline_output_files/test25_expl_output.txt b/tests/expected_cmdline_output_files/test25_expl_output.txt index 5bd20d4b9..53ff38ac5 100644 --- a/tests/expected_cmdline_output_files/test25_expl_output.txt +++ b/tests/expected_cmdline_output_files/test25_expl_output.txt @@ -3,6 +3,7 @@ src_ns: [default] src_pods: [my-test-deployment-C] dst_ns: [default] dst_pods: [ Explainability results: + Configurations affecting default/my-test-deployment-A(Deployment): diff --git a/tests/expected_runtime/k8s_tests_expected_runtime.csv b/tests/expected_runtime/k8s_tests_expected_runtime.csv index 10878f154..37a0e2cc5 100644 --- a/tests/expected_runtime/k8s_tests_expected_runtime.csv +++ b/tests/expected_runtime/k8s_tests_expected_runtime.csv @@ -55,7 +55,6 @@ test,run_time(seconds) "k8s_cmdline_tests.yaml, query name: test25-expl",1.08 "k8s_cmdline_tests.yaml, query name: test4-expl",1.08 "k8s_cmdline_tests.yaml, query name: istio-ingress-expl",4.1 -"k8s_cmdline_tests.yaml, query name: bad-yamls",0.05 fw_rules_tests/policies/cyclonus-simple-example-scheme.yaml,0.08 fw_rules_tests/policies/label_expr_test_1-scheme.yaml,0.38 fw_rules_tests/policies/poc1-scheme.yaml,0.64 @@ -140,5 +139,3 @@ k8s_testcases/example_policies/withIpBlock2/withIpBlock2-scheme.yaml,5.51 k8s_testcases/example_policies/workload-resources-test/file-system-resource-test-scheme.yaml,0.07 k8s_testcases/example_policies/workload-resources-test/git-resource-test-scheme.yaml,7.66 k8s_testcases/network-policy-checks-bad-path/network-policy-check-bad-path-scheme.yaml,0.80 -k8s_testcases/ingress-bad-path-test/test-ingress-bad-port-scheme.yaml,0.06 -k8s_testcases/example_policies/sample-app/sample-all-captured-scheme.yaml,0.08 \ No newline at end of file diff --git a/tests/fw_rules_tests/policies/expected_output/poc1-scheme_output.dot b/tests/fw_rules_tests/policies/expected_output/poc1-scheme_output.dot index 371568899..3a169d5d2 100644 --- a/tests/fw_rules_tests/policies/expected_output/poc1-scheme_output.dot +++ b/tests/fw_rules_tests/policies/expected_output/poc1-scheme_output.dot @@ -6,18 +6,24 @@ digraph { subgraph cluster_map_explanation { dict_box [label=<
Connectivity legend
tcp3550 TCP 3550
tcp50051 TCP 50051
tcp5050 TCP 5050
tcp6379 TCP 6379
tcp7000 TCP 7000
tcp7070 TCP 7070
tcp8080 TCP 8080
tcp9555 TCP 9555
> shape=box] "0.0.0.0/0" [label=<
0.0.0.0/0
> shape=box fontcolor=red2 tooltip="IP Block"] - "default/adservice(Deployment)" [label=<
default/adservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] - "default/cartservice(Deployment)" [label=<
default/cartservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] - "default/checkoutservice(Deployment)" [label=<
default/checkoutservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] - "default/currencyservice(Deployment)" [label=<
default/currencyservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] - "default/emailservice(Deployment)" [label=<
default/emailservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] - "default/frontend(Deployment)" [label=<
default/frontend(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] - "default/loadgenerator(Deployment)" [label=<
default/loadgenerator(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] - "default/paymentservice(Deployment)" [label=<
default/paymentservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] - "default/productcatalogservice(Deployment)" [label=<
default/productcatalogservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] - "default/recommendationservice(Deployment)" [label=<
default/recommendationservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] - "default/redis-cart(Deployment)" [label=<
default/redis-cart(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] - "default/shippingservice(Deployment)" [label=<
default/shippingservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] +subgraph cluster_default_namespace{ + label="default" + fontsize=20 + fontcolor=blue + tooltip="Namespace" + "default/adservice(Deployment)" [label=<
adservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] + "default/cartservice(Deployment)" [label=<
cartservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] + "default/checkoutservice(Deployment)" [label=<
checkoutservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] + "default/currencyservice(Deployment)" [label=<
currencyservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] + "default/emailservice(Deployment)" [label=<
emailservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] + "default/frontend(Deployment)" [label=<
frontend(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] + "default/loadgenerator(Deployment)" [label=<
loadgenerator(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] + "default/paymentservice(Deployment)" [label=<
paymentservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] + "default/productcatalogservice(Deployment)" [label=<
productcatalogservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] + "default/recommendationservice(Deployment)" [label=<
recommendationservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] + "default/redis-cart(Deployment)" [label=<
redis-cart(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] + "default/shippingservice(Deployment)" [label=<
shippingservice(Deployment)
> shape=box fontcolor=blue tooltip="Workload"] +} "0.0.0.0/0" -> "default/frontend(Deployment)"[label="tcp8080" labeltooltip="TCP 8080" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] "default/cartservice(Deployment)" -> "default/redis-cart(Deployment)"[label="tcp6379" labeltooltip="TCP 6379" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] "default/checkoutservice(Deployment)" -> "default/cartservice(Deployment)"[label="tcp7070" labeltooltip="TCP 7070" color=darkorange4 fontcolor=darkgreen dir=both arrowhead=normal arrowtail=none] diff --git a/tests/fw_rules_tests/policies/poc1-scheme.yaml b/tests/fw_rules_tests/policies/poc1-scheme.yaml index eef86f6e1..3ecfb10fd 100644 --- a/tests/fw_rules_tests/policies/poc1-scheme.yaml +++ b/tests/fw_rules_tests/policies/poc1-scheme.yaml @@ -31,7 +31,6 @@ queries: outputConfiguration: outputFormat: dot outputPath: null - simplifyGraph: true fwRulesRunInTestMode: false expectedOutput: expected_output/poc1-scheme_output.dot - name: connectivity_map_csv diff --git a/tests/istio_testcases/example_policies/bookinfo-demo/sidecar_examples/containments-w-sidecar-and-service-entry-scheme.yaml b/tests/istio_testcases/example_policies/bookinfo-demo/sidecar_examples/containments-w-sidecar-and-service-entry-scheme.yaml index 673d2390a..719ea2d82 100644 --- a/tests/istio_testcases/example_policies/bookinfo-demo/sidecar_examples/containments-w-sidecar-and-service-entry-scheme.yaml +++ b/tests/istio_testcases/example_policies/bookinfo-demo/sidecar_examples/containments-w-sidecar-and-service-entry-scheme.yaml @@ -62,9 +62,9 @@ queries: twoWayContainment: - sidecar-with-selector-allows-any - sidecar-with-selector-registery-only -# outputConfiguration: # TODO - uncomment after updating expected results according to optimized solution -# fullExplanation: True -# expectedOutput: ../../../expected_output/two_way_containment_configs_w_sidecars_different_outbound_mode.txt + outputConfiguration: + fullExplanation: True + expectedOutput: ../../../expected_output/two_way_containment_configs_w_sidecars_different_outbound_mode.txt expected: 1 # first config contains all conns in the second config but the second does not contain all conns of the first config - name: permits-on-configs-with-different-outbound-modes diff --git a/tests/istio_testcases/example_policies/istio-ingress-test/istio-ingress-test-scheme.yaml b/tests/istio_testcases/example_policies/istio-ingress-test/istio-ingress-test-scheme.yaml index de0a5452b..3bedae228 100644 --- a/tests/istio_testcases/example_policies/istio-ingress-test/istio-ingress-test-scheme.yaml +++ b/tests/istio_testcases/example_policies/istio-ingress-test/istio-ingress-test-scheme.yaml @@ -15,5 +15,4 @@ queries: outputConfiguration: outputFormat: txt fwRulesRunInTestMode: false -# temporarily commenting out expected results, until new Ingress/Egress implementation is completed. -# expectedOutput: ../../expected_output/istio_ingress_test_connectivity_map.txt \ No newline at end of file + expectedOutput: ../../expected_output/istio_ingress_test_connectivity_map.txt \ No newline at end of file diff --git a/tests/istio_testcases/example_policies/istio-ingress-test/resources/gateway.yaml b/tests/istio_testcases/example_policies/istio-ingress-test/resources/gateway.yaml index 5f0847c97..09f85d4b5 100644 --- a/tests/istio_testcases/example_policies/istio-ingress-test/resources/gateway.yaml +++ b/tests/istio_testcases/example_policies/istio-ingress-test/resources/gateway.yaml @@ -5,7 +5,7 @@ metadata: namespace: default spec: selector: - istio: ingressgateway + app: my-gateway-controller servers: - port: number: 80 diff --git a/tests/istio_testcases/example_policies/sidecar_examples_w_onlineboutique/invalid_sidecars/sidecar-invalid-host-10.yaml b/tests/istio_testcases/example_policies/sidecar_examples_w_onlineboutique/invalid_sidecars/sidecar-invalid-host-10.yaml deleted file mode 100644 index 95ee313b0..000000000 --- a/tests/istio_testcases/example_policies/sidecar_examples_w_onlineboutique/invalid_sidecars/sidecar-invalid-host-10.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: networking.istio.io/v1beta1 -kind: Sidecar -metadata: - name: frontend-host-too-long -spec: - workloadSelector: - labels: - app: frontend - egress: - - hosts: - - "*/abc.abc09-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ab.cd------------------------------------------------------------------------------------------------------------------kl.badfqdn" \ No newline at end of file diff --git a/tests/istio_testcases/example_policies/sidecar_examples_w_onlineboutique/sidecars-syntax-tests-scheme.yaml b/tests/istio_testcases/example_policies/sidecar_examples_w_onlineboutique/sidecars-syntax-tests-scheme.yaml index b0347bfe8..bce700537 100644 --- a/tests/istio_testcases/example_policies/sidecar_examples_w_onlineboutique/sidecars-syntax-tests-scheme.yaml +++ b/tests/istio_testcases/example_policies/sidecar_examples_w_onlineboutique/sidecars-syntax-tests-scheme.yaml @@ -73,11 +73,6 @@ networkConfigList: - invalid_sidecars/sidecar-invalid-host-9.yaml expectedError: 1 - - name: invalid_host_10 - networkPolicyList: - - invalid_sidecars/sidecar-invalid-host-10.yaml - expectedError: 1 - - name: invalid_spec networkPolicyList: - invalid_sidecars/sidecar-missing-spec.yaml diff --git a/tests/k8s_cmdline_tests.yaml b/tests/k8s_cmdline_tests.yaml index b00423bcc..1cf9adfa5 100644 --- a/tests/k8s_cmdline_tests.yaml +++ b/tests/k8s_cmdline_tests.yaml @@ -498,12 +498,5 @@ --explain ALL -opt=true -d -# temporarily commenting out expected results, until new Ingress/Egress implementation is completed. -# --expected_output expected_cmdline_output_files/istio-ingress_expl_output.txt + --expected_output expected_cmdline_output_files/istio-ingress_expl_output.txt expected: 0 - -- name: bad-yamls - args: > - --connectivity - -r bad_yamls - expected: 4 diff --git a/tests/k8s_testcases/example_policies/sample-app/sample-all-captured-scheme.yaml b/tests/k8s_testcases/example_policies/sample-app/sample-all-captured-scheme.yaml deleted file mode 100644 index 1df151cfe..000000000 --- a/tests/k8s_testcases/example_policies/sample-app/sample-all-captured-scheme.yaml +++ /dev/null @@ -1,10 +0,0 @@ -resourceList: [] -networkConfigList: - - name: sample - resourceList: - - ./** -queries: - - name: sample-all-captured - allCaptured: - - sample - expected: 2 \ No newline at end of file diff --git a/tests/k8s_testcases/example_policies/sample-app/sample.yaml b/tests/k8s_testcases/example_policies/sample-app/sample.yaml deleted file mode 100644 index 771e3b636..000000000 --- a/tests/k8s_testcases/example_policies/sample-app/sample.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: my-service-ingress - namespace: sample -spec: - rules: - - host: my-app.domain - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: my-service - port: - number: 8080 ---- - -apiVersion: v1 -kind: Service -metadata: - name: my-service - namespace: sample - labels: - app: my-app -spec: - type: ClusterIP - ports: - - port: 8080 - targetPort: 8080 - selector: - app: my-app - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: my-app - namespace: sample -spec: - replicas: 1 - selector: - matchLabels: - app: my-app - template: - metadata: - labels: - app: my-app - spec: - serviceAccountName: sample - containers: - - name: my-app - image: webapp:1 - ports: - - containerPort: 8080 \ No newline at end of file diff --git a/tests/k8s_testcases/example_policies/testcase7/testcase7-scheme.yaml b/tests/k8s_testcases/example_policies/testcase7/testcase7-scheme.yaml index a02b9badc..d868ad652 100644 --- a/tests/k8s_testcases/example_policies/testcase7/testcase7-scheme.yaml +++ b/tests/k8s_testcases/example_policies/testcase7/testcase7-scheme.yaml @@ -55,29 +55,29 @@ queries: containment: - np2 - np1 -# outputConfiguration: # TODO - uncomment after updating expected results according to optimized solution -# fullExplanation: true -# expectedOutput: ../../expected_output/containment-np2-and-np1-all-pairs.txt + outputConfiguration: + fullExplanation: true + expectedOutput: ../../expected_output/containment-np2-and-np1-all-pairs.txt expected: 0 - name: containment_np2_and_np1_print_all_pairs containment: - np2 - np1 -# outputConfiguration: # TODO - uncomment after updating expected results according to optimized solution -# fullExplanation: true -# outputFormat: yaml -# expectedOutput: ../../expected_output/containment-np2-and-np1-all-pairs.yaml + outputConfiguration: + fullExplanation: true + outputFormat: yaml + expectedOutput: ../../expected_output/containment-np2-and-np1-all-pairs.yaml expected: 0 - name: containment_np2_and_np1_print_all_pairs_json containment: - np2 - np1 -# outputConfiguration: # TODO - uncomment after updating expected results according to optimized solution -# fullExplanation: true -# outputFormat: json -# expectedOutput: ../../expected_output/containment-np2-and-np1-all-pairs.json + outputConfiguration: + fullExplanation: true + outputFormat: json + expectedOutput: ../../expected_output/containment-np2-and-np1-all-pairs.json expected: 0 - name: connectivity_map diff --git a/tests/k8s_testcases/example_policies/tests-different-topologies/containment-permits-different-topologies-scheme.yaml b/tests/k8s_testcases/example_policies/tests-different-topologies/containment-permits-different-topologies-scheme.yaml index 77efbdc57..586716896 100644 --- a/tests/k8s_testcases/example_policies/tests-different-topologies/containment-permits-different-topologies-scheme.yaml +++ b/tests/k8s_testcases/example_policies/tests-different-topologies/containment-permits-different-topologies-scheme.yaml @@ -104,29 +104,29 @@ queries: twoWayContainment: - config_a_with_ipBlock - config_a_with_b_ipBlock -# outputConfiguration: # TODO - uncomment after updating expected results according to optimized solution -# fullExplanation: true -# expectedOutput: ../../expected_output/two-way-containment-a-with-different-ipblock-policies-all-pairs.txt + outputConfiguration: + fullExplanation: true + expectedOutput: ../../expected_output/two-way-containment-a-with-different-ipblock-policies-all-pairs.txt expected: 0 - name: twoWayContainment_a_with_different_ipBlock_policies_print_all_pairs twoWayContainment: - config_a_with_ipBlock - config_a_with_b_ipBlock -# outputConfiguration: # TODO - uncomment after updating expected results according to optimized solution -# fullExplanation: true -# outputFormat: yaml -# expectedOutput: ../../expected_output/two-way-containment-a-with-different-ipblock-policies-all-pairs.yaml + outputConfiguration: + fullExplanation: true + outputFormat: yaml + expectedOutput: ../../expected_output/two-way-containment-a-with-different-ipblock-policies-all-pairs.yaml expected: 0 - name: twoWayContainment_a_with_different_ipBlock_policies_print_all_pairs_json twoWayContainment: - config_a_with_ipBlock - config_a_with_b_ipBlock -# outputConfiguration: # TODO - uncomment after updating expected results according to optimized solution -# fullExplanation: true -# outputFormat: json -# expectedOutput: ../../expected_output/two-way-containment-a-with-different-ipblock-policies-all-pairs.json + outputConfiguration: + fullExplanation: true + outputFormat: json + expectedOutput: ../../expected_output/two-way-containment-a-with-different-ipblock-policies-all-pairs.json expected: 0 - name: twoWayContainment_np1_np2 diff --git a/tests/k8s_testcases/expected_output/semantic_diff_online_boutique_new_synthesized_vs_orig_synthesized.txt b/tests/k8s_testcases/expected_output/semantic_diff_online_boutique_new_synthesized_vs_orig_synthesized.txt index 13e23ba37..c6aa85ebe 100644 --- a/tests/k8s_testcases/expected_output/semantic_diff_online_boutique_new_synthesized_vs_orig_synthesized.txt +++ b/tests/k8s_testcases/expected_output/semantic_diff_online_boutique_new_synthesized_vs_orig_synthesized.txt @@ -2,9 +2,6 @@ orig_online_boutique_synthesis_res and new_online_synthesis_res are not semantic Lost connections between removed peers and persistent peers (based on topology from config: orig_online_boutique_synthesis_res) : src_ns: [default] src_pods: [cartservice] dst_ns: [default] dst_pods: [redis-cart] conn: TCP 6379 -src_ns: [kube-system] src_pods: [*] dst: *.googleapis.com conn: All connections -src_ns: [kube-system] src_pods: [*] dst: accounts.google.com conn: All connections -src_ns: [kube-system] src_pods: [*] dst: metadata.google.internal conn: All connections Removed connections between persistent peers (based on topology from config: orig_online_boutique_synthesis_res) : src_ns: [default] src_pods: [cartservice] dst_ns: [kube-system] dst_pods: [*] conn: UDP 53 diff --git a/tests/k8s_testcases/ingress-bad-path-test/sample-app/sample.yaml b/tests/k8s_testcases/ingress-bad-path-test/sample-app/sample.yaml deleted file mode 100644 index e5f8f3848..000000000 --- a/tests/k8s_testcases/ingress-bad-path-test/sample-app/sample.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: my-service-ingress - namespace: sample -spec: - rules: - - host: my-app.domain - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: my-service - port: - number: 8080 ---- - -apiVersion: v1 -kind: Service -metadata: - name: my-service - namespace: sample - labels: - app: my-app -spec: - type: ClusterIP - ports: - - port: 80 - targetPort: 8080 - selector: - app: my-app - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: my-app - namespace: sample -spec: - replicas: 1 - selector: - matchLabels: - app: my-app - template: - metadata: - labels: - app: my-app - spec: - serviceAccountName: sample - containers: - - name: my-app - image: webapp:1 - ports: - - containerPort: 8080 \ No newline at end of file diff --git a/tests/k8s_testcases/ingress-bad-path-test/test-ingress-bad-port-scheme.yaml b/tests/k8s_testcases/ingress-bad-path-test/test-ingress-bad-port-scheme.yaml deleted file mode 100644 index c3c7abab4..000000000 --- a/tests/k8s_testcases/ingress-bad-path-test/test-ingress-bad-port-scheme.yaml +++ /dev/null @@ -1,10 +0,0 @@ -resourceList: [] -networkConfigList: - - name: sample - resourceList: - - ./sample-app/** - expectedWarnings: 1 # Warning: Ingress rule redirects traffic to my-service:8080, but port 8080 is not exposed by Service my-service -queries: - - name: connectivity_map - connectivityMap: - - sample diff --git a/tests/run_all_tests.py b/tests/run_all_tests.py index f1aab3eb5..ccb26c280 100644 --- a/tests/run_all_tests.py +++ b/tests/run_all_tests.py @@ -112,10 +112,9 @@ def run_all_test_flow(self, all_results): tmp_opt = [i for i in self.test_queries_obj.args_obj.args if '-opt=' in i] opt = tmp_opt[0].split('=')[1] if tmp_opt else 'false' if isinstance(self.test_queries_obj, CliQuery) and (opt == 'debug' or opt == 'true'): - implemented_opt_queries = {'--connectivity', '--equiv', '--permits', '--interferes', '--forbids', - '--sanity', '--semantic_diff'} + implemented_opt_queries = ['--connectivity'] # TODO - update/remove the optimization below when all queries are supported in optimized implementation - if not implemented_opt_queries.intersection(set(self.test_queries_obj.args_obj.args)): + if not set(implemented_opt_queries).intersection(set(self.test_queries_obj.args_obj.args)): print(f'Skipping {self.test_queries_obj.test_name} since it does not have optimized implementation yet') return 0, 0