DPDK: Allow arbitrary rdma-core source build

mcgov · mcgov · commit 27fd8c7b83f5 · 2023-12-19T19:35:55.000-08:00
Fulfilling an ask to be able to test multiple rdma-core
versions to allow development and regression tests.

Setting rdma_core_source to either a git or tar.gz link allows you to
build from source. For git either set rdma_core_ref or it will pick the
latest version sorted tag.
diff --git a/lisa/util/__init__.py b/lisa/util/__init__.py
@@ -24,6 +24,7 @@
     Union,
     cast,
 )
+from urllib.parse import urlparse
 
 import paramiko
 import pluggy
@@ -47,7 +48,7 @@
 # source -
 # https://github.com/django/django/blob/stable/1.3.x/django/core/validators.py#L45
 __url_pattern = re.compile(
-    r"^(?:http|ftp)s?://"  # http:// or https://
+    r"^(?:http|s?ftp)s?://"  # http:// or https://
     r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)"
     r"+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|"  # ...domain
     r"localhost|"  # localhost...
@@ -563,7 +564,9 @@ def find_group_in_lines(
     return result
 
 
-def deep_update_dict(src: Dict[str, Any], dest: Dict[str, Any]) -> Dict[str, Any]:
+def deep_update_dict(
+    src: Dict[str, Any], dest: Optional[Dict[str, Any]]
+) -> Dict[str, Any]:
     if (
         dest is None
         or isinstance(dest, int)
@@ -577,7 +580,7 @@ def deep_update_dict(src: Dict[str, Any], dest: Dict[str, Any]) -> Dict[str, Any
 
     if isinstance(result, dict):
         for key, value in src.items():
-            if isinstance(value, dict) and key in dest:
+            if dest and isinstance(value, dict) and key in dest:
                 value = deep_update_dict(value, dest[key])
             result[key] = value
     elif isinstance(src, dict):
@@ -598,6 +601,58 @@ def is_valid_url(url: str, raise_error: bool = True) -> bool:
     return is_url
 
 
+def is_valid_source_code_package(
+    source_url: str,
+    expected_package_name_pattern: Pattern[str],
+    allowed_protocols: Optional[List[str]] = None,
+    expected_domains: Optional[List[str]] = None,
+) -> bool:
+    # avoid using a mutable default parameter
+    if not allowed_protocols:
+        allowed_protocols = [
+            "https",
+            "sftp",
+        ]
+    # first, check if it's a url.
+    if not is_valid_url(url=source_url, raise_error=False):
+        return False
+
+    # NOTE: urllib might not work as you'd expect.
+    # It doesn't throw on lots of things you wouldn't expect to be urls.
+    # You must verify the parts on your own, some of them may be empty, some null.
+    # check: https://docs.python.org/3/library/urllib.parse.html#url-parsing
+
+    try:
+        parts = urlparse(source_url)
+    except ValueError:
+        return False
+
+    # ex: from https://www.com/path/to/file.tar
+    # scheme : https
+    # netloc : www.com
+    # path   : path/to/file.tar
+
+    # get the filename from the path portion of the url
+    file_path = parts.path.split("/")[-1]
+    full_match = expected_package_name_pattern.match(file_path)
+    if not full_match:
+        return False
+
+    # check the expected domain is correct if present
+    valid_netloc = not expected_domains or any(
+        [domain.endswith(parts.netloc) for domain in expected_domains]
+    )
+
+    # optional but default is check access is via sftp/https
+    valid_scheme = any([parts.scheme == x for x in allowed_protocols])
+    return (
+        valid_scheme
+        and parts.netloc != ""
+        and valid_netloc
+        and (full_match.group(0) == file_path)
+    )
+
+
 def filter_ansi_escape(content: str) -> str:
     return __ansi_escape.sub("", content)
 
diff --git a/microsoft/testsuites/dpdk/dpdktestpmd.py b/microsoft/testsuites/dpdk/dpdktestpmd.py
@@ -4,6 +4,7 @@
 import re
 from pathlib import PurePosixPath
 from typing import Any, List, Tuple, Type, Union
+from urllib.parse import urlparse
 
 from assertpy import assert_that, fail
 from semver import VersionInfo
@@ -19,7 +20,6 @@
     Kill,
     Lscpu,
     Lspci,
-    Make,
     Modprobe,
     Pidof,
     Pkgconfig,
@@ -40,6 +40,7 @@
     is_ubuntu_latest_or_prerelease,
     is_ubuntu_lts_version,
 )
+from microsoft.testsuites.dpdk.rdma_core import RdmaCoreManager
 
 PACKAGE_MANAGER_SOURCE = "package_manager"
 
@@ -136,22 +137,6 @@ def command(self) -> str:
         _rx_pps_key: r"Rx-pps:\s+([0-9]+)",
     }
 
-    def get_rdma_core_package_name(self) -> str:
-        distro = self.node.os
-        package = ""
-        # check if rdma-core is installed already...
-        if self.node.tools[Pkgconfig].package_info_exists("libibuverbs"):
-            return package
-        if isinstance(distro, Debian):
-            package = "rdma-core ibverbs-providers libibverbs-dev"
-        elif isinstance(distro, Suse):
-            package = "rdma-core-devel librdmacm1"
-        elif isinstance(distro, Fedora):
-            package = "librdmacm-devel"
-        else:
-            fail("Invalid OS for rdma-core source installation.")
-        return package
-
     @property
     def can_install(self) -> bool:
         for _os in [Debian, Fedora, Suse]:
@@ -465,8 +450,17 @@ def add_sample_apps_to_build_list(self, apps: Union[List[str], None]) -> None:
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
+        # set source args for builds if needed, first for dpdk
         self._dpdk_source = kwargs.pop("dpdk_source", PACKAGE_MANAGER_SOURCE)
         self._dpdk_branch = kwargs.pop("dpdk_branch", "main")
+        # then for rdma-core
+        rdma_core_source = kwargs.pop("rdma_core_source", "")
+        rdma_core_ref = kwargs.pop("rdma_core_ref", "")
+        self.rdma_core = RdmaCoreManager(
+            node=self.node,
+            rdma_core_source=rdma_core_source,
+            rdma_core_ref=rdma_core_ref,
+        )
         self._sample_apps_to_build = kwargs.pop("sample_apps", [])
         self._dpdk_version_info = VersionInfo(0, 0)
         self._testpmd_install_path: str = ""
@@ -522,54 +516,6 @@ def _check_pps_data(self, rx_or_tx: str) -> None:
             f"empty or all zeroes for dpdktestpmd.{rx_or_tx.lower()}_pps_data."
         ).is_true()
 
-    def _install_upstream_rdma_core_for_mana(self) -> None:
-        node = self.node
-        wget = node.tools[Wget]
-        make = node.tools[Make]
-        tar = node.tools[Tar]
-        distro = node.os
-
-        if isinstance(distro, Debian):
-            distro.install_packages(
-                "cmake libudev-dev "
-                "libnl-3-dev libnl-route-3-dev ninja-build pkg-config "
-                "valgrind python3-dev cython3 python3-docutils pandoc "
-                "libssl-dev libelf-dev python3-pip libnuma-dev"
-            )
-        elif isinstance(distro, Fedora):
-            distro.group_install_packages("Development Tools")
-            distro.install_packages(
-                "cmake gcc libudev-devel "
-                "libnl3-devel pkg-config "
-                "valgrind python3-devel python3-docutils  "
-                "openssl-devel unzip "
-                "elfutils-devel python3-pip libpcap-devel  "
-                "tar wget dos2unix psmisc kernel-devel-$(uname -r)  "
-                "librdmacm-devel libmnl-devel kernel-modules-extra numactl-devel  "
-                "kernel-headers elfutils-libelf-devel meson ninja-build libbpf-devel "
-            )
-        else:
-            # check occcurs before this function
-            return
-
-        tar_path = wget.get(
-            url=(
-                "https://github.com/linux-rdma/rdma-core/"
-                "releases/download/v46.0/rdma-core-46.0.tar.gz"
-            ),
-            file_path=str(node.working_path),
-        )
-
-        tar.extract(tar_path, dest_dir=str(node.working_path), gzip=True, sudo=True)
-        source_path = node.working_path.joinpath("rdma-core-46.0")
-        node.execute(
-            "cmake -DIN_PLACE=0 -DNO_MAN_PAGES=1 -DCMAKE_INSTALL_PREFIX=/usr",
-            shell=True,
-            cwd=source_path,
-            sudo=True,
-        )
-        make.make_install(source_path)
-
     def _set_backport_repo_args(self) -> None:
         distro = self.node.os
         # skip attempting to use backports for latest/prerlease
@@ -597,6 +543,18 @@ def _install(self) -> bool:
         self._testpmd_output_during_rescind = ""
         self._last_run_output = ""
         node = self.node
+        distro = node.os
+        if not (
+            isinstance(distro, Fedora)
+            or isinstance(distro, Debian)
+            or isinstance(distro, Suse)
+        ):
+            raise SkippedException(
+                UnsupportedDistroException(
+                    distro, "DPDK tests not implemented for this OS."
+                )
+            )
+
         # before doing anything: determine if backport repo needs to be enabled
         self._set_backport_repo_args()
 
@@ -608,42 +566,50 @@ def _install(self) -> bool:
             self._load_drivers_for_dpdk()
             return True
 
-        # otherwise, install from package manager, git, or tar
-
-        self._install_dependencies()
+        # if this is mana VM, we don't support other distros yet
+        is_mana_test_supported = isinstance(distro, Ubuntu) or isinstance(
+            distro, Fedora
+        )
+        if self.is_mana and not is_mana_test_supported:
+            raise SkippedException("MANA DPDK test is not supported on this OS")
 
-        # if this is mana VM, we need an upstream rdma-core package (for now)
-        if self.is_mana:
-            if not (isinstance(node.os, Ubuntu) or isinstance(node.os, Fedora)):
-                raise SkippedException("MANA DPDK test is not supported on this OS")
+        # if we need an rdma-core source install, do it now.
+        if self.rdma_core.can_install_from_source() or (
+            is_mana_test_supported and self.is_mana
+        ):
+            # ensure no older version is installed
+            distro.uninstall_packages("rdma-core")
+            self.rdma_core.do_source_install()
 
-            # ensure no older dependency is installed
-            node.os.uninstall_packages("rdma-core")
-            self._install_upstream_rdma_core_for_mana()
+        # otherwise, install kernel and dpdk deps from package manager, git, or tar
+        self._install_dependencies()
+        # install any missing rdma-core packages
+        rdma_packages = self.rdma_core.get_missing_distro_packages()
+        distro.install_packages(rdma_packages)
 
         # installing from distro package manager
         if self.use_package_manager_install():
             self.node.log.info(
                 "Installing dpdk and dev package from package manager..."
             )
-            if isinstance(node.os, Debian):
-                node.os.install_packages(
+            if isinstance(distro, Debian):
+                distro.install_packages(
                     ["dpdk", "dpdk-dev"],
                     extra_args=self._backport_repo_args,
                 )
-            elif isinstance(node.os, (Fedora, Suse)):
-                node.os.install_packages(["dpdk", "dpdk-devel"])
+            elif isinstance(distro, (Fedora, Suse)):
+                distro.install_packages(["dpdk", "dpdk-devel"])
             else:
                 raise NotImplementedError(
                     "Dpdk package names are missing in dpdktestpmd.install"
-                    f" for os {node.os.name}"
+                    f" for os {distro.name}"
                 )
             self.node.log.info(
                 f"Installed DPDK version {str(self._dpdk_version_info)} "
                 "from package manager"
             )
 
-            self._dpdk_version_info = node.os.get_package_information("dpdk")
+            self._dpdk_version_info = distro.get_package_information("dpdk")
             self.find_testpmd_binary()
             self._load_drivers_for_dpdk()
             return True
@@ -867,9 +833,6 @@ def _install_suse_dependencies(self) -> None:
             suse.install_packages(self._suse_packages)
             if not self.use_package_manager_install():
                 self._install_ninja_and_meson()
-            rdma_core_packages = self.get_rdma_core_package_name()
-            if rdma_core_packages:
-                suse.install_packages(rdma_core_packages.split())
 
     def _install_ubuntu_dependencies(self) -> None:
         node = self.node
@@ -904,9 +867,6 @@ def _install_ubuntu_dependencies(self) -> None:
             # MANA tests use linux-modules-extra-azure, install if it's available.
             if self.is_mana and ubuntu.is_package_in_repo("linux-modules-extra-azure"):
                 ubuntu.install_packages("linux-modules-extra-azure")
-        rdma_core_packages = self.get_rdma_core_package_name()
-        if rdma_core_packages:
-            ubuntu.install_packages(rdma_core_packages.split())
 
     def _install_fedora_dependencies(self) -> None:
         node = self.node
@@ -919,7 +879,7 @@ def _install_fedora_dependencies(self) -> None:
             return  # appease the type checker
 
         # DPDK is very sensitive to rdma-core/kernel mismatches
-        # update to latest kernel before instaling dependencies
+        # update to latest kernel before installing dependencies
         rhel.install_packages("kernel")
         node.reboot()
 
@@ -935,9 +895,7 @@ def _install_fedora_dependencies(self) -> None:
 
         # RHEL 8 doesn't require special cases for installed packages.
         # TODO: RHEL9 may require updates upon release
-        rdma_core_packages = self.get_rdma_core_package_name()
-        if rdma_core_packages:
-            self._fedora_packages += rdma_core_packages.split()
+        if not self.rdma_core.is_installed_from_source:
             rhel.group_install_packages("Infiniband Support")
 
         rhel.group_install_packages("Development Tools")
diff --git a/microsoft/testsuites/dpdk/dpdkutil.py b/microsoft/testsuites/dpdk/dpdkutil.py
@@ -312,6 +312,8 @@ def initialize_node_resources(
     _set_forced_source_by_distro(node, variables)
     dpdk_source = variables.get("dpdk_source", PACKAGE_MANAGER_SOURCE)
     dpdk_branch = variables.get("dpdk_branch", "")
+    rdma_core_source = variables.get("rdma_core_source", "")
+    rdma_core_ref = variables.get("rdma_core_git_ref", "")
     force_net_failsafe_pmd = variables.get("dpdk_force_net_failsafe_pmd", False)
     log.info(
         "Dpdk initialize_node_resources running"
@@ -348,6 +350,8 @@ def initialize_node_resources(
         dpdk_branch=dpdk_branch,
         sample_apps=sample_apps,
         force_net_failsafe_pmd=force_net_failsafe_pmd,
+        rdma_core_source=rdma_core_source,
+        rdma_core_ref=rdma_core_ref,
     )
 
     # init and enable hugepages (required by dpdk)
diff --git a/microsoft/testsuites/dpdk/rdma_core.py b/microsoft/testsuites/dpdk/rdma_core.py
