Reproducible and faster generation

bazel_ros2_rules/lib/private/ros2bzl/scraping/__init__.py

@@ -1,3 +1,4 @@
+import copy
 import os
 
 from cmake_tools import get_packages_with_prefixes
@@ -10,6 +11,8 @@
     get_packages_with_prefixes as get_python_packages_with_prefixes
 )
 
+from ros2bzl.utilities import ordered_set
+
 
 def list_all_executables():
     # Delay import to allow testing most of ros2bzl without a ros2 workspace
@@ -53,20 +56,20 @@ def index_all_packages():
 
 
 def build_dependency_graph(packages, include=None, exclude=None):
-    package_set = set(packages)
+    package_set = ordered_set(packages)
     if include:
-        include = set(include)
-        if not package_set.issuperset(include):
-            unknown_packages = tuple(include.difference(package_set))
+        include = ordered_set(include)
+        if not all(p in package_set for p in include):
+            unknown_packages = [p for p in include if p not in package_set]
             msg = 'Cannot find package'
             if len(unknown_packages) == 1:
                 msg += ' ' + repr(unknown_packages[0])
             else:
                 msg += 's ' + repr(unknown_packages)
             raise RuntimeError(msg)
-        package_set &= include
+        package_set = include
     if exclude:
-        package_set -= exclude
+        package_set = [p for p in package_set if p not in exclude]
 
     groups = {}
     for name, metadata in packages.items():
@@ -80,22 +83,22 @@ def build_dependency_graph(packages, include=None, exclude=None):
     while package_set:
         name = package_set.pop()
         metadata = packages[name]
-        dependencies = set(metadata.get('build_export_dependencies', []))
-        dependencies.update(metadata.get('run_dependencies', []))
+        dependencies = copy.deepcopy(metadata.get('build_export_dependencies', []))
+        dependencies += [dep for dep in metadata.get('run_dependencies', []) if dep not in dependencies]
         if 'group_dependencies' in metadata:
             for group_name in metadata['group_dependencies']:
-                dependencies.update(groups[group_name])
+                dependencies += [dep for dep in groups[group_name] if dep not in dependencies]
         if exclude:
-            dependencies -= exclude
+            dependencies = [dep for dep in dependencies if dep not in exclude]
         # Ignore system, non-ROS dependencies
         # NOTE(hidmic): shall we sandbox those too?
-        dependencies = {
+        dependencies = ordered_set([
             dependency_name
             for dependency_name in dependencies
             if dependency_name in packages
-        }
+        ])
         dependency_graph[name] = dependencies
-        package_set.update(dependencies)
+        package_set += [d for d in dependencies if d not in package_set and d not in dependency_graph]
 
     packages = {name: packages[name] for name in dependency_graph}
 

bazel_ros2_rules/lib/private/ros2bzl/scraping/metadata.py

@@ -1,6 +1,8 @@
 import os
 import xml.etree.ElementTree as ET
 
+from ros2bzl.utilities import ordered_set
+
 
 # Remove elements that have a condition attribute on ROS1
 def remove_ros1_elements(root):
@@ -24,26 +26,26 @@ def parse_package_xml(path_to_package_xml):
 
     remove_ros1_elements(tree.getroot())
 
-    depends = set([
+    depends = ordered_set([
         tag.text for tag in tree.findall('./depend')
     ])
-    exec_depends = set([
+    exec_depends = ordered_set([
         tag.text for tag in tree.findall('./exec_depend')
-    ])
-    build_export_depends = set([
+    ] + depends)
+    build_export_depends = ordered_set([
         tag.text for tag in tree.findall('./build_export_depend')
-    ])
-    group_depends = set([
+    ] + depends)
+    group_depends = ordered_set([
         tag.text for tag in tree.findall('./group_depend')
     ])
-    member_of_groups = set([
+    member_of_groups = ordered_set([
         tag.text for tag in tree.findall('./member_of_group')
     ])
     build_type = tree.find('./export/build_type').text
 
     return dict(
-        build_export_dependencies=build_export_depends | depends,
-        run_dependencies=exec_depends | depends,
+        build_export_dependencies=build_export_depends,
+        run_dependencies=exec_depends,
         group_dependencies=group_depends,
         groups=member_of_groups,
         build_type=build_type

bazel_ros2_rules/lib/private/ros2bzl/scraping/system.py

@@ -9,11 +9,14 @@
 import subprocess
 import sys
 
+from ros2bzl.utilities import ordered_set
+
 # Standard include files' search paths for compilers in Linux systems.
 # Useful to detect system includes in package exported configuration.
 DEFAULT_INCLUDE_DIRECTORIES = ['/usr/include', '/usr/local/include']
 
 
+@lru_cache(maxsize=None)
 def is_system_include(include_path):
     """
     Checks whether `include_path` is in a system include directory
@@ -65,6 +68,7 @@ def system_shared_lib_dirs():
     return tuple([d for d in lib_dirs if d])
 
 
+@lru_cache(maxsize=None)
 def is_system_library(library_path):
     """
     Checks whether `library_path` is in a system library directory
@@ -94,8 +98,8 @@ def find_library_path(library_name, link_directories=None, link_flags=None):
     paths = []
     if link_directories:
         paths.extend(link_directories)
-    paths.extend(set(os.environ.get('LIBRARY_PATH', '').split(':')))
-    paths.extend(set(os.environ.get('LD_LIBRARY_PATH', '').split(':')))
+    paths.extend(ordered_set(os.environ.get('LIBRARY_PATH', '').split(':')))
+    paths.extend(ordered_set(os.environ.get('LD_LIBRARY_PATH', '').split(':')))
     paths.extend(system_link_dirs())
     paths.extend(system_shared_lib_dirs())
 
@@ -126,6 +130,7 @@ def find_library_path(library_name, link_directories=None, link_flags=None):
 LDD_LINE_PATTERN = re.compile(r' => (/(?:[^\(\)\s]*/)*lib[^\(\)\s]*)')
 
 
+@lru_cache(maxsize=None)
 def find_library_dependencies(library_path):
     """
     Lists all shared library dependencies of a given library.
@@ -144,10 +149,11 @@ def find_library_dependencies(library_path):
             stderr=subprocess.DEVNULL,
             encoding='utf8'
         ).stdout.strip().split('\n')
+        ret = []
         for line in lines:
             match = LDD_LINE_PATTERN.search(line.strip())
             if match:
-                yield match.group(1)
+                ret.append(match.group(1))
+        return ret
     except Exception:
-        pass
-    return
+        return []

bazel_ros2_rules/lib/private/ros2bzl/templates.py

@@ -188,8 +188,8 @@ def configure_package_py_library(
             template = 'templates/package_py_library_with_cc_libs.bazel.tpl'
             config.update({
                 'cc_name': c_name("_" + target_name, metadata),
-                'cc_libs': [
-                    sandbox(lib) for lib in properties.cc_libraries],
+                'cc_libs': sorted([
+                    sandbox(lib) for lib in properties.cc_libraries]),
                 'cc_deps': cc_deps
             })
             data.append(c_label("_" + target_name, metadata))

bazel_ros2_rules/lib/private/ros2bzl/utilities.py

@@ -23,3 +23,13 @@ def interpolate(template, config):
     for key, value in config.items():
         content = content.replace('@{}@'.format(key), value)
     return content
+
+def ordered_set(l: list) -> list:
+    """
+    Returns an ordered set from a list of values
+
+    Works for Python 3.7 and up
+
+    From https://stackoverflow.com/a/53657523
+    """
+    return list(dict.fromkeys(l))

bazel_ros2_rules/lib/private/ros_py.bzl

@@ -64,11 +64,11 @@ def ros_import_binary(
         srcs = [shim_name],
         main = shim_name,
         tags = ["nolint"] + kwargs.get("tags", []),
-        data = [executable] + kwargs.get("data", []),
-        deps = kwargs.get("deps", []) + [
+        data = sorted([executable] + kwargs.get("data", [])),
+        deps = sorted(kwargs.get("deps", []) + [
             "@bazel_ros2_rules//lib/dynamic_load:dload_shim_py",
             "@bazel_ros2_rules//lib/network_isolation:network_isolation_py",
-        ],
+        ]),
     )
     py_binary_rule(name = name, **kwargs)
 

bazel_ros2_rules/lib/private/scripts/scrape_distribution.py

@@ -17,6 +17,7 @@
 
 from ros2bzl.scraping import scrape_distribution
 
+from ros2bzl.utilities import ordered_set
 
 def parse_arguments():
     parser = argparse.ArgumentParser(
@@ -44,8 +45,8 @@ def main():
     args = parse_arguments()
 
     distro = scrape_distribution(
-        set(args.include_packages),
-        set(args.exclude_packages))
+        ordered_set(args.include_packages),
+        ordered_set(args.exclude_packages))
 
     json.dump(distro, args.output, default=list)