Make package layout in the published repository configurable

closes #3874

Author: dralley

CHANGES/3874.feature

@@ -0,0 +1 @@
+Make the layout of the packages in the published repository configurable.

functest_requirements.txt

@@ -1,12 +1,13 @@
+dictdiffer
 django  # TODO: test_sync.py has a dependency on date parsing functions
+lxml
 productmd>=1.25
-pytest<8
-dictdiffer
-xmltodict
+pydantic
 pyyaml
-lxml
-pyzstd
-requests
+pytest<8
 pytest-xdist
 pytest-timeout
 pytest-custom_exit_code
+pyzstd
+requests
+xmltodict

pulp_rpm/app/constants.py

@@ -11,6 +11,14 @@
     (COMPRESSION_TYPES.GZ, COMPRESSION_TYPES.GZ),
 )
 
+# publication layout
+LAYOUT_TYPES = SimpleNamespace(NESTED_ALPHABETICALLY="nested_alphabetically", FLAT="flat")
+
+LAYOUT_CHOICES = (
+    (LAYOUT_TYPES.NESTED_ALPHABETICALLY, LAYOUT_TYPES.NESTED_ALPHABETICALLY),
+    (LAYOUT_TYPES.FLAT, LAYOUT_TYPES.FLAT),
+)
+
 CHECKSUM_TYPES = SimpleNamespace(
     UNKNOWN="unknown",
     MD5="md5",

pulp_rpm/app/migrations/0063_rpmpublication_layout_rpmrepository_layout.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.10 on 2025-02-12 19:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rpm', '0062_rpmpackagesigningservice_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='rpmpublication',
+            name='layout',
+            field=models.TextField(choices=[('nested_alphabetically', 'nested_alphabetically'), ('flat', 'flat')], null=True),
+        ),
+        migrations.AddField(
+            model_name='rpmrepository',
+            name='layout',
+            field=models.TextField(choices=[('nested_alphabetically', 'nested_alphabetically'), ('flat', 'flat')], null=True),
+        ),
+    ]

pulp_rpm/app/models/repository.py

@@ -30,7 +30,7 @@
     validate_version_paths,
 )
 
-from pulp_rpm.app.constants import CHECKSUM_CHOICES, COMPRESSION_CHOICES
+from pulp_rpm.app.constants import CHECKSUM_CHOICES, COMPRESSION_CHOICES, LAYOUT_CHOICES
 from pulp_rpm.app.downloaders import RpmDownloader, RpmFileDownloader, UlnDownloader
 from pulp_rpm.app.exceptions import DistributionTreeConflict
 from pulp_rpm.app.models import (
@@ -208,6 +208,8 @@ class RpmRepository(Repository, AutoAddObjPermsMixin):
         repo_config (JSON): repo configuration that will be served by distribution
         compression_type(pulp_rpm.app.constants.COMPRESSION_TYPES):
             Compression type to use for metadata files.
+        layout(pulp_rpm.app.constants.LAYOUT_TYPES):
+            How to layout the package files within the publication (flat, nested, etc.)
     """
 
     TYPE = "rpm"
@@ -240,6 +242,7 @@ class RpmRepository(Repository, AutoAddObjPermsMixin):
     autopublish = models.BooleanField(default=False)
     checksum_type = models.TextField(null=True, choices=CHECKSUM_CHOICES)
     compression_type = models.TextField(null=True, choices=COMPRESSION_CHOICES)
+    layout = models.TextField(null=True, choices=LAYOUT_CHOICES)
     metadata_checksum_type = models.TextField(null=True, choices=CHECKSUM_CHOICES)
     package_checksum_type = models.TextField(null=True, choices=CHECKSUM_CHOICES)
     repo_config = models.JSONField(default=dict)
@@ -267,6 +270,7 @@ def on_new_version(self, version):
                 },
                 repo_config=self.repo_config,
                 compression_type=self.compression_type,
+                layout=self.layout,
             )
 
     @property
@@ -500,6 +504,7 @@ class RpmPublication(Publication, AutoAddObjPermsMixin):
     compression_type = models.TextField(null=True, choices=COMPRESSION_CHOICES)
     metadata_checksum_type = models.TextField(choices=CHECKSUM_CHOICES)
     package_checksum_type = models.TextField(choices=CHECKSUM_CHOICES)
+    layout = models.TextField(null=True, choices=LAYOUT_CHOICES)
     repo_config = models.JSONField(default=dict)
 
     class Meta:

pulp_rpm/app/serializers/repository.py

@@ -29,6 +29,7 @@
     ALLOWED_PUBLISH_CHECKSUMS,
     CHECKSUM_CHOICES,
     COMPRESSION_CHOICES,
+    LAYOUT_CHOICES,
     SKIP_TYPES,
     SYNC_POLICY_CHOICES,
 )
@@ -117,6 +118,12 @@ class RpmRepositorySerializer(RepositorySerializer):
         required=False,
         allow_null=True,
     )
+    layout = serializers.ChoiceField(
+        help_text=_("How to layout the packages within the published repository."),
+        choices=LAYOUT_CHOICES,
+        required=False,
+        allow_null=True,
+    )
     gpgcheck = serializers.IntegerField(
         max_value=1,
         min_value=0,
@@ -278,6 +285,7 @@ class Meta:
             "sqlite_metadata",
             "repo_config",
             "compression_type",
+            "layout",
         )
         model = RpmRepository
 
@@ -395,6 +403,12 @@ class RpmPublicationSerializer(PublicationSerializer):
         choices=COMPRESSION_CHOICES,
         required=False,
     )
+    layout = serializers.ChoiceField(
+        help_text=_("How to layout the packages within the published repository."),
+        choices=LAYOUT_CHOICES,
+        required=False,
+        allow_null=True,
+    )
     gpgcheck = serializers.IntegerField(
         max_value=1,
         min_value=0,
@@ -476,6 +490,7 @@ class Meta:
             "sqlite_metadata",
             "repo_config",
             "compression_type",
+            "layout",
         )
         model = RpmPublication
 

pulp_rpm/app/tasks/publishing.py

@@ -26,6 +26,7 @@
     CHECKSUM_TYPES,
     COMPRESSION_TYPES,
     PACKAGES_DIRECTORY,
+    LAYOUT_TYPES,
 )
 from pulp_rpm.app.kickstart.treeinfo import PulpTreeInfo, TreeinfoData
 from pulp_rpm.app.models import (
@@ -328,6 +329,7 @@ def publish(
     checksum_type=None,
     repo_config=None,
     compression_type=COMPRESSION_TYPES.GZ,
+    layout=LAYOUT_TYPES.NESTED_ALPHABETICALLY,
     *args,
     **kwargs,
 ):
@@ -342,6 +344,8 @@ def publish(
         repo_config (JSON): repo config that will be served by distribution
         compression_type(pulp_rpm.app.constants.COMPRESSION_TYPES):
             Compression type to use for metadata files.
+        layout(pulp_rpm.app.constants.LAYOUT_TYPES):
+            How to layout the package files within the publication (flat, nested, etc.)
 
     """
     repository_version = RepositoryVersion.objects.get(pk=repository_version_pk)
@@ -369,6 +373,7 @@ def publish(
             publication.metadata_checksum_type = checksum_type
             publication.package_checksum_type = checksum_types.get("package") or checksum_type
             publication.compression_type = compression_type
+            publication.layout = layout
             publication.repo_config = repo_config
 
             publication_data = PublicationData(publication, checksum_types)
@@ -422,6 +427,7 @@ def generate_repo_metadata(
     sub_folder=None,
     metadata_signing_service=None,
     compression_type=COMPRESSION_TYPES.GZ,
+    layout=LAYOUT_TYPES.NESTED_ALPHABETICALLY,
 ):
     """
     Creates a repomd.xml file.
@@ -435,6 +441,8 @@ def generate_repo_metadata(
             A reference to an associated signing service.
         compression_type(pulp_rpm.app.constants.COMPRESSION_TYPES):
             Compression type to use for metadata files.
+        layout(pulp_rpm.app.constants.LAYOUT_TYPES):
+            How to layout the package files within the publication (flat, nested, etc.)
 
     """
     cwd = os.getcwd()
@@ -565,11 +573,16 @@ def generate_repo_metadata(
             pkg.pkgId = pkgId
 
             pkg_filename = os.path.basename(package.location_href)
-            # this can cause an issue when two same RPM package names appears
-            # a/name1.rpm b/name1.rpm
-            pkg.location_href = os.path.join(
-                PACKAGES_DIRECTORY, pkg_filename[0].lower(), pkg_filename
-            )
+            if layout == LAYOUT_TYPES.NESTED_ALPHABETICALLY:
+                # this can cause an issue when two same RPM package names appears
+                # a/name1.rpm b/name1.rpm
+                pkg_path = os.path.join(PACKAGES_DIRECTORY, pkg_filename[0].lower(), pkg_filename)
+            elif layout == LAYOUT_TYPES.FLAT:
+                pkg_path = os.path.join(PACKAGES_DIRECTORY, pkg_filename)
+            else:
+                raise ValueError("Layout value is unsupported by this version")
+
+            pkg.location_href = pkg_path
 
             if settings.RPM_METADATA_USE_REPO_PACKAGE_TIME:
                 pkg.time_file = repo_pkg_times[package.pk]

pulp_rpm/tests/functional/api/test_publish.py

@@ -224,6 +224,81 @@ def test_publish_references_update(assert_created_publication):
     assert_created_publication(RPM_REFERENCES_UPDATEINFO_URL)
 
 
+def get_metadata_content_helper(base_url, repomd_elem, meta_type):
+    """Return the text contents of metadata file.
+
+    Provided a url, a repomd root element, and a metadata type, locate the metadata
+    file's location href, download it from the provided url, un-gzip it, parse it, and
+    return the root element node.
+
+    Don't use this with large repos because it will blow up.
+    """
+    # <ns0:repomd xmlns:ns0="http://linux.duke.edu/metadata/repo">
+    #     <ns0:data type="primary">
+    #         <ns0:checksum type="sha256">[…]</ns0:checksum>
+    #         <ns0:location href="repodata/[…]-primary.xml.gz" />
+    #         …
+    #     </ns0:data>
+    #     …
+    xpath = "{{{}}}data".format(RPM_NAMESPACES["metadata/repo"])
+    data_elems = [elem for elem in repomd_elem.findall(xpath) if elem.get("type") == meta_type]
+    if not data_elems:
+        return None
+
+    xpath = "{{{}}}location".format(RPM_NAMESPACES["metadata/repo"])
+    location_href = data_elems[0].find(xpath).get("href")
+
+    return download_and_decompress_file(os.path.join(base_url, location_href))
+
+
+@pytest.mark.parametrize("layout", ["flat", "nested_alphabetically"])
+def test_repo_layout(
+    layout,
+    init_and_sync,
+    rpm_publication_api,
+    gen_object_with_cleanup,
+    rpm_distribution_api,
+    rpm_distribution_factory,
+    monitor_task,
+    delete_orphans_pre,
+    tmpdir,
+    wget_recursive_download_on_host,
+):
+    """Test that using the "layout" option for publication produces the correct package layouts"""
+
+    # create repo and remote
+    repo, _ = init_and_sync(url=RPM_UNSIGNED_FIXTURE_URL, policy="on_demand")
+
+    # publish
+    publish_data = RpmRpmPublication(repository=repo.pulp_href, layout=layout)
+    publish_response = rpm_publication_api.create(publish_data)
+    created_resources = monitor_task(publish_response.task).created_resources
+    publication_href = created_resources[0]
+
+    # distribute
+    distribution = rpm_distribution_factory(publication=publication_href)
+
+    # Download and parse the metadata.
+    repomd = ElementTree.fromstring(
+        requests.get(os.path.join(distribution.base_url, "repodata/repomd.xml")).text
+    )
+
+    # Convert the metadata into a more workable form and then compare.
+    primary = get_metadata_content_helper(distribution.base_url, repomd, "primary")
+
+    packages = xmltodict.parse(primary, dict_constructor=collections.OrderedDict)["metadata"][
+        "package"
+    ]
+
+    for package in packages:
+        if layout == "flat":
+            assert package["location"]["@href"].startswith("Packages/{}".format(package["name"][0]))
+        elif layout == "nested_alphabetically":
+            assert package["location"]["@href"].startswith(
+                "Packages/{}/".format(package["name"][0])
+            )
+
+
 @pytest.mark.parametrize("repo_url", [RPM_COMPLEX_FIXTURE_URL, RPM_MODULAR_FIXTURE_URL])
 def test_complex_repo_core_metadata(
     distribution_base_url,
@@ -265,45 +340,19 @@ def test_complex_repo_core_metadata(
         ).text
     )
 
-    def get_metadata_content(base_url, repomd_elem, meta_type):
-        """Return the text contents of metadata file.
-
-        Provided a url, a repomd root element, and a metadata type, locate the metadata
-        file's location href, download it from the provided url, un-gzip it, parse it, and
-        return the root element node.
-
-        Don't use this with large repos because it will blow up.
-        """
-        # <ns0:repomd xmlns:ns0="http://linux.duke.edu/metadata/repo">
-        #     <ns0:data type="primary">
-        #         <ns0:checksum type="sha256">[…]</ns0:checksum>
-        #         <ns0:location href="repodata/[…]-primary.xml.gz" />
-        #         …
-        #     </ns0:data>
-        #     …
-        xpath = "{{{}}}data".format(RPM_NAMESPACES["metadata/repo"])
-        data_elems = [elem for elem in repomd_elem.findall(xpath) if elem.get("type") == meta_type]
-        if not data_elems:
-            return None
-
-        xpath = "{{{}}}location".format(RPM_NAMESPACES["metadata/repo"])
-        location_href = data_elems[0].find(xpath).get("href")
-
-        return download_and_decompress_file(os.path.join(base_url, location_href))
-
     # Convert the metadata into a more workable form and then compare.
     for metadata_file in ["primary", "filelists", "other"]:
-        original_metadata = get_metadata_content(repo_url, original_repomd, metadata_file)
-        generated_metadata = get_metadata_content(
+        original_metadata = get_metadata_content_helper(repo_url, original_repomd, metadata_file)
+        generated_metadata = get_metadata_content_helper(
             distribution_base_url(distribution.base_url), reproduced_repomd, metadata_file
         )
 
         _compare_xml_metadata_file(original_metadata, generated_metadata, metadata_file)
 
     # =================
 
-    original_modulemds = get_metadata_content(repo_url, original_repomd, "modules")
-    generated_modulemds = get_metadata_content(
+    original_modulemds = get_metadata_content_helper(repo_url, original_repomd, "modules")
+    generated_modulemds = get_metadata_content_helper(
         distribution_base_url(distribution.base_url), reproduced_repomd, "modules"
     )
 
@@ -319,8 +368,8 @@ def get_metadata_content(base_url, repomd_elem, meta_type):
     # ===================
 
     # TODO: make this deeper
-    original_updateinfo = get_metadata_content(repo_url, original_repomd, "updateinfo")
-    generated_updateinfo = get_metadata_content(
+    original_updateinfo = get_metadata_content_helper(repo_url, original_repomd, "updateinfo")
+    generated_updateinfo = get_metadata_content_helper(
         distribution_base_url(distribution.base_url), reproduced_repomd, "updateinfo"
     )
     assert bool(original_updateinfo) == bool(generated_updateinfo)