Add support for buildpack.toml files

src/packagedcode/__init__.py

@@ -12,6 +12,7 @@
 from packagedcode import alpine
 from packagedcode import bower
 from packagedcode import build
+from packagedcode import buildpack
 from packagedcode import build_gradle
 from packagedcode import cargo
 from packagedcode import chef
@@ -62,6 +63,8 @@
     build.BuckMetadataBzlHandler,
     build.BuckPackageHandler,
 
+    buildpack.BuildpackHandler,
+
     cargo.CargoLockHandler,
     cargo.CargoTomlHandler,
 

src/packagedcode/buildpack.py

@@ -0,0 +1,210 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import toml
+from packagedcode import models
+from packageurl import PackageURL
+import yaml
+
+class BuildpackHandler(models.NonAssemblableDatafileHandler):
+    """
+    Handle buildpack.toml manifests.
+    See https://buildpacks.io/ for details on buildpack format.
+    """
+    datasource_id = "buildpack_toml"
+    path_patterns = ("*buildpack.toml",)
+    default_package_type = "generic"
+    description = "Cloud Native Buildpack manifest"
+    documentation_url = "https://buildpacks.io/"
+
+    @classmethod
+    def parse(cls, location, package_only=False):
+        """
+        Parse the buildpack.toml file at `location` and yield PackageData.
+        """
+        with open(location, "r", encoding="utf-8") as f:
+            data = toml.load(f)
+
+        api_version = data.get("api")
+        buildpack = data.get("buildpack", {})
+        if not buildpack:
+            return
+
+        buildpack_id = buildpack.get("id")
+        name = buildpack.get("name")
+        namespace = None
+
+        if buildpack_id and "/" in buildpack_id:
+            namespace, name = buildpack_id.split("/", 1)
+        version = buildpack.get("version")
+        if version and "{{" in version and "}}" in version:
+            version = None
+
+        # Initialize common package data
+        package_data = dict(
+            datasource_id=cls.datasource_id,
+            type=cls.default_package_type,
+            name=name,
+            namespace=namespace,
+            version=version,
+            description=buildpack.get("description"),
+            homepage_url=buildpack.get("homepage"),
+            keywords=buildpack.get("keywords", []),
+            extracted_license_statement=None,
+            extra_data={},
+        )
+
+        if api_version:
+            package_data["extra_data"]["api_version"] = api_version
+
+        if buildpack_id:
+            package_data["extra_data"]["id"] = buildpack_id
+
+        metadata = data.get("metadata", {})
+        include_files = metadata.get("include-files", [])
+        if include_files:
+            package_data["extra_data"]["include_files"] = include_files
+
+        # Handle Paketo-specific fields if present
+        if "api" in data:
+            cls.handle_paketo_buildpack(data, buildpack, package_data)
+
+        # Handle Heroku-specific fields if present
+        elif "publish" in data and "Ignore" in data["publish"]:
+            cls.handle_heroku_buildpack(data, buildpack, package_data)
+
+        yield models.PackageData.from_data(package_data, package_only)
+
+    @staticmethod
+    def handle_paketo_buildpack(data, buildpack, package_data):
+        package_data.update({
+            "description": buildpack.get("description"),
+            "homepage_url": buildpack.get("homepage"),
+            "keywords": buildpack.get("keywords", []),
+        })
+
+        licenses = buildpack.get("licenses", [])
+        if licenses:
+            license_statements = [
+                yaml.dump({"type": license_entry.get("type")}).strip() 
+                for license_entry in licenses
+                if license_entry.get("type")  
+            ]
+            package_data["extracted_license_statement"] = "\n".join(license_statements)
+
+        dependencies = []
+        metadata = data.get("metadata", {})
+        metadata_dependencies = metadata.get("dependencies", [])
+        for dep in metadata_dependencies:
+            dep_purl = dep.get("purl")
+            dep_name = dep.get("name")
+            dep_version = dep.get("version")
+            dep_cpes = dep.get("cpes", [])
+
+            resolved_package = {}
+
+            for field in ["id", "name", "sha256", "stacks", "uri", "licenses", "homepage"]:
+                if field in dep:
+                    resolved_package[field] = dep[field]
+
+            if dep_cpes:
+                resolved_package["cpes"] = dep_cpes
+
+            if "arch" in dep:
+                resolved_package["arch"] = dep["arch"]
+
+            if "license" in dep:
+                resolved_package["license"] = dep["license"]
+
+            if "uri" in dep:
+                resolved_package["download_url"] = dep["uri"]
+
+            for checksum_type in ["sha256", "sha512", "md5"]:
+                if checksum_type in dep:
+                    resolved_package[checksum_type] = dep[checksum_type]
+
+            extra_data = {}
+            for key, value in dep.items():
+                if key not in ["purl", "name", "version", "cpes", "id", "sha256", 
+                              "stacks", "uri", "licenses", "license", "homepage", 
+                              "arch", "sha512", "md5"]:
+                    extra_data[key] = value
+
+            if not dep_purl and dep_name and dep_version:
+                qualifiers = {}
+                if "arch" in dep:
+                    qualifiers["arch"] = dep["arch"]
+
+                dep_purl = PackageURL(
+                    type="generic", 
+                    name=dep_name, 
+                    version=dep_version,
+                    qualifiers=qualifiers if qualifiers else None
+                ).to_string()
+
+            if dep_purl:
+                dependencies.append(
+                    models.DependentPackage(
+                        purl=dep_purl,
+                        scope="runtime",
+                        is_runtime=True,
+                        is_optional=False,
+                        is_pinned=True if "sha256" in dep else False,
+                        is_direct=True,
+                        resolved_package=resolved_package,
+                        extra_data=extra_data if extra_data else None,
+                    )
+                )
+
+        orders = data.get("order", [])
+        for order in orders:
+            for group in order.get("group", []):
+                group_id = group.get("id")
+                group_version = group.get("version")
+                if group_id and group_version:
+                    resolved_package = {
+                        "id": group_id,
+                        "version": group_version
+                    }
+
+                    for key, value in group.items():
+                        if key not in ["id", "version", "optional"]:
+                            resolved_package[key] = value
+
+                    dependencies.append(
+                        models.DependentPackage(
+                            purl=PackageURL(type="buildpack", name=group_id, version=group_version).to_string(),
+                            scope="runtime",
+                            is_runtime=True,
+                            is_optional=group.get("optional", False),
+                            is_pinned=False, 
+                            is_direct=True,   
+                            resolved_package=resolved_package,
+                        )
+                    )
+
+        package_data["dependencies"] = dependencies
+
+        targets = data.get("targets", [])
+        if targets:
+            package_data["extra_data"]["targets"] = targets
+
+    @staticmethod
+    def handle_heroku_buildpack(data, buildpack, package_data):
+        publish_section = data.get("publish", {})
+        if "Ignore" in publish_section:
+            ignore_files = publish_section["Ignore"].get("files", [])
+            if ignore_files:  
+                package_data["extra_data"]["ignore_files"] = ignore_files
+            else:
+                package_data["extra_data"]["ignore_files"] = []
+        else:
+            package_data["extra_data"]["ignore_files"] = []
+
+        package_data["description"] = f"Heroku buildpack for {buildpack.get('name')}"

src/packagedcode/models.py

@@ -582,7 +582,7 @@ class PackageData(IdentifiablePackageData):
     download_url = String(
         label='Download URL',
         help='A direct download URL.')
-
+    
     size = Integer(
         default=None,
         label='download size',

tests/packagedcode/data/buildpack/heroku-buildpacks/heroku-buildpack-java/buildpack.toml

@@ -0,0 +1,14 @@
+[buildpack]
+name = "Java"
+
+  [publish.Ignore]
+  files = [
+    "etc/",
+    "spec/",
+    "test/",
+    ".gitignore",
+    ".github/",
+    "hatchet.json",
+    "Gemfile",
+    "Gemfile.lock"
+  ]

tests/packagedcode/data/buildpack/heroku-buildpacks/heroku-buildpack-java/expectedoutput.json

@@ -0,0 +1,57 @@
+[
+  {
+    "type": "generic",
+    "namespace": null,
+    "name": "Java",
+    "version": null,
+    "qualifiers": {},
+    "subpath": null,
+    "primary_language": null,
+    "description": "Heroku buildpack for Java",
+    "release_date": null,
+    "parties": [],
+    "keywords": [],
+    "homepage_url": null,
+    "download_url": null,
+    "size": null,
+    "sha1": null,
+    "md5": null,
+    "sha256": null,
+    "sha512": null,
+    "bug_tracking_url": null,
+    "code_view_url": null,
+    "vcs_url": null,
+    "copyright": null,
+    "holder": null,
+    "declared_license_expression": null,
+    "declared_license_expression_spdx": null,
+    "license_detections": [],
+    "other_license_expression": null,
+    "other_license_expression_spdx": null,
+    "other_license_detections": [],
+    "extracted_license_statement": null,
+    "notice_text": null,
+    "source_packages": [],
+    "file_references": [],
+    "is_private": false,
+    "is_virtual": false,
+    "extra_data": {
+      "ignore_files": [
+        "etc/",
+        "spec/",
+        "test/",
+        ".gitignore",
+        ".github/",
+        "hatchet.json",
+        "Gemfile",
+        "Gemfile.lock"
+      ]
+    },
+    "dependencies": [],
+    "repository_homepage_url": null,
+    "repository_download_url": null,
+    "api_data_url": null,
+    "datasource_id": "buildpack_toml",
+    "purl": "pkg:generic/Java"
+  }
+]

tests/packagedcode/data/buildpack/heroku-buildpacks/heroku-buildpack-php/buildpack.toml

@@ -0,0 +1,17 @@
+[buildpack]
+name = "PHP"
+
+	[publish.Ignore]
+	files = [
+		".github/",
+		".gitignore",
+		".rspec_parallel",
+		"support/build/",
+		"support/devcenter/",
+		"test/",
+		"Gemfile",
+		"Gemfile.lock",
+		"hatchet.json",
+		"hatchet.lock",
+		"requirements.txt",
+	]

tests/packagedcode/data/buildpack/heroku-buildpacks/heroku-buildpack-php/expectedoutput.json

@@ -0,0 +1,60 @@
+[
+  {
+    "type": "generic",
+    "namespace": null,
+    "name": "PHP",
+    "version": null,
+    "qualifiers": {},
+    "subpath": null,
+    "primary_language": null,
+    "description": "Heroku buildpack for PHP",
+    "release_date": null,
+    "parties": [],
+    "keywords": [],
+    "homepage_url": null,
+    "download_url": null,
+    "size": null,
+    "sha1": null,
+    "md5": null,
+    "sha256": null,
+    "sha512": null,
+    "bug_tracking_url": null,
+    "code_view_url": null,
+    "vcs_url": null,
+    "copyright": null,
+    "holder": null,
+    "declared_license_expression": null,
+    "declared_license_expression_spdx": null,
+    "license_detections": [],
+    "other_license_expression": null,
+    "other_license_expression_spdx": null,
+    "other_license_detections": [],
+    "extracted_license_statement": null,
+    "notice_text": null,
+    "source_packages": [],
+    "file_references": [],
+    "is_private": false,
+    "is_virtual": false,
+    "extra_data": {
+      "ignore_files": [
+        ".github/",
+        ".gitignore",
+        ".rspec_parallel",
+        "support/build/",
+        "support/devcenter/",
+        "test/",
+        "Gemfile",
+        "Gemfile.lock",
+        "hatchet.json",
+        "hatchet.lock",
+        "requirements.txt"
+      ]
+    },
+    "dependencies": [],
+    "repository_homepage_url": null,
+    "repository_download_url": null,
+    "api_data_url": null,
+    "datasource_id": "buildpack_toml",
+    "purl": "pkg:generic/PHP"
+  }
+]