Update melange convert python to have tests and produce subpackages for each python version.

.gitignore

@@ -15,6 +15,16 @@
 # vendor/
 .vscode/*
 
+# emacs
+*~
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+*.elc
+auto-save-list
+tramp
+.\#*
+
 melange
 melange.rsa
 melange.rsa.pub

e2e-tests/numpy-test.yaml

@@ -12,7 +12,7 @@ test:
     # TODO(pnasrat): fix to use multiple python
     contents:
       packages:
-        - python-3.12
+        - python-3.12-base
   pipeline:
     # Test import with command (python -c "import numpy")
     - uses: python/test

e2e-tests/py3-pandas-test.yaml

@@ -12,7 +12,7 @@ test:
   environment:
     contents:
       packages:
-        - busybox
+        - python3
   pipeline:
     - uses: python/import
       with:

pkg/convert/python/e2e_test.go

@@ -49,7 +49,8 @@ func TestGenerateManifest(t *testing.T) {
 		assert.Equal(t, got.Package.Version, "1.29.78")
 		assert.EqualValues(t, got.Package.Epoch, 0)
 		assert.Equal(t, got.Package.Description, "Low-level, data-driven core of boto 3.")
-		assert.Equal(t, got.Package.Dependencies.Runtime, []string{"py" + versions[i] + "-jmespath", "py" + versions[i] + "-python-dateutil", "py" + versions[i] + "-urllib3", "python-" + versions[i]})
+		assert.Equal(t, []string(nil), got.Package.Dependencies.Runtime)
+		assert.Equal(t, "0", got.Package.Dependencies.ProviderPriority)
 
 		// Check Package.Copyright
 		assert.Equal(t, len(got.Package.Copyright), 1)
@@ -60,17 +61,40 @@ func TestGenerateManifest(t *testing.T) {
 			"build-base",
 			"busybox",
 			"ca-certificates-bundle",
+			"py3-supported-build-base-dev",
 			"wolfi-base",
 		})
 
 		// Check Pipeline
-		assert.Equal(t, len(got.Pipeline), 3)
+		assert.Equal(t, 1, len(got.Pipeline))
 
 		// Check Pipeline - fetch
 		assert.Equal(t, got.Pipeline[0].Uses, "fetch")
 
+		// Check Subpackages
+		assert.Equal(t, "py-versions", got.Subpackages[0].Range)
+		assert.Equal(t, "py3-${{vars.pypi-package}}", got.Subpackages[0].Dependencies.Provides[0])
+		assert.Equal(t, "py/pip-build-install", got.Subpackages[0].Pipeline[0].Uses)
+		var expectedRuntimeDeps []string = []string{
+			"py3.10-jmespath",
+			"py3.10-python-dateutil",
+			"py3.10-urllib3",
+			"python-3.10",
+		}
+		// Subpackages aren't added to this array in the same order every time causing spurious
+		// test failues. To fix this Just check what is seen and replace the deps with the version string.
+		var replacementPyVersion = got.Subpackages[0].Dependencies.Runtime[0][2:6]
+		for idx, dep := range expectedRuntimeDeps {
+			expectedRuntimeDeps[idx] = strings.Replace(dep, "3.10", replacementPyVersion, 1)
+		}
+
+		assert.Equal(t, expectedRuntimeDeps, got.Subpackages[0].Dependencies.Runtime)
+		assert.Equal(t, "${{range.value}}", got.Subpackages[0].Dependencies.ProviderPriority)
 		releases, ok := pythonctx.Package.Releases[pythonctx.PackageVersion]
 
+		assert.Equal(t, "python/import", got.Subpackages[0].Test.Pipeline[0].Uses)
+		assert.Equal(t, "${{vars.module_name}}", got.Subpackages[0].Test.Pipeline[0].With["import"])
+
 		// If the key exists
 		assert.True(t, ok)
 
@@ -89,9 +113,6 @@ func TestGenerateManifest(t *testing.T) {
 			"uri":             strings.ReplaceAll(tempURI, pythonctx.PackageVersion, "${{package.version}}"),
 		})
 
-		// Check Pipeline - runs
-		assert.Equal(t, got.Pipeline[1].Uses, "python/build-wheel")
-		assert.Equal(t, got.Pipeline[2].Uses, "strip")
 	}
 }
 
@@ -117,11 +138,8 @@ func TestGenerateManifestPreserveURI(t *testing.T) {
 		assert.Equal(t, got.Package.Description,
 			"Backported and Experimental Type Hints for Python 3.8+",
 		)
-		assert.Equal(t, got.Package.Dependencies.Runtime,
-			[]string{
-				"python-" + versions[i],
-			},
-		)
+		assert.Equal(t, []string(nil), got.Package.Dependencies.Runtime)
+		assert.Equal(t, "0", got.Package.Dependencies.ProviderPriority)
 
 		// Check Package.Copyright
 		assert.Equal(t, len(got.Package.Copyright), 1)
@@ -132,11 +150,12 @@ func TestGenerateManifestPreserveURI(t *testing.T) {
 			"build-base",
 			"busybox",
 			"ca-certificates-bundle",
+			"py3-supported-build-base-dev",
 			"wolfi-base",
 		})
 
 		// Check Pipeline
-		assert.Equal(t, len(got.Pipeline), 3)
+		assert.Equal(t, 1, len(got.Pipeline))
 
 		// Check Pipeline - fetch
 		assert.Equal(t, got.Pipeline[0].Uses, "fetch")
@@ -160,8 +179,8 @@ func TestGenerateManifestPreserveURI(t *testing.T) {
 			"uri":             "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-${{package.version}}.tar.gz",
 		})
 
-		// Check Pipeline - runs
-		assert.Equal(t, got.Pipeline[1].Uses, "python/build-wheel")
-		assert.Equal(t, got.Pipeline[2].Uses, "strip")
+		// Check Tests
+		assert.Equal(t, "python/import", got.Test.Pipeline[0].Uses)
+		assert.Equal(t, "${{vars.module_name}}", got.Test.Pipeline[0].With["import"])
 	}
 }

pkg/convert/python/python.go

@@ -244,7 +244,7 @@ func (c *PythonContext) findDep(ctx context.Context) error {
 		if err != nil {
 			return err
 		}
-		p.Dependencies = append(p.Dependencies, "py"+c.PythonVersion+"-"+dep)
+		p.Dependencies = append(p.Dependencies, "py${{range.key}}-"+dep)
 		// if dep is not already visited then check if it has deps
 		_, found := c.ToGenerate[dep]
 		if !found {
@@ -273,7 +273,11 @@ func (c *PythonContext) generateManifest(ctx context.Context, pack Package, vers
 	// Generate each field in the manifest
 	generated.GeneratedFromComment = pack.Info.ProjectURL
 	generated.Package = c.generatePackage(ctx, pack, version)
+	generated.Data = c.generateRange(ctx)
+	generated.Vars = c.generateVars(pack)
+	generated.Subpackages = c.generateSubpackages(ctx, pack)
 	generated.Environment = c.generateEnvironment(ctx, pack)
+	generated.Test = c.generateTest(ctx, pack)
 
 	pipelines, err := c.generatePipeline(ctx, pack, version, ghVersions)
 	if err != nil {
@@ -333,16 +337,14 @@ func (c *PythonContext) generatePackage(ctx context.Context, pack Package, versi
 
 	log.Infof("[%s] Run time Deps %v", pack.Info.Name, pack.Dependencies)
 
-	pack.Dependencies = append(pack.Dependencies, "python-"+c.PythonVersion)
-
 	pkg := config.Package{
 		Name:        fmt.Sprintf("py%s-%s", c.PythonVersion, pack.Info.Name),
 		Version:     version,
 		Epoch:       0,
 		Description: pack.Info.Summary,
 		Copyright:   []config.Copyright{},
 		Dependencies: config.Dependencies{
-			Runtime: pack.Dependencies,
+			ProviderPriority: "0",
 		},
 	}
 
@@ -363,6 +365,7 @@ func (c *PythonContext) generateEnvironment(ctx context.Context, pack Package) a
 		"build-base",
 		"busybox",
 		"ca-certificates-bundle",
+		"py3-supported-build-base-dev",
 		"wolfi-base",
 	}
 
@@ -455,16 +458,91 @@ func (c *PythonContext) generatePipeline(ctx context.Context, pack Package, vers
 		})
 	}
 
-	pythonBuild := config.Pipeline{
-		Name: "Python Build",
-		Uses: "python/build-wheel",
+	return pipeline, nil
+}
+
+// generateVars handles generated variables for multi version python generateSubpackages
+func (c *PythonContext) generateRange(ctx context.Context) []config.RangeData {
+	return []config.RangeData{{
+		Name: "py-versions",
+		Items: map[string]string{
+			"3.10": "310",
+			"3.11": "311",
+			"3.12": "312",
+			"3.13": "300", // Update this when 3.13 goes live
+		}},
 	}
+}
 
-	strip := config.Pipeline{
-		Uses: "strip",
+// Generate the vars for pypi package name and pip
+// Set pypi-package and module_name to the same value because it's the most common case.
+// Someone else can fix it up if the build fails
+func (c *PythonContext) generateVars(pack Package) map[string]string {
+	return map[string]string{
+		"pypi-package": pack.Info.Name,
+		"module_name":  pack.Info.Name,
 	}
-	pipeline = append(pipeline, pythonBuild)
-	pipeline = append(pipeline, strip)
+}
 
-	return pipeline, nil
+// generateSubpackages handles generating suibpackages field of the melange manifest
+func (c *PythonContext) generateSubpackages(ctx context.Context, pack Package) []config.Subpackage {
+	log := clog.FromContext(ctx)
+
+	log.Infof("[%s] Generating Subpackages", pack.Info.Name)
+
+	importTest := config.Test{
+		Pipeline: []config.Pipeline{config.Pipeline{
+			Name: "Import Test",
+			Uses: "python/import",
+			With: map[string]string{
+				"python": "python${{range.key}}",
+				"import": "${{vars.module_name}}",
+			},
+		},
+		},
+	}
+
+	pythonSubpackages := config.Subpackage{
+		Range: "py-versions",
+		Name:  "py${{range.key}}-${{vars.pypi-package}}",
+		Dependencies: config.Dependencies{
+			Runtime:          pack.Dependencies,
+			Provides:         []string{"py3-${{vars.pypi-package}}"},
+			ProviderPriority: "${{range.value}}",
+		},
+		Pipeline: []config.Pipeline{config.Pipeline{
+			Name: "Python Build",
+			Uses: "py/pip-build-install",
+			With: map[string]string{
+				"python": "python${{range.key}}",
+			},
+		},
+		},
+		Test: &importTest,
+	}
+
+	return []config.Subpackage{pythonSubpackages}
+}
+
+// generate file-level package test.  When building python packages for multiple
+// python versions we want to ensure that we don't generate -support packages with
+// contents in /bin as well as ensuring that people installing the unversioned package
+// receive on and only one version of the library
+func (c *PythonContext) generateTest(ctx context.Context, pack Package) *config.Test {
+	log := clog.FromContext(ctx)
+
+	log.Infof("[%s] Generating Tests", pack.Info.Name)
+
+	importTest := config.Test{
+		Pipeline: []config.Pipeline{config.Pipeline{
+			Name: "Import Test",
+			Uses: "python/import",
+			With: map[string]string{
+				"import": "${{vars.module_name}}",
+			},
+		},
+		},
+	}
+
+	return &importTest
 }

pkg/convert/python/python_test.go

@@ -136,7 +136,7 @@ func TestFindDependencies(t *testing.T) {
 	}
 }
 
-// TestGeneratePackage tests when a gem has multiple licenses
+// TestGeneratePackage tests when a python package has multiple licenses
 func TestGeneratePackage(t *testing.T) {
 	for i := range versions {
 		pythonctxs, err := SetupContext(versions[i])
@@ -157,7 +157,7 @@ func TestGeneratePackage(t *testing.T) {
 				},
 			},
 			Dependencies: config.Dependencies{
-				Runtime: []string{"py" + versions[i] + "-jmespath", "py" + versions[i] + "-python-dateutil", "py" + versions[i] + "-urllib3", "python-" + versions[i]},
+				ProviderPriority: "0",
 			},
 		}
 
@@ -177,7 +177,7 @@ func SetupContext(version string) ([]*PythonContext, error) {
 	botocorepythonctx.PackageVersion = "1.29.78"
 	botocorepythonctx.PythonVersion = version
 
-	// Read the gem meta into
+	// Read the pypi meta into
 	data, err := os.ReadFile(filepath.Join(botocoreMeta, "json"))
 	if err != nil {
 		return nil, err
@@ -190,7 +190,7 @@ func SetupContext(version string) ([]*PythonContext, error) {
 	}
 
 	botocorepythonctx.Package = botocorePackageMeta
-	botocorepythonctx.Package.Dependencies = []string{"py" + version + "-jmespath", "py" + version + "-python-dateutil", "py" + version + "-urllib3"}
+	botocorepythonctx.Package.Dependencies = []string{"py" + version + "-jmespath", "py" + version + "-python-dateutil", "py" + version + "-urllib3", "python-" + version}
 
 	jsonschemapythonctx, err := New("jsonschema")
 	if err != nil {
@@ -203,7 +203,7 @@ func SetupContext(version string) ([]*PythonContext, error) {
 	jsonschemapythonctx.PackageVersion = "4.17.3"
 	jsonschemapythonctx.PythonVersion = version
 
-	// Read the gem meta into
+	// Read the pypi meta into
 	data, err = os.ReadFile(filepath.Join(jsonschemaMeta, "json"))
 	if err != nil {
 		return nil, err
@@ -238,7 +238,7 @@ func SetupContextPreserveURI(version string) ([]*PythonContext, error) {
 	typingextctx.PythonVersion = version
 	typingextctx.PreserveBaseURI = true
 
-	// Read the gem meta into
+	// Read the pypi package meta into
 	data, err := os.ReadFile(filepath.Join(typingextMeta, "json"))
 	if err != nil {
 		return nil, err
@@ -291,6 +291,7 @@ func TestGenerateEnvironment(t *testing.T) {
 				"build-base",
 				"busybox",
 				"ca-certificates-bundle",
+				"py3-supported-build-base-dev",
 				"wolfi-base",
 			},
 		},
@@ -314,6 +315,7 @@ func TestGenerateEnvironment(t *testing.T) {
 				"build-base",
 				"busybox",
 				"ca-certificates-bundle",
+				"py3-supported-build-base-dev",
 				"wolfi-base",
 			},
 		},