fix!: also find tests under outputs, ensure all outputs have tests, ensure outputs names are different from package name (#1057)

This pull request is motivated by trying to add a subpackage to an
existing package and trying to get the syntax right:
bioconda/bioconda-recipes#58141

I think I have to add both the main and the subpackage under an
`outputs:` section, and the linter then complains that it doesn't find
any global `test:` section. So this PR makes the linter go through each
subpackage if an `outputs:` section exists, and checks whether all of
them have a valid `test:` section specified.

Author: dlaehnemann

bioconda_utils/lint/check_completeness.py

@@ -4,7 +4,7 @@
 """
 
 import os
-from . import LintCheck, ERROR, WARNING, INFO
+from . import LintCheck, ERROR, WARNING, INFO, _recipe
 
 
 class missing_build_number(LintCheck):
@@ -81,19 +81,44 @@ class missing_tests(LintCheck):
     and/or any file named ``run_test.py`, ``run_test.sh`` or
     ``run_test.pl`` executing tests.
 
+    You can either add the test section on the top level of
+    your meta.yaml file; or if you use an ``outputs:`` section
+    to specify multiple outputs, add a test to each of your
+    ``outputs:`` entries.
     """
     test_files = ['run_test.py', 'run_test.sh', 'run_test.pl']
 
     def check_recipe(self, recipe):
         if any(os.path.exists(os.path.join(recipe.dir, f))
-               for f in self.test_files):
+                for f in self.test_files):
             return
-        if recipe.get('test/commands', '') or recipe.get('test/imports', ''):
-            return
-        if recipe.get('test', False) is not False:
-            self.message(section='test')
+        # if multiple `outputs:` are specified, we check that
+        # all subpackages have `test:` specified, but ignore
+        # top-level tests, as top-level package outputs will
+        # become incompatible with multiple `outputs:` in the
+        # future, see:
+        # https://conda.org/learn/ceps/cep-0014/#outputs-section
+        if recipe.get('outputs', ''):
+            packages = recipe.get('outputs')
         else:
-            self.message()
+            packages = [ recipe, ]
+        # can't use the Recipe.get function, as we might have plain dicts
+        # here; so we need to go one level of the resulting dicts at
+        # a time and check that all of them have a test specified
+        tests_specified = [
+            t.get('commands', '') or t.get('imports', '')
+            for t in [ p.get('test', {}) for p in packages ]
+        ]
+        if all(tests_specified):
+            return
+        for i in range(len(packages)):
+            if not tests_specified[i]:
+                if not isinstance(packages[i], _recipe.Recipe) and packages[i].get(f'outputs/{i}/test', ''):
+                    self.message(section=f'outputs/{i}/test')
+                elif packages[i].get('test', ''):
+                    self.message(section='test')
+                else:
+                    self.message()
 
 
 class missing_hash(LintCheck):

bioconda_utils/lint/check_policy.py

@@ -124,6 +124,25 @@ def check_deps(self, deps):
                    self.message()
 
 
+class outputs_name_same_as_package_name(LintCheck):
+    """Output names must differ from main package name
+    
+    If multiple outputs are specified, their names must be different from the
+    main package name.
+
+    This enforces accepted CEP 0014: https://conda.org/learn/ceps/cep-0014/#outputs-section
+    And it prevents hard to debug issues: https://github.com/conda/conda-build/pull/5767
+
+    """
+    def check_recipe(self, recipe):
+        name = recipe.get('package', {}).get('name', '')
+        outputs = recipe.get('outputs', '')
+        if outputs:
+            for o in outputs:
+                if o.get('name', '') == name:
+                    self.message()
+
+
 class version_starts_with_v(LintCheck):
     """The version string should not start with a "v" character
 

test/conftest.py

@@ -139,13 +139,23 @@ def recipe_dir(recipes_folder: py.path.local, tmpdir: py.path.local,
         for remove in utils.ensure_list(case['remove']):
             path = remove.split('/')
             cont = recipe
+            select_by_name = False
             for p in path[:-1]:
-                cont = cont[p]
+                if select_by_name:
+                    for subpackage in cont:
+                        if subpackage['name'] == p:
+                            cont = subpackage
+                    select_by_name = False
+                else:
+                    cont = cont.get(p, {})
+                    if p == "outputs":
+                        select_by_name = True
             if isinstance(cont, list):
                 for n in range(len(cont)):
                     del cont[n][path[-1]]
             else:
-                del cont[path[-1]]
+                if cont.get(path[-1], ""):
+                    del cont[path[-1]]
     if 'add' in case:
         dict_merge(recipe, case['add'])
 

test/lint_cases.yaml

@@ -52,6 +52,56 @@ setup:
           license: BSD
           home: https://elsewhere
           summary: the_summary
+    meta:
+      folder: meta_package
+      meta.yaml:
+        package:
+          name: meta_package
+          version: 0.1
+        source:
+          - url: https://somewhere
+            sha256: 123
+          - url: https://elsewhere
+            sha256: abc
+        build:
+          noarch: True
+          number: 0
+          run_exports:
+            # add a run export
+            # (note that jinja templating as in reality is not
+            # supported by our test system to the same
+            # extend as in conda build)
+            - "{{ pin_subpackage('meta_package') }}"
+        outputs:
+          - name: meta_one
+            build:
+              noarch: True
+              run_exports:
+                # add a run export
+                # (note that jinja templating as in reality is not
+                # supported by our test system to the same
+                # extend as in conda build)
+                - "{{ pin_subpackage('meta_one') }}"
+            test:
+              commands:
+                - do nothing
+          - name: meta_two
+            build:
+              noarch: True
+              number: 0
+              run_exports:
+                # add a run export
+                # (note that jinja templating as in reality is not
+                # supported by our test system to the same
+                # extend as in conda build)
+                - "{{ pin_subpackage('meta_two') }}"
+            test:
+              imports:
+                - no imports to see here
+        about:
+          license: MIT
+          home: https://elsewhere
+          summary: another_summary
   repodata:
     bioconda:
       one:
@@ -60,6 +110,8 @@ setup:
         - version: 0.0.1
       four:
         - version: 0.0.1
+      meta_package:
+        - version: 0.0.1
     conda-forge:
       three:
         - version: 1.0
@@ -123,11 +175,14 @@ tests:
     add: { about: { license: "" } }
     expect: missing_license
   - name: missing_tests
-    remove: test
+    remove:
+      - test
+      - outputs/meta_one/test
     expect: missing_tests
   - name: missing_tests_empty_section
     remove:
       - test/commands
+      - outputs/meta_two/test/imports
     expect: missing_tests
   - name: missing_tests_but_runtest_py
     remove: test
@@ -142,7 +197,10 @@ tests:
     add_files:
       run_test.pl: ""
   - name: missing_tests_but_runtst_sh
-    remove: test
+    remove:
+      - test
+      - outputs/meta_one/test
+      - outputs/meta_two/test
     add_files:
       run_tst.sh: ""
     expect: missing_tests
@@ -364,12 +422,22 @@ tests:
   - name: build_number_needs_bump
     expect: build_number_needs_bump
     add: { package: { version: 0.0.1 } }
+  - name: outputs_name_same_as_package_name
+    expect: outputs_name_same_as_package_name
+    add:
+      outputs:
+       - name: "meta_package"
+       - name: "one"
+       - name: "two"
+      extra:
+        skip-lints:
+          - missing_tests
   - name: version_starts_with_v
     expect: version_starts_with_v
     add: { package: { version: "v0.1" } }
   - name: in_other_channels
     expect: in_other_channels
-    repodata: { conda-forge: { one: [{ version: 0.1 }], two: [version: 0.1] } }
+    repodata: { conda-forge: { one: [{ version: 0.1 }], two: [version: 0.1], meta_package: [version: 0.1] } }
   - name: long_summary
     expect: long_summary
     add:
@@ -383,6 +451,7 @@ tests:
       blacklist: |
         recipes/one
         recipes/two
+        recipes/meta_package
     config:
       blacklists:
         - blacklist