Explore metadata in hidden files and directories (#264)

Add support for a simple config where users can specify which
hidden files and directories should be included in the search.

Fix #183.

docs/concept.rst

@@ -89,6 +89,8 @@ collisions between similar attributes. For example:
 * test_description, requirement_description
 
 
+.. _trees:
+
 Trees
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -99,6 +101,28 @@ contains at least a ``version`` file with a single integer number
 defining version of the format.
 
 
+.. _config:
+
+Config
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, all hidden files are ignored when exploring metadata
+on the disk. If a specific file or directory should be included in
+the search, create a simple config file ``.fmf/config`` with the
+following format:
+
+.. code-block:: yaml
+
+    explore:
+        include:
+          - .plans
+          - .tests
+
+In the example above files or directories named ``.plans`` or
+``.tests`` will be included in the discovered metadata. Note that
+the ``.fmf`` directory cannot be used for storing metadata.
+
+
 Names
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/features.rst

@@ -79,6 +79,7 @@ directive::
     /:
         inherit: false
 
+
 .. _merging:
 
 Merging

docs/releases.rst

@@ -4,7 +4,16 @@
     Releases
 ======================
 
-fmf-1.4
+
+fmf-1.5.0
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The fmf :ref:`trees` can now be built from hidden files and
+directories as well. Use a simple :ref:`config` file to specify
+names which should be included in the search.
+
+
+fmf-1.4.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 New :ref:`merging<merging>` suffixes ``~`` and ``-~`` can be used

examples/hidden/.fmf/config

@@ -0,0 +1,3 @@
+explore:
+    include:
+      - .plans

examples/hidden/.fmf/version

@@ -0,0 +1 @@
+1

examples/hidden/.plans/basic.fmf

@@ -0,0 +1,4 @@
+discover:
+    how: fmf
+execute:
+    how: tmt

fmf/base.py

@@ -5,6 +5,7 @@
 import re
 import subprocess
 from io import open
+from pathlib import Path
 from pprint import pformat as pretty
 from typing import Any, Dict, Optional, Protocol
 
@@ -71,6 +72,7 @@ def __init__(self, data, name=None, parent=None):
         self.data = dict()
         self.sources = list()
         self.root = None
+        self.config = {}
         self.version = utils.VERSION
         self.original_data = dict()
         self._commit = None
@@ -98,6 +100,7 @@ def __init__(self, data, name=None, parent=None):
         # Handle child node creation
         else:
             self.root = self.parent.root
+            self.config = self.parent.config
             self.name = os.path.join(self.parent.name, name)
 
         # Update data from a dictionary (handle empty nodes)
@@ -145,7 +148,8 @@ def __str__(self):
         return self.name
 
     def _initialize(self, path):
-        """ Find metadata tree root, detect format version """
+        """ Find metadata tree root, detect format version, check for config """
+
         # Find the tree root
         root = os.path.abspath(path)
         try:
@@ -159,6 +163,7 @@ def _initialize(self, path):
             raise utils.FileError("Invalid directory path: {0}".format(root))
         log.info("Root directory found: {0}".format(root))
         self.root = root
+
         # Detect format version
         try:
             with open(os.path.join(self.root, ".fmf", "version")) as version:
@@ -170,6 +175,16 @@ def _initialize(self, path):
         except ValueError:
             raise utils.FormatError("Invalid version format")
 
+        # Check for the config file
+        config_file_path = Path(self.root) / ".fmf/config"
+        try:
+            self.config = YAML(typ="safe").load(config_file_path.read_text())
+            log.debug(f"Config file '{config_file_path}' loaded.")
+        except FileNotFoundError:
+            log.debug("Config file not found.")
+        except YAMLError as error:
+            raise utils.FileError(f"Failed to parse '{config_file_path}'.\n{error}")
+
     def _merge_plus(self, data, key, value, prepend=False):
         """ Handle extending attributes using the '+' suffix """
 
@@ -593,6 +608,21 @@ def child(self, name, data, source=None):
             self.children[name].sources.append(source)
             self.children[name]._raw_data = copy.deepcopy(data)
 
+    @property
+    def explore_include(self):
+        """ Additional filenames to be explored """
+        try:
+            explore_include = self.config["explore"]["include"]
+            if not isinstance(explore_include, list):
+                raise utils.GeneralError(
+                    f"The 'include' config section should be a list, found '{explore_include}'.")
+            if ".fmf" in explore_include:
+                raise utils.GeneralError(
+                    "The '.fmf' directory cannot be used for storing fmf metadata.")
+        except KeyError:
+            explore_include = []
+        return explore_include
+
     def grow(self, path):
         """
         Grow the metadata tree for the given directory path
@@ -613,16 +643,18 @@ def grow(self, path):
         except StopIteration:
             log.debug("Skipping '{0}' (not accessible).".format(path))
             return
+
         # Investigate main.fmf as the first file (for correct inheritance)
         filenames = sorted(
             [filename for filename in filenames if filename.endswith(SUFFIX)])
         try:
             filenames.insert(0, filenames.pop(filenames.index(MAIN)))
         except ValueError:
             pass
+
         # Check every metadata file and load data (ignore hidden)
         for filename in filenames:
-            if filename.startswith("."):
+            if filename.startswith(".") and filename not in self.explore_include:
                 continue
             fullpath = os.path.abspath(os.path.join(dirpath, filename))
             log.info("Checking file {0}".format(fullpath))
@@ -643,9 +675,10 @@ def grow(self, path):
             # Handle other *.fmf files as children
             else:
                 self.child(os.path.splitext(filename)[0], data, fullpath)
+
         # Explore every child directory (ignore hidden dirs and subtrees)
         for dirname in sorted(dirnames):
-            if dirname.startswith("."):
+            if dirname.startswith(".") and dirname not in self.explore_include:
                 continue
             fulldir = os.path.join(dirpath, dirname)
             if os.path.islink(fulldir):
@@ -665,6 +698,7 @@ def grow(self, path):
                 log.debug("Ignoring metadata tree '{0}'.".format(dirname))
                 continue
             self.child(dirname, os.path.join(path, dirname))
+
         # Ignore directories with no metadata (remove all child nodes which
         # do not have children and their data haven't been updated)
         for name in list(self.children.keys()):

tests/unit/test_base.py

@@ -48,6 +48,9 @@ def test_basic(self):
     def test_hidden(self):
         """ Hidden files and directories """
         assert ".hidden" not in self.wget.children
+        hidden = Tree(EXAMPLES + "hidden")
+        plan = hidden.find("/.plans/basic")
+        assert plan.get("discover") == {"how": "fmf"}
 
     def test_inheritance(self):
         """ Inheritance and data types """