Format docs directory with black (NVIDIA#5214)

This PR applies Black to the docs/ directory and enables linter on this path.

The `examples/use_cases` subdirectory is excluded - it contains a lot of 3rd party
code that was forked by DALI and has numerous linter issues. 
Linting and formatting that part should be done as a separate step (if at all).

 NVIDIA#5226 enabled black to be used within jupyter files on our CI.

Notable changes:
1. The index file generation for operator references gets a header 
   with imports and a license.
   This solves a lot of `F821 undefined name XXXX`.
2. Some too-long strings and too-long comments are broken into additional lines
3. Several one-letter variables are replaced with more meaningful ones 
4. noqa is added around mxnet imports that appear to be no-op,
   but it looks like the importing has side effects (for example path
   adjustment to look up the library).
5. Some fixes in the conf.py

Signed-off-by: Krzysztof Lecki <[email protected]>

Author: klecki

.flake8

@@ -10,4 +10,5 @@ extend-ignore = E203,W503
 # W504 - Line break occurred after a binary operator
 extend-select = W504
 # Filter out the AutoGraph from regular linter - it has a separate .flake8.ag config.
-exclude = *nvidia/dali/_autograph/*,*test/python/autograph/*
+# Filter out use_cases directory, with forked 3rd party examples that include DALI modifications
+exclude = *nvidia/dali/_autograph/*,*test/python/autograph/*,*docs/examples/use_cases/*

cmake/lint.cmake

@@ -21,6 +21,7 @@ add_custom_target(lint-cpp
 
 set(PYTHON_LINT_PATHS
         ${PROJECT_SOURCE_DIR}/dali
+        ${PROJECT_SOURCE_DIR}/docs
         ${PROJECT_SOURCE_DIR}/tools
         ${PROJECT_SOURCE_DIR}/dali_tf_plugin
         ${PROJECT_SOURCE_DIR}/qa

docs/autodoc_submodules.py

@@ -1,40 +1,50 @@
-from nvidia.dali import backend as b
-import nvidia.dali.ops as ops
+# Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import nvidia.dali.plugin.pytorch
 import nvidia.dali.plugin.numba
 import inspect
 import sys
-from pathlib import Path
 
 import operations_table
 
 # Dictionary with modules that can have registered Ops
 ops_modules = {
-    'nvidia.dali.ops': nvidia.dali.ops,
-    'nvidia.dali.plugin.numba.experimental': nvidia.dali.plugin.numba.experimental,
+    "nvidia.dali.ops": nvidia.dali.ops,
+    "nvidia.dali.plugin.numba.experimental": nvidia.dali.plugin.numba.experimental,
 }
 
 
-exclude_ops_members = {
-    'nvidia.dali.ops': ["PythonFunctionBase"]
-}
+exclude_ops_members = {"nvidia.dali.ops": ["PythonFunctionBase"]}
 
 
 fn_modules = {
-    'nvidia.dali.fn': nvidia.dali.fn,
-    'nvidia.dali.plugin.pytorch.fn': nvidia.dali.plugin.pytorch.fn,
-    'nvidia.dali.plugin.numba.fn.experimental': nvidia.dali.plugin.numba.fn.experimental,
+    "nvidia.dali.fn": nvidia.dali.fn,
+    "nvidia.dali.plugin.pytorch.fn": nvidia.dali.plugin.pytorch.fn,
+    "nvidia.dali.plugin.numba.fn.experimental": nvidia.dali.plugin.numba.fn.experimental,
 }
 
 
-exclude_fn_members = {
-}
+exclude_fn_members = {}
 
 
 mod_aditional_doc = {
-    'nvidia.dali.fn.transforms' : "All operators in this module support only CPU device as they are meant " +
-"to be provided as an input to named keyword operator arguments. Check for more details the relevant " +
-":ref:`pipeline documentation section<Processing Graph Structure>`."
+    "nvidia.dali.fn.transforms": (
+        "All operators in this module support only CPU device as they are meant to be provided"
+        " as an input to named keyword operator arguments. Check for more details the relevant"
+        " :ref:`pipeline documentation section<Processing Graph Structure>`."
+    )
 }
 
 
@@ -47,8 +57,11 @@ def get_modules(top_modules):
     modules = []
     for module in sys.modules.keys():
         for doc_module in top_modules:
-            if (module.startswith(doc_module) and not module.endswith('hidden')
-                    and not _is_private(module)):
+            if (
+                module.startswith(doc_module)
+                and not module.endswith("hidden")
+                and not _is_private(module)
+            ):
                 modules += [module]
     return sorted(modules)
 
@@ -66,6 +79,7 @@ def get_functions(module):
             result.append(member_name)
     return result
 
+
 def get_schema_names(module, functions):
     return [getattr(sys.modules[module], fun)._schema_name for fun in functions]
 
@@ -85,7 +99,7 @@ def op_autodoc(out_filename):
             excluded = exclude_ops_members[module]
             s += "   :exclude-members: {}\n".format(", ".join(excluded))
         s += "\n\n"
-    with open(out_filename, 'w') as f:
+    with open(out_filename, "w") as f:
         f.write(s)
 
 
@@ -99,18 +113,18 @@ def get_references(name, references):
             result += f"   * `{desc} <../{url}>`_\n"
     return result
 
+
 def single_fun_file(full_name, references):
-    """Generate stub page for documentation of given function from fn api.
-    """
+    """Generate stub page for documentation of given function from fn api."""
     result = f"{full_name}\n"
     result += "-" * len(full_name) + "\n\n"
     result += f".. autofunction:: {full_name}\n\n"
     result += get_references(full_name, references)
     return result
 
+
 def single_module_file(module, funs_in_module, references):
-    """Generate stub page for documentation of given module
-    """
+    """Generate stub page for documentation of given module"""
     result = f"{module}\n"
     result += "~" * len(module) + "\n\n"
 
@@ -123,7 +137,6 @@ def single_module_file(module, funs_in_module, references):
     result += operations_table.operations_table_str(get_schema_names(module, funs_in_module))
     result += "\n\n"
 
-
     result += ".. toctree::\n   :hidden:\n\n"
 
     for fun in funs_in_module:
@@ -133,6 +146,7 @@ def single_module_file(module, funs_in_module, references):
         result += f"   {full_name}\n"
     return result
 
+
 def fn_autodoc(out_filename, generated_path, references):
     all_modules_str = ".. toctree::\n   :hidden:\n\n"
     all_modules = get_modules(fn_modules)
@@ -149,7 +163,7 @@ def fn_autodoc(out_filename, generated_path, references):
         all_modules_str += f"   {generated_path / module}\n"
 
         single_module_str = single_module_file(module, funs_in_module, references)
-        with open(generated_path / (module + ".rst"), 'w') as module_file:
+        with open(generated_path / (module + ".rst"), "w") as module_file:
             module_file.write(single_module_str)
 
         for fun in funs_in_module:
@@ -160,5 +174,5 @@ def fn_autodoc(out_filename, generated_path, references):
                 single_file_str = single_fun_file(full_name, references)
                 function_file.write(single_file_str)
 
-    with open(out_filename, 'w') as f:
+    with open(out_filename, "w") as f:
         f.write(all_modules_str)