Adjust configuration for autoformatting with black (NVIDIA#5168)

Temporarily disable the linter in this commit, before the whole
repository is adjusted to new formatting.

Add black to third party deps listing to include it in the deps update
procedure.

Use latest version but require only the one for given year according to
the compatibility policy.

Remove obsolete configs related to previous linter or exceptions related
to the formatting of some parts of DALI.

Keep the separate config with exceptions for autograph to limit the code
changes.

Signed-off-by: Krzysztof Lecki <[email protected]>

Author: klecki

.flake8

@@ -2,8 +2,12 @@
 color = auto
 count = True
 doctests = False
-max-doc-length = 120
 max-line-length = 100
 statistics = True
+# E203 - whitespace before ':' - PEP8 non-compliant warning in slicing expressions
+# W503 - Line break occurred before a binary operator, replaced by W504
+extend-ignore = E203,W503
+# W504 - Line break occurred after a binary operator
+extend-select = W504
 # Filter out the AutoGraph from regular linter - it has a separate .flake8.ag config.
 exclude = *nvidia/dali/_autograph/*,*test/python/autograph/*

.flake8.ag

@@ -1,13 +1,16 @@
-# AutoGraph uses 2 spaces, thus it requires a separate flake8 config
+# per-file-ignore do not work well with absolute path (or more complex paths),
+# so just disable them by a special config
 [flake8]
-indent-size = 2
 color = auto
 count = True
 doctests = False
-max-doc-length = 100
 max-line-length = 100
 statistics = True
+# E203 - whitespace before ':' - PEP8 non-compliant warning in slicing expressions
+# W503 - Line break occurred before a binary operator, replaced by W504
 # E741 - ambiguous variable name 'l'
 # E731 - do not assign a lambda expression, use a def
 # F401 - 'autograph.operators....' imported but unused - used to bring types into another module
-extend-ignore = E741,E731,F401
+extend-ignore = E203,E741,E731,F401
+# W504 - Line break occurred after a binary operator
+extend-select = W504

.style.yapf

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -31,15 +31,26 @@ set(AUTOGRAPH_LINT_PATHS
         ${PROJECT_SOURCE_DIR}/dali/test/python/autograph/
 )
 
+add_custom_target(lint-python-black
+        COMMAND
+          black --check --config ${PROJECT_SOURCE_DIR}/pyproject.toml ${PYTHON_LINT_PATHS} ${AUTOGRAPH_LINT_PATHS}
+        COMMENT
+          "Performing black Python formatting check"
+)
 
-add_custom_target(lint-python
+
+add_custom_target(lint-python-flake
         COMMAND
           flake8 --config=${PROJECT_SOURCE_DIR}/.flake8 ${PYTHON_LINT_PATHS}
         COMMAND
           flake8 --config=${PROJECT_SOURCE_DIR}/.flake8.ag ${AUTOGRAPH_LINT_PATHS}
         COMMENT
           "Performing Python linter check"
 )
+add_dependencies(lint-python-flake lint-python-black)
+
+add_custom_target(lint-python)
+add_dependencies(lint-python lint-python-flake)
 
 add_custom_target(lint)
-add_dependencies(lint lint-cpp lint-python)
+add_dependencies(lint lint-cpp)

cmake/lint.cmake

@@ -73,7 +73,7 @@ requirements:
     - astunparse >=1.6.0
     - gast >=0.3.3
     - dm-tree >=0.1.8
-    - black
+    - black =23.11.0
   host:
     - python
     - future

conda/recipe/meta.yaml

@@ -20,7 +20,7 @@ ENV PYVER=${PYVER} \
 RUN ln -s /opt/python/cp${PYV}* /opt/python/v
 
 # install Python bindings and patch it to use the clang we have here
-RUN pip install future setuptools wheel clang==14.0 flake8 astunparse gast dm-tree black && \
+RUN pip install future setuptools wheel clang==14.0 flake8 astunparse gast dm-tree black==23.11.0 && \
     PY_CLANG_PATH=$(echo $(pip show clang) | sed 's/.*Location: \(.*\) Requires.*/\1/')/clang/cindex.py && \
     LIBCLANG_PATH=/usr/local/lib/libclang.so && \
     sed -i "s|library_file = None|library_file = \"${LIBCLANG_PATH}\"|" ${PY_CLANG_PATH} && \

dali/python/nvidia/dali/_autograph/.style.yapf

@@ -0,0 +1,5 @@
+[tool.black]
+line-length = 100
+target-version = ['py38', 'py39', 'py310', 'py311']
+required-version = '23'
+workers = 32

dali/test/python/autograph/.style.yapf

@@ -23,6 +23,8 @@ This part of the repository contains extra dependencies required to build DALI,
 +-----------------+---------------------+---------------------+
 | |rapidjson|_    | |rapidjsonver|_     | |rapidjsonlic|_     |
 +-----------------+---------------------+---------------------+
+| |black|_        | |blackver|_         | |blacklic|_         |
++-----------------+---------------------+---------------------+
 
 .. |benchmark| replace:: Google Benchmark
 .. _benchmark: https://github.com/google/benchmark
@@ -86,3 +88,10 @@ This part of the repository contains extra dependencies required to build DALI,
 .. _rapidjsonver: https://github.com/Tencent/rapidjson/commit/f9d53419e912910fd8fa57d5705fa41425428c35
 .. |rapidjsonlic| replace:: MIT License, BSD 3-Clause License, JSON License
 .. _rapidjsonlic: https://github.com/Tencent/rapidjson/blob/master/license.txt
+
+.. |black| replace:: black
+.. _black: https://github.com/psf/black
+.. |blackver| replace:: 23.11.0
+.. _blackver: https://pypi.org/project/black/23.11.0/
+.. |blacklic| replace:: MIT License
+.. _blacklic: https://github.com/psf/black/blob/main/LICENSE