pyproject.toml

[build-system]
requires = ["setuptools", "wheel"]
# uncomment to enable pep517 after versioneer problem is fixed.
# https://github.com/python-versioneer/python-versioneer/issues/193
# build-backend = "setuptools.build_meta"

[tool.black]
extend_excludes = '''(docs/.*|tests/.*.fixture|.*.ge_store_backend_id)'''

[tool.isort]
profile = "black"
skip_gitignore = true
extend_skip_glob = ['venv/*', 'docs/*']

[tool.mypy]
python_version = "3.7"
plugins = ["pydantic.mypy"]
files = [
    "great_expectations",
    "tests/experimental/datasources",
    # "contrib" # ignore entire `contrib` package
    ]
warn_unused_configs = true
ignore_missing_imports = true
# TODO: change this to 'normal' once we have 'full' type coverage
follow_imports = 'silent'
warn_redundant_casts = true
show_error_codes = true
enable_error_code = [
    'ignore-without-code'
    ]
# The following list of codes are globally ignored, do not add to this list
disable_error_code = [
    # annotation-unchecked are 'warning notes', not errors and won't cause mypy to fail
    # but it does create a lot of noise in the CI mypy step.
    # https://mypy-lang.blogspot.com/2022/11/mypy-0990-released.html
    'annotation-unchecked'
    ]
exclude = [
    # BEGIN ALWAYS EXCLUDE SECTION #####################################################
    # If pattern should always be excluded add comment explaining why and put
    'docs/',  # Docs should not be type checked with the rest of the library.
    '_version\.py',  # generated by `versioneer`
    'v012',  # legacy code
    'tests/experimental/datasources/test_metadatasource\.py', # metaprogramming leads to errors
    # END ALWAYS EXCLUDE SECTION ######################################################
    #
    # #################################################################################
    # TODO: complete typing for the following modules and remove from exclude list
    # number is the current number of typing errors for the excluded pattern
    'checkpoint/actions\.py',  # 18
    'checkpoint/checkpoint\.py',  # 22
    'checkpoint/types/checkpoint_result\.py',  # 34
    'checkpoint/util\.py',  # 5
    'cli/checkpoint\.py',  # 9
    'cli/suite\.py',  # 24
    'cli/upgrade_helpers/upgrade_helper_v11\.py',  # 59
    'cli/upgrade_helpers/upgrade_helper_v13\.py',  # 17
    'core/batch\.py', # 29
    'core/usage_statistics/anonymizers/action_anonymizer\.py', # 1
    'core/usage_statistics/anonymizers/anonymizer\.py', # 6
    'core/usage_statistics/anonymizers/base\.py', # 8
    'core/usage_statistics/anonymizers/batch_anonymizer\.py', # 10
    'core/usage_statistics/anonymizers/batch_request_anonymizer\.py', # 16
    'core/usage_statistics/anonymizers/checkpoint_anonymizer\.py', # 16
    'core/usage_statistics/anonymizers/data_connector_anonymizer\.py', # 3
    'core/usage_statistics/anonymizers/data_docs_anonymizer\.py', # 5
    'core/usage_statistics/anonymizers/datasource_anonymizer\.py', # 9
    'core/usage_statistics/anonymizers/expectation_anonymizer\.py', # 6
    'core/usage_statistics/anonymizers/profiler_anonymizer\.py', # 2
    'core/usage_statistics/anonymizers/store_anonymizer\.py', # 6
    'core/usage_statistics/anonymizers/store_backend_anonymizer\.py', # 5
    'core/usage_statistics/anonymizers/validation_operator_anonymizer\.py',  # 5
    'core/usage_statistics/usage_statistics\.py', # 19
    'dataset/sparkdf_dataset\.py', # 3
    'dataset/sqlalchemy_dataset\.py', # 16
    'datasource/data_connector/configured_asset_sql_data_connector\.py',  # 47
    'expectations/core/expect_column_values_to_be_of_type\.py', # 12
    'expectations/core/expect_column_values_to_not_match_regex_list\.py', # 2
    'expectations/core/expect_column_values_to_not_match_regex\.py', # 2
    'expectations/core/expect_column_values_to_not_match_like_pattern_list\.py', # 3
    'expectations/core/expect_column_values_to_not_match_like_pattern\.py', # 2
    'expectations/core/expect_column_values_to_not_be_null\.py', # 2
    'expectations/core/expect_column_values_to_not_be_in_set\.py', # 2
    'expectations/core/expect_column_values_to_match_strftime_format\.py', # 2
    'expectations/core/expect_column_values_to_match_regex_list\.py', # 2
    'expectations/core/expect_column_values_to_match_regex\.py', # 1
    'expectations/core/expect_column_values_to_match_like_pattern_list\.py', # 3
    'expectations/core/expect_column_values_to_match_like_pattern\.py', # 2
    'expectations/core/expect_column_values_to_match_json_schema\.py', # 1
    'expectations/core/expect_column_values_to_be_unique\.py', # 1
    'expectations/core/expect_column_values_to_be_null\.py', # 3
    'expectations/core/expect_column_values_to_be_json_parseable\.py', # 1
    'expectations/core/expect_column_values_to_be_increasing\.py', # 1
    'expectations/core/expect_column_values_to_be_in_type_list\.py', # 11
    'expectations/core/expect_column_values_to_be_in_set\.py', # 1
    'expectations/core/expect_column_values_to_be_decreasing\.py', # 1
    'expectations/core/expect_column_values_to_be_dateutil_parseable\.py', # 1
    'expectations/core/expect_column_values_to_be_between\.py', # 3
    'expectations/core/expect_column_unique_value_count_to_be_between\.py', # 1
    'expectations/core/expect_column_to_exist\.py', # 4
    'expectations/core/expect_column_stdev_to_be_between\.py', # 1
    'expectations/core/expect_column_quantile_values_to_be_between\.py', # 15
    'expectations/core/expect_column_value_lengths_to_equal\.py', # 1
    'expectations/core/expect_column_value_lengths_to_be_between\.py', # 5
    'expectations/core/expect_column_proportion_of_unique_values_to_be_between\.py', # 1
    'expectations/core/expect_column_values_to_be_in_set\.py', # 2
    'expectations/core/expect_column_values_to_be_equal\.py', # 3
    'expectations/core/expect_column_values_a_to_be_greater_than_b\.py', # 3
    'expectations/core/expect_column_pair_cramers_phi_value_to_be_less_than\.py', # 7
    'expectations/core/expect_column_most_common_value_to_be_in_set\.py', # 3
    'expectations/core/expect_column_min_to_be_between\.py', # 1
    'expectations/core/expect_column_median_to_be_between\.py', # 1
    'expectations/core/expect_column_mean_to_be_between\.py', # 1
    'expectations/core/expect_column_max_to_be_between\.py', # 1
    'expectations/core/expect_column_kl_divergence_to_be_less_than\.py', # 22
    'expectations/core/expect_column_pair_values_to_be_in_set\.py', # 2
    'expectations/core/expect_column_pair_values_to_be_equal\.py', # 3
    'expectations/core/expect_column_pair_values_a_to_be_greater_than_b\.py', # 3
    'expectations/core/expect_column_distinct_values_to_equal_set\.py', # 2
    'expectations/core/expect_column_distinct_values_to_contain_set\.py', # 4
    'expectations/core/expect_column_distinct_values_to_be_in_set\.py', # 1
    'expectations/core/expect_compound_columns_to_be_unique\.py', # 3
    'expectations/core/expect_multicolumn_sum_to_equal\.py', # 4
    'expectations/core/expect_multicolumn_values_to_be_unique\.py', # 3
    'expectations/core/expect_select_column_values_to_be_unique_within_record\.py', # 3
    'expectations/core/expect_table_columns_to_match_set\.py', # 8
    'expectations/core/expect_table_columns_to_match_ordered_list\.py', # 11
    'expectations/core/expect_table_column_count_to_equal\.py', # 5
    'expectations/core/expect_table_column_count_to_be_between\.py', # 3
    'expectations/core/expect_table_row_count_to_equal_other_table\.py', # 11
    'expectations/metrics/column_aggregate_metrics/column_value_counts\.py', # 6
    'expectations/metrics/column_aggregate_metrics/column_quantile_values\.py', # 1
    'expectations/metrics/column_aggregate_metrics/column_partition\.py', # 10
    'expectations/metrics/column_aggregate_metrics/column_most_common_value\.py', # 4
    'expectations/metrics/column_aggregate_metrics/column_median\.py', # 1
    'expectations/metrics/map_metric_provider\.py',  # 57
    'expectations/regex_based_column_map_expectation\.py',  # 3
    'expectations/row_conditions\.py',  # 4
    'expectations/set_based_column_map_expectation\.py',  # 3
    'render/renderer/checkpoint_new_notebook_renderer\.py',  # 9
    'render/renderer/content_block/content_block\.py',  # 5
    'render/renderer/content_block/exception_list_content_block\.py',  # 4
    'render/renderer/datasource_new_notebook_renderer\.py',  #  4
    'render/renderer/notebook_renderer\.py',  # 2
    'render/renderer/page_renderer\.py',  # 10
    'render/renderer/profiling_results_overview_section_renderer\.py',  # 2
    'render/renderer/site_builder\.py',  # 3
    'render/renderer/slack_renderer\.py',  # 9
    'render/renderer/suite_edit_notebook_renderer\.py',  # 7
    'render/renderer/suite_scaffold_notebook_renderer\.py',  # 7
    'render/renderer/v3/suite_edit_notebook_renderer\.py',  # 11
    'render/renderer/v3/suite_profile_notebook_renderer\.py',  # 4
    'render/util\.py',  # 17
    'render/view/view\.py',  # 11
    'rule_based_profiler/data_assistant_result/data_assistant_result\.py',  # 63
    'rule_based_profiler/data_assistant_result/plot_components\.py', # 12
    'rule_based_profiler/domain_builder/categorical_column_domain_builder\.py',  # 18
    'rule_based_profiler/domain_builder/map_metric_column_domain_builder\.py',  # 8
    'rule_based_profiler/estimators/bootstrap_numeric_range_estimator\.py',  # 8
    'rule_based_profiler/estimators/kde_numeric_range_estimator\.py',  # 7
    'rule_based_profiler/expectation_configuration_builder',  # 13
    'rule_based_profiler/helpers/cardinality_checker\.py',  # 9
    'rule_based_profiler/helpers/util\.py',  # 46
    'rule_based_profiler/parameter_builder/mean_unexpected_map_metric_multi_batch_parameter_builder\.py',  # 19
    'rule_based_profiler/parameter_builder/metric_multi_batch_parameter_builder\.py',  # 15
    'rule_based_profiler/parameter_builder/numeric_metric_range_multi_batch_parameter_builder\.py',  # 27
    'rule_based_profiler/parameter_builder/parameter_builder\.py',  # 40
    'rule_based_profiler/parameter_builder/partition_parameter_builder\.py',  # 9
    'rule_based_profiler/parameter_builder/regex_pattern_string_parameter_builder\.py',  # 21
    'rule_based_profiler/parameter_builder/simple_date_format_string_parameter_builder\.py',  # 20
    'rule_based_profiler/rule_based_profiler\.py',  # 40
    'validation_operators/types/validation_operator_result\.py', # 35
    'validation_operators/validation_operators\.py',  # 16
    'validator/validator\.py', # 54
    # tests
    'tests/experimental/datasources/tasks\.py',
    'tests/experimental/datasources/integration/integration_test_utils\.py',
    'tests/experimental/datasources/test_spark_datasource\.py',
]

[tool.pydantic-mypy]
# https://pydantic-docs.helpmanual.io/mypy_plugin/#plugin-settings
init_typed = true
warn_required_dynamic_aliases = true
warn_untyped_fields = true

[tool.ruff]
target-version = "py37"
line-length = 88
select = [
    # https://github.com/charliermarsh/ruff#pyflakes-f
    "F", # Pyflakes
    # https://github.com/charliermarsh/ruff#pycodestyle-e-w
    "E", # pycodestyle
    "W", # Warning
    # https://github.com/charliermarsh/ruff#flake8-comprehensions-c4
    # https://github.com/charliermarsh/ruff#mccabe-c90
    "C", # Complexity (mccabe+) & comprehensions
    # https://github.com/charliermarsh/ruff#pyupgrade-up
    "UP", # pyupgrade
    # https://github.com/charliermarsh/ruff#isort-i
    # "I", # isort # TODO: replace isort usage with this
]
ignore = [
    # https://github.com/charliermarsh/ruff#pyflakes-f
    "F842", # variable annotated but unused # TODO enable
    # https://github.com/charliermarsh/ruff#pycodestyle-e-w
    "E501", # line-length # TODO: too many violations
    "E402", # module level import not at top of file
    # https://github.com/charliermarsh/ruff#flake8-comprehensions-c4
    "C400", # TODO enable
    "C408", # TODO enable
    "C409", # TODO enable
    "C413", # TODO enable
    "C414", # TODO enable
    "C416", # TODO enable
    "C417", # TODO enable
    # https://github.com/charliermarsh/ruff#pyupgrade-up
    "UP006", # use-pep585-annotation
    "UP007", # use-pep604-annotation
]
extend-exclude = [
    "scripts/*",
    "docs/*",
    "build/*",
    "assets/*",
    "versioneer*",
    "examples/*",
    # TODO: remove the items below and fix linting issues
    "great_expectations/dataset", # 52 warnings
    "tests/checkpoint", # 15 warnings
    "tests/cli", # 34 warnings
    "tests/data_asset", # 10 warnings
    "tests/data_context", # 127 warnings
    "tests/dataset", # 10 warnings
    "tests/datasource", # 81 warnings
    "tests/execution_engine",  # 23 warnings
    "tests/expectations", # 56 warnings
    "tests/integration", # 114 warnings
    "tests/rule_based_profiler", # 28 warnings
    "tests/test_fixtures/notebook_assets/suite_edit/footer.py",  # 7 warnings
]

[tool.ruff.per-file-ignores]
"__init__.py" = [
    "F401",  # unused import
    "F403",  # star imports
    ]

[tool.ruff.mccabe]
max-complexity = 15

[tool.ruff.pydocstyle]
convention = "google"

[tool.pytest.ini_options]
filterwarnings = [
    # This warning is common during testing where we intentionally use a COMPLETE format even in cases that would
    # be potentially overly resource intensive in standard operation
    "ignore:Setting result format to COMPLETE for a SqlAlchemyDataset:UserWarning",
    # This deprecation warning was fixed in moto release 1.3.15, and the filter should be removed once we migrate
    # to that minimum version
    "ignore:Using or importing the ABCs:DeprecationWarning:moto.cloudformation.parsing",
    # This deprecation warning comes from getsentry/responses, a mocking utility for requests. It is a dependency in moto.
    "ignore:stream argument is deprecated. Use stream parameter in request directly:DeprecationWarning",
    # We likely won't be updating to `marhsmallow` 4, these errors should be filtered out
    "error::marshmallow.warnings.RemovedInMarshmallow4Warning",
    # This warning is for configuring the result_format parameter at the Validator-level, which will not be persisted,
    # but is still useful for building the configuration.
    "ignore:`result_format` configured at the Validator-level will not be persisted:UserWarning",
    # This warning is for configuring the result_format parameter at the Expectation-level, which will not be persisted,
    # but is still useful for building the configuration.
    "ignore:`result_format` configured at the Expectation-level will not be persisted:UserWarning",
]
junit_family="xunit2"
markers = [
    "cloud: mark test as being relevant to Great Expectations Cloud.",
    "docs: mark a test as a docs test.",
    "e2e: mark test as an E2E test.",
    "external_sqldialect: mark test as requiring install of an external sql dialect.",
    "integration: mark test as an integration test.",
    "slow: mark tests taking longer than 1 second.",
    "unit: mark a test as a unit test.",
    "v2_api: mark test as specific to the v2 api (e.g. pre Data Connectors)",
]
testpaths = "tests"
# use `pytest-mock` drop-in replacement for `unittest.mock`
# https://pytest-mock.readthedocs.io/en/latest/configuration.html#use-standalone-mock-package
mock_use_standalone_module = false
# https://docs.pytest.org/en/7.1.x/how-to/logging.html#how-to-manage-logging
# uncomment to adjust captured log levels
# WARNING some cli v012 tests may fail if this is altered
# log_level = "info"