Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Intersphinx lookups: log warnings for ambiguous target definitions and resolutions. #12329

Open
wants to merge 19 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
c65b877
Intersphinx lookups: log warning for ambiguous target resolution.
jayaddison Apr 25, 2024
fe3a647
Refactor: use existing shared Intersphinx logger.
jayaddison Apr 26, 2024
57099f1
Code review feedback: include type, subtype and location in warning.
jayaddison Apr 26, 2024
c992329
Cleanup: remove unused import from test module.
jayaddison Apr 26, 2024
d9417e0
Tests: refactor: extract INVENTORY_V2_AMBIGUOUS_TERMS fixture data.
jayaddison Apr 26, 2024
c07c799
Merge branch 'refs/heads/master' into pr-12033-follow/intersphinx-amb…
AA-Turner Apr 29, 2024
436297c
Allow translation
AA-Turner Apr 29, 2024
ea7349b
Cleanup: remove redundant domain.name from log message line.
jayaddison Apr 29, 2024
72b266d
Intersphinx lookups: add origin-hyperlink to log warnings about ambig…
jayaddison Apr 29, 2024
4935023
Revert "Intersphinx lookups: add origin-hyperlink to log warnings abo…
jayaddison Apr 29, 2024
ab78182
Intersphinx lookups: log inventory name at both load-time and ambiguo…
jayaddison Apr 29, 2024
88a338a
Intersphinx inventories: add loading-time warning when ambiguous name…
jayaddison Apr 29, 2024
8b51aaf
linting: resolve type-inconsistency warnings by returning an empty se…
jayaddison Apr 29, 2024
897e268
linting: cleanup: remove unused variable.
jayaddison Apr 29, 2024
7f0b375
Fixup: include type namespace when checking for ambiguities.
jayaddison Apr 29, 2024
4d61b14
Revert "linting: cleanup: remove unused variable."
jayaddison Apr 29, 2024
330c2eb
Merge branch 'master' into pr-12033-follow/intersphinx-ambiguous-look…
jayaddison Apr 29, 2024
15d2416
Add CHANGES.rst entry.
jayaddison May 4, 2024
f10c9ca
Merge branch 'master' into pr-12033-follow/intersphinx-ambiguous-look…
jayaddison May 9, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGES.rst
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ Features added
* Flatten ``Union[Literal[T], Literal[U], ...]`` to ``Literal[T, U, ...]``
when turning annotations into strings.
Patch by Adam Turner.
* Add detection of ambiguous ``std:label`` and ``std:term`` references during
loading and resolution of Intersphinx targets.
Patch by James Addison.

Bugs fixed
----------
Expand Down
4 changes: 3 additions & 1 deletion sphinx/ext/intersphinx/_load.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,9 @@ def fetch_inventory_group(
# files; remote ones only if the cache time is expired
if '://' not in inv or uri not in cache or cache[uri][1] < cache_time:
safe_inv_url = _get_safe_url(inv)
LOGGER.info(__('loading intersphinx inventory from %s...'), safe_inv_url)
inv_descriptor = name or 'main_inventory'
LOGGER.info(__("loading intersphinx inventory '%s' from %s..."),
inv_descriptor, safe_inv_url)
try:
invdata = fetch_inventory(app, uri, inv)
except Exception as err:
Expand Down
5 changes: 5 additions & 0 deletions sphinx/ext/intersphinx/_resolve.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ def _resolve_reference_in_domain_by_target(
target_lower = target.lower()
insensitive_matches = list(filter(lambda k: k.lower() == target_lower,
inventory[objtype].keys()))
if len(insensitive_matches) > 1:
inv_descriptor = inv_name or 'main_inventory'
LOGGER.warning(__("inventory '%s': multiple matches found for %s:%s"),
inv_descriptor, objtype, target,
type='intersphinx', subtype='external', location=node)
if insensitive_matches:
data = inventory[objtype][insensitive_matches[0]]
else:
Expand Down
28 changes: 22 additions & 6 deletions sphinx/util/inventory.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import zlib
from typing import IO, TYPE_CHECKING, Callable

from sphinx.locale import __
from sphinx.util import logging

BUFSIZE = 16 * 1024
Expand Down Expand Up @@ -86,19 +87,23 @@ def load(
reader = InventoryFileReader(stream)
line = reader.readline().rstrip()
if line == '# Sphinx inventory version 1':
return cls.load_v1(reader, uri, joinfunc)
invdata, ambiguities = cls.load_v1(reader, uri, joinfunc)
elif line == '# Sphinx inventory version 2':
return cls.load_v2(reader, uri, joinfunc)
invdata, ambiguities = cls.load_v2(reader, uri, joinfunc)
else:
raise ValueError('invalid inventory header: %s' % line)
for ambiguity in ambiguities:
logger.warning(__("inventory <%s> contains multiple definitions for %s"),
uri, ambiguity, type='intersphinx', subtype='external')
return invdata

@classmethod
def load_v1(
cls: type[InventoryFile],
stream: InventoryFileReader,
uri: str,
join: Callable[[str, str], str],
) -> Inventory:
) -> tuple[Inventory, set[str]]:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a breaking change to public API which, without deprecation warnings, I think is not ideal.

either,

just log the warnings within load_v2 (this seems easiest)

or you should create two methods for each version, like: load_v2_with_ambiguties (new signature), load_v2 (calls load_v2_with_ambiguties, but then just drops the ambiguties and only returns the inventory)

invdata: Inventory = {}
projname = stream.readline().rstrip()[11:]
version = stream.readline().rstrip()[11:]
Expand All @@ -113,18 +118,20 @@ def load_v1(
type = 'py:' + type
location += '#' + name
invdata.setdefault(type, {})[name] = (projname, version, location, '-')
return invdata
return invdata, set()

@classmethod
def load_v2(
cls: type[InventoryFile],
stream: InventoryFileReader,
uri: str,
join: Callable[[str, str], str],
) -> Inventory:
) -> tuple[Inventory, set[str]]:
invdata: Inventory = {}
projname = stream.readline().rstrip()[11:]
version = stream.readline().rstrip()[11:]
potential_ambiguities = set()
actual_ambiguities = set()
line = stream.readline()
if 'zlib' not in line:
raise ValueError('invalid inventory header (not compressed): %s' % line)
Expand All @@ -147,12 +154,21 @@ def load_v2(
# for Python modules, and the first
# one is correct
continue
if type in {'std:label', 'std:term'}:
# Some types require case insensitive matches:
# * 'term': https://github.com/sphinx-doc/sphinx/issues/9291
# * 'label': https://github.com/sphinx-doc/sphinx/issues/12008
definition = f"{type}:{name}"
if definition.lower() in potential_ambiguities:
actual_ambiguities.add(definition)
else:
potential_ambiguities.add(definition.lower())
if location.endswith('$'):
location = location[:-1] + name
location = join(uri, location)
inv_item: InventoryItem = projname, version, location, dispname
invdata.setdefault(type, {})[name] = inv_item
return invdata
return invdata, actual_ambiguities

@classmethod
def dump(
Expand Down
24 changes: 23 additions & 1 deletion tests/test_extensions/test_ext_intersphinx.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,11 @@
from sphinx.ext.intersphinx._load import _get_safe_url, _strip_basic_auth
from sphinx.util.console import strip_colors

from tests.test_util.intersphinx_data import INVENTORY_V2, INVENTORY_V2_NO_VERSION
from tests.test_util.intersphinx_data import (
INVENTORY_V2,
INVENTORY_V2_AMBIGUOUS_TERMS,
INVENTORY_V2_NO_VERSION,
)
from tests.utils import http_server


Expand Down Expand Up @@ -247,6 +251,24 @@ def test_missing_reference_stddomain(tmp_path, app, status, warning):
assert rn.astext() == 'The Julia Domain'


def test_ambiguous_reference_warning(tmp_path, app, warning):
inv_file = tmp_path / 'inventory'
inv_file.write_bytes(INVENTORY_V2_AMBIGUOUS_TERMS)
set_config(app, {
'cmd': ('https://docs.python.org/', str(inv_file)),
})

# load the inventory
normalize_intersphinx_mapping(app, app.config)
load_mappings(app)

# term reference (case insensitive)
node, contnode = fake_node('std', 'term', 'A TERM', 'A TERM')
missing_reference(app, app.env, node, contnode)

assert 'multiple matches found for std:term:A TERM' in warning.getvalue()


@pytest.mark.sphinx('html', testroot='ext-intersphinx-cppdomain')
def test_missing_reference_cppdomain(tmp_path, app, status, warning):
inv_file = tmp_path / 'inventory'
Expand Down
10 changes: 10 additions & 0 deletions tests/test_util/intersphinx_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,3 +50,13 @@
''' + zlib.compress(b'''\
module1 py:module 0 foo.html#module-module1 Long Module desc
''')

INVENTORY_V2_AMBIGUOUS_TERMS: Final[bytes] = b'''\
# Sphinx inventory version 2
# Project: foo
# Version: 2.0
# The remainder of this file is compressed with zlib.
''' + zlib.compress(b'''\
a term std:term -1 glossary.html#term-a-term -
A term std:term -1 glossary.html#term-a-term -
''')
8 changes: 8 additions & 0 deletions tests/test_util/test_util_inventory.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from tests.test_util.intersphinx_data import (
INVENTORY_V1,
INVENTORY_V2,
INVENTORY_V2_AMBIGUOUS_TERMS,
INVENTORY_V2_NO_VERSION,
)

Expand Down Expand Up @@ -48,6 +49,13 @@ def test_read_inventory_v2_not_having_version():
('foo', '', '/util/foo.html#module-module1', 'Long Module desc')


def test_ambiguous_definition_warning(warning):
f = BytesIO(INVENTORY_V2_AMBIGUOUS_TERMS)
InventoryFile.load(f, '/util', posixpath.join)

assert 'contains multiple definitions for std:term:a' in warning.getvalue().lower()


def _write_appconfig(dir, language, prefix=None):
prefix = prefix or language
os.makedirs(dir / prefix, exist_ok=True)
Expand Down