Better comment handling to not lose URL anchors, pass thru non-pep508…

… data

src/ansible_builder/_target_scripts/introspect.py

@@ -8,9 +8,12 @@
 base_collections_path = '/usr/share/ansible/collections'
 logger = logging.getLogger(__name__)
 
+# regex for a comment at the start of a line, or embedded with leading space(s)
+COMMENT_RE = re.compile(r'(?:^|\s+)#.*$')
+
 # https://peps.python.org/pep-0503/#normalized-names
 REQ_NORM_RE = re.compile(r'[-_.]+')
-REQ_NAME_RE = re.compile(r'^([-\w.]+)')
+REQ_NAME_RE = re.compile(r'^([^-][-\w.]+)')
 
 
 def line_is_empty(line):
@@ -201,7 +204,8 @@ def get_dependency(self, entry):
 
 
 def simple_combine(reqs, exclude=None, name_only=False):
- """Given a dictionary of requirement lines keyed off collections,
+ """
+ Given a dictionary of requirement lines keyed off collections,
  return a list with the most basic of de-duplication logic,
  and comments indicating the sources based off the collection keys
  """
@@ -212,11 +216,17 @@ def simple_combine(reqs, exclude=None, name_only=False):
  fancy_lines = []
  for collection, lines in reqs.items():
  for line in lines:
- if line_is_empty(line):
+ # strip comments
+ if not (base_line := COMMENT_RE.sub('', line.strip())):
+ continue
+
+ # Did not match expected name format (maybe a pip option?), just pass thru
+ if not (name_match := REQ_NAME_RE.match(base_line)):
+ logger.warning("Passing through unparsable requirement: %s", base_line)
+ fancy_line = f'{base_line} # from collection {collection}'
+ fancy_lines.append(fancy_line)
  continue
 
- base_line = line.split('#')[0].strip()
- name_match = REQ_NAME_RE.match(base_line)
  name = REQ_NORM_RE.sub('-', name_match.group(1))
  if name in exclude and collection not in {'user', 'exclude'}:
  logger.debug('# Explicitly excluding requirement %s from %s', name, collection)

test/unit/test_introspect.py

@@ -129,3 +129,65 @@ def test_sanitize_pep508():
  ]
 
  assert simple_combine(reqs) == expected
+
+
+def test_comment_parsing():
+ """
+ Test that simple_combine() does not remove embedded URL anchors due to comment parsing.
+ """
+ reqs = {
+ 'a.b': [
+ '# comment 1',
+ 'git+https://git.repo/some_pkg.git#egg=SomePackage',
+ 'git+https://git.repo/some_pkg.git#egg=SomeOtherPackage # inline comment',
+ 'git+https://git.repo/some_pkg.git#egg=AlsoSomePackage #inline comment that hates leading spaces',
+ ' # crazy indented comment (waka waka!)',
+ '####### something informative'
+ ' ',
+ '',
+ ]
+ }
+
+ expected = [
+ 'git+https://git.repo/some_pkg.git#egg=SomePackage # from collection a.b',
+ 'git+https://git.repo/some_pkg.git#egg=SomeOtherPackage # from collection a.b',
+ 'git+https://git.repo/some_pkg.git#egg=AlsoSomePackage # from collection a.b',
+ ]
+
+ assert simple_combine(reqs) == expected
+
+
+def test_pass_thru():
+ """
+ Test that simple_combine() will pass through non-pep508 data.
+ """
+ reqs = {
+ # various VCS and URL options
+ 'a.b': [
+ 'git+https://git.repo/some_pkg.git#egg=SomePackage',
+ 'svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage',
+ 'https://example.com/foo/foo-0.26.0-py2.py3-none-any.whl',
+ 'http://my.package.repo/SomePackage-1.0.4.zip',
+ ],
+
+ # various 'pip install' options
+ 'c.d': [
+ '-i https://pypi.org/simple',
+ '--extra-index-url http://my.package.repo/simple',
+ '--no-clean',
+ '-e svn+http://svn.example.com/svn/MyProject/trunk@2019#egg=MyProject',
+ ]
+ }
+
+ expected = [
+ 'git+https://git.repo/some_pkg.git#egg=SomePackage # from collection a.b',
+ 'svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage # from collection a.b',
+ 'https://example.com/foo/foo-0.26.0-py2.py3-none-any.whl # from collection a.b',
+ 'http://my.package.repo/SomePackage-1.0.4.zip # from collection a.b',
+ '-i https://pypi.org/simple # from collection c.d',
+ '--extra-index-url http://my.package.repo/simple # from collection c.d',
+ '--no-clean # from collection c.d',
+ '-e svn+http://svn.example.com/svn/MyProject/trunk@2019#egg=MyProject # from collection c.d',
+ ]
+
+ assert simple_combine(reqs) == expected