Drop requirements-parser in favor of pep508

setup.cfg

@@ -35,7 +35,7 @@ max-line-length=160
 include_package_data = true
 install_requires =
     PyYAML
-    requirements_parser
+    packaging
     bindep
     jsonschema
 python_requires = >=3.9

src/ansible_builder/_target_scripts/introspect.py

@@ -1,16 +1,19 @@
 import argparse
-import importlib.metadata
 import logging
 import os
+import re
 import sys
 import yaml
 
-import requirements
-
+from packaging.requirements import InvalidRequirement, Requirement
+from packaging.specifiers import SpecifierSet
+from packaging.utils import canonicalize_name
 
 base_collections_path = '/usr/share/ansible/collections'
 logger = logging.getLogger(__name__)
 
+COMMENT_RE = re.compile(r'\s*#.*$')
+
 
 def line_is_empty(line):
     return bool((not line.strip()) or line.startswith('#'))
@@ -330,47 +333,34 @@ def sanitize_requirements(collection_py_reqs):
     :returns: A finalized list of sanitized Python requirements.
     """
     # de-duplication
-    consolidated = []
-    seen_pkgs = set()
+    consolidated = {}
 
     for collection, lines in collection_py_reqs.items():
-        try:
-            for req in requirements.parse('\n'.join(lines)):
-                if req.specifier:
-                    req.name = importlib.metadata.Prepared(req.name).normalized
-                req.collections = [collection]  # add backref for later
-                if req.name is None:
-                    consolidated.append(req)
-                    continue
-                if req.name in seen_pkgs:
-                    for prior_req in consolidated:
-                        if req.name == prior_req.name:
-                            prior_req.specs.extend(req.specs)
-                            prior_req.collections.append(collection)
-                            break
-                    continue
-                consolidated.append(req)
-                seen_pkgs.add(req.name)
-        except Exception as e:
-            logger.warning('Warning: failed to parse requirements from %s, error: %s', collection, e)
+        for line in lines:
+            if not (line := COMMENT_RE.sub('', line.strip())):
+                continue
+            try:
+                req = Requirement(line)
+            except InvalidRequirement as e:
+                logger.warning('Warning: failed to parse requirements from %s, error: %s', collection, e)
+                continue
+            req.name = canonicalize_name(req.name)
+            req.collections = [collection]  # add backref for later
+            if (prior_req := consolidated.get(req.name)) and prior_req.marker == req.marker:
+                specifiers = f'{prior_req.specifier},{req.specifier}'
+                prior_req.specifier = SpecifierSet(specifiers)
+                prior_req.collections.append(collection)
+                continue
+            consolidated[req.name] = req
 
     # removal of unwanted packages
     sanitized = []
-    for req in consolidated:
+    for name, req in consolidated.items():
         # Exclude packages, unless it was present in the user supplied requirements.
-        if req.name and req.name.lower() in EXCLUDE_REQUIREMENTS and 'user' not in req.collections:
+        if name.lower() in EXCLUDE_REQUIREMENTS and 'user' not in req.collections:
             logger.debug('# Excluding requirement %s from %s', req.name, req.collections)
             continue
-        if req.vcs or req.uri:
-            # Requirement like git+ or http return as-is
-            new_line = req.line
-        elif req.name:
-            specs = [f'{cmp}{ver}' for cmp, ver in req.specs]
-            new_line = req.name + ','.join(specs)
-        else:
-            raise RuntimeError(f'Could not process {req.line}')
-
-        sanitized.append(f'{new_line}  # from collection {",".join(req.collections)}')
+        sanitized.append(f'{req}  # from collection {",".join(req.collections)}')
 
     return sanitized
 

src/ansible_builder/containerfile.py

@@ -141,7 +141,7 @@ def prepare(self) -> None:
         self._insert_global_args()
 
         if image == "base":
-            self.steps.append("RUN $PYCMD -m pip install --no-cache-dir bindep pyyaml requirements-parser")
+            self.steps.append("RUN $PYCMD -m pip install --no-cache-dir bindep pyyaml packaging")
 
         self._insert_custom_steps('prepend_builder')
         self._prepare_galaxy_copy_steps()

test/data/ansible_collections/test/reqfile/requirements.txt

@@ -1,4 +1,5 @@
 pytz
 python-dateutil>=2.8.2    # intentional dash
 jinja2>=3.0    # intentional lowercase
--r extra_req.txt
+tacacs_plus
+pyvcloud>=18.0.10  # duplicate with test.metadata collection

test/integration/test_introspect_cli.py

@@ -50,9 +50,9 @@ def test_introspect_write_python_and_sanitize(cli, data_dir, tmp_path):
     assert dest_file.read_text() == '\n'.join([
         'pyvcloud>=14,>=18.0.10  # from collection test.metadata,test.reqfile',
         'pytz  # from collection test.reqfile',
-        'python_dateutil>=2.8.2  # from collection test.reqfile',
+        'python-dateutil>=2.8.2  # from collection test.reqfile',
         'jinja2>=3.0  # from collection test.reqfile',
-        'tacacs_plus  # from collection test.reqfile',
+        'tacacs-plus  # from collection test.reqfile',
         '',
     ])
 

test/unit/test_introspect.py

@@ -17,11 +17,11 @@ def test_multiple_collection_metadata(data_dir):
         'pytz  # from collection test.reqfile',
         # python-dateutil should appear only once even though referenced in
         # multiple places, once with a dash and another with an underscore in the name.
-        'python_dateutil>=2.8.2  # from collection test.reqfile',
+        'python-dateutil>=2.8.2  # from collection test.reqfile',
         # jinja2 should appear only once even though referenced in multiple
         # places, once with uppercase and another with lowercase in the name.
         'jinja2>=3.0  # from collection test.reqfile',
-        'tacacs_plus  # from collection test.reqfile'
+        'tacacs-plus  # from collection test.reqfile'
     ], 'system': [
         'subversion [platform:rpm]  # from collection test.bindep',
         'subversion [platform:dpkg]  # from collection test.bindep'