Expose a way to exclude deps

.github/workflows/ci.yml

@@ -124,6 +124,9 @@ jobs:
           - name: '3.11'
             tox_env: integration-py311
 
+          - name: '3.12'
+            tox_env: integration-py312
+
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -173,6 +176,9 @@ jobs:
           - name: '3.11'
             tox_env: unit-py311
 
+          - name: '3.12'
+            tox_env: unit-py312
+
     steps:
       - name: Checkout
         uses: actions/checkout@v3

setup.cfg

@@ -21,6 +21,7 @@ classifiers =
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
+    Programming Language :: Python :: 3.12
     Programming Language :: Python :: 3 :: Only
     Topic :: Software Development :: Build Tools
     Topic :: System :: Systems Administration
@@ -35,7 +36,7 @@ max-line-length=160
 include_package_data = true
 install_requires =
     PyYAML
-    requirements_parser
+    packaging
     bindep
     jsonschema
 python_requires = >=3.9

src/ansible_builder/_target_scripts/introspect.py

@@ -1,16 +1,19 @@
 import argparse
-import importlib.metadata
 import logging
 import os
+import re
 import sys
 import yaml
 
-import requirements
-
+from packaging.requirements import InvalidRequirement, Requirement
+from packaging.specifiers import SpecifierSet
+from packaging.utils import canonicalize_name
 
 base_collections_path = '/usr/share/ansible/collections'
 logger = logging.getLogger(__name__)
 
+COMMENT_RE = re.compile(r'\s*#.*$')
+
 
 def line_is_empty(line):
     return bool((not line.strip()) or line.startswith('#'))
@@ -75,7 +78,8 @@ def process_collection(path):
     return (pip_lines, bindep_lines)
 
 
-def process(data_dir=base_collections_path, user_pip=None, user_bindep=None):
+def process(data_dir=base_collections_path, user_pip=None, user_bindep=None,
+            user_pip_exclude=None, user_bindep_exclude=None):
     paths = []
     path_root = os.path.join(data_dir, 'ansible_collections')
 
@@ -112,10 +116,18 @@ def process(data_dir=base_collections_path, user_pip=None, user_bindep=None):
         col_pip_lines = pip_file_data(user_pip)
         if col_pip_lines:
             py_req['user'] = col_pip_lines
+    if user_pip_exclude:
+        col_pip_exclude_lines = pip_file_data(user_pip_exclude)
+        if col_pip_exclude_lines:
+            py_req['exclude'] = col_pip_exclude_lines
     if user_bindep:
         col_sys_lines = bindep_file_data(user_bindep)
         if col_sys_lines:
             sys_req['user'] = col_sys_lines
+    if user_bindep_exclude:
+        col_sys_exclude_lines = bindep_file_data(user_bindep_exclude)
+        if col_sys_exclude_lines:
+            sys_req['exclude'] = col_sys_exclude_lines
 
     return {
         'python': py_req,
@@ -190,11 +202,14 @@ def get_dependency(self, entry):
         return req_file
 
 
-def simple_combine(reqs):
+def simple_combine(reqs, exclude=None, name_only=False):
     """Given a dictionary of requirement lines keyed off collections,
     return a list with the most basic of de-duplication logic,
     and comments indicating the sources based off the collection keys
     """
+    if exclude is None:
+        exclude = []
+
     consolidated = []
     fancy_lines = []
     for collection, lines in reqs.items():
@@ -203,11 +218,20 @@ def simple_combine(reqs):
                 continue
 
             base_line = line.split('#')[0].strip()
+            pkg_name = base_line.split()[0].lower()
+            if pkg_name in exclude:
+                logger.debug('# Explicitly excluding requirement %s from %s', pkg_name, collection)
+                continue
+
             if base_line in consolidated:
                 i = consolidated.index(base_line)
-                fancy_lines[i] += f', {collection}'
+                if not name_only:
+                    fancy_lines[i] += f', {collection}'
             else:
-                fancy_line = f'{base_line}  # from collection {collection}'
+                if name_only:
+                    fancy_line = pkg_name
+                else:
+                    fancy_line = f'{base_line}  # from collection {collection}'
                 consolidated.append(base_line)
                 fancy_lines.append(fancy_line)
 
@@ -235,12 +259,19 @@ def parse_args(args=None):
 
 
 def run_introspect(args, log):
-    data = process(args.folder, user_pip=args.user_pip, user_bindep=args.user_bindep)
+    data = process(args.folder, user_pip=args.user_pip, user_bindep=args.user_bindep,
+                   user_pip_exclude=args.user_pip_exclude, user_bindep_exclude=args.user_bindep_exclude)
     if args.sanitize:
         log.info('# Sanitized dependencies for %s', args.folder)
         data_for_write = data
-        data['python'] = sanitize_requirements(data['python'])
-        data['system'] = simple_combine(data['system'])
+        data['python'] = sanitize_requirements(
+            data['python'],
+            exclude=sanitize_requirements({'exclude': data['python'].pop('exclude', {})}, name_only=True)
+        )
+        data['system'] = simple_combine(
+            data['system'],
+            exclude=simple_combine({'exclude': data['system'].pop('exclude', {})}, name_only=True)
+        )
     else:
         log.info('# Dependency data for %s', args.folder)
         data_for_write = data.copy()
@@ -287,10 +318,18 @@ def create_introspect_parser(parser):
         '--user-pip', dest='user_pip',
         help='An additional file to combine with collection pip requirements.'
     )
+    introspect_parser.add_argument(
+        '--user-pip-exclude', dest='user_pip_exclude',
+        help='An additional file to exclude specific pip requirements.'
+    )
     introspect_parser.add_argument(
         '--user-bindep', dest='user_bindep',
         help='An additional file to combine with collection bindep requirements.'
     )
+    introspect_parser.add_argument(
+        '--user-bindep-exclude', dest='user_bindep_exclude',
+        help='An additional file to exclude specific bindep requirements.'
+    )
     introspect_parser.add_argument(
         '--write-pip', dest='write_pip',
         help='Write the combined pip requirements file to this location.'
@@ -316,7 +355,7 @@ def create_introspect_parser(parser):
 ))
 
 
-def sanitize_requirements(collection_py_reqs):
+def sanitize_requirements(collection_py_reqs, exclude=None, name_only=False):
     """
     Cleanup Python requirements by removing duplicates and excluded packages.
 
@@ -329,48 +368,51 @@ def sanitize_requirements(collection_py_reqs):
 
     :returns: A finalized list of sanitized Python requirements.
     """
+    if exclude is None:
+        exclude = []
+
     # de-duplication
-    consolidated = []
-    seen_pkgs = set()
+    consolidated = {}
 
     for collection, lines in collection_py_reqs.items():
-        try:
-            for req in requirements.parse('\n'.join(lines)):
-                if req.specifier:
-                    req.name = importlib.metadata.Prepared(req.name).normalized
-                req.collections = [collection]  # add backref for later
-                if req.name is None:
-                    consolidated.append(req)
-                    continue
-                if req.name in seen_pkgs:
-                    for prior_req in consolidated:
-                        if req.name == prior_req.name:
-                            prior_req.specs.extend(req.specs)
-                            prior_req.collections.append(collection)
-                            break
-                    continue
-                consolidated.append(req)
-                seen_pkgs.add(req.name)
-        except Exception as e:
-            logger.warning('Warning: failed to parse requirements from %s, error: %s', collection, e)
+        for line in lines:
+            if not (line := COMMENT_RE.sub('', line.strip())):
+                continue
+            try:
+                req = Requirement(line)
+            except InvalidRequirement as e:
+                logger.warning('Warning: failed to parse requirements from %s, error: %s', collection, e)
+                continue
+            req.name = canonicalize_name(req.name)
+            if req.name in exclude and collection != 'user':
+                logger.debug('# Explicitly excluding requirement %s from %s', req.name, collection)
+                continue
+
+            req.collections = {collection: None}  # add backref for later
+            key = (req.name, req.marker)
+            if (prior_req := consolidated.get(key)):
+                specifiers = f'{prior_req.specifier},{req.specifier}'
+                prior_req.specifier = SpecifierSet(specifiers)
+                if not prior_req.url and req.url:
+                    # An explicit install URL is preferred over none
+                    # The first URL seen wins
+                    prior_req.url = req.url
+                prior_req.extras.update(req.extras)
+                prior_req.collections.update({collection: None})
+                continue
+            consolidated[key] = req
 
     # removal of unwanted packages
     sanitized = []
-    for req in consolidated:
+    for (name, _marker), req in consolidated.items():
         # Exclude packages, unless it was present in the user supplied requirements.
-        if req.name and req.name.lower() in EXCLUDE_REQUIREMENTS and 'user' not in req.collections:
+        if name.lower() in EXCLUDE_REQUIREMENTS and 'user' not in req.collections:
             logger.debug('# Excluding requirement %s from %s', req.name, req.collections)
             continue
-        if req.vcs or req.uri:
-            # Requirement like git+ or http return as-is
-            new_line = req.line
-        elif req.name:
-            specs = [f'{cmp}{ver}' for cmp, ver in req.specs]
-            new_line = req.name + ','.join(specs)
+        if name_only:
+            sanitized.append(f'{req.name}')
         else:
-            raise RuntimeError(f'Could not process {req.line}')
-
-        sanitized.append(f'{new_line}  # from collection {",".join(req.collections)}')
+            sanitized.append(f'{req}  # from collection {",".join(req.collections)}')
 
     return sanitized
 

src/ansible_builder/containerfile.py

@@ -144,7 +144,7 @@ def prepare(self) -> None:
         self._insert_global_args()
 
         if image == "base":
-            self.steps.append("RUN $PYCMD -m pip install --no-cache-dir bindep pyyaml requirements-parser")
+            self.steps.append("RUN $PYCMD -m pip install --no-cache-dir bindep pyyaml packaging")
         else:
             # For an EE schema earlier than v3 with a custom builder image, we always make sure pip is available.
             context_dir = Path(self.build_outputs_dir).stem
@@ -259,16 +259,19 @@ def _create_folder_copy_files(self) -> None:
             if not new_name:
                 continue
 
-            requirement_path = self.definition.get_dep_abs_path(item)
-            if requirement_path is None:
-                continue
-            dest = os.path.join(
-                self.build_context, constants.user_content_subfolder, new_name)
+            for exclude in (False, True):
+                if exclude is True:
+                    new_name = f'exclude-{new_name}'
+                requirement_path = self.definition.get_dep_abs_path(item, exclude=exclude)
+                if requirement_path is None:
+                    continue
+                dest = os.path.join(
+                    self.build_context, constants.user_content_subfolder, new_name)
 
-            # Ignore modification time of the requirement file because we could
-            # be writing it out dynamically (inline EE reqs), and we only care
-            # about the contents anyway.
-            copy_file(requirement_path, dest, ignore_mtime=True)
+                # Ignore modification time of the requirement file because we could
+                # be writing it out dynamically (inline EE reqs), and we only care
+                # about the contents anyway.
+                copy_file(requirement_path, dest, ignore_mtime=True)
 
         if self.original_galaxy_keyring:
             copy_file(
@@ -384,7 +387,12 @@ def _prepare_label_steps(self) -> None:
         ])
 
     def _prepare_build_context(self) -> None:
-        if any(self.definition.get_dep_abs_path(thing) for thing in ('galaxy', 'system', 'python')):
+        deps: list[str] = []
+        for exclude in (False, True):
+            deps.extend(
+                self.definition.get_dep_abs_path(thing, exclude=exclude) for thing in ('galaxy', 'system', 'python')
+            )
+        if any(deps):
             self.steps.extend([
                 f"COPY {constants.user_content_subfolder} /build",
                 "WORKDIR /build",
@@ -423,14 +431,17 @@ def _prepare_galaxy_install_steps(self) -> None:
 
     def _prepare_introspect_assemble_steps(self) -> None:
         # The introspect/assemble block is valid if there are any form of requirements
-        if any(self.definition.get_dep_abs_path(thing) for thing in ('galaxy', 'system', 'python')):
-
+        deps: list[str] = []
+        for exclude in (False, True):
+            deps.extend(
+                self.definition.get_dep_abs_path(thing, exclude=exclude) for thing in ('galaxy', 'system', 'python')
+            )
+        if any(deps):
             introspect_cmd = "RUN $PYCMD /output/scripts/introspect.py introspect --sanitize"
 
             requirements_file_exists = os.path.exists(os.path.join(
                 self.build_outputs_dir, constants.CONTEXT_FILES['python']
             ))
-
             if requirements_file_exists:
                 relative_requirements_path = os.path.join(
                     constants.user_content_subfolder,
@@ -439,12 +450,35 @@ def _prepare_introspect_assemble_steps(self) -> None:
                 self.steps.append(f"COPY {relative_requirements_path} {constants.CONTEXT_FILES['python']}")
                 # WORKDIR is /build, so we use the (shorter) relative paths there
                 introspect_cmd += f" --user-pip={constants.CONTEXT_FILES['python']}"
+
+            pip_exclude_exists = os.path.exists(os.path.join(
+                self.build_outputs_dir, f"exclude-{constants.CONTEXT_FILES['python']}"
+            ))
+            if pip_exclude_exists:
+                relative_pip_exclude_path = os.path.join(
+                    constants.user_content_subfolder,
+                    f"exclude-{constants.CONTEXT_FILES['python']}"
+                )
+                self.steps.append(f"COPY {relative_pip_exclude_path} exclude-{constants.CONTEXT_FILES['python']}")
+                introspect_cmd += f" --user-pip-exclude=exclude-{constants.CONTEXT_FILES['python']}"
+
             bindep_exists = os.path.exists(os.path.join(self.build_outputs_dir, constants.CONTEXT_FILES['system']))
             if bindep_exists:
                 relative_bindep_path = os.path.join(constants.user_content_subfolder, constants.CONTEXT_FILES['system'])
                 self.steps.append(f"COPY {relative_bindep_path} {constants.CONTEXT_FILES['system']}")
                 introspect_cmd += f" --user-bindep={constants.CONTEXT_FILES['system']}"
 
+            exclude_bindep_exists = os.path.exists(os.path.join(
+                self.build_outputs_dir, f"exclude-{constants.CONTEXT_FILES['system']}"
+            ))
+            if exclude_bindep_exists:
+                relative_exclude_bindep_path = os.path.join(
+                    constants.user_content_subfolder,
+                    f"exclude-{constants.CONTEXT_FILES['system']}"
+                )
+                self.steps.append(f"COPY {relative_exclude_bindep_path} exclude-{constants.CONTEXT_FILES['system']}")
+                introspect_cmd += f" --user-bindep-exclude=exclude-{constants.CONTEXT_FILES['system']}"
+
             introspect_cmd += " --write-bindep=/tmp/src/bindep.txt --write-pip=/tmp/src/requirements.txt"
 
             self.steps.append(introspect_cmd)