Add bundle_replacements parameter to regenerate_bundle API

Adding support towards the specification of specific bundle replacements
to perform when calling the `regenerate_bundle` API.

Refers to CLOUDDST-14679

Signed-off-by: arewm <arewm@users.noreply.github.com>

iib/web/api_v1.py

@@ -525,6 +525,13 @@ def patch_request(request_id):
             for bundle in value:
                 if not isinstance(bundle, str):
                     raise ValidationError(exc_msg)
+        elif key == 'bundle_replacements':
+            exc_msg = f'The "{key}" key must be a dictionary object mapping from strings to strings'
+            if not isinstance(value, dict):
+                raise ValidationError(exc_msg)
+            for k, v in value.items():
+                if not isinstance(v, str) or not isinstance(k, str):
+                    raise ValidationError(exc_msg)
         elif not value or not isinstance(value, str):
             raise ValidationError(f'The value for "{key}" must be a non-empty string')
 
@@ -667,6 +674,7 @@ def regenerate_bundle():
         payload.get('organization'),
         request.id,
         payload.get('registry_auths'),
+        payload.get('bundle_replacements'),
     ]
     safe_args = _get_safe_args(args, payload)
 
@@ -731,6 +739,7 @@ def regenerate_bundle_batch():
                 build_request.get('organization'),
                 request.id,
                 build_request.get('registry_auths'),
+                build_request.get('bundle_replacements'),
             ]
             safe_args = _get_safe_args(args, build_request)
             error_callback = failed_request_callback.s(request.id)

iib/web/migrations/versions/a0eadb516360_update_regenerate_bundle_request_.py

@@ -0,0 +1,26 @@
+"""Update regenerate_bundle_request endpoint.
+
+Revision ID: a0eadb516360
+Revises: 625fba6081be
+Create Date: 2022-09-06 15:00:55.115536
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'a0eadb516360'
+down_revision = '625fba6081be'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('request_regenerate_bundle', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('bundle_replacements', sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table('request_regenerate_bundle', schema=None) as batch_op:
+        batch_op.drop_column('bundle_replacements')

iib/web/models.py

@@ -1178,12 +1178,34 @@ class RequestRegenerateBundle(Request):
     )
     # The name of the organization the bundle should be regenerated for
     organization = db.Column(db.String, nullable=True)
+    # The mapping of bundle replacements to apply to the regeneration request
+    _bundle_replacements = db.Column('bundle_replacements', db.Text, nullable=True)
 
     __mapper_args__ = {
         'polymorphic_identity': RequestTypeMapping.__members__['regenerate_bundle'].value
     }
     build_tags = None
 
+    @property
+    def bundle_replacements(self):
+        """Return the Python representation of the JSON bundle_replacements."""
+        return json.loads(self._bundle_replacements) if self._bundle_replacements else None
+
+    @bundle_replacements.setter
+    def bundle_replacements(self, bundle_replacements):
+        """
+        Set the bundle_replacements column to the input bundle_replacements as a JSON string.
+
+        If ``None`` is provided, it will be simply set to ``None`` and not be converted to JSON.
+
+        :param dict bundle_replacements: the dictionary of the bundle_replacements or ``None``
+        """
+        self._bundle_replacements = (
+            json.dumps(bundle_replacements, sort_keys=True)
+            if bundle_replacements is not None
+            else None
+        )
+
     @classmethod
     def from_json(cls, kwargs, batch=None):
         """
@@ -1199,8 +1221,17 @@ def from_json(cls, kwargs, batch=None):
         validate_request_params(
             request_kwargs,
             required_params={'from_bundle_image'},
-            optional_params={'organization', 'registry_auths'},
+            optional_params={'bundle_replacements', 'organization', 'registry_auths'},
         )
+        # Validate bundle_replacements is correctly provided
+        bundle_replacements = request_kwargs.get('bundle_replacements', None)
+        if bundle_replacements is not None:
+            if not isinstance(bundle_replacements, dict):
+                raise ValidationError('The value of "bundle_replacements" must be a JSON object')
+
+            for key, value in bundle_replacements.items():
+                if not isinstance(value, str) or not isinstance(key, str):
+                    raise ValidationError(f'The key and value of "{key}" must be a string')
 
         # Validate organization is correctly provided
         organization = request_kwargs.get('organization')
@@ -1271,6 +1302,7 @@ def get_mutable_keys(self):
         rv = super().get_mutable_keys()
         rv.add('bundle_image')
         rv.add('from_bundle_image_resolved')
+        rv.add('bundle_replacements')
         return rv
 
 

iib/web/static/api_v1.yaml

@@ -1075,6 +1075,16 @@ components:
             auths:
               registry.example.io:
                 auth: "my-secret-token-base-64-encoded"
+        bundle_replacements:
+          type: object
+          description: >
+            A dictionary of bundle image pullspecs to replace in the CSV. The dictionary
+            should contain a mapping of the original pullspec to the target pullspec. Since
+            replacement happens according to the location of the `perform_bundle_replacements`
+            customization type, the original pullspecs need to be accurate for where the
+            customization is run.
+          example:
+            quay.io/foo/bar@sha256:123456: quay.io/foo/bar@sha256:654321
         related_bundles:
           type: object
           properties:

iib/workers/tasks/build_regenerate_bundle.py

@@ -4,7 +4,7 @@
 import os
 import tempfile
 import textwrap
-from typing import Any, Dict, List, Optional, Set, Tuple
+from typing import Any, Dict, List, Optional, Set, Tuple, TypedDict
 
 from operator_manifest.operator import ImageName, OperatorManifest, OperatorCSV
 import ruamel.yaml
@@ -55,15 +55,18 @@ def handle_regenerate_bundle_request(
     organization: str,
     request_id: int,
     registry_auths: Optional[Dict[str, Any]] = None,
+    bundle_replacements: Optional[TypedDict[str, str]] = None,
 ) -> None:
     """
     Coordinate the work needed to regenerate the operator bundle image.
 
     :param str from_bundle_image: the pull specification of the bundle image to be regenerated.
     :param str organization: the name of the organization the bundle should be regenerated for.
-    :param int request_id: the ID of the IIB build request
+    :param int request_id: the ID of the IIB build request.
     :param dict registry_auths: Provide the dockerconfig.json for authentication to private
       registries, defaults to ``None``.
+    :param dict bundle_replacements: Dictionary mapping from original bundle pullspecs to rebuilt
+      bundle pullspecs.
     :raises IIBError: if the regenerate bundle image build fails.
     """
     _cleanup()

iib/workers/tasks/iib_static_types.py

@@ -39,6 +39,7 @@ class PrebuildInfo(TypedDict):
     binary_image: str
     binary_image_resolved: str
     bundle_mapping: NotRequired[Dict[str, List[str]]]
+    bundle_replacements: NotRequired[Dict[str, str]]
     distribution_scope: str
     extra: NotRequired[str]
     from_index_resolved: NotRequired[str]
@@ -67,6 +68,7 @@ class UpdateRequestPayload(TypedDict, total=False):
     binary_image_resolved: NotRequired[str]
     bundle_image: NotRequired[str]
     bundle_mapping: NotRequired[Dict[str, List[str]]]
+    bundle_replacements: NotRequired[Dict[str, str]]
     distribution_scope: NotRequired[str]
     from_bundle_image_resolved: NotRequired[str]
     from_index_resolved: NotRequired[str]

tests/test_web/test_api_v1.py

@@ -1145,6 +1145,7 @@ def test_patch_request_regenerate_bundle_success(
         'state_reason': 'All done!',
         'bundle_image': 'bundle:image',
         'from_bundle_image_resolved': 'from-bundle-image:resolved',
+        'bundle_replacements': {'foo': 'bar:baz'},
     }
 
     response_json = {
@@ -1398,6 +1399,20 @@ def test_regenerate_bundle_success(mock_smfsc, mock_hrbr, db, auth_env, client):
             {'from_bundle_image': 'registry.example.com/bundle-image:latest', 'spam': 'maps'},
             'The following parameters are invalid: spam',
         ),
+        (
+            {
+                'from_bundle_image': 'registry.example.com/bundle-image:latest',
+                'bundle_replacements': 'bundle_replacements',
+            },
+            'The value of "bundle_replacements" must be a JSON object',
+        ),
+        (
+            {
+                'from_bundle_image': 'registry.example.com/bundle-image:latest',
+                'bundle_replacements': {'bundle': ['replacements']},
+            },
+            'The key and value of "bundle" must be a string',
+        ),
     ),
 )
 @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change')
@@ -1469,6 +1484,7 @@ def test_regenerate_bundle_batch_success(
             {
                 'from_bundle_image': 'registry.example.com/bundle-image:latest',
                 'registry_auths': {'auths': {'registry2.example.com': {'auth': 'dummy_auth'}}},
+                'bundle_replacements': {'foo': 'bar:baz'},
             },
             {'from_bundle_image': 'registry.example.com/bundle-image2:latest'},
         ]
@@ -1487,14 +1503,18 @@ def test_regenerate_bundle_batch_success(
                     None,
                     1,
                     {'auths': {'registry2.example.com': {'auth': 'dummy_auth'}}},
+                    {'foo': 'bar:baz'},
                 ],
-                argsrepr="['registry.example.com/bundle-image:latest', None, 1, '*****']",
+                argsrepr=(
+                    "['registry.example.com/bundle-image:latest', None, 1, '*****', "
+                    "{'foo': 'bar:baz'}]"
+                ),
                 link_error=mock.ANY,
                 queue=expected_queue,
             ),
             mock.call(
-                args=['registry.example.com/bundle-image2:latest', None, 2, None],
-                argsrepr="['registry.example.com/bundle-image2:latest', None, 2, None]",
+                args=['registry.example.com/bundle-image2:latest', None, 2, None, None],
+                argsrepr="['registry.example.com/bundle-image2:latest', None, 2, None, None]",
                 link_error=mock.ANY,
                 queue=expected_queue,
             ),