Skip to content

Commit

Permalink
Add bundle_replacements parameter to regenerate_bundle worker
Browse files Browse the repository at this point in the history 
Continuing to add support towards the specification of specific bundle
replacements to perform when calling the `regenerate_bundle` API.

In order to apply the bundle_replacements, a `perform_bundle_replacements`
typed customization needs to be set for the organization.

Refers to CLOUDDST-14790

Signed-off-by: arewm <arewm@users.noreply.github.com>
  • Loading branch information
@arewm
arewm committed Sep 15, 2022
1 parent 86b6c19 commit df21a4c
Show file tree
Hide file tree
Showing 4 changed files with 149 additions and 7 deletions.
4 changes: 4 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,10 @@ The custom configuration options for the Celery workers are listed below:
question to their digests. If this customization is not specified in the config for an
organization, the pinning will not be done. If no organization is specified, IIB will try and
pin the pull specs in the CSV files to their digests.
* The `perform_bundle_replacements` customization type is a dictionary with no additional
arguments. It enables bundle replacements to be passed to the bundle regeneration APIs with
the `bundle_replacements` parameter. If the customization type is not set, and bundle
replacements specified will be ignored.

Here is an example that ties this all together:

Expand Down
1 change: 1 addition & 0 deletions iib/workers/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ class DevelopmentConfig(Config):
'company-marketplace': [
IIBOrganizationCustomizations({'type': 'resolve_image_pullspecs'}),
IIBOrganizationCustomizations({'type': 'related_bundles'}),
IIBOrganizationCustomizations({'type': 'perform_bundle_replacements'}),
CSVAnnotations(
{
'type': 'csv_annotations',
Expand Down
18 changes: 17 additions & 1 deletion iib/workers/tasks/build_regenerate_bundle.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,12 @@ def handle_regenerate_bundle_request(
metadata_path = os.path.join(temp_dir, 'metadata')
_copy_files_from_image(from_bundle_image_resolved, '/metadata', metadata_path)
new_labels = _adjust_operator_bundle(
manifests_path, metadata_path, request_id, organization, pinned_by_iib
manifests_path,
metadata_path,
request_id,
organization=organization,
pinned_by_iib=pinned_by_iib,
bundle_replacements=bundle_replacements,
)

with open(os.path.join(temp_dir, 'Dockerfile'), 'w') as dockerfile:
Expand Down Expand Up @@ -202,6 +207,7 @@ def _adjust_operator_bundle(
organization: Optional[str] = None,
pinned_by_iib: bool = False,
recursive_related_bundles: bool = False,
bundle_replacements: Optional[TypedDict[str, str]] = {},
) -> Dict[str, str]:
"""
Apply modifications to the operator manifests at the given location.
Expand All @@ -225,6 +231,8 @@ def _adjust_operator_bundle(
IIB to perform image pinning of related images.
:param bool recursive_related_bundles: whether or not the call is from a
recursive_related_bundles request.
:param dict bundle_replacements: mapping between original pullspecs and rebuilt bundles,
allowing the updating of digests if any bundles have been rebuilt.
:raises IIBError: if the operator manifest has invalid entries
:return: a dictionary of labels to set on the bundle
:rtype: dict
Expand All @@ -244,6 +252,7 @@ def _adjust_operator_bundle(
{'type': 'related_bundles'},
{'type': 'package_name_suffix'},
{'type': 'registry_replacements'},
{'type': 'perform_bundle_replacements'},
{'type': 'image_name_from_labels'},
{'type': 'csv_annotations'},
{'type': 'enclose_repo'},
Expand Down Expand Up @@ -325,6 +334,13 @@ def _adjust_operator_bundle(
log.info('Resolving image pull specs')
bundle_metadata = _get_bundle_metadata(operator_manifest, pinned_by_iib)
_resolve_image_pull_specs(bundle_metadata, labels, pinned_by_iib)
elif customization_type == 'perform_bundle_replacements':
log.info('Performing bundle replacements')
bundle_metadata = _get_bundle_metadata(operator_manifest, pinned_by_iib)
replacement_pullspecs = {}
for old, new in bundle_replacements.items():
replacement_pullspecs[ImageName.parse(old)] = ImageName.parse(new)
_replace_csv_pullspecs(bundle_metadata, replacement_pullspecs)

return labels

Expand Down
133 changes: 127 additions & 6 deletions tests/test_workers/test_tasks/test_build_regenerate_bundle.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,14 @@


@pytest.mark.parametrize(
'pinned_by_iib_label, pinned_by_iib_bool',
(('true', True), ('True', True), (None, False), ('false', False), ('False', False)),
'pinned_by_iib_label, pinned_by_iib_bool, bundle_replacements',
(
('true', True, None),
('True', True, {}),
(None, False, {'a': 'b'}),
('false', False, None),
('False', False, {'a@b', 'c@d'}),
),
)
@pytest.mark.parametrize(
'iib_index_image_output_registry, expected_bundle_image',
Expand Down Expand Up @@ -56,6 +62,7 @@ def test_handle_regenerate_bundle_request(
expected_bundle_image,
pinned_by_iib_label,
pinned_by_iib_bool,
bundle_replacements,
tmpdir,
):
arches = ['amd64', 's390x']
Expand All @@ -77,7 +84,7 @@ def test_handle_regenerate_bundle_request(
mock_gil.return_value = pinned_by_iib_label

build_regenerate_bundle.handle_regenerate_bundle_request(
from_bundle_image, organization, request_id
from_bundle_image, organization, request_id, bundle_replacements=bundle_replacements
)

mock_cleanup.assert_called_once()
Expand All @@ -104,6 +111,7 @@ def test_handle_regenerate_bundle_request(
request_id,
'acme',
pinned_by_iib_bool,
bundle_replacements,
)

assert mock_bi.call_count == len(arches)
Expand Down Expand Up @@ -638,7 +646,7 @@ def test_adjust_operator_bundle_already_pinned_by_iib(
# worker configuration.
registry='registry.access.company.com',
ref='@sha256:765432',
related_name=f'operator/image-765432-annotation',
related_name='operator/image-765432-annotation',
related_ref='@sha256:765432',
operator='operator',
image='/image',
Expand All @@ -654,15 +662,128 @@ def test_adjust_operator_bundle_already_pinned_by_iib(
assert csv1.read_text('utf-8') == csv_related_images_template.format(
registry='quay.io',
ref='@sha256:654321',
related_name=f'image-654321-annotation',
related_name='image-654321-annotation',
related_ref='@sha256:654321',
operator='namespace/reponame',
image='-rhel-8-final',
)
assert csv2.read_text('utf-8') == csv_related_images_template.format(
registry='registry.marketplace.company.com',
ref='@sha256:765432',
related_name=f'operator/image-765432-annotation',
related_name='operator/image-765432-annotation',
related_ref='@sha256:765432',
operator='namespace/reponame',
image='-rhel-8-final',
)
mock_aca.assert_called_once_with(mock.ANY, 'amqstreams', annotations)
mock_gri.assert_not_called()


@mock.patch('iib.workers.tasks.build_regenerate_bundle._get_package_annotations')
@mock.patch('iib.workers.tasks.build_regenerate_bundle._apply_package_name_suffix')
@mock.patch('iib.workers.tasks.build_regenerate_bundle.get_resolved_image')
@mock.patch('iib.workers.tasks.build_regenerate_bundle._adjust_csv_annotations')
@mock.patch('iib.workers.tasks.build_regenerate_bundle.get_image_labels')
@mock.patch('iib.workers.tasks.build_regenerate_bundle.get_related_bundle_images')
@mock.patch('iib.workers.tasks.build_regenerate_bundle.write_related_bundles_file')
def test_adjust_operator_bundle_perform_bundle_replacements(
mock_wrbf, mock_grbi, mock_gil, mock_aca, mock_gri, mock_apns, mock_gpa, tmpdir
):
annotations = {
'marketplace.company.io/remote-workflow': (
'https://marketplace.company.com/en-us/operators/{package_name}/pricing'
),
'marketplace.company.io/support-workflow': (
'https://marketplace.company.com/en-us/operators/{package_name}/support'
),
}
mock_gpa.return_value = {
'annotations': {'operators.operatorframework.io.bundle.package.v1': 'amqstreams'}
}
mock_apns.return_value = (
'amqstreams',
{'operators.operatorframework.io.bundle.package.v1': 'amqstreams-cmp'},
)
mock_gil.return_value = {'name': 'namespace/reponame', 'version': 'rhel-8'}
manifests_dir = tmpdir.mkdir('manifests')
metadata_dir = tmpdir.mkdir('metadata')
csv1 = manifests_dir.join('2.clusterserviceversion.yaml')
csv2 = manifests_dir.join('3.clusterserviceversion.yaml')

# NOTE: The OperatorManifest class is capable of modifying pull specs found in
# various locations within the CSV file. Since IIB relies on this class to do
# such modifications, this test only verifies that at least one of the locations
# is being handled properly. This is to ensure IIB is using OperatorManifest
# correctly.
csv_template = textwrap.dedent(
"""\
apiVersion: operators.example.com/v1
kind: ClusterServiceVersion
metadata:
name: amqstreams.v1.0.0
namespace: placeholder
annotations:
containerImage: {registry}/{operator}{image}{ref}
"""
)
csv_related_images_template = csv_template + textwrap.dedent(
"""\
spec:
relatedImages:
- name: {related_name}
image: {registry}/{operator}{image}{related_ref}
"""
)
csv1.write(
csv_related_images_template.format(
registry='quay.io',
ref='@sha256:654321',
related_name='image-654321-annotation',
related_ref='@sha256:654321',
operator='operator',
image='/image',
)
)
csv2.write(
csv_related_images_template.format(
# This registry for the company-marketplace will be replaced based on
# worker configuration.
registry='registry.access.company.com',
ref='@sha256:765432',
related_name='operator/image-765432-annotation',
related_ref='@sha256:765432',
operator='operator',
image='/image',
)
)

bundle_replacements = {
'quay.io/operator/image@sha256:654321': 'quay.io/operator/image@sha256:123456',
}

labels = build_regenerate_bundle._adjust_operator_bundle(
str(manifests_dir),
str(metadata_dir),
1,
organization='company-marketplace',
pinned_by_iib=True,
bundle_replacements=bundle_replacements,
)

# The com.redhat.iib.pinned label is not explicitly set, but inherited from the original image
assert labels == {'operators.operatorframework.io.bundle.package.v1': 'amqstreams-cmp'}
assert csv1.read_text('utf-8') == csv_related_images_template.format(
registry='quay.io',
ref='@sha256:123456',
related_name='image-654321-annotation',
related_ref='@sha256:123456',
operator='namespace/reponame',
image='-rhel-8-final',
)
assert csv2.read_text('utf-8') == csv_related_images_template.format(
registry='registry.marketplace.company.com',
ref='@sha256:765432',
related_name='operator/image-765432-annotation',
related_ref='@sha256:765432',
operator='namespace/reponame',
image='-rhel-8-final',
Expand Down

0 comments on commit df21a4c

Please sign in to comment.