Merge branch 'master' into non_tag_index_image_in_graph_update_mode_list

release-engineering · Apr 23, 2024 · 0e6fd2a · 0e6fd2a
2 parents 6a6ef7c + e5b5d71
commit 0e6fd2a
Show file tree

Hide file tree

Showing 8 changed files with 182 additions and 90 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## 8.7.0
+* Adding support for new ocp_version_ranges by @lipoja in https://github.com/release-engineering/iib/pull/653
+* Fix comma bundle version validation by @xDaile in https://github.com/release-engineering/iib/pull/654
+* Update python Docker tag to v3.12.2 by @renovate in https://github.com/release-engineering/iib/pull/620
+* Use multiple OPM versions in IIB by @chandwanitulsi in https://github.com/release-engineering/iib/pull/637
+* Update actions/setup-python action to v5 by @renovate in https://github.com/release-engineering/iib/pull/648
+* Update dependency importlib-resources to v6.4.0 by @renovate in https://github.com/release-engineering/iib/pull/645
+* Update dependency botocore to v1.34.79 by @renovate in https://github.com/release-engineering/iib/pull/658
+
 
 ## 8.6.0
 * Fixing logging in tracing.py by @lipoja in https://github.com/release-engineering/iib/pull/628

diff --git a/iib/workers/tasks/fbc_utils.py b/iib/workers/tasks/fbc_utils.py
@@ -3,13 +3,18 @@
 import os
 import logging
 import shutil
+import json
+from pathlib import Path
+from typing import Tuple, List
+
+import ruamel.yaml
 
-from typing import Tuple
 from iib.exceptions import IIBError
 from iib.workers.config import get_worker_config
 from iib.common.tracing import instrument_tracing
 
 log = logging.getLogger(__name__)
+yaml = ruamel.yaml.YAML()
 
 
 def is_image_fbc(image: str) -> bool:
@@ -81,6 +86,8 @@ def merge_catalogs_dirs(src_config: str, dest_config: str):
     :param str src_config: source config directory
     :param str dest_config: destination config directory
     """
+    from iib.workers.tasks.opm_operations import opm_validate
+
     for conf_dir in (src_config, dest_config):
         if not os.path.isdir(conf_dir):
             msg = f"config directory does not exist: {conf_dir}"
@@ -89,15 +96,17 @@ def merge_catalogs_dirs(src_config: str, dest_config: str):
 
     log.info("Merging config folders: %s to %s", src_config, dest_config)
     shutil.copytree(src_config, dest_config, dirs_exist_ok=True)
+    enforce_json_config_dir(conf_dir)
+    opm_validate(conf_dir)
 
 
-def extract_fbc_fragment(temp_dir: str, fbc_fragment: str) -> Tuple[str, str]:
+def extract_fbc_fragment(temp_dir: str, fbc_fragment: str) -> Tuple[str, List[str]]:
     """
-    Extract operator package from the fbc_fragment image.
+    Extract operator packages from the fbc_fragment image.
 
     :param str temp_dir: base temp directory for IIB request.
     :param str fbc_fragment: pull specification of fbc_fragment in the IIB request.
-    :return: fbc_fragment path, fbc_operator_package.
+    :return: fbc_fragment path, fbc_operator_packages.
     :rtype: tuple
     """
     from iib.workers.tasks.build import _copy_files_from_image
@@ -111,10 +120,29 @@ def extract_fbc_fragment(temp_dir: str, fbc_fragment: str) -> Tuple[str, str]:
 
     log.info("fbc_fragment extracted at %s", fbc_fragment_path)
     operator_packages = os.listdir(fbc_fragment_path)
-    log.info("fbc_fragment contains package %s", operator_packages)
+    log.info("fbc_fragment contains packages %s", operator_packages)
     if not operator_packages:
         raise IIBError("No operator packages in fbc_fragment %s", fbc_fragment)
-    if len(operator_packages) > 1:
-        raise IIBError("More than 1 package is present in fbc_fragment %s", fbc_fragment)
 
-    return fbc_fragment_path, operator_packages[0]
+    return fbc_fragment_path, operator_packages
+
+
+def enforce_json_config_dir(config_dir: str) -> None:
+    """
+    Ensure the files from config dir are in JSON format.
+
+    It will walk recursively and convert any YAML files to the JSON format.
+
+    :param str config_dir: The config dir to walk recursively converting any YAML to JSON.
+    """
+    log.info("Enforcing JSON content on config_dir: %s", config_dir)
+    for dirpath, _, filenames in os.walk(config_dir):
+        for file in filenames:
+            in_file = os.path.join(dirpath, file)
+            if in_file.lower().endswith(".yaml"):
+                out_file = os.path.join(dirpath, f"{Path(in_file).stem}.json")
+                log.debug(f"Converting {in_file} to {out_file}.")
+                with open(in_file, 'r') as yaml_in, open(out_file, 'w') as json_out:
+                    data = yaml.load(yaml_in)
+                    json.dump(data, json_out)
+                os.remove(in_file)
diff --git a/iib/workers/tasks/opm_operations.py b/iib/workers/tasks/opm_operations.py
@@ -366,6 +366,7 @@ def opm_migrate(
 
     run_cmd(cmd, {'cwd': base_dir}, exc_msg='Failed to migrate index.db to file-based catalog')
     log.info("Migration to file-based catalog was completed.")
+    opm_validate(fbc_dir_path)
 
     if generate_cache:
         # Remove outdated cache before generating new one
@@ -819,57 +820,56 @@ def opm_registry_add_fbc_fragment(
     """
     set_request_state(request_id, 'in_progress', 'Extracting operator package from fbc_fragment')
     # fragment path will look like /tmp/iib-**/fbc-fragment
-    fragment_path, fragment_operator = extract_fbc_fragment(
+    fragment_path, fragment_operators = extract_fbc_fragment(
         temp_dir=temp_dir, fbc_fragment=fbc_fragment
     )
 
-    is_operator_in_db, index_db_path = verify_operator_exists(
+    # the dir where all the configs from from_index are stored
+    # this will look like /tmp/iib-**/configs
+    from_index_configs_dir = get_catalog_dir(from_index=from_index, base_dir=temp_dir)
+    log.info("The content of from_index configs located at %s", from_index_configs_dir)
+
+    operators_in_db, index_db_path = verify_operators_exists(
         from_index=from_index,
         base_dir=temp_dir,
-        operator_package=fragment_operator,
+        operator_packages=fragment_operators,
         overwrite_from_index_token=overwrite_from_index_token,
     )
 
-    # the dir where all the configs from from_index is stored
-    # this will look like /tmp/iib-**/configs
-    from_index_configs_dir = get_catalog_dir(from_index=from_index, base_dir=temp_dir)
+    if operators_in_db:
+        log.info('Removing %s from %s index.db ', operators_in_db, from_index)
+        _opm_registry_rm(index_db_path=index_db_path, operators=operators_in_db, base_dir=temp_dir)
 
-    log.info("The content of from_index configs located at %s", from_index_configs_dir)
-
-    if is_operator_in_db:
-        log.info('Removing %s from %s index.db ', fragment_operator, from_index)
-        _opm_registry_rm(
-            index_db_path=index_db_path, operators=[fragment_operator], base_dir=temp_dir
-        )
-        # migated_catalog_dir path will look like /tmp/iib-**/catalog
-        migated_catalog_dir, _ = opm_migrate(
+        # migrated_catalog_dir path will look like /tmp/iib-**/catalog
+        migrated_catalog_dir, _ = opm_migrate(
             index_db=index_db_path,
             base_dir=temp_dir,
             generate_cache=False,
         )
-        log.info("Migated catalog after removing from db at %s", migated_catalog_dir)
+        log.info("Migrated catalog after removing from db at %s", migrated_catalog_dir)
 
         # copy the content of migrated_catalog to from_index's config
-        log.info("Copying content of %s to %s", migated_catalog_dir, from_index_configs_dir)
-        for operator_package in os.listdir(migated_catalog_dir):
+        log.info("Copying content of %s to %s", migrated_catalog_dir, from_index_configs_dir)
+        for operator_package in os.listdir(migrated_catalog_dir):
             shutil.copytree(
-                os.path.join(migated_catalog_dir, operator_package),
+                os.path.join(migrated_catalog_dir, operator_package),
                 os.path.join(from_index_configs_dir, operator_package),
                 dirs_exist_ok=True,
             )
 
-    # copy fragment_operator to from_index configs
-    set_request_state(request_id, 'in_progress', 'Adding fbc_fragment to from_index')
-    fragment_opr_src_path = os.path.join(fragment_path, fragment_operator)
-    fragment_opr_dest_path = os.path.join(from_index_configs_dir, fragment_operator)
-    if os.path.exists(fragment_opr_dest_path):
-        shutil.rmtree(fragment_opr_dest_path)
-    log.info(
-        "Copying content of %s to %s",
-        fragment_opr_src_path,
-        fragment_opr_dest_path,
-    )
-    shutil.copytree(fragment_opr_src_path, fragment_opr_dest_path)
+    for fragment_operator in fragment_operators:
+        # copy fragment_operator to from_index configs
+        set_request_state(request_id, 'in_progress', 'Adding fbc_fragment to from_index')
+        fragment_opr_src_path = os.path.join(fragment_path, fragment_operator)
+        fragment_opr_dest_path = os.path.join(from_index_configs_dir, fragment_operator)
+        if os.path.exists(fragment_opr_dest_path):
+            shutil.rmtree(fragment_opr_dest_path)
+        log.info(
+            "Copying content of %s to %s",
+            fragment_opr_src_path,
+            fragment_opr_dest_path,
+        )
+        shutil.copytree(fragment_opr_src_path, fragment_opr_dest_path)
 
     local_cache_path = os.path.join(temp_dir, 'cache')
     generate_cache_locally(
@@ -886,32 +886,32 @@ def opm_registry_add_fbc_fragment(
     )
 
 
-def verify_operator_exists(
+def verify_operators_exists(
     from_index: str,
     base_dir: str,
-    operator_package: str,
+    operator_packages: List[str],
     overwrite_from_index_token: Optional[str],
 ):
     """
-    Check if operator exists in index image.
+    Check if operators exists in index image.
 
     :param str from_index: index in which operator existence is checked
     :param str base_dir: base temp directory for IIB request
-    :param str operator_package: operator_package to check
+    :param list(str) operator_packages: operator_package to check
     :param str overwrite_from_index_token: token used to access the image
-    :return: is_package_in_index, index_db_path
-    :rtype: (str, str)
+    :return: packages_in_index, index_db_path
+    :rtype: (list, str)
     """
     from iib.workers.tasks.build import terminate_process, get_bundle_json
     from iib.workers.tasks.iib_static_types import BundleImage
     from iib.workers.tasks.utils import run_cmd
     from iib.workers.tasks.utils import set_registry_token
 
-    is_package_in_index = False
+    packages_in_index: List[str] = []
 
-    log.info("Verifying if operator package %s exists in index %s", operator_package, from_index)
+    log.info("Verifying if operator packages %s exists in index %s", operator_packages, from_index)
 
-    # check if operater package exists in hidden index.db
+    # check if operator packages exists in hidden index.db
     with set_registry_token(overwrite_from_index_token, from_index, append=True):
         index_db_path = get_hidden_index_database(from_index=from_index, base_dir=base_dir)
 
@@ -924,10 +924,13 @@ def verify_operator_exists(
     present_bundles: List[BundleImage] = get_bundle_json(bundles)
 
     for bundle in present_bundles:
-        if bundle['packageName'] == operator_package:
-            is_package_in_index = True
-            log.info("operator package %s found in index_db %s", operator_package, index_db_path)
-    return is_package_in_index, index_db_path
+        if bundle['packageName'] in operator_packages:
+            packages_in_index.append(bundle['packageName'])
+
+    if packages_in_index:
+        log.info("operator packages found in index_db %s:  %s", index_db_path, packages_in_index)
+
+    return packages_in_index, index_db_path
 
 
 @retry(
@@ -1116,6 +1119,20 @@ def deprecate_bundles(
         run_cmd(cmd, {'cwd': base_dir}, exc_msg='Failed to deprecate the bundles')
 
 
+def opm_validate(config_dir: str) -> None:
+    """
+    Validate the declarative config files in a given directory.
+
+    :param str config_dir: directory containing the declarative config files.
+    :raises IIBError: if the validation fails
+    """
+    from iib.workers.tasks.utils import run_cmd
+
+    log.info("Validating files under %s", config_dir)
+    cmd = [Opm.opm_version, 'validate', config_dir]
+    run_cmd(cmd, exc_msg=f'Failed to validate the content from config_dir {config_dir}')
+
+
 class Opm:
     """A class to store the opm version for the IIB operation."""
 

diff --git a/requirements-test.txt b/requirements-test.txt
@@ -49,13 +49,13 @@ blinker==1.7.0 \
     # via
     #   -r requirements.txt
     #   flask
-boto3==1.34.58 \
-    --hash=sha256:09e3d17c718bc938a76774f31bc557b20733c0f5f9135a3e7782b55f3459cbdd \
-    --hash=sha256:d213a6fea9db6d537b1e65924133d8279ada79a40bc840d4930e1b64be869d4c
+boto3==1.34.85 \
+    --hash=sha256:135f1358fbc7d7dc89ad1a4346cb8da621fdc2aea69deb7b20c71ffec7cde111 \
+    --hash=sha256:de73d0f2dec1819074caf3f0888e18f6e13a9fb75ef5f17b1bdd9d1acc127b33
     # via -r requirements.txt
-botocore==1.34.79 \
-    --hash=sha256:6b59b0f7de219d383a2a633f6718c2600642ebcb707749dc6c67a6a436474b7a \
-    --hash=sha256:a42a014d3dbaa9ef123810592af69f9e55b456c5be3ac9efc037325685519e83
+botocore==1.34.85 \
+    --hash=sha256:18548525d4975bbe982f393f6470ba45249919a93f5dc6a69e37e435dd2cf579 \
+    --hash=sha256:9abae3f7925a8cc2b91b6ff3f09e631476c74826d45dc44fb30d1d15960639db
     # via
     #   -r requirements.txt
     #   boto3
@@ -542,9 +542,9 @@ gssapi==1.8.3 \
     # via
     #   -r requirements.txt
     #   pyspnego
-idna==3.6 \
-    --hash=sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca \
-    --hash=sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f
+idna==3.7 \
+    --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \
+    --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0
     # via
     #   -r requirements.txt
     #   requests

diff --git a/requirements.txt b/requirements.txt
@@ -40,13 +40,13 @@ blinker==1.7.0 \
     --hash=sha256:c3f865d4d54db7abc53758a01601cf343fe55b84c1de4e3fa910e420b438d5b9 \
     --hash=sha256:e6820ff6fa4e4d1d8e2747c2283749c3f547e4fee112b98555cdcdae32996182
     # via flask
-boto3==1.34.58 \
-    --hash=sha256:09e3d17c718bc938a76774f31bc557b20733c0f5f9135a3e7782b55f3459cbdd \
-    --hash=sha256:d213a6fea9db6d537b1e65924133d8279ada79a40bc840d4930e1b64be869d4c
+boto3==1.34.85 \
+    --hash=sha256:135f1358fbc7d7dc89ad1a4346cb8da621fdc2aea69deb7b20c71ffec7cde111 \
+    --hash=sha256:de73d0f2dec1819074caf3f0888e18f6e13a9fb75ef5f17b1bdd9d1acc127b33
     # via iib (setup.py)
-botocore==1.34.79 \
-    --hash=sha256:6b59b0f7de219d383a2a633f6718c2600642ebcb707749dc6c67a6a436474b7a \
-    --hash=sha256:a42a014d3dbaa9ef123810592af69f9e55b456c5be3ac9efc037325685519e83
+botocore==1.34.85 \
+    --hash=sha256:18548525d4975bbe982f393f6470ba45249919a93f5dc6a69e37e435dd2cf579 \
+    --hash=sha256:9abae3f7925a8cc2b91b6ff3f09e631476c74826d45dc44fb30d1d15960639db
     # via
     #   boto3
     #   s3transfer
@@ -450,9 +450,9 @@ gssapi==1.8.3 \
     --hash=sha256:edc8ef3a9e397dbe18bb6016f8e2209969677b534316d20bb139da2865a38efe \
     --hash=sha256:ee74b9211c977b9181ff4652d886d7712c9a221560752a35393b58e5ea07887a
     # via pyspnego
-idna==3.6 \
-    --hash=sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca \
-    --hash=sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f
+idna==3.7 \
+    --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \
+    --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0
     # via requests
 importlib-metadata==6.11.0 \
     --hash=sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443 \

diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 
 setup(
     name='iib',
-    version='8.6.0',
+    version='8.7.0',
     long_description=__doc__,
     packages=find_packages(exclude=['tests', 'tests.*']),
     include_package_data=True,