From fdf92432502d08263a7920b31b02d5396f9edff2 Mon Sep 17 00:00:00 2001 From: Janine Olear Date: Wed, 26 Jul 2023 11:56:22 +0200 Subject: [PATCH 1/3] add TransformerV2ListVersionByProvider Signed-off-by: Janine Olear --- .../transform/transform.py | 39 +++++++++++++++++++ src/cloudimagedirectory/transformer.py | 1 + 2 files changed, 40 insertions(+) diff --git a/src/cloudimagedirectory/transform/transform.py b/src/cloudimagedirectory/transform/transform.py index 88722478..1f6f2095 100644 --- a/src/cloudimagedirectory/transform/transform.py +++ b/src/cloudimagedirectory/transform/transform.py @@ -523,3 +523,42 @@ def run(self, data: type[Transformer]) -> list: # NOTE: Add /list suffix to prevent collision with "provider" folder. results.append(connection.DataEntry(f"v2/os/{os}/provider/list", provider_map)) return results + + +class TransformerV2ListVersionByProvider(Transformer): + """Generate a list for all available versions for a specific provider.""" + + @no_type_check + def run(self, data: type[Transformer]) -> list: + # TODO: check that its the v2 data entries. + entries = [x for x in data if x.is_API("v2")] + + results = [] + versions = {} + + for e in entries: + entry = copy.deepcopy(e) + filename = entry.filename.split("/") + os = filename[2] + provider = filename[4] + version = filename[6] + + if os not in versions: + versions[os] = {provider : {}} + + if provider not in versions[os]: + versions[os][provider] = {version : 1} + continue + + if version not in versions[os][provider]: + versions[os][provider][version] = 1 + continue + + # NOTE: Counter of how many images are available in this explicit provider. + versions[os][provider][version] += 1 + + for os, version_map in versions.items(): + for provider in version_map: + # NOTE: Add /list suffix to prevent collision with "version" folder. + results.append(connection.DataEntry(f"v2/os/{os}/provider/{provider}/version/list", version_map[provider])) + return results diff --git a/src/cloudimagedirectory/transformer.py b/src/cloudimagedirectory/transformer.py index 0d37a612..2766cbd9 100644 --- a/src/cloudimagedirectory/transformer.py +++ b/src/cloudimagedirectory/transformer.py @@ -99,6 +99,7 @@ def run(origin_path: str, destination_path: str, arg_files: str, filter_until: s transform.TransformerV2All, transform.TransformerV2ListOS, transform.TransformerV2ListProviderByOS, + transform.TransformerV2ListVersionByProvider, ], ) print("run pipeline v2") From 757d2a5135f10f26a15b1bcd502ae7fb9911c2ca Mon Sep 17 00:00:00 2001 From: Janine Olear Date: Wed, 26 Jul 2023 12:33:32 +0200 Subject: [PATCH 2/3] add tests for TransformerV2ListVersionByProvider Signed-off-by: Janine Olear --- tests/transformer/test_list_version.py | 86 +++++++++++++++++++ .../v2/os/rhel/provider/google/version/list | 1 + 2 files changed, 87 insertions(+) create mode 100644 tests/transformer/test_list_version.py create mode 100644 tests/transformer/testdata/expected/v2/os/rhel/provider/google/version/list diff --git a/tests/transformer/test_list_version.py b/tests/transformer/test_list_version.py new file mode 100644 index 00000000..66ad86da --- /dev/null +++ b/tests/transformer/test_list_version.py @@ -0,0 +1,86 @@ +"""Test for the list version endpoint.""" +import filecmp +import os + +from cloudimagedirectory import transformer + + +def test_V2ListVersion(runner, tmp_path): + """Run transformer end to end and generate a list with all available + versions for the specific provider.""" + result = runner.invoke( + transformer.run, + [ + "-f", + "tests/transformer/testdata/input/raw/google/all.json,tests/transformer/testdata/input/raw/aws/af-south-1.json,tests/transformer/testdata/input/raw/azure/eastus.json", + "-op=.", + f"-dp={tmp_path}", + "--filter.until=none", + ], + ) + + assert result.exit_code == 0, f"expected no error, but got code {result.exit_code} and output:\n{result.output}" + + # Get current directory + pwd = os.getcwd() + + # Check image data by comparing the expected file and the output file byte by byte. + assert filecmp.cmp( + f"{pwd}/tests/transformer/testdata/expected/v2/os/rhel/provider/google/version/list", + f"{tmp_path}/v2/os/rhel/provider/google/version/list", + ) + + +def test_transformerV2ListVersion(tmpdir): + """Test list version run method.""" + temporary_directory = tmpdir.mkdir("test") + connection = transformer.connection.ConnectionFS(temporary_directory, []) + runner = transformer.transform.TransformerV2ListVersionByProvider(connection) + chunk_size = 2 + runner.chunk_size = chunk_size + data = [ + transformer.connection.DataEntry( + "v2/os/rhel/provider/google/version/9/region/global/image/9054fbe0b622c638224d50d20824d2ff6782e308", + { + "date": "2023-03-06T12:57:17.827-08:00", + "name": "test2", + "arch": "ARM64", + "region": "global", + }, + ), + transformer.connection.DataEntry( + "v2/os/rhel/provider/google/version/8/region/global/image/dba7673010f19a94af4345453005933fd511bea9", + { + "date": "2019-01-01", + "name": "test1", + "arch": "arch1", + "region": "global", + }, + ), + transformer.connection.DataEntry( + "v2/os/rhel/provider/aws/version/8/region/ap-south-2/image/9054fbe0b622c638224d50d20824d2ff6782e308", + { + "date": "2020-01-01", + "name": "test2", + "arch": "arch2", + "region": "ap-south-2", + }, + ), + transformer.connection.DataEntry( + "v2/os/unkown/provider/aws/version/7/region/some-region-1/image/9054fbe0b622c638224d50d20824d2ff6782e308", + { + "date": "2020-01-01", + "name": "test2", + "arch": "arch2", + "region": "region-1", + }, + ), + ] + results = runner.run(data) + expected = transformer.connection.DataEntry( + "v2/os/rhel/provider/google/version/list", + {"8": 1, "9": 1}, + ) + + assert expected.filename == results[0].filename + assert expected.content == results[0].content diff --git a/tests/transformer/testdata/expected/v2/os/rhel/provider/google/version/list b/tests/transformer/testdata/expected/v2/os/rhel/provider/google/version/list new file mode 100644 index 00000000..7adc243a --- /dev/null +++ b/tests/transformer/testdata/expected/v2/os/rhel/provider/google/version/list @@ -0,0 +1 @@ +{"7": 1} From 0ac79b4241555f295d5df66c64e760b73ead542c Mon Sep 17 00:00:00 2001 From: Janine Olear Date: Wed, 26 Jul 2023 12:35:01 +0200 Subject: [PATCH 3/3] resolve linter suggestions Signed-off-by: Janine Olear --- src/cloudimagedirectory/transform/transform.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/cloudimagedirectory/transform/transform.py b/src/cloudimagedirectory/transform/transform.py index 1f6f2095..3954d270 100644 --- a/src/cloudimagedirectory/transform/transform.py +++ b/src/cloudimagedirectory/transform/transform.py @@ -530,7 +530,7 @@ class TransformerV2ListVersionByProvider(Transformer): @no_type_check def run(self, data: type[Transformer]) -> list: - # TODO: check that its the v2 data entries. + # NOTE: check that its the v2 data entries. entries = [x for x in data if x.is_API("v2")] results = [] @@ -544,10 +544,10 @@ def run(self, data: type[Transformer]) -> list: version = filename[6] if os not in versions: - versions[os] = {provider : {}} + versions[os] = {provider: {}} if provider not in versions[os]: - versions[os][provider] = {version : 1} + versions[os][provider] = {version: 1} continue if version not in versions[os][provider]: @@ -560,5 +560,7 @@ def run(self, data: type[Transformer]) -> list: for os, version_map in versions.items(): for provider in version_map: # NOTE: Add /list suffix to prevent collision with "version" folder. - results.append(connection.DataEntry(f"v2/os/{os}/provider/{provider}/version/list", version_map[provider])) + results.append( + connection.DataEntry(f"v2/os/{os}/provider/{provider}/version/list", version_map[provider]) + ) return results