Merge pull request ansible-collections#724 from tremble/boto3/redshif…

…t_subnet_group

Migrate redshift_subnet_group to boto3

SUMMARY
Migrate redshift_subnet_group to boto3
note: while there additional features (tagging springs to mind) that could be added, I'm trying to avoid scope creep and mostly just want to knock out migrations of the remaining old-boto modules.  That said, I am converting tags using the boto3_tag_list_to_ansible_dict helper, in the return values this is just to avoid needing to change formats in future
ISSUE TYPE

Feature Pull Request

COMPONENT NAME
redshift_subnet_group
ADDITIONAL INFORMATION
Depends-On: ansible-collections#719

Reviewed-by: Markus Bergholz <git@osuv.de>
Reviewed-by: Alina Buzachis <None>
Reviewed-by: None <None>

changelogs/fragments/724-redshift_subnet_group-boto3.yml

@@ -0,0 +1,7 @@
+minor_changes:
+- redshift_subnet_group - the module has been migrated to the boto3 AWS SDK (https://github.com/ansible-collections/community.aws/pull/724).
+- redshift_subnet_group - added support for check_mode (https://github.com/ansible-collections/community.aws/pull/724).
+- redshift_subnet_group - the ``group_description`` option has been renamed to ``description`` and is now optional.
+  The old parameter name will continue to work (https://github.com/ansible-collections/community.aws/pull/724).
+- redshift_subnet_group - the ``group_subnets`` option has been renamed to ``subnets`` and is now only required when creating a new group.
+  The old parameter name will continue to work (https://github.com/ansible-collections/community.aws/pull/724).

plugins/modules/redshift_subnet_group.py

@@ -9,8 +9,6 @@
 
 DOCUMENTATION = r'''
 ---
-author:
-  - "Jens Carl (@j-carl), Hothead Games Inc."
 module: redshift_subnet_group
 version_added: 1.0.0
 short_description: manage Redshift cluster subnet groups
@@ -19,34 +17,33 @@
 options:
   state:
     description:
-      - Specifies whether the subnet should be present or absent.
-    required: true
+      - Specifies whether the subnet group should be present or absent.
+    default: 'present'
     choices: ['present', 'absent' ]
     type: str
-  group_name:
+  name:
     description:
       - Cluster subnet group name.
     required: true
-    aliases: ['name']
+    aliases: ['group_name']
     type: str
-  group_description:
+  description:
     description:
-      - Database subnet group description.
-      - Required when I(state=present).
-    aliases: ['description']
+      - Cluster subnet group description.
+    aliases: ['group_description']
     type: str
-  group_subnets:
+  subnets:
     description:
       - List of subnet IDs that make up the cluster subnet group.
-      - Required when I(state=present).
-    aliases: ['subnets']
+      - At least one subnet must be provided when creating a cluster subnet group.
+    aliases: ['group_subnets']
     type: list
     elements: str
 extends_documentation_fragment:
 - amazon.aws.aws
 - amazon.aws.ec2
-requirements:
-- boto >= 2.49.0
+author:
+  - "Jens Carl (@j-carl), Hothead Games Inc."
 '''
 
 EXAMPLES = r'''
@@ -66,113 +63,209 @@
 '''
 
 RETURN = r'''
-group:
-    description: dictionary containing all Redshift subnet group information
+cluster_subnet_group:
+    description: A dictionary containing information about the Redshift subnet group.
     returned: success
-    type: complex
+    type: dict
     contains:
         name:
-            description: name of the Redshift subnet group
-            returned: success
+            description: Name of the Redshift subnet group.
+            returned: when the cache subnet group exists
             type: str
             sample: "redshift_subnet_group_name"
         vpc_id:
-            description: Id of the VPC where the subnet is located
-            returned: success
+            description: Id of the VPC where the subnet is located.
+            returned: when the cache subnet group exists
             type: str
             sample: "vpc-aabb1122"
+        description:
+            description: The description of the cache subnet group.
+            returned: when the cache subnet group exists
+            type: str
+            sample: Redshift subnet
+        subnet_ids:
+            description: The IDs of the subnets beloging to the Redshift subnet group.
+            returned: when the cache subnet group exists
+            type: list
+            elements: str
+            sample:
+              - subnet-aaaaaaaa
+              - subnet-bbbbbbbb
 '''
 
 try:
-    import boto
-    import boto.redshift
+    import botocore
 except ImportError:
-    pass  # Handled by HAS_BOTO
+    pass  # Handled by AnsibleAWSModule
+
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
 
 from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import HAS_BOTO
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import connect_to_aws
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import get_aws_connection_info
+from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict
+
+
+def get_subnet_group(name):
+    try:
+        groups = client.describe_cluster_subnet_groups(
+            aws_retry=True,
+            ClusterSubnetGroupName=name,
+        )['ClusterSubnetGroups']
+    except is_boto3_error_code('ClusterSubnetGroupNotFoundFault'):
+        return None
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
+        module.fail_json_aws(e, msg="Failed to describe subnet group")
+
+    if not groups:
+        return None
+
+    if len(groups) > 1:
+        module.fail_aws(
+            msg="Found multiple matches for subnet group",
+            cluster_subnet_groups=camel_dict_to_snake_dict(groups),
+        )
+
+    # No support for managing tags yet, but make sure that we don't need to
+    # change the return value structure after it's been available in a release.
+    tags = boto3_tag_list_to_ansible_dict(groups[0]['Tags'])
+
+    subnet_group = camel_dict_to_snake_dict(groups[0])
+
+    subnet_group['tags'] = tags
+    subnet_group['name'] = subnet_group['cluster_subnet_group_name']
+
+    subnet_ids = list(s['subnet_identifier'] for s in subnet_group['subnets'])
+    subnet_group['subnet_ids'] = subnet_ids
+
+    return subnet_group
+
+
+def create_subnet_group(name, description, subnets):
+
+    if not subnets:
+        module.fail_json(msg='At least one subnet must be provided when creating a subnet group')
+
+    if module.check_mode:
+        return True
+
+    try:
+        if not description:
+            description = name
+        client.create_cluster_subnet_group(
+            aws_retry=True,
+            ClusterSubnetGroupName=name,
+            Description=description,
+            SubnetIds=subnets,
+        )
+        return True
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Failed to create subnet group")
+
+
+def update_subnet_group(subnet_group, name, description, subnets):
+    update_params = dict()
+    if description and subnet_group['description'] != description:
+        update_params['Description'] = description
+    if subnets:
+        old_subnets = set(subnet_group['subnet_ids'])
+        new_subnets = set(subnets)
+        if old_subnets != new_subnets:
+            update_params['SubnetIds'] = list(subnets)
+
+    if not update_params:
+        return False
+
+    if module.check_mode:
+        return True
+
+    # Description is optional, SubnetIds is not
+    if 'SubnetIds' not in update_params:
+        update_params['SubnetIds'] = subnet_group['subnet_ids']
+
+    try:
+        client.modify_cluster_subnet_group(
+            aws_retry=True,
+            ClusterSubnetGroupName=name,
+            **update_params,
+        )
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Failed to update subnet group")
+
+    return True
+
+
+def delete_subnet_group(name):
+
+    if module.check_mode:
+        return True
+
+    try:
+        client.delete_cluster_subnet_group(
+            aws_retry=True,
+            ClusterSubnetGroupName=name,
+        )
+        return True
+    except is_boto3_error_code('ClusterSubnetGroupNotFoundFault'):
+        # AWS is "eventually consistent", cope with the race conditions where
+        # deletion hadn't completed when we ran describe
+        return False
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
+        module.fail_json_aws(e, msg="Failed to delete subnet group")
 
 
 def main():
     argument_spec = dict(
-        state=dict(required=True, choices=['present', 'absent']),
-        group_name=dict(required=True, aliases=['name']),
-        group_description=dict(required=False, aliases=['description']),
-        group_subnets=dict(required=False, aliases=['subnets'], type='list', elements='str'),
+        state=dict(default='present', choices=['present', 'absent']),
+        name=dict(required=True, aliases=['group_name']),
+        description=dict(required=False, aliases=['group_description']),
+        subnets=dict(required=False, aliases=['group_subnets'], type='list', elements='str'),
     )
-    module = AnsibleAWSModule(argument_spec=argument_spec, check_boto3=False)
 
-    if not HAS_BOTO:
-        module.fail_json(msg='boto v2.9.0+ required for this module')
+    global module
+    global client
+
+    module = AnsibleAWSModule(
+        argument_spec=argument_spec,
+        supports_check_mode=True,
+    )
 
     state = module.params.get('state')
-    group_name = module.params.get('group_name')
-    group_description = module.params.get('group_description')
-    group_subnets = module.params.get('group_subnets')
+    name = module.params.get('name')
+    description = module.params.get('description')
+    subnets = module.params.get('subnets')
 
-    if state == 'present':
-        for required in ('group_name', 'group_description', 'group_subnets'):
-            if not module.params.get(required):
-                module.fail_json(msg=str("parameter %s required for state='present'" % required))
-    else:
-        for not_allowed in ('group_description', 'group_subnets'):
-            if module.params.get(not_allowed):
-                module.fail_json(msg=str("parameter %s not allowed for state='absent'" % not_allowed))
+    client = module.client('redshift', retry_decorator=AWSRetry.jittered_backoff())
 
-    region, ec2_url, aws_connect_params = get_aws_connection_info(module)
-    if not region:
-        module.fail_json(msg=str("Region must be specified as a parameter, in EC2_REGION or AWS_REGION environment variables or in boto configuration file"))
+    subnet_group = get_subnet_group(name)
+    changed = False
 
-    # Connect to the Redshift endpoint.
-    try:
-        conn = connect_to_aws(boto.redshift, region, **aws_connect_params)
-    except boto.exception.JSONResponseError as e:
-        module.fail_json(msg=str(e))
+    if state == 'present':
+        if not subnet_group:
+            result = create_subnet_group(name, description, subnets)
+            changed |= result
+        else:
+            result = update_subnet_group(subnet_group, name, description, subnets)
+            changed |= result
+        subnet_group = get_subnet_group(name)
+    else:
+        if subnet_group:
+            result = delete_subnet_group(name)
+            changed |= result
+        subnet_group = None
 
-    try:
-        changed = False
-        exists = False
-        group = None
-
-        try:
-            matching_groups = conn.describe_cluster_subnet_groups(group_name, max_records=100)
-            exists = len(matching_groups) > 0
-        except boto.exception.JSONResponseError as e:
-            if e.body['Error']['Code'] != 'ClusterSubnetGroupNotFoundFault':
-                # if e.code != 'ClusterSubnetGroupNotFoundFault':
-                module.fail_json(msg=str(e))
-
-        if state == 'absent':
-            if exists:
-                conn.delete_cluster_subnet_group(group_name)
-                changed = True
+    compat_results = dict()
+    if subnet_group:
+        compat_results['group'] = dict(
+            name=subnet_group['name'],
+            vpc_id=subnet_group['vpc_id'],
+        )
 
-        else:
-            if not exists:
-                new_group = conn.create_cluster_subnet_group(group_name, group_description, group_subnets)
-                group = {
-                    'name': new_group['CreateClusterSubnetGroupResponse']['CreateClusterSubnetGroupResult']
-                    ['ClusterSubnetGroup']['ClusterSubnetGroupName'],
-                    'vpc_id': new_group['CreateClusterSubnetGroupResponse']['CreateClusterSubnetGroupResult']
-                    ['ClusterSubnetGroup']['VpcId'],
-                }
-            else:
-                changed_group = conn.modify_cluster_subnet_group(group_name, group_subnets, description=group_description)
-                group = {
-                    'name': changed_group['ModifyClusterSubnetGroupResponse']['ModifyClusterSubnetGroupResult']
-                    ['ClusterSubnetGroup']['ClusterSubnetGroupName'],
-                    'vpc_id': changed_group['ModifyClusterSubnetGroupResponse']['ModifyClusterSubnetGroupResult']
-                    ['ClusterSubnetGroup']['VpcId'],
-                }
-
-            changed = True
-
-    except boto.exception.JSONResponseError as e:
-        module.fail_json(msg=str(e))
-
-    module.exit_json(changed=changed, group=group)
+    module.exit_json(
+        changed=changed,
+        cluster_subnet_group=subnet_group,
+        **compat_results,
+    )
 
 
 if __name__ == '__main__':