diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 00000000..840bb2e1 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,28 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: pre-commit + +on: + pull_request: + push: + branches: [main, devel] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/publish_docs.yml b/.github/workflows/publish_docs.yml index 5edc636f..53e98007 100644 --- a/.github/workflows/publish_docs.yml +++ b/.github/workflows/publish_docs.yml @@ -19,8 +19,8 @@ name: Publish documentation on: push: branches: - - 'main' - + - 'main' + workflow_dispatch: jobs: diff --git a/.github/workflows/validate_pr.yml b/.github/workflows/validate_pr.yml index 87de5003..2af68cb0 100644 --- a/.github/workflows/validate_pr.yml +++ b/.github/workflows/validate_pr.yml @@ -21,7 +21,7 @@ on: branches: - 'release/**' - 'devel' - + jobs: validate: runs-on: ubuntu-latest @@ -34,7 +34,7 @@ jobs: with: python-version: '3.9' cache: 'pip' - + - name: Set up Ansible and Ansible collections and roles run: | sudo update-alternatives --install /usr/bin/python python $(which python3) 1 @@ -49,16 +49,16 @@ jobs: ansible --version ansible-galaxy collection list ansible-galaxy role list - + - name: Set up Ansible collection dependencies run: | ansible-builder introspect --write-pip final_python.txt --write-bindep final_bindep.txt /usr/share/ansible/collections [[ -f final_python.txt ]] && pip install -r final_python.txt || echo "No Python dependencies found." [[ -f final_bindep.txt ]] && bindep --file final_bindep.txt || echo "No system dependencies found." - + - name: Report installed Python dependencies run: pip freeze - + - name: Validate collection run: | pushd /usr/share/ansible/collections/ansible_collections/cloudera/cluster @@ -75,7 +75,7 @@ jobs: run: | mkdir -p ./pr echo $PR_NUMBER > ./pr/pr_number - + - name: Upload the PR number uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/validate_pr_docs.yml b/.github/workflows/validate_pr_docs.yml index b82112a5..0cd7c701 100644 --- a/.github/workflows/validate_pr_docs.yml +++ b/.github/workflows/validate_pr_docs.yml @@ -21,7 +21,7 @@ on: branches: - 'release/**' - 'devel' - + workflow_dispatch: jobs: @@ -31,4 +31,4 @@ jobs: with: antsibull-log-upload: true collection-namespace: cloudera - collection-name: cluster \ No newline at end of file + collection-name: cluster diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..9cda92b6 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,26 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5d849b35..5692abc7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ Thank you for considering contributions to the `cloudera.cluster` Ansible collec ## Submitting a pull request -You can start work on issues that are not yet part of a [Milestone](https://github.com/cloudera-labs/cloudera.cluster/milestones) -- anything in our issue tracker that isn't assigned to a Milestone is considered the [backlog](https://github.com/cloudera-labs/cloudera.cluster/issues?q=is%3Aopen+is%3Aissue+no%3Amilestone). +You can start work on issues that are not yet part of a [Milestone](https://github.com/cloudera-labs/cloudera.cluster/milestones) -- anything in our issue tracker that isn't assigned to a Milestone is considered the [backlog](https://github.com/cloudera-labs/cloudera.cluster/issues?q=is%3Aopen+is%3Aissue+no%3Amilestone). Before you start working, please announce that you want to do so by commenting on the issue. _([Create an issue](https://github.com/cloudera-labs/cloudera.cluster/issues/new?labels=enhancement) if there isn't one yet, and you can also check out our [Discussions](https://github.com/cloudera-labs/cloudera.cluster/discussions) for ideas.)_ We try to ensure that all active work is assigned to a Milestone in order to keep our backlog accurate. diff --git a/README.md b/README.md index 41a9a398..6cf98608 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ If you have any questions, want to chat about the collection's capabilities and ## API -See the [API documentation](https://cloudera-labs.github.io/cloudera.cluster/) for details for each plugin and role within the collection. +See the [API documentation](https://cloudera-labs.github.io/cloudera.cluster/) for details for each plugin and role within the collection. ## Roadmap @@ -105,7 +105,7 @@ The collection also requires the following Python libraries to operate its modul The collection's Python dependencies alone, _not_ the required Python libraries of its collection dependencies, are in `requirements.txt`. -All collection dependencies, required and optional, can be found in `requirements.yml`; only the _required_ dependencies are in `galaxy.yml`. `ansible-galaxy` will install only the _required_ collection dependencies; you will need to add the _optional_ collection dependencies as needed (see above). +All collection dependencies, required and optional, can be found in `requirements.yml`; only the _required_ dependencies are in `galaxy.yml`. `ansible-galaxy` will install only the _required_ collection dependencies; you will need to add the _optional_ collection dependencies as needed (see above). `ansible-builder` can discover and install all Python dependencies - current collection and dependencies - if you wish to use that application to construct your environment. Otherwise, you will need to read each collection and role dependency and follow its installation instructions. @@ -154,7 +154,7 @@ For example, here we use the To create a local collection tarball, run: ```bash -ansible-galaxy collection build +ansible-galaxy collection build ``` ## Building the API Documentation diff --git a/builder/requirements.yml b/builder/requirements.yml index 31595fcb..c0d8a994 100644 --- a/builder/requirements.yml +++ b/builder/requirements.yml @@ -23,4 +23,4 @@ roles: # geerlingguy.mysql with fix for issue #332 - src: https://github.com/dbeech/ansible-role-mysql - version: master \ No newline at end of file + version: master diff --git a/docs/inventories.md b/docs/inventories.md index 2245f26f..1b7542d5 100644 --- a/docs/inventories.md +++ b/docs/inventories.md @@ -13,7 +13,7 @@ host-1.example.com ### Cluster Nodes -A group named `cluster` is **required**. This is the set of nodes which will have a Cloudera Manager agent and receive pre-requisite configurations like OS tuning, database client and JDK installation and Kerberos configs. +A group named `cluster` is **required**. This is the set of nodes which will have a Cloudera Manager agent and receive pre-requisite configurations like OS tuning, database client and JDK installation and Kerberos configs. Usually, `cluster` will be composed of child groups like this: @@ -40,7 +40,7 @@ host-5.example.com host-6.example.com ``` -The names of the cluster sub-groups are arbitrary. These are only for convenience when assigning host template names and to make the inventory easier to read and understand. +The names of the cluster sub-groups are arbitrary. These are only for convenience when assigning host template names and to make the inventory easier to read and understand. #### Assigning Host Templates @@ -114,9 +114,9 @@ cdsw ### HDFS Encryption (KMS / Key Trustee Server) -Configuring HDFS encryption requires two extra groups `kts_active` and `kms_servers` with a third, optional (but **recommended**) group `kts_passive` to enable Key Trustee Server high availability. +Configuring HDFS encryption requires two extra groups `kts_active` and `kms_servers` with a third, optional (but **recommended**) group `kts_passive` to enable Key Trustee Server high availability. -The `kts_active` and `kts_passive` groups must contain a single node each. The KMS group `kms_servers` must have at least one host but can have as many as desired. +The `kts_active` and `kts_passive` groups must contain a single node each. The KMS group `kms_servers` must have at least one host but can have as many as desired. ```ini [kms_servers] @@ -181,10 +181,10 @@ host-10.example.com ## Multiple Clusters -It is possible to define multiple clusters in the inventory. The key point is that the `cluster` group must contain **all** servers which will be under Cloudera Manager's control, regardless of which cluster they belong to. +It is possible to define multiple clusters in the inventory. The key point is that the `cluster` group must contain **all** servers which will be under Cloudera Manager's control, regardless of which cluster they belong to. -The inventory group names `cluster1` and `cluster2` are arbitrary. This is just a convenience to make the inventory easier to understand. +The inventory group names `cluster1` and `cluster2` are arbitrary. This is just a convenience to make the inventory easier to understand. ```ini [cluster1] @@ -202,4 +202,4 @@ host-9.example.com host_template=Cluster2-Worker [cluster:children] cluster1 cluster2 -``` \ No newline at end of file +``` diff --git a/docs/security.md b/docs/security.md index 1a1f3204..a8a0fb07 100644 --- a/docs/security.md +++ b/docs/security.md @@ -43,7 +43,7 @@ tls=True ca-server-1.example.com ``` -3b) If you wish to manually sign certificates against an +3b) If you wish to manually sign certificates against an external CA, like Active Directory, add the path where signed certificates will be stored and root CA certificate details in `defintion.yml` ```yaml @@ -103,4 +103,4 @@ kms-3.example.com If you do not want Key Trustee Server to be highly available, you can omit the `kts_passive` group (this is **not** recommended). -The KMS group `kms_servers` must have at least one host but can have as many as desired. Unlike KTS, KMS nodes are all active and load balanced. \ No newline at end of file +The KMS group `kms_servers` must have at least one host but can have as many as desired. Unlike KTS, KMS nodes are all active and load balanced. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index ba95289e..6fdb9842 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -12,9 +12,9 @@ or fatal: [host.example.com]: UNREACHABLE! => {"changed": false, "msg": "SSH password was requested, but none specified", "unreachable": true} ``` -**Problem**: +**Problem**: - - Passwordless SSH connectivity is not correctly set up between your Ansible server and the remote servers in the inventory. + - Passwordless SSH connectivity is not correctly set up between your Ansible server and the remote servers in the inventory. **Solutions**: @@ -32,7 +32,7 @@ fatal: [host.example.com]: UNREACHABLE! => {"changed": false, "msg": "SSH passwo fatal: [host.example.com]: UNREACHABLE! => {"changed": false, "msg": "Host key checking is enabled, and SSH reported an unrecognized or mismatching host key.", "unreachable": true} ``` -**Problem**: +**Problem**: - SSH host key checking is enabled (Ansible default) but the servers you are trying to connect to are not present in `known_hosts` (or their key has changed) @@ -57,7 +57,7 @@ fatal: [host.example.com]: UNREACHABLE! => {"changed": false, "msg": "Host key c The cause is likely to be one of the following: -1) Inconsistent hostname resolution. The Cloudera Manager is reporting itself with a different hostname to that contained in your inventory file. +1) Inconsistent hostname resolution. The Cloudera Manager is reporting itself with a different hostname to that contained in your inventory file. 2) Cloudera Manager agent(s) are not heartbeating correctly. @@ -65,13 +65,13 @@ The cause is likely to be one of the following: By this stage of the playbook execution, Cloudera Manager server will be running. Log into Cloudera Manager and view the **Hosts** page: -- If hosts appear, check the list to ensure that the hostnames shown match your inventory file. If they do not match, either update your inventory file with these hostnames or update the cluster DNS so that the same names can be resolved consistently everywhere. +- If hosts appear, check the list to ensure that the hostnames shown match your inventory file. If they do not match, either update your inventory file with these hostnames or update the cluster DNS so that the same names can be resolved consistently everywhere. - If no hosts appear, log into the server indicated in the error message, and: - - Check that the `cloudera-manager-agent` service is running. + - Check that the `cloudera-manager-agent` service is running. - - Check the Cloudera Manager agent log file `/var/log/cloudera-scm-agent/cloudera-scm-agent.log`. Any error message there should give a clue as to why communication with the Cloudera Manager server is failing. + - Check the Cloudera Manager agent log file `/var/log/cloudera-scm-agent/cloudera-scm-agent.log`. Any error message there should give a clue as to why communication with the Cloudera Manager server is failing. ## Common issue #4 @@ -79,11 +79,11 @@ By this stage of the playbook execution, Cloudera Manager server will be running ERROR! couldn't resolve module/action 'cm_api'. This often indicates a misspelling, missing collection, or incorrect module path. ``` -**Problem**: +**Problem**: - The `cm_api` action is not available in a custom role. -**Solution**: +**Solution**: - Add the following into the role's `meta/main.yml` file. @@ -91,4 +91,4 @@ ERROR! couldn't resolve module/action 'cm_api'. This often indicates a misspelli --- dependencies: - role: cloudera_manager/api_client - ``` \ No newline at end of file + ``` diff --git a/docsbuild/.gitignore b/docsbuild/.gitignore index 12166213..8baa63af 100644 --- a/docsbuild/.gitignore +++ b/docsbuild/.gitignore @@ -3,4 +3,4 @@ /temp-rst /build -/rst \ No newline at end of file +/rst diff --git a/docsbuild/antsibull-docs.cfg b/docsbuild/antsibull-docs.cfg index 8f263ca6..3a72f36b 100644 --- a/docsbuild/antsibull-docs.cfg +++ b/docsbuild/antsibull-docs.cfg @@ -47,4 +47,4 @@ logging_cfg = { output_name = stderr } } -} \ No newline at end of file +} diff --git a/docsbuild/cloudera.css b/docsbuild/cloudera.css index c48334e7..676d9ca8 100644 --- a/docsbuild/cloudera.css +++ b/docsbuild/cloudera.css @@ -1,16 +1,15 @@ /** * Copyright 2024 Cloudera, Inc. - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - diff --git a/docsbuild/conf.py b/docsbuild/conf.py index 21534fa8..51992467 100644 --- a/docsbuild/conf.py +++ b/docsbuild/conf.py @@ -1,23 +1,22 @@ - # Created with antsibull-docs 2.3.1.post0 # This file only contains a selection of the most common options. For a full list see the # documentation: # http://www.sphinx-doc.org/en/master/config -project = 'cloudera.cluster' -copyright = 'Cloudera, Inc.' +project = "cloudera.cluster" +copyright = "Cloudera, Inc." -title = 'Cloudera Labs' -html_short_title = 'Cloudera Labs' +title = "Cloudera Labs" +html_short_title = "Cloudera Labs" -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx_antsibull_ext'] +extensions = ["sphinx.ext.autodoc", "sphinx.ext.intersphinx", "sphinx_antsibull_ext"] -pygments_style = 'ansible' +pygments_style = "ansible" -highlight_language = 'YAML+Jinja' +highlight_language = "YAML+Jinja" -html_theme = 'sphinx_ansible_theme' +html_theme = "sphinx_ansible_theme" html_show_sphinx = False display_version = False @@ -29,36 +28,40 @@ # See https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#confval-intersphinx_mapping for the syntax intersphinx_mapping = { - 'python': ('https://docs.python.org/2/', (None, '../python2.inv')), - 'python3': ('https://docs.python.org/3/', (None, '../python3.inv')), - 'jinja2': ('http://jinja.palletsprojects.com/', (None, '../jinja2.inv')), - 'ansible_devel': ('https://docs.ansible.com/ansible/devel/', (None, '../ansible_devel.inv')), + "python": ("https://docs.python.org/2/", (None, "../python2.inv")), + "python3": ("https://docs.python.org/3/", (None, "../python3.inv")), + "jinja2": ("http://jinja.palletsprojects.com/", (None, "../jinja2.inv")), + "ansible_devel": ( + "https://docs.ansible.com/ansible/devel/", + (None, "../ansible_devel.inv"), + ), # If you want references to resolve to a released Ansible version (say, `5`), uncomment and replace X by this version: # 'ansibleX': ('https://docs.ansible.com/ansible/X/', (None, '../ansibleX.inv')), } -default_role = 'any' +default_role = "any" nitpicky = True html_css_files = [ - 'css/cloudera.css', + "css/cloudera.css", ] -html_last_updated_fmt = '%b %d, %Y' +html_last_updated_fmt = "%b %d, %Y" html_theme_options = { - 'vcs_pageview_mode': 'edit', - 'documentation_home_url': 'https://github.com/cloudera-labs/', - 'topbar_links': { - 'Cloudera Data Platform (CDP)': 'https://www.cloudera.com/products/cloudera-data-platform.html', - 'Documentation': 'https://docs.cloudera.com/', - 'Downloads': 'https://www.cloudera.com/downloads.html', - 'Training': 'https://www.cloudera.com/about/training.html', - 'Certification': 'https://www.cloudera.com/about/training/certification.html', + "vcs_pageview_mode": "edit", + "documentation_home_url": "https://github.com/cloudera-labs/", + "topbar_links": { + "Cloudera Data Platform (CDP)": "https://www.cloudera.com/products/cloudera-data-platform.html", + "Documentation": "https://docs.cloudera.com/", + "Downloads": "https://www.cloudera.com/downloads.html", + "Training": "https://www.cloudera.com/about/training.html", + "Certification": "https://www.cloudera.com/about/training/certification.html", }, + "analytics_id": "G-YMR2P5DEWR", } html_content = { - 'display_github': 'True', + "display_github": "True", } diff --git a/meta/runtime.yml b/meta/runtime.yml index 3dc9b998..51339f2d 100644 --- a/meta/runtime.yml +++ b/meta/runtime.yml @@ -15,3 +15,23 @@ # limitations under the License. requires_ansible: ">=2.10" + +action_groups: + cm: + - cm_config_info + - cm_config + - cm_license_info + - cm_license + - cm_resource_info + - cm_resource + - cm_service_info + - cm_service + - cm_trial_license + - cm_version_info + - cm_endpoint_info + cluster: + - metadata: + extend_group: + - cm + - cluster_info + - cluster diff --git a/plugins/action/assemble_cluster_template.py b/plugins/action/assemble_cluster_template.py index 33c81b80..432cd830 100644 --- a/plugins/action/assemble_cluster_template.py +++ b/plugins/action/assemble_cluster_template.py @@ -29,14 +29,23 @@ from ansible.plugins.action import ActionBase from ansible.utils.hashing import checksum_s -from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ClusterTemplate +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + ClusterTemplate, +) + class ActionModule(ActionBase): TRANSFERS_FILES = True - - def __init__(self, task, connection, play_context, loader, templar, shared_loader_obj): - super().__init__(task, connection, play_context, loader, templar, shared_loader_obj) - self.TEMPLATE = ClusterTemplate(warn_fn=self._display.warning, error_fn=self._display.error) + + def __init__( + self, task, connection, play_context, loader, templar, shared_loader_obj + ): + super().__init__( + task, connection, play_context, loader, templar, shared_loader_obj + ) + self.TEMPLATE = ClusterTemplate( + warn_fn=self._display.warning, error_fn=self._display.error + ) self.MERGED = {} def assemble_fragments( @@ -67,7 +76,9 @@ def assemble_fragments( if not self.MERGED: self.MERGED = json.loads(fragment_file.read()) else: - self.TEMPLATE.merge(self.MERGED, json.loads(fragment_file.read())) + self.TEMPLATE.merge( + self.MERGED, json.loads(fragment_file.read()) + ) except json.JSONDecodeError as e: raise AnsibleActionFail( message=f"JSON parsing error: {to_text(e.msg)}", diff --git a/plugins/action/cm_api.py b/plugins/action/cm_api.py index 7465208b..658ab3a3 100644 --- a/plugins/action/cm_api.py +++ b/plugins/action/cm_api.py @@ -23,98 +23,134 @@ display = Display() -class ActionModule(ActionBase): - def build_url(self, api_base, api_endpoint): - if not api_endpoint.startswith("/"): - api_endpoint = "/" + api_endpoint - return api_base + api_endpoint - - def build_args(self, task_vars, additional_args=dict()): - args = dict( - body_format = "json", - force_basic_auth=True, - url_username=task_vars['cloudera_manager_api_user'], - url_password=task_vars['cloudera_manager_api_password'], - return_content=True, - validate_certs=task_vars['cloudera_manager_tls_validate_certs'] - ) - args.update(additional_args) - return args - - def get_api_base_url(self, task_vars): - # If there's already a value in task vars, just use that - if 'cloudera_manager_api' in task_vars: - api_base = task_vars['cloudera_manager_api'] - result = None - else: - # Call /api/version endpoint to find the correct API version number. - url = self.build_url(task_vars['cloudera_manager_url'], '/api/version') - args = self.build_args(task_vars, dict(url=url)) - result = self._execute_module('uri', module_args=args, task_vars=task_vars, wrap_async=self._task.async_val) - # We use the URL returned in the result rather than the one we defined originally. - # This has the benefit of allowing to use TLS-enabled endpoints later if the call was redirected. - api_base = result["url"].replace("version", result['content']) if result['status'] == 200 else None - return (api_base, result) - - def poll_command_status(self, task_vars, api_base_url, command_id): - args = self.build_args(task_vars, additional_args=dict( - url = self.build_url(api_base_url, "/commands/" + str(command_id)) - )) - result = self._execute_module('uri', module_args=args, task_vars=task_vars, wrap_async=self._task.async_val) - return result - - def run(self, tmp=None, task_vars=None): - - result = super(ActionModule, self).run(tmp, task_vars) - - # Get Cloudera Manager API base url from task vars, or work it out ourselves - api_base_url, api_base_result = self.get_api_base_url(task_vars) - if not api_base_url: - result.update(api_base_result) - return result - - # Add endpoint and request method to base args containing creds etc - uri_module_args = self.build_args(task_vars, additional_args=dict( - url = self.build_url(api_base_url, self._task.args['endpoint']), - method = self._task.args.get('method') or 'GET', - status_code = self._task.args.get('status_code') or '200', - timeout = self._task.args.get('timeout') or '30' - )) - - poll_duration = int(self._task.args.get('poll_duration') or 10) - poll_max_failed_retries = int(self._task.args.get('poll_max_failed_retries') or 3) - - # Add request body if necessary - if 'body' in self._task.args: - uri_module_args.update(body = self._task.args['body']) - - # Send request to CM API - uri_result = self._execute_module('uri', module_args=uri_module_args, task_vars=task_vars, wrap_async=self._task.async_val) - result.update(uri_result) - - # If we get ApiCommand response, and it is active, wait for completion - failed_polls = 0 - if 'json' in uri_result: - response = uri_result['json'] - if 'id' in response and 'active' in response: - command_id = response['id'] - command_name = response['name'] - command_active = response['active'] - while command_active and failed_polls < poll_max_failed_retries: - time.sleep(poll_duration) - display.vv("Waiting for {} command ({}) to complete...".format(command_name, command_id)) - command_status = self.poll_command_status(task_vars, api_base_url, command_id) - if 'json' in command_status: - failed_polls = 0 - response = command_status['json'] - command_active = response['active'] - else: - failed_polls += 1 - response = {'success': False} - display.vv("Failed to poll command ({}) for status (attempt {} of {})...".format( - command_id, failed_polls, poll_max_failed_retries)) - result.update(command_status) - result['failed'] = not response['success'] - - return result +class ActionModule(ActionBase): + def build_url(self, api_base, api_endpoint): + if not api_endpoint.startswith("/"): + api_endpoint = "/" + api_endpoint + return api_base + api_endpoint + + def build_args(self, task_vars, additional_args=dict()): + args = dict( + body_format="json", + force_basic_auth=True, + url_username=task_vars["cloudera_manager_api_user"], + url_password=task_vars["cloudera_manager_api_password"], + return_content=True, + validate_certs=task_vars["cloudera_manager_tls_validate_certs"], + ) + args.update(additional_args) + return args + + def get_api_base_url(self, task_vars): + # If there's already a value in task vars, just use that + if "cloudera_manager_api" in task_vars: + api_base = task_vars["cloudera_manager_api"] + result = None + else: + # Call /api/version endpoint to find the correct API version number. + url = self.build_url(task_vars["cloudera_manager_url"], "/api/version") + args = self.build_args(task_vars, dict(url=url)) + result = self._execute_module( + "uri", + module_args=args, + task_vars=task_vars, + wrap_async=self._task.async_val, + ) + # We use the URL returned in the result rather than the one we defined originally. + # This has the benefit of allowing to use TLS-enabled endpoints later if the call was redirected. + api_base = ( + result["url"].replace("version", result["content"]) + if result["status"] == 200 + else None + ) + return (api_base, result) + + def poll_command_status(self, task_vars, api_base_url, command_id): + args = self.build_args( + task_vars, + additional_args=dict( + url=self.build_url(api_base_url, "/commands/" + str(command_id)) + ), + ) + result = self._execute_module( + "uri", + module_args=args, + task_vars=task_vars, + wrap_async=self._task.async_val, + ) + return result + + def run(self, tmp=None, task_vars=None): + + result = super(ActionModule, self).run(tmp, task_vars) + + # Get Cloudera Manager API base url from task vars, or work it out ourselves + api_base_url, api_base_result = self.get_api_base_url(task_vars) + if not api_base_url: + result.update(api_base_result) + return result + + # Add endpoint and request method to base args containing creds etc + uri_module_args = self.build_args( + task_vars, + additional_args=dict( + url=self.build_url(api_base_url, self._task.args["endpoint"]), + method=self._task.args.get("method") or "GET", + status_code=self._task.args.get("status_code") or "200", + timeout=self._task.args.get("timeout") or "30", + ), + ) + + poll_duration = int(self._task.args.get("poll_duration") or 10) + poll_max_failed_retries = int( + self._task.args.get("poll_max_failed_retries") or 3 + ) + + # Add request body if necessary + if "body" in self._task.args: + uri_module_args.update(body=self._task.args["body"]) + + # Send request to CM API + uri_result = self._execute_module( + "uri", + module_args=uri_module_args, + task_vars=task_vars, + wrap_async=self._task.async_val, + ) + result.update(uri_result) + + # If we get ApiCommand response, and it is active, wait for completion + failed_polls = 0 + if "json" in uri_result: + response = uri_result["json"] + if "id" in response and "active" in response: + command_id = response["id"] + command_name = response["name"] + command_active = response["active"] + while command_active and failed_polls < poll_max_failed_retries: + time.sleep(poll_duration) + display.vv( + "Waiting for {} command ({}) to complete...".format( + command_name, command_id + ) + ) + command_status = self.poll_command_status( + task_vars, api_base_url, command_id + ) + if "json" in command_status: + failed_polls = 0 + response = command_status["json"] + command_active = response["active"] + else: + failed_polls += 1 + response = {"success": False} + display.vv( + "Failed to poll command ({}) for status (attempt {} of {})...".format( + command_id, failed_polls, poll_max_failed_retries + ) + ) + result.update(command_status) + result["failed"] = not response["success"] + + return result diff --git a/plugins/doc_fragments/cm_endpoint.py b/plugins/doc_fragments/cm_endpoint.py index b67f2f3c..e1d16143 100644 --- a/plugins/doc_fragments/cm_endpoint.py +++ b/plugins/doc_fragments/cm_endpoint.py @@ -15,8 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. + class ModuleDocFragment(object): - DOCUMENTATION = r''' + DOCUMENTATION = r""" options: url: description: @@ -27,4 +28,4 @@ class ModuleDocFragment(object): aliases: - endpoint - cm_endpoint_url - ''' + """ diff --git a/plugins/doc_fragments/cm_resource.py b/plugins/doc_fragments/cm_resource.py index 2fae2671..6485618e 100644 --- a/plugins/doc_fragments/cm_resource.py +++ b/plugins/doc_fragments/cm_resource.py @@ -15,8 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. + class ModuleDocFragment(object): - DOCUMENTATION = r''' + DOCUMENTATION = r""" options: path: description: @@ -38,4 +39,4 @@ class ModuleDocFragment(object): default: 'items' aliases: - return_field - ''' + """ diff --git a/plugins/doc_fragments/message.py b/plugins/doc_fragments/message.py index 00fc6697..75ad35f0 100644 --- a/plugins/doc_fragments/message.py +++ b/plugins/doc_fragments/message.py @@ -15,8 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. + class ModuleDocFragment(object): - DOCUMENTATION = r''' + DOCUMENTATION = r""" options: message: description: @@ -26,4 +27,4 @@ class ModuleDocFragment(object): default: "Managed by Ansible" aliases: - msg - ''' + """ diff --git a/plugins/doc_fragments/purge.py b/plugins/doc_fragments/purge.py index e4d27592..05027563 100644 --- a/plugins/doc_fragments/purge.py +++ b/plugins/doc_fragments/purge.py @@ -15,8 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. + class ModuleDocFragment(object): - DOCUMENTATION = r''' + DOCUMENTATION = r""" options: purge: description: @@ -24,4 +25,4 @@ class ModuleDocFragment(object): type: bool required: False default: False - ''' + """ diff --git a/plugins/filter/filters.py b/plugins/filter/filters.py index 6927e396..5eb17638 100644 --- a/plugins/filter/filters.py +++ b/plugins/filter/filters.py @@ -23,30 +23,29 @@ class FilterModule(object): - def filters(self): return { - 'flatten_dict_list': self.flatten_dict_list, - 'extract_custom_roles': self.extract_custom_roles, - 'extract_custom_role_groups': self.extract_custom_role_groups, - 'extract_products_from_manifests': self.extract_products_from_manifests, - 'extract_role_and_group': self.extract_role_and_group, - 'format_database_type': self.format_database_type, - 'get_product_version': self.get_product_version, - 'get_major_version': self.get_major_version, # Unused - 'append_database_port': self.append_database_port, - 'default_database_port': self.default_database_port, - 'get_database_encoding_mysql': self.get_database_encoding_mysql, - 'get_database_collation_mysql': self.get_database_collation_mysql, - 'filter_null_configs': self.filter_null_configs, - 'to_ldap_type_enum': self.to_ldap_type_enum, - 'extract_parcel_urls': self.extract_parcel_urls, - 'cluster_service_role_hosts': self.cluster_service_role_hosts, - 'find_clusters': self.find_clusters + "flatten_dict_list": self.flatten_dict_list, + "extract_custom_roles": self.extract_custom_roles, + "extract_custom_role_groups": self.extract_custom_role_groups, + "extract_products_from_manifests": self.extract_products_from_manifests, + "extract_role_and_group": self.extract_role_and_group, + "format_database_type": self.format_database_type, + "get_product_version": self.get_product_version, + "get_major_version": self.get_major_version, # Unused + "append_database_port": self.append_database_port, + "default_database_port": self.default_database_port, + "get_database_encoding_mysql": self.get_database_encoding_mysql, + "get_database_collation_mysql": self.get_database_collation_mysql, + "filter_null_configs": self.filter_null_configs, + "to_ldap_type_enum": self.to_ldap_type_enum, + "extract_parcel_urls": self.extract_parcel_urls, + "cluster_service_role_hosts": self.cluster_service_role_hosts, + "find_clusters": self.find_clusters, } - def flatten_dict_list(self, item, level=2, sep='_', show_index=False): - """ flatten a structure of dicts and lists into a flat array + def flatten_dict_list(self, item, level=2, sep="_", show_index=False): + """flatten a structure of dicts and lists into a flat array e.g. { "a": [1, 2, 3], "b": { "c": "d", "e": "f" } } with level=2 @@ -78,7 +77,9 @@ def _flatten_dict_list(i, l, parents): return state - def extract_products_from_manifests(self, manifests, os_distribution: Optional[str] = None): + def extract_products_from_manifests( + self, manifests, os_distribution: Optional[str] = None + ): products = dict() for manifest in manifests: for parcel in manifest["parcels"]: @@ -87,7 +88,7 @@ def extract_products_from_manifests(self, manifests, os_distribution: Optional[s # the parcel OS distribution is between the last "-" and the ".parcel" extension parcel_os_distribution = full_parcel_name[ full_parcel_name.rindex("-") - + 1: full_parcel_name.rindex(".parcel") + + 1 : full_parcel_name.rindex(".parcel") ] # take first parcel, strip off OS name and file extension parcel_name = re.sub(r"-[a-z0-9]+\.parcel$", "", full_parcel_name) @@ -97,17 +98,17 @@ def extract_products_from_manifests(self, manifests, os_distribution: Optional[s os_distribution == parcel_os_distribution or os_distribution is None ): # the version string is everything after the first dash - version = parcel_name[parcel_name.index("-") + 1:] + version = parcel_name[parcel_name.index("-") + 1 :] products[product] = version return products def extract_parcel_urls(self, manifest_results): parcels = list() for result in manifest_results: - manifest_url = result['invocation']['module_args']['url'] - base_url = '/'.join(manifest_url.rsplit('/')[:-1]) - parcel_names = [x['parcelName'] for x in result['json']['parcels']] - parcels += ['/'.join([str(base_url), str(y)]) for y in parcel_names] + manifest_url = result["invocation"]["module_args"]["url"] + base_url = "/".join(manifest_url.rsplit("/")[:-1]) + parcel_names = [x["parcelName"] for x in result["json"]["parcels"]] + parcels += ["/".join([str(base_url), str(y)]) for y in parcel_names] return parcels def format_database_type(self, database_type): @@ -117,14 +118,14 @@ def format_database_type(self, database_type): def get_product_version(self, products, product_name): for product in products: - if product['product'] == product_name: - version = product['version'] - return version[:version.index('-')] if "-" in version else version + if product["product"] == product_name: + version = product["version"] + return version[: version.index("-")] if "-" in version else version def get_major_version(self, products, product_name): version = self.get_product_version(products, product_name) if version: - return version.split('.')[0] + return version.split(".")[0] def append_database_port(self, database_host, database_port=None): if ":" not in database_host and database_port: @@ -171,8 +172,8 @@ def to_ldap_type_enum(self, s): def cluster_service_role_hosts(self, cluster, hostvars, service, roles=None): candidate_templates = [] - if 'host_templates' in cluster: - templates = cluster['host_templates'] + if "host_templates" in cluster: + templates = cluster["host_templates"] if roles: for role in roles: @@ -192,24 +193,22 @@ def cluster_service_role_hosts(self, cluster, hostvars, service, roles=None): host for host, hostvar in hostvars.items() if host not in hosts - if hostvar.get('host_template') == t_name] + if hostvar.get("host_template") == t_name + ] hosts = hosts + t_hosts return hosts def find_clusters(self, clusters, name): - return [ - cluster - for cluster in clusters - if cluster.get('name') == name] + return [cluster for cluster in clusters if cluster.get("name") == name] def extract_role_and_group(self, role_spec): role = None template_group = "BASE" - if '/' in role_spec: - role = role_spec[:role_spec.index('/')] - template_group = role_spec[role_spec.index('/')+1:] + if "/" in role_spec: + role = role_spec[: role_spec.index("/")] + template_group = role_spec[role_spec.index("/") + 1 :] else: role = role_spec return (role, template_group) @@ -218,7 +217,7 @@ def extract_custom_roles(self, host_templates, service): custom_roles = set([]) for role_mapping in host_templates.values(): if service in role_mapping: - for custom_role in filter(lambda x: '/' in x, role_mapping[service]): + for custom_role in filter(lambda x: "/" in x, role_mapping[service]): custom_roles.add(custom_role) return list(custom_roles) @@ -226,6 +225,8 @@ def extract_custom_role_groups(self, host_templates): custom_role_groups = set([]) for role_mapping in host_templates.values(): for (service, roles) in role_mapping.items(): - for custom_role in filter(lambda x: '/' in x, roles): - custom_role_groups.add("-".join([service.lower()] + custom_role.split("/"))) + for custom_role in filter(lambda x: "/" in x, roles): + custom_role_groups.add( + "-".join([service.lower()] + custom_role.split("/")) + ) return list(custom_role_groups) diff --git a/plugins/lookup/cm_service.py b/plugins/lookup/cm_service.py index 7b054ba9..657a0ce6 100644 --- a/plugins/lookup/cm_service.py +++ b/plugins/lookup/cm_service.py @@ -89,7 +89,7 @@ agent_header: description: Header string to identify the connection. type: string - default: cm_service + default: cm_service notes: - Requires C(cm_client). """ diff --git a/plugins/module_utils/cm_controller_utils.py b/plugins/module_utils/cm_controller_utils.py index da7b2be0..2d53f274 100644 --- a/plugins/module_utils/cm_controller_utils.py +++ b/plugins/module_utils/cm_controller_utils.py @@ -2,13 +2,13 @@ # -*- coding: utf-8 -*- # Copyright 2023 Cloudera, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,6 +42,7 @@ A common Ansible Lookup plugin for API access to Cloudera Manager. """ + class ClouderaManagerLookupBase(LookupBase): def initialize_client(self): # Set up core CM API client parameters diff --git a/plugins/module_utils/cm_utils.py b/plugins/module_utils/cm_utils.py index 4cdc23c9..17000e72 100644 --- a/plugins/module_utils/cm_utils.py +++ b/plugins/module_utils/cm_utils.py @@ -35,6 +35,7 @@ ApiClient, ApiCommand, ApiConfigList, + ApiParcel, ApiRole, ApiRoleConfigGroup, ApiService, @@ -91,6 +92,16 @@ # "service_ref", ] +PARCEL = [ + "product", + "version", + "stage", + # "cluster_ref", + "state", + "display_name", + "description", +] + def _parse_output(entity: dict, filter: list) -> dict: output = {} @@ -121,13 +132,20 @@ def parse_role_result(role: ApiRole) -> dict: return output -def parse_role_config_group_result(role_config_group: ApiRoleConfigGroup): +def parse_role_config_group_result(role_config_group: ApiRoleConfigGroup) -> dict: # Retrieve only the service identifier output = dict(service_name=role_config_group.service_ref.service_name) output.update(_parse_output(role_config_group.to_dict(), ROLE_CONFIG_GROUP)) return output +def parse_parcel_result(parcel: ApiParcel) -> dict: + # Retrieve only the cluster identifier + output = dict(cluster_name=parcel.cluster_ref.cluster_name) + output.update(_parse_output(parcel.to_dict(), PARCEL)) + return output + + def normalize_values(add: dict) -> dict: """Normalize whitespace of parameter values. diff --git a/plugins/module_utils/parcel_utils.py b/plugins/module_utils/parcel_utils.py new file mode 100644 index 00000000..e01319e2 --- /dev/null +++ b/plugins/module_utils/parcel_utils.py @@ -0,0 +1,120 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A common functions for Cloudera Manager parcel management +""" + +import time + +from enum import IntEnum + +from cm_client import ParcelResourceApi + + +class Parcel(object): + + STAGE = IntEnum( + "STAGE", + "AVAILABLE_REMOTELY DOWNLOADING DOWNLOADED UNDISTRIBUTING DISTRIBUTING DISTRIBUTED ACTIVATING ACTIVATED", + ) + + def __init__( + self, + parcel_api: ParcelResourceApi, + product: str, + version: str, + cluster: str, + delay: int = 15, + timeout: int = 600, + ) -> None: + self.parcel_api = parcel_api + self.product = product + self.version = version + self.cluster = cluster + self.delay = delay + self.timeout = timeout + + self.current = Parcel.STAGE[ + str( + self.parcel_api.read_parcel( + cluster_name=self.cluster, + product=self.product, + version=self.version, + ).stage + ).upper() + ] + + @property + def stage(self) -> str: + return self.current.name + + def _wait(self, stage: STAGE) -> None: + end_time = time.time() + self.timeout + + while end_time > time.time(): + parcel_status = self.parcel_api.read_parcel( + cluster_name=self.cluster, product=self.product, version=self.version + ) + if parcel_status.stage == stage.name: + return + else: + time.sleep(self.delay) + + return Exception(f"Failed to reach {stage.name}: timeout ({self.timeout} secs)") + + def _exec(self, stage: STAGE, func) -> None: + func( + cluster_name=self.cluster, + product=self.product, + version=self.version, + ) + self._wait(stage) + + def remove(self): + if self.current > self.STAGE.AVAILABLE_REMOTELY: + self.download(self.STAGE.AVAILABLE_REMOTELY) + self._exec( + self.STAGE.AVAILABLE_REMOTELY, self.parcel_api.remove_download_command + ) + + def download(self, target: STAGE = STAGE.DOWNLOADED): + if self.current > target: + self.distribute(target) + self._exec( + self.STAGE.DOWNLOADED, + self.parcel_api.start_removal_of_distribution_command, + ) + elif self.current == self.STAGE.DOWNLOADING: + self._wait(self.STAGE.DOWNLOADED) + elif self.current < self.STAGE.DOWNLOADING: + self._exec(self.STAGE.DOWNLOADED, self.parcel_api.start_download_command) + + def distribute(self, target: STAGE = STAGE.DISTRIBUTED): + if self.current > target: + self._exec(self.STAGE.DISTRIBUTED, self.parcel_api.deactivate_command) + elif self.current == self.STAGE.DISTRIBUTING: + self._wait(self.STAGE.DISTRIBUTED) + elif self.current < self.STAGE.DISTRIBUTING: + self.download(target) + self._exec( + self.STAGE.DISTRIBUTED, self.parcel_api.start_distribution_command + ) + + def activate(self): + if self.current == self.STAGE.ACTIVATING: + self._wait(self.STAGE.ACTIVATED) + elif self.current < self.STAGE.ACTIVATED: + self.distribute(self.STAGE.ACTIVATED) + self._exec(self.STAGE.ACTIVATED, self.parcel_api.activate_command) diff --git a/plugins/modules/assemble_cluster_template.py b/plugins/modules/assemble_cluster_template.py index 857ef8f4..0acfad66 100644 --- a/plugins/modules/assemble_cluster_template.py +++ b/plugins/modules/assemble_cluster_template.py @@ -25,8 +25,8 @@ description: - Merge multiple Cloudera Manager cluster template files into a single cluster template file. - Often a cluster template file is composed of several services, host templates, - and other parameters from multiple sources and/or configurations. - M(cloudera.cluster.assemble_cluster_template) will take a directory of + and other parameters from multiple sources and/or configurations. + M(cloudera.cluster.assemble_cluster_template) will take a directory of cluster template configuration files that can be local or have already been transferred to the system and merge them together to produce a single, composite cluster template configuration file. @@ -123,7 +123,7 @@ src: examples dest: /opt/cloudera/cluster-template.json regexp: "base|nifi" - + - name: Assemble a cluster template from files on the target host cloudera.cluster.assemble_cluster_template: src: /tmp/examples @@ -193,11 +193,11 @@ def assemble_fragments(self, assembled_file): with open(fragment, "r", encoding="utf-8") as fragment_file: try: if not self.merged: - self.merged = json.loads(fragment_file.read()) + self.merged = json.loads(fragment_file.read()) else: - self.template.merge( - self.merged, json.loads(fragment_file.read()) - ) + self.template.merge( + self.merged, json.loads(fragment_file.read()) + ) except json.JSONDecodeError as e: self.module.fail_json( msg=f"JSON parsing error for file, {fragment}: {to_text(e.msg)}", diff --git a/plugins/modules/cluster.py b/plugins/modules/cluster.py new file mode 100644 index 00000000..e24c148f --- /dev/null +++ b/plugins/modules/cluster.py @@ -0,0 +1,909 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from ansible.module_utils.common.text.converters import to_text, to_native + +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + ClouderaManagerModule, + ClusterTemplate, +) + +from ansible_collections.cloudera.cluster.plugins.module_utils.parcel_utils import ( + Parcel, +) + +from cm_client import ( + ApiCluster, + ApiClusterList, + ApiClusterTemplate, + ApiConfig, + ApiConfigList, + ApiDataContext, + ApiHostRef, + ApiHostRefList, + ApiHostTemplate, + ApiHostTemplateList, + ApiRole, + ApiRoleList, + ApiRoleConfigGroup, + ApiRoleConfigGroupRef, + ApiRoleNameList, + ApiService, + ApiServiceConfig, + ClouderaManagerResourceApi, + ClustersResourceApi, + HostsResourceApi, + HostTemplatesResourceApi, + ParcelResourceApi, + ServicesResourceApi, + RoleConfigGroupsResourceApi, + RolesResourceApi, +) +from cm_client.rest import ApiException + +ANSIBLE_METADATA = { + "metadata_version": "1.1", + "status": ["preview"], + "supported_by": "community", +} + +DOCUMENTATION = r""" +--- +module: cluster +short_description: Manage the lifecycle and state of a cluster +description: + - Enables cluster management, cluster creation, deletion, and unified control of all services of a cluster. + - Create or delete cluster in Cloudera Manager. + - Start or stop all services inside the cluster. + - If provided the C(template) parameter, the module will create a cluster based on the template. +author: + - "Ronald Suplina (@rsuplina)" + - "Webster Mudge (@wmudge)" +requirements: + - cm_client +""" + +EXAMPLES = r""" +--- +- name: Create an ECS cluster + cloudera.cluster.cluster: + host: example.cloudera.com + username: "jane_smith" + password: "S&peR4Ec*re" + port: 7180 + clusterName: example-cluster + cluster_version: "1.5.1-b626.p0.42068229" + cluster_type: EXPERIENCE_CLUSTER + state: present + +- name: Create a cluster from a cluster template + cloudera.cluster.cm_cluster: + host: example.cloudera.com + username: "jane_smith" + password: "S&peR4Ec*re" + port: "7180" + clusterName: example-cluster + template: "./files/cluster-template.json" + add_repositories: yes + +- name: Start all services on a cluster + cloudera.cluster.cluster: + host: example.cloudera.com + port: "7180" + username: "jane_smith" + password: "S&peR4Ec*re" + clusterName: example-cluster + state: started + +- name: Delete a Cluster + cloudera.cluster.cm_cluster: + host: example.cloudera.com + port: "7180" + username: "jane_smith" + password: "S&peR4Ec*re" + clusterName: example-cluster + state: absent +""" + +RETURN = r""" +--- +cloudera_manager: + description: Details about Cloudera Manager Cluster + type: dict + contains: + cluster_type: + description: The type of cluster created from template. + type: str + returned: optional + cluster_url: + description: Url of Cloudera Manager cluster. + type: str + returned: optional + display_name: + description: The name of the cluster displayed on the site. + type: str + returned: optional + entity_status: + description: Health status of the cluster. + type: str + returned: optional + full_version: + description: Version of the cluster installed. + type: str + returned: optional + hosts_url: + description: Url of all the hosts on which cluster is installed. + type: str + returned: optional + maintenance_mode: + description: Maintance mode of Cloudera Manager Cluster. + type: bool + returned: optional + maintenance_owners: + description: List of Maintance owners for Cloudera Manager Cluster. + type: list + returned: optional + name: + description: The name of the cluster created. + type: str + returned: optional + tags: + description: List of tags for Cloudera Manager Cluster. + type: list + returned: optional + uuid: + description: Unique ID of created cluster + type: bool + returned: optional +""" + + +class ClouderaCluster(ClouderaManagerModule): + def __init__(self, module): + super(ClouderaCluster, self).__init__(module) + + self.name = self.get_param("name") + self.cluster_version = self.get_param("cluster_version") + self.type = self.get_param("type") + self.state = self.get_param("state") + self.template = self.get_param("template") + self.add_repositories = self.get_param("add_repositories") + self.maintenance = self.get_param("maintenance") + self.hosts = self.get_param("hosts") + self.host_templates = self.get_param("host_templates") + self.services = self.get_param("services") + self.parcels = self.get_param("parcels") + self.tags = self.get_param("tags") + self.display_name = self.get_param("display_name") + self.contexts = self.get_param("contexts") + self.auto_assign = self.get_param("auto_assign") + + self.changed = False + self.output = {} + + self.delay = 15 # TODO Parameterize + self.timeout = 7200 # TODO Parameterize + self.message = "Ansible-powered" # TODO Parameterize + + self.process() + + @ClouderaManagerModule.handle_process + def process(self): + self.cm_api = ClouderaManagerResourceApi(self.api_client) + self.cluster_api = ClustersResourceApi(self.api_client) + self.service_api = ServicesResourceApi(self.api_client) + self.host_template_api = HostTemplatesResourceApi(self.api_client) + self.host_api = HostsResourceApi(self.api_client) + self.role_group_api = RoleConfigGroupsResourceApi(self.api_client) + self.role_api = RolesResourceApi(self.api_client) + + refresh = True + + # TODO manage services following the ClusterTemplateService data model + # TODO manage host and host template assignments following the ClusterTemplateHostInfo data model + # TODO manage host templates following the ClusterTemplateHostTemplate data model + # TODO manage role config groups following the ClusterTemplateRoleConfigGroupInfo data model (-CREATE-, MODIFY) + # TODO cluster template change management + # TODO auto assign roles (xCREATEx, MODIFY) + # TODO auto configure services and roles + # TODO auto-TLS (separate module for full credential lifecycle) + # TODO configure KRB (separate module for full KRB lifecycle) + # TODO deploy client configs (services) + # TODO auto-upgrade (bool: False) including prechecks + # TODO refresh configurations (bool: True, set False to suppress restarts) + # TODO restart arguments (selective restarts) + # TODO rolling restart arguments + # TODO rolling upgrade arguments + # TODO cluster object return + + # Retrieve existing cluster + existing = None + try: + existing = self.cluster_api.read_cluster(cluster_name=self.name) + except ApiException as ex: + if ex.status != 404: + raise ex + + # Prepare any cluster template content + template_contents = None + if self.template: + try: + with open(self.template, "r", encoding="utf-8") as file: + template_contents = json.load(file) + except OSError as oe: + self.module.fail_json( + msg=f"Error reading cluster template file, '{to_text(self.template)}'", + err=to_native(oe), + ) + + if self.state == "present": + # Modify cluster + if existing: + self.module.warn( + "Module currently does not support reconcilation of cluster templates with existing clusters." + ) + refresh = False + + # Reconcile the existing vs. the incoming values into a set of diffs + # then process via the PUT /clusters/{clusterName} endpoint + + if self.auto_assign: + self.changed = True + if not self.module.check_mode: + self.cluster_api.auto_assign_roles(cluster_name=self.name) + refresh = True + # Create cluster + else: + # TODO import_cluster_template appears to construct and first run the cluster, which is NOT what present should do + # That would mean import_cluster_template should only be executed on a fresh cluster with 'started' or 'restarted' states + if template_contents: + self.create_cluster_from_template(template_contents) + else: + self.create_cluster_from_parameters() + + elif self.state == "absent": + # Delete cluster + + refresh = False + + # TODO Check for status when deleting + # if existing and existing.entity_status == "": + # self.wait_for_active_cmd(cluster_api, self.cluster_name) + # elif existing: + if existing: + self.changed = True + if not self.module.check_mode: + self.cluster_api.delete_cluster(cluster_name=self.name) + self.wait_for_active_cmd(self.name) + + elif self.state == "started": + # TODO NONE seems to be fresh cluster, never run before + # Already started + if existing and existing.entity_status == "GOOD_HEALTH": + refresh = False + pass + # Start underway + elif existing and existing.entity_status == "STARTING": + self.wait_for_active_cmd(self.name) + # Needs starting + else: + # Create if needed + if not existing: + if template_contents: + self.create_cluster_from_template(template_contents) + else: + self.create_cluster_from_parameters() + + self.changed = True + if not self.module.check_mode: + # If newly created or created by not yet initialize + if not existing or existing.entity_status == "NONE": + first_run = self.cluster_api.first_run(cluster_name=self.name) + self.wait_for_composite_cmd(first_run.id) + # Start the existing and previously initialized cluster + else: + start = self.cluster_api.start_command(cluster_name=self.name) + self.wait_for_composite_cmd(start.id) + + if self.state == "stopped": + # Already stopped + if existing and existing.entity_status == "STOPPED": + refresh = False + pass + # Stop underway + elif existing and existing.entity_status == "STOPPING": + self.wait_for_active_cmd(self.name) + # Needs stopping + else: + # Create if needed + if not existing: + if template_contents: + self.create_cluster_from_template(template_contents) + else: + self.create_cluster_from_parameters() + # Stop an existing cluster + else: + self.changed = True + if not self.module.check_mode: + stop = self.cluster_api.stop_command(cluster_name=self.name) + self.wait_for_composite_cmd(stop.id) + + if self.state == "restarted": + # Start underway + if existing and existing.entity_status == "STARTING": + self.wait_for_active_cmd(self.name) + # Needs restarting + else: + # Create if needed + if not existing: + if template_contents: + self.create_cluster_from_template(template_contents) + else: + self.create_cluster_from_parameters() + + self.changed = True + if not self.module.check_mode: + restart = self.cluster_api.restart_command(cluster_name=self.name) + self.wait_for_composite_cmd(restart.id) + + if refresh: + # Retrieve the updated cluster details + self.output = self.cluster_api.read_cluster( + cluster_name=self.name + ).to_dict() + elif existing: + self.output = existing.to_dict() + + def wait_for_composite_cmd(self, command_id: str): + cmd = self.wait_for_command_state( + command_id=command_id, + polling_interval=self.delay, + ) + if not cmd[0].success: + collected_msgs = [ + f"[{c.name}] {c.result_message}" + for c in cmd[0].children.items + if not c.success + ] + self.module.fail_json(msg="\n".join(collected_msgs)) + elif not cmd: + self.module.fail_json(msg="Invalid command result", cmd=to_native(cmd)) + + return cmd + + def wait_for_active_cmd(self, cluster_name: str): + active_cmd = None + try: + active_cmd = next( + iter( + self.cluster_api.list_active_commands( + cluster_name=cluster_name + ).items + ), + None, + ) + except ApiException as e: + if e.status != 404: + raise e + + if active_cmd: + self.wait_for_command_state( + command_id=active_cmd.id, + polling_interval=self.delay, + ) + + def create_cluster_from_template(self, template_contents: dict): + payload = dict() + + # Construct import template payload from the template and/or explicit parameters + explicit_params = dict() + + # Set up 'instantiator' parameters + explicit_params.update(instantiator=dict(clusterName=self.name)) + + # Merge/overlay any explicit parameters over the template + TEMPLATE = ClusterTemplate( + warn_fn=self.module.warn, error_fn=self.module.fail_json + ) + TEMPLATE.merge(template_contents, explicit_params) + payload.update(body=ApiClusterTemplate(**template_contents)) + + # Update to include repositories + if self.add_repositories: + payload.update(add_repositories=True) + + # Execute the import + self.changed = True + if not self.module.check_mode: + import_template_request = self.cm_api.import_cluster_template( + **payload + ).to_dict() + + command_id = import_template_request["id"] + self.wait_for_command_state( + command_id=command_id, polling_interval=self.delay + ) + + def create_cluster_from_parameters(self): + if self.cluster_version is None: + self.module.fail_json( + msg=f"Cluster must be created. Missing required parameter: cluster_version" + ) + + # Configure the core cluster + cluster = ApiCluster( + name=self.name, + full_version=self.cluster_version, + cluster_type=self.type, + ) + + # Configure cluster services + if self.services: + cluster.services = [self.marshal_service(s) for s in self.services] + + # Configure cluster contexts + if self.contexts: + cluster.data_context_refs = [ApiDataContext(name=d) for d in self.contexts] + + # Execute the creation + self.changed = True + + if not self.module.check_mode: + # Validate any incoming host membership to fail fast + if self.hosts: + hostrefs = self.marshal_hostrefs({h["name"]: h for h in self.hosts}) + hostref_by_host_id = {h.host_id: h for h in hostrefs} + hostref_by_hostname = {h.hostname: h for h in hostrefs} + + # Create the cluster configuration + self.cluster_api.create_clusters(body=ApiClusterList(items=[cluster])) + self.wait_for_active_cmd(self.name) + + # Add host templates to cluster + if self.host_templates: + + # Prepare host template role group assignments + # and discover base role groups if needed + templates = [ + ApiHostTemplate( + name=ht["name"], + role_config_group_refs=[ + ApiRoleConfigGroupRef( + rcg["name"] + if rcg["name"] + else self.find_base_role_group_name( + service_type=rcg["service"], role_type=rcg["type"] + ) + ) + for rcg in ht["role_groups"] + ], + ) + for ht in self.host_templates + ] + + self.host_template_api.create_host_templates( + cluster_name=self.name, + body=ApiHostTemplateList(items=templates), + ) + + # Add hosts to cluster and set up assignments + template_map = {} + role_group_list = [] + role_list = [] + + if self.hosts: + # Add the hosts + self.cluster_api.add_hosts( + cluster_name=self.name, + body=ApiHostRefList(items=hostrefs), + ) + + for h in self.hosts: + # Normalize hostref + hostref = ( + hostref_by_host_id[h["name"]] + if h["name"] in hostref_by_host_id + else hostref_by_hostname[h["name"]] + ) + + # Prepare host template assignments + if h["host_template"]: + if h["host_template"] in template_map: + template_map[h["host_template"]].append(hostref) + else: + template_map[h["host_template"]] = [hostref] + + # Prepare role group assignments + if h["role_groups"]: + role_group_list.append((hostref, h["role_groups"])) + + # Prepare role overrides + if h["roles"]: + role_list.append((hostref, h["roles"])) + + # Activate parcels + if self.parcels: + parcel_api = ParcelResourceApi(self.api_client) + for p, v in self.parcels.items(): + parcel = Parcel( + parcel_api=parcel_api, + product=p, + version=v, + cluster=self.name, + delay=self.delay, + timeout=self.timeout, + ) + parcel.activate() + + # Apply host templates + for ht, refs in template_map.items(): + self.host_template_api.apply_host_template( + cluster_name=self.name, + host_template_name=ht, + start_roles=False, + body=ApiHostRefList(items=refs), + ) + + # Configure direct role group assignments + if role_group_list: + # Gather all the RCGs for all services, as the host template might + # not reference RCGs that are directly configured in the service + # definition, i.e. base role groups + all_rcgs = { + rcg.name: ( + rcg.base, + s.name, + rcg.role_type, + ) + for s in self.service_api.read_services( + cluster_name=self.name + ).items # s.name + for rcg in self.role_group_api.read_role_config_groups( + cluster_name=self.name, service_name=s.name + ).items + } + + for hostref, host_rcgs in role_group_list: + for rcg in host_rcgs: + rcg_ref = None + + if rcg["name"]: + # Use the declared RCG name + if rcg["name"] not in all_rcgs: + self.module.fail_json( + msg="Role config group '%s' not found on cluster." + % rcg["name"] + ) + else: + rcg_ref = all_rcgs[rcg["name"]] + else: + # Or discover the role group + rcg_name = next( + iter( + [ + name + for name, refs in all_rcgs.items() + if refs[0] + and refs[1] == rcg["service"] + and refs[2] == rcg["type"] + ] + ), + None, + ) + + if rcg_name is None: + self.module.fail_json( + msg="Unable to find base role group, '%s [%s]', on cluster, '%s'" + % (rcg["service"], rcg["type"], self.name) + ) + + rcg_ref = all_rcgs[rcg_name] + + # Add the role of that type to the host (generating a name) + direct_roles = self.role_api.create_roles( + cluster_name=self.name, + service_name=rcg_ref[1], + body=ApiRoleList( + items=[ApiRole(type=rcg_ref[2], host_ref=hostref)] + ), + ) + + # Move the newly-created role to the RCG if it is not a base/default group + if not rcg_ref[0]: + self.role_group_api.move_roles( + cluster_name=self.name, + role_config_group_name=rcg["name"], + service_name=rcg_ref[1], + body=ApiRoleNameList( + items=[direct_roles.items[0].name] + ), + ) + + # Configure per-host role overrides + for ( + hostref, + overrides, + ) in role_list: + for override in overrides: + # Discover the role on the host + host_role = next( + iter( + self.role_api.read_roles( + cluster_name=self.name, + service_name=override["service"], + filter="type==%s;hostId==%s" + % (override["type"], hostref.host_id), + ).items + ), + None, + ) + + if host_role is not None: + self.role_api.update_role_config( + cluster_name=self.name, + service_name=override["service"], + role_name=host_role.name, + message=self.message, + body=ApiConfigList( + items=[ + ApiConfig(name=k, value=v) + for k, v in override["config"].items() + ] + ), + ) + else: + self.module.fail_json( + msg="Role not found. No role type '%s' for service '%s' found on host '%s'" + % (override["type"], override["service"], hostref.hostname) + ) + + # Execute auto-role assignments + if self.auto_assign: + self.cluster_api.auto_assign_roles(cluster_name=self.name) + + def marshal_service(self, options: str) -> ApiService: + service = ApiService(name=options["name"], type=options["type"]) + + if "display_name" in options: + service.display_name = options["display_name"] + + # Service-wide configuration + if options["config"]: + service.config = ApiServiceConfig( + items=[ApiConfig(name=k, value=v) for k, v in options["config"].items()] + ) + + if options["role_groups"]: + rcg_list = [] + + for body in options["role_groups"]: + rcg = ApiRoleConfigGroup(role_type=body["type"]) + + # Either a defined role group or a default/base group + if body["name"]: + rcg.name = body["name"] + else: + rcg.base = True + + if body["display_name"]: + rcg.display_name = body["display_name"] + + if body["config"]: + rcg.config = ApiConfigList( + items=[ + ApiConfig(name=k, value=v) + for k, v in body["config"].items() + ] + ) + + rcg_list.append(rcg) + + service.role_config_groups = rcg_list + + if "tags" in options: + pass + + if "version" in options: + pass + + return service + + def marshal_hostrefs(self, hosts: dict) -> list[ApiHostRef]: + results = [] + hosts_query = self.host_api.read_hosts().items + for h in hosts_query: + if h.host_id in hosts.keys() or h.hostname in hosts.keys(): + if ( + h.cluster_ref is not None + and h.cluster_ref.cluster_name != self.name + ): + self.module.fail_json( + msg=f"Invalid host reference! Host {h.hostname} ({h.host_id}) already in use with cluster '{h.cluster_ref.cluster_name}'!" + ) + results.append(ApiHostRef(host_id=h.host_id, hostname=h.hostname)) + if len(results) != len(hosts.keys()): + self.module.fail_json( + msg="Did not find the following hosts: " + + ", ".join(set(hosts.keys() - set(results))) + ) + return results + + def find_base_role_group_name( + self, service_type: str, role_type: str + ) -> ApiRoleConfigGroup: + rcgs = [ + rcg + for s in self.service_api.read_services(cluster_name=self.name).items + for rcg in self.role_group_api.read_role_config_groups( + cluster_name=self.name, service_name=s.name + ).items + if s.type == service_type + ] + + base = next( + iter([rcg for rcg in rcgs if rcg.base and rcg.role_type == role_type]), + None, + ) + + if base is None: + self.module.fail_json( + "Invalid role group; unable to discover base role group for service role, %s[%s]" + % (role_type, service_type) + ) + else: + return base.name + + +def main(): + module = ClouderaManagerModule.ansible_module( + argument_spec=dict( + name=dict(required=True, aliases=["cluster_name"]), + cluster_version=dict(), + type=dict( + aliases=["cluster_type"], + choices=["BASE_CLUSTER", "COMPUTE_CLUSTER", "EXPERIENCE_CLUSTER"], + ), + state=dict( + default="present", + choices=["present", "absent", "stopped", "started", "restarted"], + ), + # A cluster template used as the "baseline" for the parameters + template=dict(type="path", aliases=["cluster_template"]), + # Only valid if using 'template' + add_repositories=dict(type="bool", default=False), + # Flag for warning suppression + maintenance=dict(type="bool", aliases=["maintenance_enabled"]), + # Services, service configs, role config groups + services=dict( + type="list", + elements="dict", + options=dict( + name=dict(aliases=["ref", "ref_name"]), + type=dict(required=True), + version=dict(), + # Service-level config + config=dict(type="dict"), + # Role config groups (RCG) + role_groups=dict( + type="list", + elements="dict", + options=dict( + name=dict(aliases=["ref", "ref_name"]), + type=dict(required=True, aliases=["role_type"]), + display_name=dict(), + config=dict(type="dict"), + ), + aliases=["role_config_groups"], + ), + display_name=dict(), + tags=dict(type="dict"), + ), + ), + # Hosts, host template assignments, explicit role group assignments, and per-host role overrides + hosts=dict( + type="list", + elements="dict", + options=dict( + name=dict(required=True, aliases=["host_id", "hostname"]), + config=dict(), + host_template=dict(), + role_groups=dict( + type="list", + elements="dict", + options=dict( + name=dict(aliases=["ref", "ref_name"]), + service=dict(required=True, aliases=["service_name"]), + type=dict(aliases=["role_type"]), + ), + aliases=["role_config_groups"], + mutually_exclusive=[ + ("name", "type"), + ], + ), + roles=dict( + type="list", + elements="dict", + options=dict( + service=dict( + required=True, aliases=["service_name", "service_ref"] + ), + type=dict(required=True, aliases=["role_type"]), + config=dict(type="dict", required=True), + ), + ), + tags=dict(), + ), + ), + # Host templates + host_templates=dict( + type="list", + elements="dict", + options=dict( + name=dict(required=True), + role_groups=dict( + type="list", + elements="dict", + required=True, + options=dict( + name=dict(aliases=["ref", "ref_name"]), + service=dict(required=True, aliases=["service_name"]), + type=dict(aliases=["role_type"]), + ), + aliases=["role_config_groups"], + mutually_exclusive=[ + ("name", "type"), + ], + ), + ), + ), + # Parcels is a dict of product:version of the cluster + parcels=dict(type="dict", aliases=["products"]), + # Tags is a dict of key:value assigned to the cluster + tags=dict(), + # Optional UI name + display_name=dict(), + # Optional data contexts, required for compute-type clusters + contexts=dict(type="list", elements="str", aliases=["data_contexts"]), + # Optional enable/disable TLS for the cluster + tls=dict(type="bool", aliases=["tls_enabled", "cluster_tls"]), + # Optional auto-assign roles on cluster (honors existing assignments) + auto_assign=dict(type="bool", default=False, aliases=["auto_assign_roles"]), + ), + supports_check_mode=True, + mutually_exclusive=[ + ("cdh_version", "cdh_short_version"), + ], + required_if=[("type", "COMPUTE_CLUSTER", ("contexts"))], + ) + + result = ClouderaCluster(module) + + changed = result.changed + + output = dict( + changed=changed, + cloudera_manager=result.output, + ) + + if result.debug: + log = result.log_capture.getvalue() + output.update(debug=log, debug_lines=log.split("\n")) + + module.exit_json(**output) + + +if __name__ == "__main__": + main() diff --git a/plugins/modules/cm_cluster_info.py b/plugins/modules/cluster_info.py similarity index 89% rename from plugins/modules/cm_cluster_info.py rename to plugins/modules/cluster_info.py index c71a3b1a..c22b1904 100644 --- a/plugins/modules/cm_cluster_info.py +++ b/plugins/modules/cluster_info.py @@ -27,7 +27,7 @@ DOCUMENTATION = r""" --- -module: cm_cluster_info +module: cluster_info short_description: Retrieve details about one or more clusters description: - Retrieves details about one or more clusters managed by Cloudera Manager @@ -47,15 +47,15 @@ EXAMPLES = r""" --- - name: Get information about the cluster - cloudera.cluster.cm_cluster_info: + cloudera.cluster.cluster_info: host: example.cloudera.com username: "jane_smith" name: "OneNodeCluster" password: "S&peR4Ec*re" port: "7180" - + - name: Get information about all clusters - cloudera.cluster.cm_cluster_info: + cloudera.cluster.cluster_info: host: example.cloudera.com username: "jane_smith" password: "S&peR4Ec*re" @@ -129,30 +129,34 @@ def process(self): try: cluster_api_instance = ClustersResourceApi(self.api_client) if self.name: - self.output = [cluster_api_instance.read_cluster(cluster_name=self.name).to_dict()] + self.output = [ + cluster_api_instance.read_cluster(cluster_name=self.name).to_dict() + ] else: - self.output = cluster_api_instance.read_clusters().to_dict()['items'] - + self.output = cluster_api_instance.read_clusters().to_dict()["items"] + except ApiException as e: if e.status == 404: pass else: raise e except KeyError as ke: - self.module.fail_json(msg='Invalid result object from Cloudera Manager API', error=to_native(ke)) + self.module.fail_json( + msg="Invalid result object from Cloudera Manager API", + error=to_native(ke), + ) def main(): module = ClouderaManagerModule.ansible_module( - argument_spec=dict( - name=dict(aliases=["cluster_name","cluster"]), + name=dict(aliases=["cluster_name", "cluster"]), ), - supports_check_mode=True - ) + supports_check_mode=True, + ) + + result = ClusterInfo(module) - result = ClusterInfo(module) - output = dict( changed=False, clusters=result.output, diff --git a/plugins/modules/cm_cluster.py b/plugins/modules/cm_cluster.py deleted file mode 100644 index c3762764..00000000 --- a/plugins/modules/cm_cluster.py +++ /dev/null @@ -1,254 +0,0 @@ -# Copyright 2024 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ansible.module_utils.common.text.converters import to_text, to_native - -from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( - ClouderaManagerModule, ClusterTemplate -) -from cm_client.rest import ApiException -from cm_client import ClouderaManagerResourceApi -from cm_client import ClustersResourceApi -import json - -ANSIBLE_METADATA = { - "metadata_version": "1.1", - "status": ["preview"], - "supported_by": "community", -} - -DOCUMENTATION = r""" ---- -module: cm_cluster -short_description: Create a cluster based on the provided cluster template -description: - - Searches for a template file. - - The search for the file starts at the root directory where the Ansible playbook is executed. By default, the template is expected to be placed inside the './files' directory. - - Imports the template file and uses it to create the cluster. - - This module ensures that the cluster is created according to the specified template. -author: - - "Ronald Suplina (@rsuplina)" -options: - template: - description: - - Path to template file which defines the cluster - type: path - elements: str - required: True - add_repositories: - description: - - Install parcel repositories in parcel directory - type: bool - required: False - default: False - clusterName: - description: - - Name of Cloudera Manager Cluster - type: str - required: False - default: False -requirements: - - cm_client -""" - -EXAMPLES = r""" ---- -- name: Create a cluster on Cloudera Manager host - cloudera.cluster.cm_cluster: - host: example.cloudera.com - username: "jane_smith" - clusterName: "OneNodeCluster" - password: "S&peR4Ec*re" - port: "7180" - template: "./files/cluster-template.json" - -- name: Create a cluster and install the repositories defined in template - cloudera.cluster.cm_cluster: - host: example.cloudera.com - username: "jane_smith" - password: "S&peR4Ec*re" - port: "7180" - template: "./files/cluster-template.json" - add_repositories: "True" -""" - -RETURN = r""" ---- -cloudera_manager: - description: Details about Cloudera Manager Cluster - type: dict - contains: - cluster_type: - description: The type of cluster created from template. - type: str - returned: optional - cluster_url: - description: Url of Cloudera Manager cluster. - type: str - returned: optional - display_name: - description: The name of the cluster displayed on the site. - type: str - returned: optional - entity_status: - description: Health status of the cluster. - type: str - returned: optional - full_version: - description: Version of the cluster installed. - type: str - returned: optional - hosts_url: - description: Url of all the hosts on which cluster is installed. - type: str - returned: optional - maintenance_mode: - description: Maintance mode of Cloudera Manager Cluster. - type: bool - returned: optional - maintenance_owners: - description: List of Maintance owners for Cloudera Manager Cluster. - type: list - returned: optional - name: - description: The name of the cluster created. - type: str - returned: optional - tags: - description: List of tags for Cloudera Manager Cluster. - type: list - returned: optional - uuid: - description: Unique ID of created cluster - type: bool - returned: optional -""" - - -class ClusterModule(ClouderaManagerModule): - def __init__(self, module): - super(ClusterModule, self).__init__(module) - - self.template = self.get_param("template") - self.add_repositories = self.get_param("add_repositories") - self.cluster_name = self.get_param("name") - self.state = self.get_param("state") - - self.changed = False - self.output = dict() - - self.process() - - @ClouderaManagerModule.handle_process - def process(self): - - api_instance = ClouderaManagerResourceApi(self.api_client) - cluster_api_instance = ClustersResourceApi(self.api_client) - - template_contents = dict() - - if self.template: - try: - with open(self.template, 'r') as file: - template_contents = json.load(file) - except OSError as oe: - self.module.fail_json(msg=f"Error reading file '{to_text(self.template)}'", err=to_native(oe)) - # Need to catch malformed JSON, etc. - - if not self.cluster_name: - if template_contents: - self.cluster_name = template_contents['instantiator']['clusterName'] - else: - self.module.fail_json(msg="No cluster name found in template.") - - try: - self.existing = cluster_api_instance.read_cluster(cluster_name=self.cluster_name).to_dict() - except ApiException: - self.existing = dict() - - if self.state == "present": - if self.existing: - pass - # Reconcile the existing vs. the incoming values into a set of diffs - # then process via the PUT /clusters/{clusterName} endpoint - else: - payload = dict() - - # Construct import template payload from the template and/or explicit parameters - explicit_params = dict() - - # Set up 'instantiator' parameters - explicit_params.update(instantiator=dict( - clusterName=self.cluster_name - )) - - if template_contents: - TEMPLATE = ClusterTemplate(warn_fn=self.module.warn, error_fn=self.module.fail_json) - TEMPLATE.merge(template_contents, explicit_params) - payload.update(body=template_contents) - else: - payload.update(body=explicit_params) - - # Update to include repositories - if self.add_repositories: - payload.update(add_repositories=True) - - # Execute the import - if not self.module.check_mode: - self.changed = True - import_template_request = api_instance.import_cluster_template(**payload).to_dict() - - command_id = import_template_request['id'] - self.wait_for_command_state(command_id=command_id,polling_interval=60) - - # Retrieve the newly-minted cluster - self.output = cluster_api_instance.read_cluster(cluster_name=self.cluster_name).to_dict() - elif self.state == "absent": - if self.existing: - pass - # Delete the cluster via DELETE /clusters/{clusterName} - else: - self.module.fail_json(msg=f"Invalid state, ${self.state}") - - -def main(): - module = ClouderaManagerModule.ansible_module( - argument_spec=dict( - template=dict(type="path", aliases=["cluster_template"]), - add_repositories=dict(type="bool", default=False), - name=dict(aliases=["cluster_name"]), - state=dict(default="present", choices=["present", "absent"]) - ), - required_one_of=[ - ["name", "template"] - ], - supports_check_mode=True - ) - - result = ClusterModule(module) - - output = dict( - changed=result.changed, - cloudera_manager=result.existing, - ) - - if result.debug: - log = result.log_capture.getvalue() - output.update(debug=log, debug_lines=log.split("\n")) - - module.exit_json(**output) - - -if __name__ == "__main__": - main() diff --git a/plugins/modules/cm_config.py b/plugins/modules/cm_config.py index 240cf7c4..7d9ca44b 100644 --- a/plugins/modules/cm_config.py +++ b/plugins/modules/cm_config.py @@ -28,7 +28,7 @@ DOCUMENTATION = r""" --- module: cm_config -short_description: Manage the configuration of Cloudera Manager +short_description: Manage the configuration of Cloudera Manager description: - Manage Cloudera Manager configuration settings. author: diff --git a/plugins/modules/cm_endpoint_info.py b/plugins/modules/cm_endpoint_info.py index 3acf59b5..540b8023 100644 --- a/plugins/modules/cm_endpoint_info.py +++ b/plugins/modules/cm_endpoint_info.py @@ -40,7 +40,7 @@ EXAMPLES = r""" --- # This will first try 'http://example.cloudera.com:7180' and will -# follow any redirects +# follow any redirects - name: Gather details using auto-discovery cloudera.cluster.cm_endpoint_info: host: example.cloudera.com diff --git a/plugins/modules/cm_license.py b/plugins/modules/cm_license.py index 26292b01..4aca0bdc 100644 --- a/plugins/modules/cm_license.py +++ b/plugins/modules/cm_license.py @@ -51,7 +51,7 @@ RETURN = r""" --- cloudera_manager: - description: Details about a active license + description: Details about a active license type: dict contains: owner: @@ -91,10 +91,10 @@ def __init__(self, module): def process(self): try: - api_instance = ClouderaManagerResourceApi(self.api_client) + api_instance = ClouderaManagerResourceApi(self.api_client) - self.cm_license_output = api_instance.read_license().to_dict() - self.changed = False + self.cm_license_output = api_instance.read_license().to_dict() + self.changed = False except ApiException as e: if e.status == 404: @@ -105,22 +105,18 @@ def process(self): self.changed = True except FileNotFoundError: - self.cm_license_output = (f"Error: File '{self.license}' not found.") - self.module.fail_json(msg=str(self.cm_license_output)) + self.cm_license_output = f"Error: File '{self.license}' not found." + self.module.fail_json(msg=str(self.cm_license_output)) + def main(): module = ClouderaManagerModule.ansible_module( - - argument_spec= - dict( - license=dict(required=True, type="path") - ), - supports_check_mode=True - ) + argument_spec=dict(license=dict(required=True, type="path")), + supports_check_mode=True, + ) result = ClouderaLicense(module) - output = dict( changed=result.changed, cloudera_manager=result.cm_license_output, diff --git a/plugins/modules/cm_license_info.py b/plugins/modules/cm_license_info.py index 3c436a53..3e3fdaf4 100644 --- a/plugins/modules/cm_license_info.py +++ b/plugins/modules/cm_license_info.py @@ -96,15 +96,15 @@ def process(self): self.cm_license_output = api_instance.read_license().to_dict() except ApiException as e: if e.status == 404: - self.cm_cluster_info = (f"Error: License not found.") - self.module.fail_json(msg=str(self.cm_license_output)) + self.cm_cluster_info = f"Error: License not found." + self.module.fail_json(msg=str(self.cm_license_output)) + def main(): module = ClouderaManagerModule.ansible_module(supports_check_mode=False) result = ClouderaLicenseInfo(module) - output = dict( changed=False, cloudera_manager=result.cm_license_output, diff --git a/plugins/modules/cm_resource.py b/plugins/modules/cm_resource.py index 9dca4400..a50e768f 100644 --- a/plugins/modules/cm_resource.py +++ b/plugins/modules/cm_resource.py @@ -79,7 +79,7 @@ body: authRoles: - name: "ROLE_LIMITED" - + - name: Delete a Cloudera Manager user using a custom SSL certificate host: example.cloudera.com username: "jane_smith" diff --git a/plugins/modules/cm_resource_info.py b/plugins/modules/cm_resource_info.py index 7989cba7..39960c04 100644 --- a/plugins/modules/cm_resource_info.py +++ b/plugins/modules/cm_resource_info.py @@ -49,7 +49,7 @@ password: "S&peR4Ec*re" path: "/users" register: cm_users - + - name: Retrieve details for all running commands on a cluster using a custom SSL certificate cloudera.cluster.cm_resource_info: host: example.cloudera.com diff --git a/plugins/modules/cm_service.py b/plugins/modules/cm_service.py index 5247a8db..921981cf 100644 --- a/plugins/modules/cm_service.py +++ b/plugins/modules/cm_service.py @@ -14,7 +14,6 @@ from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( ClouderaManagerModule, - ) from cm_client.rest import ApiException @@ -32,7 +31,7 @@ DOCUMENTATION = r""" --- module: cm_service -short_description: Manage Cloudera Manager service roles +short_description: Manage Cloudera Manager service roles description: - Create or remove one or more Cloudera Manager service roles. - Start, stop or restart one or more Cloudera Manager service roles. @@ -81,7 +80,7 @@ role: [ "SERVICEMONITOR" , "HOSTMONITOR", "EVENTSERVER", "ALERTPUBLISHER" ] register: cm_output -- name: Purge all roles then create and start new roles +- name: Purge all roles then create and start new roles cloudera.cluster.cm_version: host: "10.10.10.10" username: "jane_smith" @@ -101,7 +100,7 @@ state: "stopped" role: [ "EVENTSERVER", "ALERTPUBLISHER" ] register: cm_output - + - name: Remove Cloudera Manager service role cloudera.cluster.cm_version: host: "10.10.10.10" @@ -206,181 +205,232 @@ def process(self): host_api_instance = HostsResourceApi(self.api_client) get_host_infomation = host_api_instance.read_hosts().to_dict() - for item in get_host_infomation['items']: - if self.host == item['hostname']: - host_id = item['host_id'] + for item in get_host_infomation["items"]: + if self.host == item["hostname"]: + host_id = item["host_id"] if not self.purge: - available_roles_info = role_api_instance.read_roles().to_dict() - existing_roles = [] - for item in available_roles_info['items']: - existing_roles.append(item['type']) - - - if self.state in ['present']: - not_existing_roles = [] - for role in self.role: - if role not in existing_roles: - not_existing_roles.append(role) - if not_existing_roles: - body = {"items": [{ "type": role, "hostRef" : { "hostId" : host_id }} for role in not_existing_roles]} - role_api_instance.create_roles(body=body) - self.cm_service_output = role_api_instance.read_roles().to_dict() - self.changed = True - - - elif self.state in ['absent']: - roles_to_remove = [role for role in self.role if role in existing_roles] - roles_to_remove_extended_info = [] - for role in roles_to_remove: - for item in available_roles_info['items']: - if role == item['type']: - roles_to_remove_extended_info.append(item['name']) - if not roles_to_remove_extended_info: - self.cm_service_output = role_api_instance.read_roles().to_dict() - self.changed = False - else: - for role in roles_to_remove_extended_info: - role_api_instance.delete_role(role_name=role) + available_roles_info = role_api_instance.read_roles().to_dict() + existing_roles = [] + for item in available_roles_info["items"]: + existing_roles.append(item["type"]) + + if self.state in ["present"]: + not_existing_roles = [] + for role in self.role: + if role not in existing_roles: + not_existing_roles.append(role) + if not_existing_roles: + body = { + "items": [ + {"type": role, "hostRef": {"hostId": host_id}} + for role in not_existing_roles + ] + } + role_api_instance.create_roles(body=body) self.cm_service_output = role_api_instance.read_roles().to_dict() self.changed = True - - elif self.state in ['started']: - - matching_roles = [] - new_roles = [] - for role in self.role: - if role in existing_roles: - matching_roles.append(role) - else: - new_roles.append(role) - - new_roles_to_start = [] - if new_roles: - body = {"items": [{"type": role, "hostRef": {"hostId": host_id}} for role in new_roles]} - newly_added_roles=role_api_instance.create_roles(body=body).to_dict() - - for role in newly_added_roles['items']: - new_roles_to_start.append(role['name']) - body = { "items" : new_roles_to_start} - - existing_roles_state = [] - for role in matching_roles: - for item in available_roles_info['items']: - if role == item['type']: - existing_roles_state.append({'type': item['type'], 'role_state': item['role_state'].lower(),'name': item['name'] }) - - existing_roles_to_start = [] - for role in existing_roles_state: - if role['role_state'] == 'stopped': - existing_roles_to_start.append(role['name']) - - all_roles_to_start = new_roles_to_start + existing_roles_to_start - body = { "items" : all_roles_to_start} - - if all_roles_to_start: - start_roles_request = role_cmd_api_instance.start_command(body=body).to_dict() - command_id = start_roles_request['items'][0]['id'] - self.wait_for_command_state(command_id=command_id,polling_interval=5) - self.cm_service_output = role_api_instance.read_roles().to_dict() - self.changed = True - else: - self.cm_service_output = role_api_instance.read_roles().to_dict() - self.changed = False - - - elif self.state in ['stopped']: - matching_roles = [] - for role in self.role: - if role in existing_roles: - matching_roles.append(role) - - matching_roles_state = [] - for role in matching_roles: - for item in available_roles_info['items']: - if role == item['type']: - matching_roles_state.append({'type': item['type'], 'role_state': item['role_state'].lower(),'name': item['name'] }) - - roles_to_stop = [] - for role in matching_roles_state: - if role['role_state'] == 'started': - roles_to_stop.append(role['name']) - body = { "items" : roles_to_stop } - - if roles_to_stop: - role_cmd_api_instance.stop_command(body=body) - self.cm_service_output = role_api_instance.read_roles().to_dict() - self.changed = True - else: - self.cm_service_output = role_api_instance.read_roles().to_dict() - self.changed = False - - elif self.state in ['restarted']: - matching_roles = [] - for role in self.role: - if role in existing_roles: - matching_roles.append(role) - - matching_roles_state = [] - for role in matching_roles: - for item in available_roles_info['items']: - if role == item['type']: - matching_roles_state.append({'type': item['type'], 'role_state': item['role_state'].lower(),'name': item['name'] }) - - roles_to_restart = [] - for role in matching_roles_state: - roles_to_restart.append(role['name']) - body = { "items" : roles_to_restart} - - if roles_to_restart: - role_cmd_api_instance.restart_command(body=body) - self.cm_service_output = role_api_instance.read_roles().to_dict() - self.changed = True - - + elif self.state in ["absent"]: + roles_to_remove = [ + role for role in self.role if role in existing_roles + ] + roles_to_remove_extended_info = [] + for role in roles_to_remove: + for item in available_roles_info["items"]: + if role == item["type"]: + roles_to_remove_extended_info.append(item["name"]) + if not roles_to_remove_extended_info: + self.cm_service_output = ( + role_api_instance.read_roles().to_dict() + ) + self.changed = False + else: + for role in roles_to_remove_extended_info: + role_api_instance.delete_role(role_name=role) + self.cm_service_output = ( + role_api_instance.read_roles().to_dict() + ) + self.changed = True + + elif self.state in ["started"]: + + matching_roles = [] + new_roles = [] + for role in self.role: + if role in existing_roles: + matching_roles.append(role) + else: + new_roles.append(role) + + new_roles_to_start = [] + if new_roles: + body = { + "items": [ + {"type": role, "hostRef": {"hostId": host_id}} + for role in new_roles + ] + } + newly_added_roles = role_api_instance.create_roles( + body=body + ).to_dict() + + for role in newly_added_roles["items"]: + new_roles_to_start.append(role["name"]) + body = {"items": new_roles_to_start} + + existing_roles_state = [] + for role in matching_roles: + for item in available_roles_info["items"]: + if role == item["type"]: + existing_roles_state.append( + { + "type": item["type"], + "role_state": item["role_state"].lower(), + "name": item["name"], + } + ) + + existing_roles_to_start = [] + for role in existing_roles_state: + if role["role_state"] == "stopped": + existing_roles_to_start.append(role["name"]) + + all_roles_to_start = new_roles_to_start + existing_roles_to_start + body = {"items": all_roles_to_start} + + if all_roles_to_start: + start_roles_request = role_cmd_api_instance.start_command( + body=body + ).to_dict() + command_id = start_roles_request["items"][0]["id"] + self.wait_for_command_state( + command_id=command_id, polling_interval=5 + ) + self.cm_service_output = ( + role_api_instance.read_roles().to_dict() + ) + self.changed = True + else: + self.cm_service_output = ( + role_api_instance.read_roles().to_dict() + ) + self.changed = False + + elif self.state in ["stopped"]: + matching_roles = [] + for role in self.role: + if role in existing_roles: + matching_roles.append(role) + + matching_roles_state = [] + for role in matching_roles: + for item in available_roles_info["items"]: + if role == item["type"]: + matching_roles_state.append( + { + "type": item["type"], + "role_state": item["role_state"].lower(), + "name": item["name"], + } + ) + + roles_to_stop = [] + for role in matching_roles_state: + if role["role_state"] == "started": + roles_to_stop.append(role["name"]) + body = {"items": roles_to_stop} + + if roles_to_stop: + role_cmd_api_instance.stop_command(body=body) + self.cm_service_output = ( + role_api_instance.read_roles().to_dict() + ) + self.changed = True + else: + self.cm_service_output = ( + role_api_instance.read_roles().to_dict() + ) + self.changed = False + + elif self.state in ["restarted"]: + matching_roles = [] + for role in self.role: + if role in existing_roles: + matching_roles.append(role) + + matching_roles_state = [] + for role in matching_roles: + for item in available_roles_info["items"]: + if role == item["type"]: + matching_roles_state.append( + { + "type": item["type"], + "role_state": item["role_state"].lower(), + "name": item["name"], + } + ) + + roles_to_restart = [] + for role in matching_roles_state: + roles_to_restart.append(role["name"]) + body = {"items": roles_to_restart} + + if roles_to_restart: + role_cmd_api_instance.restart_command(body=body) + self.cm_service_output = ( + role_api_instance.read_roles().to_dict() + ) + self.changed = True + if self.purge: - mgmt_service_api_instance.delete_cms() - body = {"roles": [{"type": role} for role in self.role]} - mgmt_service_api_instance.setup_cms(body=body) - self.cm_service_output = role_api_instance.read_roles().to_dict() - - if self.state in ['started']: - start_roles_request=api_instance.start_command().to_dict() - command_id = start_roles_request['id'] - self.wait_for_command_state(command_id=command_id,polling_interval=5) + mgmt_service_api_instance.delete_cms() + body = {"roles": [{"type": role} for role in self.role]} + mgmt_service_api_instance.setup_cms(body=body) + self.cm_service_output = role_api_instance.read_roles().to_dict() + + if self.state in ["started"]: + start_roles_request = api_instance.start_command().to_dict() + command_id = start_roles_request["id"] + self.wait_for_command_state( + command_id=command_id, polling_interval=5 + ) self.cm_service_output = role_api_instance.read_roles().to_dict() - self.changed = True - - + self.changed = True + except ApiException as e: if e.status == 404 or 400: roles_dict = {"roles": [{"type": role} for role in self.role]} api_instance.setup_cms(body=roles_dict) - if self.state in ['started']: - start_roles_request=api_instance.start_command().to_dict() - command_id = start_roles_request['id'] - self.wait_for_command_state(command_id=command_id,polling_interval=5) + if self.state in ["started"]: + start_roles_request = api_instance.start_command().to_dict() + command_id = start_roles_request["id"] + self.wait_for_command_state( + command_id=command_id, polling_interval=5 + ) self.cm_service_output = role_api_instance.read_roles().to_dict() else: - self.cm_service_output = role_api_instance.read_roles().to_dict() + self.cm_service_output = role_api_instance.read_roles().to_dict() self.changed = True + def main(): module = ClouderaManagerModule.ansible_module( - argument_spec=dict( role=dict(required=True, type="list"), purge=dict(required=False, type="bool", default=False), - state=dict(type='str', default='started', choices=['started', 'stopped','absent','present','restarted']), - ), - - supports_check_mode=False - ) + state=dict( + type="str", + default="started", + choices=["started", "stopped", "absent", "present", "restarted"], + ), + ), + supports_check_mode=False, + ) result = ClouderaService(module) - + changed = result.changed output = dict( diff --git a/plugins/modules/cm_trial_license.py b/plugins/modules/cm_trial_license.py index e686e794..71f1e3b1 100644 --- a/plugins/modules/cm_trial_license.py +++ b/plugins/modules/cm_trial_license.py @@ -97,7 +97,7 @@ def process(self): if get_trial_state_request: self.cm_trial_output = get_trial_state_request self.changed = False - + except ApiException as e: if e.status == 404: api_instance.begin_trial() @@ -105,13 +105,11 @@ def process(self): self.cm_trial_output = get_trial_state_request self.changed = True + def main(): module = ClouderaManagerModule.ansible_module( - - argument_spec= - dict(), - supports_check_mode=True - ) + argument_spec=dict(), supports_check_mode=True + ) result = ClouderaTrial(module) diff --git a/plugins/modules/cm_version_info.py b/plugins/modules/cm_version_info.py index f45f679c..5017e41e 100644 --- a/plugins/modules/cm_version_info.py +++ b/plugins/modules/cm_version_info.py @@ -48,9 +48,9 @@ username: "jane_smith" password: "S&peR4Ec*re" register: cm_output - + # This will first try 'http://example.cloudera.com:7180' and will -# follow any redirects +# follow any redirects - name: Gather details using auto-discovery cloudera.cluster.cm_version: host: example.cloudera.com diff --git a/plugins/modules/host.py b/plugins/modules/host.py index 74ab71b9..218109b5 100644 --- a/plugins/modules/host.py +++ b/plugins/modules/host.py @@ -15,7 +15,7 @@ from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( ClouderaManagerModule, ) -from cm_client import ApiHost,ApiHostList +from cm_client import ApiHost, ApiHostList from cm_client import ClustersResourceApi from cm_client import HostsResourceApi from cm_client.rest import ApiException @@ -106,7 +106,7 @@ cluster_hostname: "Ecs_node_01" state: "detached" -- name: Remove a host +- name: Remove a host cloudera.cluster.host: host: "example.cloudera.host" username: "will_jordan" @@ -225,14 +225,16 @@ def process(self): existing = None try: - existing = host_api_instance.read_host(host_id=self.cluster_hostname).to_dict() + existing = host_api_instance.read_host( + host_id=self.cluster_hostname + ).to_dict() except ApiException as ex: if ex.status != 404: - raise ex - - if self.state == 'present': + raise ex + + if self.state == "present": if existing: - host_id = existing['host_id'] + host_id = existing["host_id"] else: host_params = { "hostname": self.cluster_hostname, @@ -247,13 +249,15 @@ def process(self): self.changed = True self.host_output = host_api_instance.read_host(host_id=host_id).to_dict() - elif self.state == 'absent': + elif self.state == "absent": if existing: if not self.module.check_mode: - self.host_output = host_api_instance.delete_host(host_id=existing['host_id']).to_dict() + self.host_output = host_api_instance.delete_host( + host_id=existing["host_id"] + ).to_dict() self.changed = True - elif self.state in ['attached','detached']: + elif self.state in ["attached", "detached"]: try: cluster_api_instance.read_cluster(cluster_name=self.name).to_dict() @@ -261,13 +265,22 @@ def process(self): if ex.status == 404: self.module.fail_json(msg=f"Cluster does not exist: {self.name}") - if self.state == 'attached': + if self.state == "attached": if existing: try: if not self.module.check_mode: - host_list = ApiHostList(items=[ApiHost(hostname=self.cluster_hostname, host_id=existing['host_id'])]) - cluster_api_instance.add_hosts(cluster_name=self.name,body=host_list) - host_id=existing['host_id'] + host_list = ApiHostList( + items=[ + ApiHost( + hostname=self.cluster_hostname, + host_id=existing["host_id"], + ) + ] + ) + cluster_api_instance.add_hosts( + cluster_name=self.name, body=host_list + ) + host_id = existing["host_id"] self.changed = True except ApiException as ex: if ex.status == 400: @@ -281,38 +294,62 @@ def process(self): host_params["rack_id"] = self.rack_id if not self.module.check_mode: new_host_param = ApiHostList(items=[ApiHost(**host_params)]) - create_host = host_api_instance.create_hosts(body=new_host_param) - host_list = ApiHostList(items=[ApiHost(hostname=self.cluster_hostname, host_id=create_host.items[0].host_id)]) - add_host = cluster_api_instance.add_hosts(cluster_name=self.name,body=host_list) + create_host = host_api_instance.create_hosts( + body=new_host_param + ) + host_list = ApiHostList( + items=[ + ApiHost( + hostname=self.cluster_hostname, + host_id=create_host.items[0].host_id, + ) + ] + ) + add_host = cluster_api_instance.add_hosts( + cluster_name=self.name, body=host_list + ) host_id = add_host.items[0].host_id self.changed = True - elif self.state == 'detached': - if existing and existing.get('cluster_ref') and existing['cluster_ref'].get('cluster_name'): - if not self.module.check_mode: - cluster_api_instance.remove_host(cluster_name=existing['cluster_ref']['cluster_name'],host_id=existing['host_id']) - host_id=existing['host_id'] - self.changed = True + elif self.state == "detached": + if ( + existing + and existing.get("cluster_ref") + and existing["cluster_ref"].get("cluster_name") + ): + if not self.module.check_mode: + cluster_api_instance.remove_host( + cluster_name=existing["cluster_ref"]["cluster_name"], + host_id=existing["host_id"], + ) + host_id = existing["host_id"] + self.changed = True + + self.host_output = host_api_instance.read_host( + host_id=self.cluster_hostname + ).to_dict() + - self.host_output = host_api_instance.read_host(host_id=self.cluster_hostname).to_dict() - def main(): module = ClouderaManagerModule.ansible_module( - argument_spec=dict( + argument_spec=dict( cluster_hostname=dict(required=True, type="str"), name=dict(required=False, type="str"), - host_ip=dict(required=False, type="str",aliases=["cluster_host_ip"]), + host_ip=dict(required=False, type="str", aliases=["cluster_host_ip"]), rack_id=dict(required=False, type="str"), - state=dict(type='str', default='present', choices=['present','absent','attached','detached']), - ), + state=dict( + type="str", + default="present", + choices=["present", "absent", "attached", "detached"], + ), + ), supports_check_mode=True, - - required_if=[ - ('state', 'attached', ('name', 'host_ip'), False), - ('state', 'detached', ('name',), False), - ('state', 'present', ('host_ip',), False) - ] - ) + required_if=[ + ("state", "attached", ("name", "host_ip"), False), + ("state", "detached", ("name",), False), + ("state", "present", ("host_ip",), False), + ], + ) result = ClouderaHost(module) @@ -329,5 +366,6 @@ def main(): module.exit_json(**output) + if __name__ == "__main__": main() diff --git a/plugins/modules/host_info.py b/plugins/modules/host_info.py index f4ec7182..0f479872 100644 --- a/plugins/modules/host_info.py +++ b/plugins/modules/host_info.py @@ -167,7 +167,6 @@ def __init__(self, module): # Execute the logic self.process() - @ClouderaManagerModule.handle_process def process(self): @@ -177,9 +176,13 @@ def process(self): if self.cluster_hostname or self.host_id: try: if self.cluster_hostname: - self.host_output = host_api_instance.read_host(host_id=self.cluster_hostname).to_dict() + self.host_output = host_api_instance.read_host( + host_id=self.cluster_hostname + ).to_dict() else: - self.host_output = host_api_instance.read_host(host_id=self.host_id).to_dict() + self.host_output = host_api_instance.read_host( + host_id=self.host_id + ).to_dict() except ApiException as ex: if ex.status != 404: raise ex @@ -189,11 +192,12 @@ def process(self): def main(): module = ClouderaManagerModule.ansible_module( - argument_spec=dict( + argument_spec=dict( cluster_hostname=dict(required=False, type="str"), - host_id=dict(required=False, type="str")), + host_id=dict(required=False, type="str"), + ), supports_check_mode=True, - ) + ) result = ClouderaHostInfo(module) @@ -210,5 +214,6 @@ def main(): module.exit_json(**output) + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/plugins/modules/parcel.py b/plugins/modules/parcel.py index 164d212e..206837e8 100644 --- a/plugins/modules/parcel.py +++ b/plugins/modules/parcel.py @@ -14,11 +14,15 @@ from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( ClouderaManagerModule, + parse_parcel_result, +) + +from ansible_collections.cloudera.cluster.plugins.module_utils.parcel_utils import ( + Parcel, ) from cm_client import ClustersResourceApi, ParcelResourceApi from cm_client.rest import ApiException -import time ANSIBLE_METADATA = { "metadata_version": "1.1", @@ -29,9 +33,11 @@ DOCUMENTATION = r""" --- module: parcel -short_description: Manage the state of parcels on a Cluster +short_description: Manage the state of parcels on a cluster description: - - Facilitates the management of parcels on a Cluster by downloading, distributing, and activating them according to the specified state. + - Facilitates the management of parcels of a CDP cluster according to the specified state. + - States lie on a continuum from I(absent), i.e. C(available remotely), I(downloaded), I(distributed), and I(activated)/I(present). + - The module manages the transitions between these states, e.g. if a parcel is I(distributed) and I(state=downloaded), the module will deactivate the parcel from the cluster hosts. author: - "Ronald Suplina (@rsuplina)" requirements: @@ -49,95 +55,97 @@ required: yes version: description: - - The version of the product, e.g. 1.1.0, 2.3.0. + - The semantic version of the product, e.g. 1.1.0, 2.3.0. type: str required: yes state: description: - State of the parcel. + - I(present) is the same as I(activated). type: str - default: 'activated' + default: 'present' choices: - 'downloaded' - 'distributed' - 'activated' - - 'removed' - - 'undistributed' - - 'deactivated' + - 'present' + - 'absent' required: False """ EXAMPLES = r""" --- -- name: Download, distribute and activate a parcel on a cluster +- name: Download, distribute and activate a parcel on a cluster cloudera.cluster.parcel: host: example.cloudera.com username: "jane_smith" password: "S&peR4Ec*re" - cluster_name: "OneNodeECS" + cluster_name: "Example_Cluster" product: "ECS" parcel_version: "1.5.1-b626-ecs-1.5.1-b626.p0.42068229" state: "activated" -- name: Downloand and distribute a parcel on a cluster +- name: Downloand and distribute a parcel on a cluster cloudera.cluster.parcel: host: example.cloudera.com username: "jane_smith" password: "S&peR4Ec*re" - cluster_name: "OneNodeECS" + cluster_name: "Example_Cluster" product: "ECS" parcel_version: "1.5.1-b626-ecs-1.5.1-b626.p0.42068229" state: "distributed" -- name: Remove the parcel on a specified cluster +- name: Remove the parcel on a specified cluster cloudera.cluster.parcel: host: example.cloudera.com username: "jane_smith" password: "S&peR4Ec*re" - cluster_name: "OneNodeECS" + cluster_name: "Example_Cluster" product: "ECS" parcel_version: "1.5.1-b626-ecs-1.5.1-b626.p0.42068229" - state: "removed" + state: "absent" -- name: Undistribute the parcel on a specified cluster +- name: Undistribute the parcel on a specified cluster cloudera.cluster.parcel: host: example.cloudera.com username: "jane_smith" password: "S&peR4Ec*re" - cluster_name: "OneNodeECS" + cluster_name: "Example_Cluster" product: "ECS" parcel_version: "1.5.1-b626-ecs-1.5.1-b626.p0.42068229" - state: "undistributed" + state: "downloaded" # Assuming the current state is "distributed" or "activated" """ RETURN = r""" --- -cloudera_manager: - description: Returns details about specific parcel +parcel: + description: Details about the parcel type: dict contains: product: product: The name of the product. type: str - returned: optional - parcel_version: - description: The version of the product + returned: always + version: + description: The version of the product. type: str - returned: optional + returned: always stage: description: Current stage of the parcel. type: str - returned: optional + returned: always state: - description: The state of the parcel. This shows the progress of state transitions and if there were any errors. + description: + - The state of the parcel. + - This shows the progress of state transitions and if there were any errors. type: dict returned: optional - clusterRef: - description: A reference to the enclosing cluster. + cluster_name: + description: The name of the enclosing cluster. type: dict - returned: optional - displayName: + returned: always + display_name: description: Display name of the parcel. type: str returned: optional @@ -149,176 +157,111 @@ class ClouderaParcel(ClouderaManagerModule): + + FUNCTION_MAP = { + "ACTIVATED": "activate", + "AVAILABLE_REMOTELY": "remove", + "DISTRIBUTED": "distribute", + "DOWNLOADED": "download", + } + def __init__(self, module): super(ClouderaParcel, self).__init__(module) - self.cluster_name = self.get_param("cluster_name") - self.product = self.get_param("product") + # Set parameters + self.cluster = self.get_param("cluster") + self.parcel_name = self.get_param("parcel") self.parcel_version = self.get_param("parcel_version") self.state = self.get_param("state") - self.polling_interval = self.get_param("polling_interval") - self.process() - + self.delay = self.get_param("delay") + self.timeout = self.get_param("timeout") - def deactivate_parcel(self, parcel_api_instance, cluster_name, product, parcel_version, polling_interval, timeout=600): - start_time = time.time() - parcel_api_instance.deactivate_command(cluster_name=cluster_name, product=product, version=parcel_version) - while True: - elapsed_time = time.time() - start_time - if elapsed_time > timeout: - self.module.fail_json(msg="Timeout exceeded while waiting for parcel state to complete.") - parcel_status = parcel_api_instance.read_parcel(cluster_name=cluster_name, product=product, version=parcel_version) - if parcel_status.stage == 'ACTIVATED': - time.sleep(polling_interval) - elif parcel_status.stage == "DISTRIBUTED": - break - - def undistribute_parcel(self, parcel_api_instance, cluster_name, product, parcel_version, polling_interval, timeout=600): - start_time = time.time() - parcel_api_instance.start_removal_of_distribution_command(cluster_name=cluster_name, product=product, version=parcel_version) - while True: - elapsed_time = time.time() - start_time - if elapsed_time > timeout: - self.module.fail_json(msg="Timeout exceeded while waiting for parcel state to complete.") - parcel_status = parcel_api_instance.read_parcel(cluster_name=cluster_name, product=product, version=parcel_version) - if parcel_status.stage == 'UNDISTRIBUTING': - time.sleep(polling_interval) - elif parcel_status.stage == "DOWNLOADED": - break - - def remove_parcel(self, parcel_api_instance, cluster_name, product, parcel_version, polling_interval, timeout=600): - start_time = time.time() - parcel_api_instance.remove_download_command(cluster_name=cluster_name, product=product, version=parcel_version) - while True: - elapsed_time = time.time() - start_time - if elapsed_time > timeout: - self.module.fail_json(msg="Timeout exceeded while waiting for parcel state to complete.") - parcel_status = parcel_api_instance.read_parcel(cluster_name=cluster_name, product=product, version=parcel_version) - if parcel_status.stage == 'DOWNLOADED': - time.sleep(polling_interval) - elif parcel_status.stage == "AVAILABLE_REMOTELY": - break - - def download_parcel(self, parcel_api_instance, cluster_name, product, parcel_version, polling_interval, timeout=1200): - start_time = time.time() - parcel_api_instance.start_download_command(cluster_name=cluster_name, product=product, version=parcel_version) - while True: - elapsed_time = time.time() - start_time - if elapsed_time > timeout: - self.module.fail_json(msg="Timeout exceeded while waiting for parcel state to complete.") - parcel_status = parcel_api_instance.read_parcel(cluster_name=cluster_name, product=product, version=parcel_version) - if parcel_status.stage == 'DOWNLOADING': - time.sleep(polling_interval) - elif parcel_status.stage == "DOWNLOADED": - break - - def distribute_parcel(self, parcel_api_instance, cluster_name, product, parcel_version, polling_interval, timeout=1200): - start_time = time.time() - parcel_api_instance.start_distribution_command(cluster_name=cluster_name, product=product, version=parcel_version) - while True: - elapsed_time = time.time() - start_time - if elapsed_time > timeout: - self.module.fail_json(msg="Timeout exceeded while waiting for parcel state to complete.") - parcel_status = parcel_api_instance.read_parcel(cluster_name=cluster_name, product=product, version=parcel_version) - if parcel_status.stage == 'DISTRIBUTING': - time.sleep(polling_interval) - elif parcel_status.stage == "DISTRIBUTED": - break - - def activate_parcel(self, parcel_api_instance, cluster_name, product, parcel_version, polling_interval): - parcel_api_instance.activate_command(cluster_name=cluster_name, product=product, version=parcel_version) - while True: - parcel_status = parcel_api_instance.read_parcel(cluster_name=cluster_name, product=product, version=parcel_version) - if parcel_status.stage == 'ACTIVATING': - time.sleep(polling_interval) - elif parcel_status.stage == "ACTIVATED": - break + # Set outputs + self.changed = False + self.diff = {} + self.output = {} + # Execute + self.process() @ClouderaManagerModule.handle_process def process(self): - parcel_api_instance = ParcelResourceApi(self.api_client) - cluster_api_instance = ClustersResourceApi(self.api_client) - - self.parcel_output = {} - self.changed = False - parcel_actions = [] + parcel_api = ParcelResourceApi(self.api_client) + cluster_api = ClustersResourceApi(self.api_client) try: - cluster_api_instance.read_cluster(cluster_name=self.cluster_name).to_dict() + cluster_api.read_cluster(cluster_name=self.cluster).to_dict() except ApiException as ex: if ex.status == 404: - self.module.fail_json(msg=f" Cluster {self.cluster_name} {ex.reason}") + self.module.fail_json(msg=f" Cluster {self.cluster} {ex.reason}") - try: - existing_state = parcel_api_instance.read_parcel(cluster_name=self.cluster_name, product=self.product, version=self.parcel_version).stage + try: + parcel = Parcel( + parcel_api=parcel_api, + product=self.parcel_name, + version=self.parcel_version, + cluster=self.cluster, + delay=self.delay, + timeout=self.timeout, + ) except ApiException as ex: - if ex.status == 404: - self.module.fail_json(msg=f" Parcel {self.parcel_version} {ex.reason}") - - - if self.state == "downloaded": - if existing_state == 'AVAILABLE_REMOTELY': - parcel_actions.append(self.download_parcel) - - elif self.state == "distributed": - if existing_state == 'AVAILABLE_REMOTELY': - parcel_actions.extend([self.download_parcel, self.distribute_parcel]) - elif existing_state == 'DOWNLOADED': - parcel_actions.append(self.distribute_parcel) - - elif self.state == "activated": - if existing_state == 'AVAILABLE_REMOTELY': - parcel_actions.extend([self.download_parcel, self.distribute_parcel, self.activate_parcel]) - elif existing_state == 'DOWNLOADED': - parcel_actions.extend([self.distribute_parcel, self.activate_parcel]) - elif existing_state == 'DISTRIBUTED': - parcel_actions.append(self.activate_parcel) - - - if self.state == "removed": - if existing_state == 'DOWNLOADED': - parcel_actions.append(self.remove_parcel) - if existing_state == 'DISTRIBUTED': - parcel_actions.extend([self.undistribute_parcel, self.remove_parcel]) - if existing_state == 'ACTIVATED': - parcel_actions.extend([self.deactivate_parcel, self.undistribute_parcel, self.remove_parcel]) - - if self.state == "undistributed": - if existing_state == 'DISTRIBUTED': - parcel_actions.extend([self.undistribute_parcel]) - if existing_state == 'ACTIVATED': - parcel_actions.extend([self.deactivate_parcel, self.undistribute_parcel]) - - if self.state == "deactivated": - if existing_state == 'ACTIVATED': - parcel_actions.append(self.deactivate_parcel) + if ex.status == 404: + self.module.fail_json( + msg=f"Parcel {self.parcel_name} (version: {self.parcel_version}) not found on cluster '{self.cluster}'" + ) + # Normalize self.state + if self.state == "present": + self.state = "ACTIVATED" + elif self.state == "absent": + self.state = "AVAILABLE_REMOTELY" + else: + self.state = str(self.state).upper() + if self.state != parcel.stage: + self.changed = True - if existing_state not in ['AVAILABLE_REMOTELY','DOWNLOADED','DISTRIBUTED','ACTIVATED']: - error_msg = parcel_api_instance.read_parcel(cluster_name=self.cluster_name, product=self.product, version=self.parcel_version).state.errors[0] - self.module.fail_json(msg=error_msg) + if self.module._diff: + self.diff = dict(before=parcel.stage, after=self.state) - if not self.module.check_mode: - for action in parcel_actions: - action(parcel_api_instance, self.cluster_name, self.product, self.parcel_version, self.polling_interval) - self.changed = True + if not self.module.check_mode: + cmd = getattr(parcel, self.FUNCTION_MAP[self.state]) + cmd() - self.parcel_output = parcel_api_instance.read_parcel(cluster_name=self.cluster_name, product=self.product, version=self.parcel_version).to_dict() + self.output = parse_parcel_result( + parcel_api.read_parcel( + cluster_name=self.cluster, + product=self.parcel_name, + version=self.parcel_version, + ) + ) def main(): module = ClouderaManagerModule.ansible_module( - argument_spec=dict( - cluster_name=dict(required=True, type="str"), - product=dict(required=True, type="str"), - polling_interval=dict(required=False, type="int",default=10), - parcel_version=dict(required=True, type="str"), - state=dict(type='str', default='activated', choices=['downloaded', 'distributed','activated','removed','undistributed','deactivated']), - ), - - supports_check_mode=True) + argument_spec=dict( + cluster=dict(required=True, aliases=["cluster_name"]), + parcel=dict(required=True, aliases=["parcel_name", "product", "name"]), + parcel_version=dict(required=True), + delay=dict( + required=False, type="int", default=10, aliases=["polling_interval"] + ), + timeout=dict( + required=False, type="int", default=600, aliases=["polling_timeout"] + ), + state=dict( + default="present", + choices=[ + "downloaded", + "distributed", + "activated", + "present", + "absent", + ], + ), + ), + supports_check_mode=True, + ) result = ClouderaParcel(module) @@ -326,7 +269,7 @@ def main(): output = dict( changed=changed, - cloudera_manager=result.parcel_output, + parcel=result.output, ) if result.debug: diff --git a/plugins/modules/parcel_info.py b/plugins/modules/parcel_info.py index 557db12b..b421ef92 100644 --- a/plugins/modules/parcel_info.py +++ b/plugins/modules/parcel_info.py @@ -64,7 +64,7 @@ product: "ECS" parcel_version: "1.5.1-b626-ecs-1.5.1-b626.p0.42068229" -- name: Gather details about all parcels on the cluster +- name: Gather details about all parcels on the cluster cloudera.cluster.parcel_info: host: example.cloudera.com username: "jane_smith" @@ -82,15 +82,15 @@ product: product: The name of the product. type: str - returned: always + returned: always version: description: The version of the product type: str - returned: always + returned: always stage: description: Current stage of the parcel. type: str - returned: always + returned: always state: description: The state of the parcel. This shows the progress of state transitions and if there were any errors. type: dict @@ -118,7 +118,6 @@ def __init__(self, module): self.parcel_version = self.get_param("parcel_version") self.process() - @ClouderaManagerModule.handle_process def process(self): parcel_api_instance = ParcelResourceApi(self.api_client) @@ -135,23 +134,30 @@ def process(self): self.module.fail_json(msg=f" Cluster {self.cluster_name} {ex.reason}") if self.product and self.parcel_version: - self.parcel_info = parcel_api_instance.read_parcel(cluster_name=self.cluster_name, product=self.product, version=self.parcel_version).to_dict() - self.parcel_output = {"items":[self.parcel_info]} + self.parcel_info = parcel_api_instance.read_parcel( + cluster_name=self.cluster_name, + product=self.product, + version=self.parcel_version, + ).to_dict() + self.parcel_output = {"items": [self.parcel_info]} else: - self.parcel_output = parcels_api_instance.read_parcels(cluster_name=self.cluster_name).to_dict() + self.parcel_output = parcels_api_instance.read_parcels( + cluster_name=self.cluster_name + ).to_dict() def main(): module = ClouderaManagerModule.ansible_module( - argument_spec=dict( + argument_spec=dict( cluster_name=dict(required=True, type="str"), product=dict(required=False, type="str"), - parcel_version=dict(required=False, type="str")), + parcel_version=dict(required=False, type="str"), + ), supports_check_mode=True, required_together=[ - ('product', 'parcel_version'), - ], - ) + ("product", "parcel_version"), + ], + ) result = ClouderaParcelInfo(module) diff --git a/plugins/modules/service.py b/plugins/modules/service.py index 47e45c00..e81a348f 100644 --- a/plugins/modules/service.py +++ b/plugins/modules/service.py @@ -38,7 +38,7 @@ DOCUMENTATION = r""" --- module: service -short_description: Manage a service in cluster +short_description: Manage a service in cluster description: - Manage a service in a cluster. author: @@ -121,7 +121,7 @@ service: example_ecs type: ECS display_name: Example ECS - + - name: Stop a cluster service cloudera.cluster.service: @@ -131,7 +131,7 @@ cluster: example_cluster service: example_ecs state: stopped - + - name: Force a restart of a cluster service cloudera.cluster.service: host: example.cloudera.com @@ -140,7 +140,7 @@ cluster: example_cluster service: example_ecs state: restarted - + - name: Set a cluster service into maintenance mode cloudera.cluster.service: host: example.cloudera.com @@ -149,7 +149,7 @@ cluster: example_cluster service: example_ecs maintenance: yes - + - name: Update (append) several tags on a cluster service cloudera.cluster.service: host: example.cloudera.com @@ -160,7 +160,7 @@ tags: tag_one: valueOne tag_two: valueTwo - + - name: Update (purge) the tags on a cluster service cloudera.cluster.service: host: example.cloudera.com @@ -171,7 +171,7 @@ tags: tag_three: value_three purge: yes - + - name: Remove all the tags on a cluster service cloudera.cluster.service: host: example.cloudera.com @@ -180,7 +180,7 @@ cluster: example_cluster service: example_ecs tags: {} - purge: yes + purge: yes - name: Remove a cluster service cloudera.cluster.service: diff --git a/plugins/modules/service_config.py b/plugins/modules/service_config.py index 769c8dc8..943c97ea 100644 --- a/plugins/modules/service_config.py +++ b/plugins/modules/service_config.py @@ -38,7 +38,7 @@ DOCUMENTATION = r""" --- module: service_config -short_description: Manage a service configuration in cluster +short_description: Manage a service configuration in cluster description: - Manage a service configuration (service-wide) in a cluster. author: @@ -123,7 +123,7 @@ config_one: ValueOne config_two: 4567 purge: yes - + - name: Reset all service-wide parameters cloudera.cluster.service_config: host: example.cloudera.com @@ -148,14 +148,14 @@ returned: always value: description: - - The user-defined value. + - The user-defined value. - When absent, the default value (if any) will be used. - Can also be absent, when enumerating allowed configs. type: str returned: always required: description: - - Whether this configuration is required for the service. + - Whether this configuration is required for the service. - If any required configuration is not set, operations on the service may not work. - Available using I(view=full). type: bool @@ -186,7 +186,7 @@ returned: when supported sensitive: description: - - Whether this configuration is sensitive, i.e. contains information such as passwords, which might affect how the value of this configuration might be shared by the caller. + - Whether this configuration is sensitive, i.e. contains information such as passwords, which might affect how the value of this configuration might be shared by the caller. type: bool returned: when supported validation_state: @@ -207,8 +207,8 @@ returned: when supported validation_warnings_suppressed: description: - - Whether validation warnings associated with this parameter are suppressed. - - In general, suppressed validation warnings are hidden in the Cloudera Manager UI. + - Whether validation warnings associated with this parameter are suppressed. + - In general, suppressed validation warnings are hidden in the Cloudera Manager UI. - Configurations that do not produce warnings will not contain this field. - Available using I(view=full). type: bool diff --git a/plugins/modules/service_config_info.py b/plugins/modules/service_config_info.py index 62ae7654..969e0f93 100644 --- a/plugins/modules/service_config_info.py +++ b/plugins/modules/service_config_info.py @@ -78,7 +78,7 @@ password: "S&peR4Ec*re" cluster: ExampleCluster service: knox - + - name: Gather the configuration details in 'full' for a cluster service cloudera.cluster.service_config_info: host: "example.cloudera.host" @@ -102,14 +102,14 @@ returned: always value: description: - - The user-defined value. + - The user-defined value. - When absent, the default value (if any) will be used. - Can also be absent, when enumerating allowed configs. type: str returned: always required: description: - - Whether this configuration is required for the service. + - Whether this configuration is required for the service. - If any required configuration is not set, operations on the service may not work. - Available using I(view=full). type: bool @@ -140,7 +140,7 @@ returned: when supported sensitive: description: - - Whether this configuration is sensitive, i.e. contains information such as passwords, which might affect how the value of this configuration might be shared by the caller. + - Whether this configuration is sensitive, i.e. contains information such as passwords, which might affect how the value of this configuration might be shared by the caller. type: bool returned: when supported validation_state: @@ -161,8 +161,8 @@ returned: when supported validation_warnings_suppressed: description: - - Whether validation warnings associated with this parameter are suppressed. - - In general, suppressed validation warnings are hidden in the Cloudera Manager UI. + - Whether validation warnings associated with this parameter are suppressed. + - In general, suppressed validation warnings are hidden in the Cloudera Manager UI. - Configurations that do not produce warnings will not contain this field. - Available using I(view=full). type: bool diff --git a/plugins/modules/service_info.py b/plugins/modules/service_info.py index 7d4369db..b2648a80 100644 --- a/plugins/modules/service_info.py +++ b/plugins/modules/service_info.py @@ -80,7 +80,7 @@ username: "jane_person" password: "S&peR4Ec*re" cluster: ExampleCluster - + - name: Gather the details with additional healthcheck information for a service cloudera.cluster.service_info: host: "example.cloudera.host" diff --git a/plugins/modules/service_role.py b/plugins/modules/service_role.py index 6eb1d2d6..f4d29949 100644 --- a/plugins/modules/service_role.py +++ b/plugins/modules/service_role.py @@ -43,7 +43,7 @@ DOCUMENTATION = r""" --- module: service_role -short_description: Manage a service role in cluster +short_description: Manage a service role in cluster description: - Manage a service role in a cluster. author: @@ -153,7 +153,7 @@ type: GATEWAY name: example-gateway cluster_hostname: worker-01.cloudera.internal - + - name: Set a service role to maintenance mode cloudera.cluster.service_role: host: example.cloudera.com @@ -163,7 +163,7 @@ service: example-hdfs name: example-gateway maintenance: yes - + - name: Update (append) tags to a service role cloudera.cluster.service_role: host: example.cloudera.com @@ -175,7 +175,7 @@ tags: tag_one: value_one tag_two: value_two - + - name: Set (purge) tags to a service role cloudera.cluster.service_role: host: example.cloudera.com @@ -186,7 +186,7 @@ tags: tag_three: value_three purge: yes - + - name: Remove all tags on a service role cloudera.cluster.service_role: host: example.cloudera.com @@ -197,7 +197,7 @@ name: example-gateway tags: {} purge: yes - + - name: Start a service role cloudera.cluster.service_role: host: example.cloudera.com @@ -207,7 +207,7 @@ service: example-hdfs name: example-gateway state: started - + - name: Force a restart to a service role cloudera.cluster.service_role: host: example.cloudera.com @@ -217,7 +217,7 @@ service: example-hdfs name: example-gateway state: restarted - + - name: Start a service role cloudera.cluster.service_role: host: example.cloudera.com @@ -227,7 +227,7 @@ service: example-hdfs name: example-gateway state: started - + - name: Remove a service role cloudera.cluster.service_role: host: example.cloudera.com diff --git a/plugins/modules/service_role_config.py b/plugins/modules/service_role_config.py index 12927a8b..9cc27ae1 100644 --- a/plugins/modules/service_role_config.py +++ b/plugins/modules/service_role_config.py @@ -39,7 +39,7 @@ DOCUMENTATION = r""" --- module: service_role_config -short_description: Manage a service role configuration in cluster +short_description: Manage a service role configuration in cluster description: - Manage a service role configuration (role-specific) in a cluster. author: @@ -117,7 +117,7 @@ service: example-service parameters: more_configuration: None - + - name: Update (purge) role parameters cloudera.cluster.cluster_service_role_config: host: example.cloudera.com @@ -129,7 +129,7 @@ config_one: None config_two: ValueTwo config_three: 2345 - + - name: Reset all role parameters cloudera.cluster.cluster_service_role_config: host: example.cloudera.com diff --git a/plugins/modules/service_role_config_group.py b/plugins/modules/service_role_config_group.py index 8c40f55c..60e77366 100644 --- a/plugins/modules/service_role_config_group.py +++ b/plugins/modules/service_role_config_group.py @@ -131,7 +131,7 @@ service: HDFS role_config_group: Example-DATANODE type: DATANODE - + - name: Create or update a role config group with role associations cloudera.cluster.service_role_config_group: host: example.cloudera.com @@ -154,7 +154,7 @@ role_config_group: Example-DATANODE roles: - hdfs-DATANODE-7f3a9da5805a46e3100bae67424355ac # Now two roles - + - name: Update (purge) role associations to a role config group cloudera.cluster.cluster_service_role_config: host: example.cloudera.com @@ -166,7 +166,7 @@ roles: - hdfs-DATANODE-7f3a9da5805a46e3100bae67424355ac # Now only one role purge: yes - + - name: Reset all role associations to a role config group cloudera.cluster.cluster_service_role_config: host: example.cloudera.com @@ -187,7 +187,7 @@ service: HDFS role_config_group: Example-DATANODE state: absent -- +- """ RETURN = r""" diff --git a/plugins/modules/service_role_config_group_config.py b/plugins/modules/service_role_config_group_config.py index d33298db..b496f5de 100644 --- a/plugins/modules/service_role_config_group_config.py +++ b/plugins/modules/service_role_config_group_config.py @@ -131,7 +131,7 @@ config_one: ValueOne config_two: 4567 purge: yes - + - name: Reset all role config group parameters cloudera.cluster.service_role_config_group_config: host: example.cloudera.com @@ -266,7 +266,7 @@ def process(self): api_instance = RoleConfigGroupsResourceApi(self.api_client) refresh = True - + try: existing = api_instance.read_config( cluster_name=self.cluster, @@ -311,7 +311,7 @@ def process(self): body=body, ).items ] - + if self.view == "full": refresh = True @@ -329,7 +329,9 @@ def main(): argument_spec=dict( cluster=dict(required=True, aliases=["cluster_name"]), service=dict(required=True, aliases=["service_name"]), - role_config_group=dict(required=True, aliases=["role_config_group", "name"]), + role_config_group=dict( + required=True, aliases=["role_config_group", "name"] + ), parameters=dict(type="dict", required=True, aliases=["params"]), purge=dict(type="bool", default=False), view=dict( diff --git a/plugins/modules/service_role_config_group_config_info.py b/plugins/modules/service_role_config_group_config_info.py index bf7f2e47..fc127dc8 100644 --- a/plugins/modules/service_role_config_group_config_info.py +++ b/plugins/modules/service_role_config_group_config_info.py @@ -88,7 +88,7 @@ cluster: ExampleCluster service: knox role_config_group: hdfs-GATEWAY-base - + - name: Gather the configuration details in 'full' for a cluster service role config group cloudera.cluster.service_role_config_group_config_info: host: "example.cloudera.internal" diff --git a/plugins/modules/service_role_config_group_info.py b/plugins/modules/service_role_config_group_info.py index a22b9ebb..46e95af4 100644 --- a/plugins/modules/service_role_config_group_info.py +++ b/plugins/modules/service_role_config_group_info.py @@ -82,7 +82,7 @@ cluster: ExampleCluster service: knox role: GATEWAY - + - name: Gather the configuration details in 'full' for a cluster service role cloudera.cluster.service_role_config_info: host: "example.cloudera.internal" diff --git a/plugins/modules/service_role_config_info.py b/plugins/modules/service_role_config_info.py index 340c35e2..c66f3979 100644 --- a/plugins/modules/service_role_config_info.py +++ b/plugins/modules/service_role_config_info.py @@ -88,7 +88,7 @@ cluster: ExampleCluster service: knox role: GATEWAY - + - name: Gather the configuration details in 'full' for a cluster service role cloudera.cluster.service_role_config_info: host: "example.cloudera.internal" diff --git a/plugins/modules/service_role_info.py b/plugins/modules/service_role_info.py index 2d6aff3e..c0e1f63f 100644 --- a/plugins/modules/service_role_info.py +++ b/plugins/modules/service_role_info.py @@ -110,7 +110,7 @@ password: "S&peR4Ec*re" cluster: ExampleCluster service: yarn - + - name: Gather the details with additional healthcheck information for the roles in the 'ecs' service cloudera.cluster.service_role_info: host: "example.cloudera.host" @@ -119,7 +119,7 @@ cluster: ExampleCluster service: ecs view: healthcheck - + - name: Gather details of the 'NODEMANAGER' roles for the 'yarn' service cloudera.cluster.service_role_info: host: "example.cloudera.host" @@ -128,7 +128,7 @@ cluster: ExampleCluster service: yarn type: NODEMANAGER - + - name: Gather details of the roles for the 'yarn' service on a particular cluster host cloudera.cluster.service_role_info: host: "example.cloudera.host" diff --git a/pytest.ini b/pytest.ini index 024207ab..03f164ec 100644 --- a/pytest.ini +++ b/pytest.ini @@ -20,5 +20,4 @@ filterwarnings = ; log_cli = 1 ; log_cli_level = INFO - -pythonpath = "../../../" \ No newline at end of file +pythonpath = "../../../" diff --git a/roles/assemble_template/tasks/main.yml b/roles/assemble_template/tasks/main.yml index f1e28183..c76ed466 100644 --- a/roles/assemble_template/tasks/main.yml +++ b/roles/assemble_template/tasks/main.yml @@ -34,7 +34,7 @@ src: "{{ __fragment.path }}" dest: "{{ fragments_temp_directory.path }}/{{ __fragment.path | basename }}" loop: "{{ fragments.files }}" - loop_control: + loop_control: loop_var: __fragment label: "{{ __fragment.path | basename }}" when: fragments.matched > 0 diff --git a/roles/cloudera_manager/api_client/action_plugins/cm_api.py b/roles/cloudera_manager/api_client/action_plugins/cm_api.py index 7465208b..658ab3a3 100644 --- a/roles/cloudera_manager/api_client/action_plugins/cm_api.py +++ b/roles/cloudera_manager/api_client/action_plugins/cm_api.py @@ -23,98 +23,134 @@ display = Display() -class ActionModule(ActionBase): - def build_url(self, api_base, api_endpoint): - if not api_endpoint.startswith("/"): - api_endpoint = "/" + api_endpoint - return api_base + api_endpoint - - def build_args(self, task_vars, additional_args=dict()): - args = dict( - body_format = "json", - force_basic_auth=True, - url_username=task_vars['cloudera_manager_api_user'], - url_password=task_vars['cloudera_manager_api_password'], - return_content=True, - validate_certs=task_vars['cloudera_manager_tls_validate_certs'] - ) - args.update(additional_args) - return args - - def get_api_base_url(self, task_vars): - # If there's already a value in task vars, just use that - if 'cloudera_manager_api' in task_vars: - api_base = task_vars['cloudera_manager_api'] - result = None - else: - # Call /api/version endpoint to find the correct API version number. - url = self.build_url(task_vars['cloudera_manager_url'], '/api/version') - args = self.build_args(task_vars, dict(url=url)) - result = self._execute_module('uri', module_args=args, task_vars=task_vars, wrap_async=self._task.async_val) - # We use the URL returned in the result rather than the one we defined originally. - # This has the benefit of allowing to use TLS-enabled endpoints later if the call was redirected. - api_base = result["url"].replace("version", result['content']) if result['status'] == 200 else None - return (api_base, result) - - def poll_command_status(self, task_vars, api_base_url, command_id): - args = self.build_args(task_vars, additional_args=dict( - url = self.build_url(api_base_url, "/commands/" + str(command_id)) - )) - result = self._execute_module('uri', module_args=args, task_vars=task_vars, wrap_async=self._task.async_val) - return result - - def run(self, tmp=None, task_vars=None): - - result = super(ActionModule, self).run(tmp, task_vars) - - # Get Cloudera Manager API base url from task vars, or work it out ourselves - api_base_url, api_base_result = self.get_api_base_url(task_vars) - if not api_base_url: - result.update(api_base_result) - return result - - # Add endpoint and request method to base args containing creds etc - uri_module_args = self.build_args(task_vars, additional_args=dict( - url = self.build_url(api_base_url, self._task.args['endpoint']), - method = self._task.args.get('method') or 'GET', - status_code = self._task.args.get('status_code') or '200', - timeout = self._task.args.get('timeout') or '30' - )) - - poll_duration = int(self._task.args.get('poll_duration') or 10) - poll_max_failed_retries = int(self._task.args.get('poll_max_failed_retries') or 3) - - # Add request body if necessary - if 'body' in self._task.args: - uri_module_args.update(body = self._task.args['body']) - - # Send request to CM API - uri_result = self._execute_module('uri', module_args=uri_module_args, task_vars=task_vars, wrap_async=self._task.async_val) - result.update(uri_result) - - # If we get ApiCommand response, and it is active, wait for completion - failed_polls = 0 - if 'json' in uri_result: - response = uri_result['json'] - if 'id' in response and 'active' in response: - command_id = response['id'] - command_name = response['name'] - command_active = response['active'] - while command_active and failed_polls < poll_max_failed_retries: - time.sleep(poll_duration) - display.vv("Waiting for {} command ({}) to complete...".format(command_name, command_id)) - command_status = self.poll_command_status(task_vars, api_base_url, command_id) - if 'json' in command_status: - failed_polls = 0 - response = command_status['json'] - command_active = response['active'] - else: - failed_polls += 1 - response = {'success': False} - display.vv("Failed to poll command ({}) for status (attempt {} of {})...".format( - command_id, failed_polls, poll_max_failed_retries)) - result.update(command_status) - result['failed'] = not response['success'] - - return result +class ActionModule(ActionBase): + def build_url(self, api_base, api_endpoint): + if not api_endpoint.startswith("/"): + api_endpoint = "/" + api_endpoint + return api_base + api_endpoint + + def build_args(self, task_vars, additional_args=dict()): + args = dict( + body_format="json", + force_basic_auth=True, + url_username=task_vars["cloudera_manager_api_user"], + url_password=task_vars["cloudera_manager_api_password"], + return_content=True, + validate_certs=task_vars["cloudera_manager_tls_validate_certs"], + ) + args.update(additional_args) + return args + + def get_api_base_url(self, task_vars): + # If there's already a value in task vars, just use that + if "cloudera_manager_api" in task_vars: + api_base = task_vars["cloudera_manager_api"] + result = None + else: + # Call /api/version endpoint to find the correct API version number. + url = self.build_url(task_vars["cloudera_manager_url"], "/api/version") + args = self.build_args(task_vars, dict(url=url)) + result = self._execute_module( + "uri", + module_args=args, + task_vars=task_vars, + wrap_async=self._task.async_val, + ) + # We use the URL returned in the result rather than the one we defined originally. + # This has the benefit of allowing to use TLS-enabled endpoints later if the call was redirected. + api_base = ( + result["url"].replace("version", result["content"]) + if result["status"] == 200 + else None + ) + return (api_base, result) + + def poll_command_status(self, task_vars, api_base_url, command_id): + args = self.build_args( + task_vars, + additional_args=dict( + url=self.build_url(api_base_url, "/commands/" + str(command_id)) + ), + ) + result = self._execute_module( + "uri", + module_args=args, + task_vars=task_vars, + wrap_async=self._task.async_val, + ) + return result + + def run(self, tmp=None, task_vars=None): + + result = super(ActionModule, self).run(tmp, task_vars) + + # Get Cloudera Manager API base url from task vars, or work it out ourselves + api_base_url, api_base_result = self.get_api_base_url(task_vars) + if not api_base_url: + result.update(api_base_result) + return result + + # Add endpoint and request method to base args containing creds etc + uri_module_args = self.build_args( + task_vars, + additional_args=dict( + url=self.build_url(api_base_url, self._task.args["endpoint"]), + method=self._task.args.get("method") or "GET", + status_code=self._task.args.get("status_code") or "200", + timeout=self._task.args.get("timeout") or "30", + ), + ) + + poll_duration = int(self._task.args.get("poll_duration") or 10) + poll_max_failed_retries = int( + self._task.args.get("poll_max_failed_retries") or 3 + ) + + # Add request body if necessary + if "body" in self._task.args: + uri_module_args.update(body=self._task.args["body"]) + + # Send request to CM API + uri_result = self._execute_module( + "uri", + module_args=uri_module_args, + task_vars=task_vars, + wrap_async=self._task.async_val, + ) + result.update(uri_result) + + # If we get ApiCommand response, and it is active, wait for completion + failed_polls = 0 + if "json" in uri_result: + response = uri_result["json"] + if "id" in response and "active" in response: + command_id = response["id"] + command_name = response["name"] + command_active = response["active"] + while command_active and failed_polls < poll_max_failed_retries: + time.sleep(poll_duration) + display.vv( + "Waiting for {} command ({}) to complete...".format( + command_name, command_id + ) + ) + command_status = self.poll_command_status( + task_vars, api_base_url, command_id + ) + if "json" in command_status: + failed_polls = 0 + response = command_status["json"] + command_active = response["active"] + else: + failed_polls += 1 + response = {"success": False} + display.vv( + "Failed to poll command ({}) for status (attempt {} of {})...".format( + command_id, failed_polls, poll_max_failed_retries + ) + ) + result.update(command_status) + result["failed"] = not response["success"] + + return result diff --git a/roles/cloudera_manager/api_client/filter_plugins/filters.py b/roles/cloudera_manager/api_client/filter_plugins/filters.py index 72a1853d..67f43635 100644 --- a/roles/cloudera_manager/api_client/filter_plugins/filters.py +++ b/roles/cloudera_manager/api_client/filter_plugins/filters.py @@ -15,13 +15,10 @@ class FilterModule(object): + def filters(self): + return {"to_ldap_type_enum": self.to_ldap_type_enum} - def filters(self): - return { - 'to_ldap_type_enum': self.to_ldap_type_enum - } - - def to_ldap_type_enum(self, s): - if s == "AD": - return "ACTIVE_DIRECTORY" - return s.replace(" ","_").upper() + def to_ldap_type_enum(self, s): + if s == "AD": + return "ACTIVE_DIRECTORY" + return s.replace(" ", "_").upper() diff --git a/roles/cloudera_manager/api_client/handlers/main.yml b/roles/cloudera_manager/api_client/handlers/main.yml index 0cd6907b..6531575d 100644 --- a/roles/cloudera_manager/api_client/handlers/main.yml +++ b/roles/cloudera_manager/api_client/handlers/main.yml @@ -18,4 +18,4 @@ cm_api: endpoint: /cm/service/commands/restart method: POST - timeout: "{{ cluster_restart_timeout | default(3000) }}" \ No newline at end of file + timeout: "{{ cluster_restart_timeout | default(3000) }}" diff --git a/roles/cloudera_manager/autotls/files/cert.py_patch b/roles/cloudera_manager/autotls/files/cert.py_patch index e3a12116..8b41aae5 100644 --- a/roles/cloudera_manager/autotls/files/cert.py_patch +++ b/roles/cloudera_manager/autotls/files/cert.py_patch @@ -6,6 +6,6 @@ generate_truststore(self.cfg.keytool, cluster_ca_jks, truststore_password, - cluster_ca_pem) + cluster_ca_pem, self.cfg.keystore_type) - + global_ca_pem = self.trust_files[GLOBAL_TLS_SET][PEM_TLS_TYPE] copied_cluster_to_global = False diff --git a/roles/cloudera_manager/autotls/tasks/main.yml b/roles/cloudera_manager/autotls/tasks/main.yml index 47533897..83e5bf35 100644 --- a/roles/cloudera_manager/autotls/tasks/main.yml +++ b/roles/cloudera_manager/autotls/tasks/main.yml @@ -67,4 +67,3 @@ - restart cloudera-scm-server - restart cloudera management service - restart cloudera-scm-agent - diff --git a/roles/cloudera_manager/autotls/tasks/patch_old_cm.yml b/roles/cloudera_manager/autotls/tasks/patch_old_cm.yml index 62182247..7db78d00 100644 --- a/roles/cloudera_manager/autotls/tasks/patch_old_cm.yml +++ b/roles/cloudera_manager/autotls/tasks/patch_old_cm.yml @@ -13,4 +13,4 @@ - name: Fix cert.py ansible.posix.patch: src: "{{ role_path}}/patch/cert.py_patch" - dest: /opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/tools/cert.py \ No newline at end of file + dest: /opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/tools/cert.py diff --git a/roles/cloudera_manager/autotls/templates/auto-tls-key.json b/roles/cloudera_manager/autotls/templates/auto-tls-key.json index f49c1cbb..c0278b26 100644 --- a/roles/cloudera_manager/autotls/templates/auto-tls-key.json +++ b/roles/cloudera_manager/autotls/templates/auto-tls-key.json @@ -6,4 +6,4 @@ "userName" : "{{ sudoerUser|default("root") }}", "privateKey": "{{ node_key_one_line|default('~/node_key') }}" "passphrase": "{{ passphrase|default("") }}" -} \ No newline at end of file +} diff --git a/roles/cloudera_manager/autotls/templates/auto-tls.json b/roles/cloudera_manager/autotls/templates/auto-tls.json index 467a0ce2..7990be97 100644 --- a/roles/cloudera_manager/autotls/templates/auto-tls.json +++ b/roles/cloudera_manager/autotls/templates/auto-tls.json @@ -5,5 +5,4 @@ {% if freeipa_activated %}"trustedCaCerts" : "/etc/ipa/ca.crt",{% endif %} "userName" : "root", "password": "{{ node_password }}" -} - \ No newline at end of file +} diff --git a/roles/cloudera_manager/cms_tls/files/cms_keystore_tls.json b/roles/cloudera_manager/cms_tls/files/cms_keystore_tls.json index 74056c7c..727aee72 100644 --- a/roles/cloudera_manager/cms_tls/files/cms_keystore_tls.json +++ b/roles/cloudera_manager/cms_tls/files/cms_keystore_tls.json @@ -13,4 +13,4 @@ "value": "true" } ] - } \ No newline at end of file + } diff --git a/roles/cloudera_manager/cms_tls/files/cms_navigator_keystore_tls.json b/roles/cloudera_manager/cms_tls/files/cms_navigator_keystore_tls.json index 3e139999..8477f865 100644 --- a/roles/cloudera_manager/cms_tls/files/cms_navigator_keystore_tls.json +++ b/roles/cloudera_manager/cms_tls/files/cms_navigator_keystore_tls.json @@ -25,4 +25,4 @@ "value": "true" } ] - } \ No newline at end of file + } diff --git a/roles/cloudera_manager/cms_tls/files/cms_navigator_metaserver_keystore_tls.json b/roles/cloudera_manager/cms_tls/files/cms_navigator_metaserver_keystore_tls.json index 249ed5be..67de3d07 100644 --- a/roles/cloudera_manager/cms_tls/files/cms_navigator_metaserver_keystore_tls.json +++ b/roles/cloudera_manager/cms_tls/files/cms_navigator_metaserver_keystore_tls.json @@ -17,4 +17,4 @@ "value": "true" } ] - } \ No newline at end of file + } diff --git a/roles/cloudera_manager/cms_tls/files/cms_truststore_tls.json b/roles/cloudera_manager/cms_tls/files/cms_truststore_tls.json index 261ae793..305dbd94 100644 --- a/roles/cloudera_manager/cms_tls/files/cms_truststore_tls.json +++ b/roles/cloudera_manager/cms_tls/files/cms_truststore_tls.json @@ -9,4 +9,4 @@ "value": "{{CM_AUTO_TLS}}" } ] - } \ No newline at end of file + } diff --git a/roles/cloudera_manager/cms_tls/meta/main.yml b/roles/cloudera_manager/cms_tls/meta/main.yml index c69299ca..d6070b30 100644 --- a/roles/cloudera_manager/cms_tls/meta/main.yml +++ b/roles/cloudera_manager/cms_tls/meta/main.yml @@ -1,3 +1,3 @@ --- dependencies: - - role: cloudera.cluster.cloudera_manager.api_client \ No newline at end of file + - role: cloudera.cluster.cloudera_manager.api_client diff --git a/roles/cloudera_manager/cms_tls/tasks/main.yml b/roles/cloudera_manager/cms_tls/tasks/main.yml index b3949cfc..f413cdfe 100644 --- a/roles/cloudera_manager/cms_tls/tasks/main.yml +++ b/roles/cloudera_manager/cms_tls/tasks/main.yml @@ -1,39 +1,39 @@ --- - name: Setup TLS for Activity Monitor cm_api: - method: PUT + method: PUT endpoint: /cm/service/roleConfigGroups/mgmt-ACTIVITYMONITOR-BASE/config body: "{{ lookup('file', 'cms_keystore_tls.json', convert_data=False) }}" - name: Setup TLS for Host Monitor cm_api: - method: PUT + method: PUT endpoint: /cm/service/roleConfigGroups/mgmt-HOSTMONITOR-BASE/config body: "{{ lookup('file', 'cms_keystore_tls.json', convert_data=False) }}" - name: Setup TLS for Service Monitor cm_api: - method: PUT + method: PUT endpoint: /cm/service/roleConfigGroups/mgmt-SERVICEMONITOR-BASE/config body: "{{ lookup('file', 'cms_keystore_tls.json', convert_data=False) }}" - name: Setup TLS for Navigator cm_api: - method: PUT + method: PUT endpoint: /cm/service/roleConfigGroups/mgmt-NAVIGATOR-BASE/config body: "{{ lookup('file', 'cms_navigator_keystore_tls.json', convert_data=False) }}" when: cloudera_manager_version is version('7.0.0','<') -- name: Setup TLS for Navigator Meta Server +- name: Setup TLS for Navigator Meta Server cm_api: - method: PUT + method: PUT endpoint: /cm/service/roleConfigGroups/mgmt-NAVIGATORMETASERVER-BASE/config body: "{{ lookup('file', 'cms_navigator_metaserver_keystore_tls.json', convert_data=False) }}" when: cloudera_manager_version is version('7.0.0','<') - name: Setup TLS for CMS cm_api: - method: PUT + method: PUT endpoint: /cm/service/config body: "{{ lookup('file', 'cms_truststore_tls.json', convert_data=False) }}" notify: diff --git a/roles/cloudera_manager/config/meta/main.yml b/roles/cloudera_manager/config/meta/main.yml index 387f099b..55b150d4 100644 --- a/roles/cloudera_manager/config/meta/main.yml +++ b/roles/cloudera_manager/config/meta/main.yml @@ -15,5 +15,3 @@ --- dependencies: - role: cloudera.cluster.cloudera_manager.api_client - - diff --git a/roles/cloudera_manager/external_auth/defaults/main.yml b/roles/cloudera_manager/external_auth/defaults/main.yml index 7ec57551..0947c1f2 100644 --- a/roles/cloudera_manager/external_auth/defaults/main.yml +++ b/roles/cloudera_manager/external_auth/defaults/main.yml @@ -14,4 +14,4 @@ --- -freeipa_activated: False \ No newline at end of file +freeipa_activated: False diff --git a/roles/cloudera_manager/external_auth/tasks/main.yml b/roles/cloudera_manager/external_auth/tasks/main.yml index 082c6a87..a9fdbaef 100644 --- a/roles/cloudera_manager/external_auth/tasks/main.yml +++ b/roles/cloudera_manager/external_auth/tasks/main.yml @@ -17,7 +17,7 @@ - name: Conditionally load in variables for initializing IPA ansible.builtin.include_vars: file: freeipa.yml - when: + when: - freeipa_activated - cloudera_manager_external_auth is undefined - cloudera_manager_version is version('6.0.0','>=') diff --git a/roles/cloudera_manager/external_auth/vars/freeipa.yml b/roles/cloudera_manager/external_auth/vars/freeipa.yml index d3fa4026..afad671b 100644 --- a/roles/cloudera_manager/external_auth/vars/freeipa.yml +++ b/roles/cloudera_manager/external_auth/vars/freeipa.yml @@ -41,4 +41,4 @@ auth_providers: group: "{{ ipa_ldap_group_search_base }}" ldap_search_filter: user: "{{ ipa_ldap_user_search_filter }}" - group: "{{ ipa_ldap_user_group_filter }}" \ No newline at end of file + group: "{{ ipa_ldap_user_group_filter }}" diff --git a/roles/cloudera_manager/external_auth/vars/main.yml b/roles/cloudera_manager/external_auth/vars/main.yml index f2c74494..ba87e6c9 100644 --- a/roles/cloudera_manager/external_auth/vars/main.yml +++ b/roles/cloudera_manager/external_auth/vars/main.yml @@ -29,7 +29,7 @@ auth_role_display_names: ROLE_OPERATOR: Operator ROLE_USER: Read-Only ROLE_USER_ADMIN: User Administrator - + default_free_ipa_role_mappings: - group: admins roles: [ ROLE_ADMIN ] @@ -37,10 +37,10 @@ default_free_ipa_role_mappings: roles: [ ROLE_AUDITOR ] - group: users roles: [ ROLE_USER ] -when: - - freeipa_activated - - cloudera_manager_external_auth is undefined - - cloudera_manager_version is version('6.0.0','>=') +# when: +# - freeipa_activated +# - cloudera_manager_external_auth is undefined +# - cloudera_manager_version is version('6.0.0','>=') cloudera_manager_external_auth: provider: "FreeIPA" @@ -48,10 +48,10 @@ cloudera_manager_external_auth: external_only: no external_set: yes role_mappings: "{{ default_free_ipa_role_mappings }}" -when: - - freeipa_activated - - cloudera_manager_external_auth is undefined - - cloudera_manager_version is version('6.0.0','>=') +# when: +# - freeipa_activated +# - cloudera_manager_external_auth is undefined +# - cloudera_manager_version is version('6.0.0','>=') auth_providers: FreeIPA: @@ -66,7 +66,7 @@ auth_providers: ldap_search_filter: user: "{{ ipa_ldap_user_search_filter }}" group: "{{ ipa_ldap_user_group_filter }}" -when: - - freeipa_activated - - cloudera_manager_external_auth is undefined - - cloudera_manager_version is version('6.0.0','>=') +# when: +# - freeipa_activated +# - cloudera_manager_external_auth is undefined +# - cloudera_manager_version is version('6.0.0','>=') diff --git a/roles/cloudera_manager/preload_parcels/defaults/main.yml b/roles/cloudera_manager/preload_parcels/defaults/main.yml index 8c601aa3..aca2a1ab 100644 --- a/roles/cloudera_manager/preload_parcels/defaults/main.yml +++ b/roles/cloudera_manager/preload_parcels/defaults/main.yml @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -preload_parcels: "{{ definition.preload_parcels | default([]) }}" \ No newline at end of file +preload_parcels: "{{ definition.preload_parcels | default([]) }}" diff --git a/roles/cloudera_manager/repo/defaults/main.yml b/roles/cloudera_manager/repo/defaults/main.yml index b1246dac..42ac1005 100644 --- a/roles/cloudera_manager/repo/defaults/main.yml +++ b/roles/cloudera_manager/repo/defaults/main.yml @@ -20,4 +20,4 @@ cloudera_manager_distro_version: "{{ ansible_distribution_major_version }}" install_repo_on_host: yes -set_custom_repo_as_archive_base_url: "{{ use_custom_repo_as_archive_base_url | default(True) }}" \ No newline at end of file +set_custom_repo_as_archive_base_url: "{{ use_custom_repo_as_archive_base_url | default(True) }}" diff --git a/roles/cloudera_manager/repo/tasks/main-Debian.yml b/roles/cloudera_manager/repo/tasks/main-Debian.yml index 6d7a78fd..679d5dbe 100644 --- a/roles/cloudera_manager/repo/tasks/main-Debian.yml +++ b/roles/cloudera_manager/repo/tasks/main-Debian.yml @@ -32,4 +32,3 @@ repo: "deb [arch=amd64] {{ cloudera_manager_repo_url_with_creds | default(cloudera_manager_repo_url) }} {{ cloudera_manager_repo_apt_codename }} {{ cloudera_manager_repo_apt_component }}" filename: cloudera-manager no_log: yes - diff --git a/roles/cloudera_manager/repo/tasks/main.yml b/roles/cloudera_manager/repo/tasks/main.yml index 2fb51f56..b957f147 100644 --- a/roles/cloudera_manager/repo/tasks/main.yml +++ b/roles/cloudera_manager/repo/tasks/main.yml @@ -52,4 +52,4 @@ - name: Install Cloudera Manager repository when: install_repo_on_host include_tasks: - file: "main-{{ ansible_os_family }}.yml" \ No newline at end of file + file: "main-{{ ansible_os_family }}.yml" diff --git a/roles/cloudera_manager/services_info/defaults/main.yml b/roles/cloudera_manager/services_info/defaults/main.yml index 66abcb01..466cf679 100644 --- a/roles/cloudera_manager/services_info/defaults/main.yml +++ b/roles/cloudera_manager/services_info/defaults/main.yml @@ -3,4 +3,4 @@ ranger_user: "{{ ranger_rangeradmin_user | default('admin') }}" ranger_password: "{{ ranger_rangeradmin_user_password | default(cloudera_manager_admin_password) }}" solr_admin_password: "{{ solr_solradmin_user_password | default(cloudera_manager_admin_password) }}" -wxm_api_port: 12022 \ No newline at end of file +wxm_api_port: 12022 diff --git a/roles/cloudera_manager/session_timeout/templates/unlog.json b/roles/cloudera_manager/session_timeout/templates/unlog.json index 40c50b9a..c4962dd6 100644 --- a/roles/cloudera_manager/session_timeout/templates/unlog.json +++ b/roles/cloudera_manager/session_timeout/templates/unlog.json @@ -5,4 +5,4 @@ "value": "{{ cloudera_manager_session_timeout }}" } ] - } \ No newline at end of file + } diff --git a/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 b/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 index 481a9aae..38ab195b 100644 --- a/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 +++ b/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 @@ -206,4 +206,4 @@ YARN: {% endif %} {% if 'QUEUEMANAGER' in cluster.services and not (cdh_cdp_upgrade|default(false)|bool) %} queuemanager_service: queuemanager -{% endif %} \ No newline at end of file +{% endif %} diff --git a/roles/config/cluster/base/templates/configs/kerberos-7.x.j2 b/roles/config/cluster/base/templates/configs/kerberos-7.x.j2 index 744efc08..46da8553 100644 --- a/roles/config/cluster/base/templates/configs/kerberos-7.x.j2 +++ b/roles/config/cluster/base/templates/configs/kerberos-7.x.j2 @@ -1,4 +1,4 @@ --- HBASE: SERVICEWIDE: - hadoop_secure_web_ui: true \ No newline at end of file + hadoop_secure_web_ui: true diff --git a/roles/config/cluster/base/templates/configs/ldap.j2 b/roles/config/cluster/base/templates/configs/ldap.j2 index 7a408408..98e58d80 100644 --- a/roles/config/cluster/base/templates/configs/ldap.j2 +++ b/roles/config/cluster/base/templates/configs/ldap.j2 @@ -2,64 +2,64 @@ {% set auth_provider = auth_providers[service_auth_provider] %} #ATLAS: # ATLAS_SERVER: -# atlas_authentication_method_ldap: -# atlas_authentication_method_ldap_ad_base_dn: -# atlas_authentication_method_ldap_ad_bind_dn: -# atlas_authentication_method_ldap_ad_bind_password: -# atlas_authentication_method_ldap_ad_default_role: -# atlas_authentication_method_ldap_ad_domain: -# atlas_authentication_method_ldap_ad_referral: -# atlas_authentication_method_ldap_ad_url: -# atlas_authentication_method_ldap_ad_user_searchfilter: -# atlas_authentication_method_ldap_base_dn: -# atlas_authentication_method_ldap_bind_dn: -# atlas_authentication_method_ldap_bind_password: -# atlas_authentication_method_ldap_default_role: -# atlas_authentication_method_ldap_groupRoleAttribute: -# atlas_authentication_method_ldap_groupSearchBase: -# atlas_authentication_method_ldap_groupSearchFilter: -# atlas_authentication_method_ldap_referral: -# atlas_authentication_method_ldap_type: -# atlas_authentication_method_ldap_ugi_groups: -# atlas_authentication_method_ldap_url: -# atlas_authentication_method_ldap_userDNpattern: -# atlas_authentication_method_ldap_user_searchfilter: +# atlas_authentication_method_ldap: +# atlas_authentication_method_ldap_ad_base_dn: +# atlas_authentication_method_ldap_ad_bind_dn: +# atlas_authentication_method_ldap_ad_bind_password: +# atlas_authentication_method_ldap_ad_default_role: +# atlas_authentication_method_ldap_ad_domain: +# atlas_authentication_method_ldap_ad_referral: +# atlas_authentication_method_ldap_ad_url: +# atlas_authentication_method_ldap_ad_user_searchfilter: +# atlas_authentication_method_ldap_base_dn: +# atlas_authentication_method_ldap_bind_dn: +# atlas_authentication_method_ldap_bind_password: +# atlas_authentication_method_ldap_default_role: +# atlas_authentication_method_ldap_groupRoleAttribute: +# atlas_authentication_method_ldap_groupSearchBase: +# atlas_authentication_method_ldap_groupSearchFilter: +# atlas_authentication_method_ldap_referral: +# atlas_authentication_method_ldap_type: +# atlas_authentication_method_ldap_ugi_groups: +# atlas_authentication_method_ldap_url: +# atlas_authentication_method_ldap_userDNpattern: +# atlas_authentication_method_ldap_user_searchfilter: #DAS: # DAS_WEBAPP: -# das_webapp_ldap_basedn: -# das_webapp_ldap_custom_ldap_query: -# das_webapp_ldap_domain: -# das_webapp_ldap_group_class_key: -# das_webapp_ldap_group_dn_pattern: -# das_webapp_ldap_group_filter: -# das_webapp_ldap_group_membership_key: -# das_webapp_ldap_guid_key: -# das_webapp_ldap_url: -# das_webapp_ldap_user_dn_pattern: -# das_webapp_ldap_user_filter: -# das_webapp_ldap_user_membership_key: +# das_webapp_ldap_basedn: +# das_webapp_ldap_custom_ldap_query: +# das_webapp_ldap_domain: +# das_webapp_ldap_group_class_key: +# das_webapp_ldap_group_dn_pattern: +# das_webapp_ldap_group_filter: +# das_webapp_ldap_group_membership_key: +# das_webapp_ldap_guid_key: +# das_webapp_ldap_url: +# das_webapp_ldap_user_dn_pattern: +# das_webapp_ldap_user_filter: +# das_webapp_ldap_user_membership_key: #HIVE: # SERVICEWIDE: # hive_metastore_enable_ldap_auth: false -# hive_metastore_ldap_basedn: -# hive_metastore_ldap_domain: -# hive_metastore_ldap_uri: +# hive_metastore_ldap_basedn: +# hive_metastore_ldap_domain: +# hive_metastore_ldap_uri: # hiveserver2_enable_ldap_auth: false -# hiveserver2_ldap_basedn: -# hiveserver2_ldap_domain: -# hiveserver2_ldap_uri: +# hiveserver2_ldap_basedn: +# hiveserver2_ldap_domain: +# hiveserver2_ldap_uri: #HIVE_LLAP: # SERVICEWIDE: # hiveserver2_enable_ldap_auth: false -# hiveserver2_ldap_basedn: -# hiveserver2_ldap_domain: -# hiveserver2_ldap_uri: +# hiveserver2_ldap_basedn: +# hiveserver2_ldap_domain: +# hiveserver2_ldap_uri: #HIVE_ON_TEZ: # SERVICEWIDE: # hiveserver2_enable_ldap_auth: false -# hiveserver2_ldap_basedn: -# hiveserver2_ldap_domain: -# hiveserver2_ldap_uri: +# hiveserver2_ldap_basedn: +# hiveserver2_ldap_domain: +# hiveserver2_ldap_uri: HUE: SERVICEWIDE: auth_backend: desktop.auth.backend.LdapBackend @@ -80,16 +80,16 @@ HUE: # SERVICEWIDE: # enable_ldap_auth: false # enable_ldap_tls: false -# impala_ldap_uri: -# ldap_baseDN: -# ldap_bind_pattern: -# ldap_domain: +# impala_ldap_uri: +# ldap_baseDN: +# ldap_bind_pattern: +# ldap_domain: # IMPALAD: -# impalad_ldap_ca_certificate: +# impalad_ldap_ca_certificate: #KAFKA: # SERVICEWIDE: -# ldap.auth.url: -# ldap.auth.user.dn.template: +# ldap.auth.url: +# ldap.auth.user.dn.template: NIFI: NIFI_NODE: nifi.ldap.authentication.strategy: LDAPS @@ -156,8 +156,8 @@ RANGER: # ranger.usersync.ldap.username.caseconversion: #SOLR: # SERVICEWIDE: -# ldap_domain: +# ldap_domain: # solr_enable_ldap_auth: false -# solr_ldap_basedn: +# solr_ldap_basedn: # solr_ldap_enable_starttls: false -# solr_ldap_uri: +# solr_ldap_uri: diff --git a/roles/config/cluster/base/templates/configs/logdirs-ranger-spooldirs-7.1.9.j2 b/roles/config/cluster/base/templates/configs/logdirs-ranger-spooldirs-7.1.9.j2 index bb551ea0..f749f851 100644 --- a/roles/config/cluster/base/templates/configs/logdirs-ranger-spooldirs-7.1.9.j2 +++ b/roles/config/cluster/base/templates/configs/logdirs-ranger-spooldirs-7.1.9.j2 @@ -61,4 +61,4 @@ SOLR: YARN: SERVICEWIDE: ranger_audit_hdfs_spool_dir: "{{ log_base }}/yarn/audit/hdfs/spool" - ranger_audit_solr_spool_dir: "{{ log_base }}/yarn/audit/solr/spool" \ No newline at end of file + ranger_audit_solr_spool_dir: "{{ log_base }}/yarn/audit/solr/spool" diff --git a/roles/config/cluster/base/templates/configs/logdirs-ranger-spooldirs.j2 b/roles/config/cluster/base/templates/configs/logdirs-ranger-spooldirs.j2 index 8006e32b..bf911ccf 100644 --- a/roles/config/cluster/base/templates/configs/logdirs-ranger-spooldirs.j2 +++ b/roles/config/cluster/base/templates/configs/logdirs-ranger-spooldirs.j2 @@ -66,4 +66,4 @@ SOLR: YARN: SERVICEWIDE: ranger_audit_hdfs_spool_dir: "{{ log_base }}/yarn/audit/hdfs/spool" - ranger_audit_solr_spool_dir: "{{ log_base }}/yarn/audit/solr/spool" \ No newline at end of file + ranger_audit_solr_spool_dir: "{{ log_base }}/yarn/audit/solr/spool" diff --git a/roles/config/cluster/base/templates/configs/logdirs.j2 b/roles/config/cluster/base/templates/configs/logdirs.j2 index b3497859..1314ef55 100644 --- a/roles/config/cluster/base/templates/configs/logdirs.j2 +++ b/roles/config/cluster/base/templates/configs/logdirs.j2 @@ -129,4 +129,3 @@ YARN: ZOOKEEPER: SERVER: zk_server_log_dir: "{{ log_base }}/zookeeper" - diff --git a/roles/config/cluster/base/templates/configs/tls-7.1.4.j2 b/roles/config/cluster/base/templates/configs/tls-7.1.4.j2 index 62e7f266..c86268e0 100644 --- a/roles/config/cluster/base/templates/configs/tls-7.1.4.j2 +++ b/roles/config/cluster/base/templates/configs/tls-7.1.4.j2 @@ -11,4 +11,4 @@ RANGER_RMS: ssl_client_truststore_password: {{ tls_truststore_password }} ssl_enabled: true ssl_server_keystore_location: {{ tls_keystore_path_generic }} - ssl_server_keystore_password: {{ tls_keystore_password }} \ No newline at end of file + ssl_server_keystore_password: {{ tls_keystore_password }} diff --git a/roles/config/cluster/base/templates/configs/varlib-7.1.0.j2 b/roles/config/cluster/base/templates/configs/varlib-7.1.0.j2 index 5ef85239..033e2211 100644 --- a/roles/config/cluster/base/templates/configs/varlib-7.1.0.j2 +++ b/roles/config/cluster/base/templates/configs/varlib-7.1.0.j2 @@ -118,4 +118,4 @@ ZEPPELIN: ZOOKEEPER: SERVER: dataDir: "{{ varlib_base }}/zookeeper" - dataLogDir: "{{ varlib_base }}/zookeeper" \ No newline at end of file + dataLogDir: "{{ varlib_base }}/zookeeper" diff --git a/roles/config/cluster/common/defaults/main.yml b/roles/config/cluster/common/defaults/main.yml index a20149f9..6966d5ff 100644 --- a/roles/config/cluster/common/defaults/main.yml +++ b/roles/config/cluster/common/defaults/main.yml @@ -109,4 +109,3 @@ cluster_services_ordered: - OOZIE - HUE - KNOX - diff --git a/roles/config/cluster/ecs/templates/configs/ecs.j2 b/roles/config/cluster/ecs/templates/configs/ecs.j2 index f8ae90a0..1d37fc21 100644 --- a/roles/config/cluster/ecs/templates/configs/ecs.j2 +++ b/roles/config/cluster/ecs/templates/configs/ecs.j2 @@ -9,4 +9,4 @@ ECS: infra_prometheus_ingress_user: cloudera-manager longhorn_replication: 1 lsoDataPath: /ecs/storage - app_domain: "{{ cluster.application_domain }}" \ No newline at end of file + app_domain: "{{ cluster.application_domain }}" diff --git a/roles/config/services/hue_ticket_lifetime/meta/main.yml b/roles/config/services/hue_ticket_lifetime/meta/main.yml index 474745ea..cb97f8d0 100644 --- a/roles/config/services/hue_ticket_lifetime/meta/main.yml +++ b/roles/config/services/hue_ticket_lifetime/meta/main.yml @@ -14,4 +14,4 @@ --- dependencies: - - role: cloudera.cluster.infrastructure.krb5_common \ No newline at end of file + - role: cloudera.cluster.infrastructure.krb5_common diff --git a/roles/config/services/hue_ticket_lifetime/tasks/main.yml b/roles/config/services/hue_ticket_lifetime/tasks/main.yml index 312b1378..43194027 100644 --- a/roles/config/services/hue_ticket_lifetime/tasks/main.yml +++ b/roles/config/services/hue_ticket_lifetime/tasks/main.yml @@ -6,4 +6,4 @@ loop: "{{ groups['all'] }}" loop_control: loop_var: __hue_ticket_item - ignore_errors: true \ No newline at end of file + ignore_errors: true diff --git a/roles/config/services/kms_tls/files/kms_tls.json b/roles/config/services/kms_tls/files/kms_tls.json index 64304f6a..fc663afd 100644 --- a/roles/config/services/kms_tls/files/kms_tls.json +++ b/roles/config/services/kms_tls/files/kms_tls.json @@ -26,4 +26,4 @@ } ] - } \ No newline at end of file + } diff --git a/roles/config/services/kms_tls/files/kms_tls_cdh.json b/roles/config/services/kms_tls/files/kms_tls_cdh.json index fd2e1ea6..9869551e 100644 --- a/roles/config/services/kms_tls/files/kms_tls_cdh.json +++ b/roles/config/services/kms_tls/files/kms_tls_cdh.json @@ -21,4 +21,4 @@ "value": "{{CM_AUTO_TLS}}" } ] - } \ No newline at end of file + } diff --git a/roles/config/services/kms_tls/files/kms_tls_cdh_kms.json b/roles/config/services/kms_tls/files/kms_tls_cdh_kms.json index b514cabe..c00b6948 100644 --- a/roles/config/services/kms_tls/files/kms_tls_cdh_kms.json +++ b/roles/config/services/kms_tls/files/kms_tls_cdh_kms.json @@ -1,10 +1,10 @@ { "items": [ - + { "name": "hadoop_kms_authentication_signer_secret_provider_zookeeper_auth_type", "value": "sasl" } ] - } \ No newline at end of file + } diff --git a/roles/config/services/mgmt/tasks/main.yml b/roles/config/services/mgmt/tasks/main.yml index 0d4ee67a..a07aae3e 100644 --- a/roles/config/services/mgmt/tasks/main.yml +++ b/roles/config/services/mgmt/tasks/main.yml @@ -35,4 +35,4 @@ # please take care when changing it - name: Merge custom configurations set_fact: - merged_configs: "{{ custom_configs | combine(definition.mgmt.configs | default({}), recursive=True) }}" \ No newline at end of file + merged_configs: "{{ custom_configs | combine(definition.mgmt.configs | default({}), recursive=True) }}" diff --git a/roles/config/services/mgmt/templates/configs/varlib-7.1.0.j2 b/roles/config/services/mgmt/templates/configs/varlib-7.1.0.j2 index 854bc8b1..0785dc55 100644 --- a/roles/config/services/mgmt/templates/configs/varlib-7.1.0.j2 +++ b/roles/config/services/mgmt/templates/configs/varlib-7.1.0.j2 @@ -11,4 +11,4 @@ REPORTSMANAGER: SERVICEMONITOR: firehose_storage_dir: "{{ varlib_base }}/cloudera-service-monitor" TELEMETRYPUBLISHER: - mgmt_data_dir: "{{ varlib_base }}/cloudera-scm-telemetrypublisher" \ No newline at end of file + mgmt_data_dir: "{{ varlib_base }}/cloudera-scm-telemetrypublisher" diff --git a/roles/config/services/mgmt/vars/main.yml b/roles/config/services/mgmt/vars/main.yml index 9217c24a..9e9c1578 100644 --- a/roles/config/services/mgmt/vars/main.yml +++ b/roles/config/services/mgmt/vars/main.yml @@ -22,4 +22,4 @@ custom_config_templates: - template: configs/tls-6.x.j2 condition: "{{ definition.mgmt.security.tls | default(False) and cloudera_manager_version is version('6.0.0', '>=') }}" - template: configs/varlib-7.1.0.j2 - condition: "{{ definition.mgmt.varlib_base is defined and cloudera_manager_version is version('7.1.3','>=') }}" \ No newline at end of file + condition: "{{ definition.mgmt.varlib_base is defined and cloudera_manager_version is version('7.1.3','>=') }}" diff --git a/roles/config/services/oozie_ui/tasks/main.yml b/roles/config/services/oozie_ui/tasks/main.yml index e8882438..e7ef38e4 100644 --- a/roles/config/services/oozie_ui/tasks/main.yml +++ b/roles/config/services/oozie_ui/tasks/main.yml @@ -1,5 +1,5 @@ --- -# From: https://docs.cloudera.com/cdp-private-cloud-base/7.1.7/configuring-oozie/topics/oozie-enabling-the-oozie-web-console-on-managed-clusters.html +# From: https://docs.cloudera.com/cdp-private-cloud-base/7.1.7/configuring-oozie/topics/oozie-enabling-the-oozie-web-console-on-managed-clusters.html - name: Download ext-2.2 file get_url: @@ -7,13 +7,13 @@ dest: /tmp/ext-2.2.zip - name: Remove /var/lib/oozie/ - file: + file: path: /var/lib/oozie/ state: absent ignore_errors: true - name: Mkdir /var/lib/oozie/ if it does not exists - file: + file: path: /var/lib/oozie/ owner: oozie group: oozie @@ -37,4 +37,3 @@ - name: Chown ext-2.2 shell: chown -R oozie:oozie /var/lib/oozie/ext-2.2 ignore_errors: true - diff --git a/roles/config/services/ranger_pvc_default_policies/policies/hdfs_hadoop.json b/roles/config/services/ranger_pvc_default_policies/policies/hdfs_hadoop.json index d66cecd6..60c8d7a2 100644 --- a/roles/config/services/ranger_pvc_default_policies/policies/hdfs_hadoop.json +++ b/roles/config/services/ranger_pvc_default_policies/policies/hdfs_hadoop.json @@ -52,39 +52,39 @@ "hadoop" ], "roles": [ - + ], "conditions": [ - + ], "delegateAdmin": true } ], "denyPolicyItems": [ - + ], "allowExceptions": [ - + ], "denyExceptions": [ - + ], "dataMaskPolicyItems": [ - + ], "rowFilterPolicyItems": [ - + ], "serviceType": "hdfs", "options": { - + }, "validitySchedules": [ - + ], "policyLabels": [ - + ], "zoneName": "", "isDenyAllElse": false - } \ No newline at end of file + } diff --git a/roles/config/services/ranger_pvc_default_policies/policies/hdfs_user.json b/roles/config/services/ranger_pvc_default_policies/policies/hdfs_user.json index cc3808d4..0feb1add 100644 --- a/roles/config/services/ranger_pvc_default_policies/policies/hdfs_user.json +++ b/roles/config/services/ranger_pvc_default_policies/policies/hdfs_user.json @@ -40,39 +40,39 @@ "groups": [ ], "roles": [ - + ], "conditions": [ - + ], "delegateAdmin": true } ], "denyPolicyItems": [ - + ], "allowExceptions": [ - + ], "denyExceptions": [ - + ], "dataMaskPolicyItems": [ - + ], "rowFilterPolicyItems": [ - + ], "serviceType": "hdfs", "options": { - + }, "validitySchedules": [ - + ], "policyLabels": [ - + ], "zoneName": "", "isDenyAllElse": false - } \ No newline at end of file + } diff --git a/roles/config/services/ranger_pvc_default_policies/policies/solr.json b/roles/config/services/ranger_pvc_default_policies/policies/solr.json index 7cf5589a..2164fcaa 100644 --- a/roles/config/services/ranger_pvc_default_policies/policies/solr.json +++ b/roles/config/services/ranger_pvc_default_policies/policies/solr.json @@ -44,42 +44,42 @@ "hue" ], "groups": [ - + ], "roles": [ - + ], "conditions": [ - + ], "delegateAdmin": true } ], "denyPolicyItems": [ - + ], "allowExceptions": [ - + ], "denyExceptions": [ - + ], "dataMaskPolicyItems": [ - + ], "rowFilterPolicyItems": [ - + ], "serviceType": "solr", "options": { - + }, "validitySchedules": [ - + ], "policyLabels": [ - + ], "zoneName": "", "isDenyAllElse": false - } \ No newline at end of file + } diff --git a/roles/config/services/ranger_pvc_default_policies/tasks/main.yml b/roles/config/services/ranger_pvc_default_policies/tasks/main.yml index e98e3b5f..828f1d02 100644 --- a/roles/config/services/ranger_pvc_default_policies/tasks/main.yml +++ b/roles/config/services/ranger_pvc_default_policies/tasks/main.yml @@ -2,7 +2,7 @@ - name: Post Ranger policies declared in policies directory register: __ranger_pol_response - uri: + uri: url: "{{ ranger_url }}/service/public/v2/api/policy" method: POST user: "{{ ranger_user }}" diff --git a/roles/config/services/solr_knox/templates/solr_knox_url.json b/roles/config/services/solr_knox/templates/solr_knox_url.json index 1267b807..09e40da2 100644 --- a/roles/config/services/solr_knox/templates/solr_knox_url.json +++ b/roles/config/services/solr_knox/templates/solr_knox_url.json @@ -5,4 +5,4 @@ "value": "{{ new_gateway_descriptor_cdp_proxy }}" } ] - } \ No newline at end of file + } diff --git a/roles/config/services/solr_knox/templates/solr_knox_url_api.json b/roles/config/services/solr_knox/templates/solr_knox_url_api.json index 6f9f7926..260704d2 100644 --- a/roles/config/services/solr_knox/templates/solr_knox_url_api.json +++ b/roles/config/services/solr_knox/templates/solr_knox_url_api.json @@ -5,4 +5,4 @@ "value": "{{ new_gateway_descriptor_cdp_proxy_api }}" } ] - } \ No newline at end of file + } diff --git a/roles/config/services/solr_ranger_plugin/templates/solr_plugin.json b/roles/config/services/solr_ranger_plugin/templates/solr_plugin.json index c3e13c15..6d796b29 100644 --- a/roles/config/services/solr_ranger_plugin/templates/solr_plugin.json +++ b/roles/config/services/solr_ranger_plugin/templates/solr_plugin.json @@ -16,4 +16,4 @@ "policy.download.auth.users": "solr", "ranger.plugin.audit.filters": "[{'accessResult':'DENIED','isAudited':true},{'users':['hive','hdfs','kafka','hbase','solr','rangerraz','knox','atlas'],'isAudited':false}]" } - } \ No newline at end of file + } diff --git a/roles/deployment/cluster/meta/main.yml b/roles/deployment/cluster/meta/main.yml index 44c08094..af65cb45 100644 --- a/roles/deployment/cluster/meta/main.yml +++ b/roles/deployment/cluster/meta/main.yml @@ -18,4 +18,3 @@ dependencies: - role: cloudera.cluster.deployment.credential - role: cloudera.cluster.infrastructure.ca_common - role: cloudera.cluster.prereqs.local_accounts_common - diff --git a/roles/deployment/cluster/tasks/create_ecs.yml b/roles/deployment/cluster/tasks/create_ecs.yml index 7113d086..55be5dc4 100644 --- a/roles/deployment/cluster/tasks/create_ecs.yml +++ b/roles/deployment/cluster/tasks/create_ecs.yml @@ -141,4 +141,3 @@ datalakeClusterName: "{{ cluster.datalake }}" valuesYaml: "{{ custom_values | to_yaml }}" register: controlplane_response - diff --git a/roles/deployment/cluster/tasks/fs2cs.yml b/roles/deployment/cluster/tasks/fs2cs.yml index e86edcf1..9acf0a9f 100644 --- a/roles/deployment/cluster/tasks/fs2cs.yml +++ b/roles/deployment/cluster/tasks/fs2cs.yml @@ -75,7 +75,7 @@ - name: Update Capacity Scheduler Config in CM cloudera.cluster.cm_api: - endpoint: /clusters/{{ cluster.name | urlencode() }}/services/yarn/config + # endpoint: /clusters/{{ cluster.name | urlencode() }}/services/yarn/config endpoint: /clusters/{{ cluster.name | urlencode() }}/services/yarn/roleConfigGroups/yarn-RESOURCEMANAGER-BASE/config method: PUT body: @@ -86,4 +86,4 @@ - name: Delete content & directory file: state: absent - path: "{{ fs2cs_temp_dir }}" \ No newline at end of file + path: "{{ fs2cs_temp_dir }}" diff --git a/roles/deployment/cluster/tasks/main.yml b/roles/deployment/cluster/tasks/main.yml index 93f4e212..b15c67e8 100644 --- a/roles/deployment/cluster/tasks/main.yml +++ b/roles/deployment/cluster/tasks/main.yml @@ -45,7 +45,7 @@ existing_clusters: "{{ clusters_response.json | json_query('items[*].name') }}" # If you get failures here check in CM to ensure you don't have empty clusters or other oddities -# Add deploy_only="base" to select base from several clusters in clusters.yml +# Add deploy_only="base" to select base from several clusters in clusters.yml - name: Create base clusters include_tasks: create_base.yml loop: "{{ definition.clusters }}" @@ -57,8 +57,8 @@ - cluster.type | default(default_cluster_type) == 'base' - cluster.name not in existing_clusters - (deploy_only is defined and 'base' in deploy_only) or deploy_only is not defined - -# Add deploy_only="base" to select base from several clusters in clusters.yml + +# Add deploy_only="base" to select base from several clusters in clusters.yml - name: Update base clusters include_tasks: update_base.yml loop: "{{ definition.clusters }}" @@ -72,7 +72,7 @@ - (cdh_cdp_upgrade|default(false)|bool) or (update_services|default(false)|bool) or (upgrade_runtime|default(false)|bool) - (deploy_only is defined and 'base' in deploy_only) or deploy_only is not defined -# Add deploy_only="base" to select base from several clusters in clusters.yml +# Add deploy_only="base" to select base from several clusters in clusters.yml - name: Create base cluster data contexts (SDX) include_tasks: create_data_context.yml loop: "{{ definition.clusters }}" @@ -98,9 +98,9 @@ - cluster.name not in existing_clusters - '"kts_active" in groups' - (deploy_only is defined and 'encryption' in deploy_only) or deploy_only is not defined - -# Add deploy_only="encryption" to select kts from several clusters in clusters.yml + +# Add deploy_only="encryption" to select kts from several clusters in clusters.yml - name: Upgrade Key Trustee server cluster include_tasks: upgrade_kts.yml loop: "{{ definition.clusters }}" @@ -114,8 +114,8 @@ - '"kts_active" in groups' - upgrade_kts_cluster | default(false) - (deploy_only is defined and 'encryption' in deploy_only) or deploy_only is not defined - -# Add deploy_only="compute" to select compute clust from several clusters in clusters.yml + +# Add deploy_only="compute" to select compute clust from several clusters in clusters.yml - name: Create compute clusters include_tasks: create_base.yml loop: "{{ definition.clusters }}" @@ -145,4 +145,4 @@ - name: Restart Cloudera Management Service cloudera.cluster.cm_api: endpoint: /cm/service/commands/restart - method: POST \ No newline at end of file + method: POST diff --git a/roles/deployment/cluster/tasks/update_base.yml b/roles/deployment/cluster/tasks/update_base.yml index 5615cf0d..208a0a0c 100644 --- a/roles/deployment/cluster/tasks/update_base.yml +++ b/roles/deployment/cluster/tasks/update_base.yml @@ -187,4 +187,4 @@ until: parcels_response.json.stage == "ACTIVATED" retries: "{{ parcel_poll_max_retries | default(30) }}" delay: "{{ parcel_poll_duration | default(60) }}" - when: activate_runtime_upgrade|default(false)|bool \ No newline at end of file + when: activate_runtime_upgrade|default(false)|bool diff --git a/roles/deployment/cluster/tasks/update_role_config_group.yml b/roles/deployment/cluster/tasks/update_role_config_group.yml index fbe50683..a5979d98 100644 --- a/roles/deployment/cluster/tasks/update_role_config_group.yml +++ b/roles/deployment/cluster/tasks/update_role_config_group.yml @@ -21,4 +21,4 @@ body: "{{ lookup('template', 'services/roleConfigGroupConfig.j2', convert_data=False) }}" loop: "{{ role_mappings[service] }}" loop_control: - loop_var: role_type \ No newline at end of file + loop_var: role_type diff --git a/roles/deployment/cluster/tasks/upgrade_kts.yml b/roles/deployment/cluster/tasks/upgrade_kts.yml index 38bad9b1..813d6e03 100644 --- a/roles/deployment/cluster/tasks/upgrade_kts.yml +++ b/roles/deployment/cluster/tasks/upgrade_kts.yml @@ -109,4 +109,4 @@ endpoint: /clusters/{{ cluster.name | urlencode() }}/commands/restart method: POST - when: (parcel_version | regex_search('([0-9]+[.]?)+')) is version(installed_parcel_version, '>') \ No newline at end of file + when: (parcel_version | regex_search('([0-9]+[.]?)+')) is version(installed_parcel_version, '>') diff --git a/roles/deployment/cluster/templates/cluster_template/ecs/clusters.j2 b/roles/deployment/cluster/templates/cluster_template/ecs/clusters.j2 index bb6cfc84..eb29b370 100644 --- a/roles/deployment/cluster/templates/cluster_template/ecs/clusters.j2 +++ b/roles/deployment/cluster/templates/cluster_template/ecs/clusters.j2 @@ -9,4 +9,4 @@ "clusterType": "EXPERIENCE_CLUSTER" } ] -} \ No newline at end of file +} diff --git a/roles/deployment/cluster/templates/cluster_template/ecs/controlPlaneValues.j2 b/roles/deployment/cluster/templates/cluster_template/ecs/controlPlaneValues.j2 index 5f5fbe01..40d46615 100644 --- a/roles/deployment/cluster/templates/cluster_template/ecs/controlPlaneValues.j2 +++ b/roles/deployment/cluster/templates/cluster_template/ecs/controlPlaneValues.j2 @@ -196,4 +196,4 @@ ApplicationDomain: "{{ cluster.application_domain }}" ## this feature built-in, it causes a permission issue due to which postgresql fails to create the data directory ## for persisting data (OPSX-1633). This flag needs to be set to false for Openshift, true otherwise. SecurityContext: - Enabled: true \ No newline at end of file + Enabled: true diff --git a/roles/deployment/cluster/templates/cluster_template/ecs/hosts.j2 b/roles/deployment/cluster/templates/cluster_template/ecs/hosts.j2 index 322f96f4..9d1ea5e8 100644 --- a/roles/deployment/cluster/templates/cluster_template/ecs/hosts.j2 +++ b/roles/deployment/cluster/templates/cluster_template/ecs/hosts.j2 @@ -8,4 +8,4 @@ } {%- endfor -%} ] -} \ No newline at end of file +} diff --git a/roles/deployment/cluster/templates/cluster_template/ecs/services.j2 b/roles/deployment/cluster/templates/cluster_template/ecs/services.j2 index e5e6aa97..7024ac21 100644 --- a/roles/deployment/cluster/templates/cluster_template/ecs/services.j2 +++ b/roles/deployment/cluster/templates/cluster_template/ecs/services.j2 @@ -53,4 +53,4 @@ ] } {%- endfor -%} -]} \ No newline at end of file +]} diff --git a/roles/deployment/cluster/templates/services/roleConfigGroupConfig.j2 b/roles/deployment/cluster/templates/services/roleConfigGroupConfig.j2 index 675c94e1..172d394a 100644 --- a/roles/deployment/cluster/templates/services/roleConfigGroupConfig.j2 +++ b/roles/deployment/cluster/templates/services/roleConfigGroupConfig.j2 @@ -1,2 +1,2 @@ {% import 'cm_api.j2' as cm_api with context %} -{{ cm_api.ApiConfigList(merged_configs[service][role_type]) }} \ No newline at end of file +{{ cm_api.ApiConfigList(merged_configs[service][role_type]) }} diff --git a/roles/deployment/cluster/templates/services/service.j2 b/roles/deployment/cluster/templates/services/service.j2 index 700b1222..cd07b5ce 100644 --- a/roles/deployment/cluster/templates/services/service.j2 +++ b/roles/deployment/cluster/templates/services/service.j2 @@ -50,4 +50,4 @@ {%- endif -%} ] } -]} \ No newline at end of file +]} diff --git a/roles/deployment/cluster/templates/services/serviceConfig.j2 b/roles/deployment/cluster/templates/services/serviceConfig.j2 index d7f1ad46..6ba829d8 100644 --- a/roles/deployment/cluster/templates/services/serviceConfig.j2 +++ b/roles/deployment/cluster/templates/services/serviceConfig.j2 @@ -1,2 +1,2 @@ {% import 'cm_api.j2' as cm_api with context %} -{{ cm_api.ApiConfigList(merged_configs[service]['SERVICEWIDE']) }} \ No newline at end of file +{{ cm_api.ApiConfigList(merged_configs[service]['SERVICEWIDE']) }} diff --git a/roles/deployment/cluster/templates/upgrade_scripts/fs2cs.j2 b/roles/deployment/cluster/templates/upgrade_scripts/fs2cs.j2 index db5a1ed7..03b10a5e 100644 --- a/roles/deployment/cluster/templates/upgrade_scripts/fs2cs.j2 +++ b/roles/deployment/cluster/templates/upgrade_scripts/fs2cs.j2 @@ -50,4 +50,4 @@ cp ${CORE_SITE} fs2cs/core-site.xml /opt/cloudera/parcels/CDH-{{ new_parcel_version }}/bin/yarn fs2cs ${RESOURCE_OPTS} --no-terminal-rule-check --yarnsiteconfig ${YARN_SITE} --fsconfig $(realpath fs2cs/fair-scheduler.xml) --output-directory {{ fs2cs_temp_dir }} -exit $? \ No newline at end of file +exit $? diff --git a/roles/deployment/cluster/templates/upgrade_scripts/nav2atlas.j2 b/roles/deployment/cluster/templates/upgrade_scripts/nav2atlas.j2 index fcca62f6..69495cac 100644 --- a/roles/deployment/cluster/templates/upgrade_scripts/nav2atlas.j2 +++ b/roles/deployment/cluster/templates/upgrade_scripts/nav2atlas.j2 @@ -10,4 +10,4 @@ export TRUSTSTORE_PASSWORD={{ tls_truststore_password }} /opt/cloudera/parcels/CDH-{{ new_parcel_version }}/lib/atlas/tools/nav2atlas/nav2atlas.sh -f '{{ nav2atlas_dir }}/{{ cluster.name | replace(' ','_') }}-navigator-data.zip' -o '{{ nav2atlas_dir }}{{ cluster.name | replace(' ','_') }}-atlas-data.zip' -clusterName 'cm' -exit $? \ No newline at end of file +exit $? diff --git a/roles/deployment/definition/defaults/main.yml b/roles/deployment/definition/defaults/main.yml index 3473c2b8..c2441025 100644 --- a/roles/deployment/definition/defaults/main.yml +++ b/roles/deployment/definition/defaults/main.yml @@ -29,7 +29,7 @@ default_database_versions: '8': 10.2 mysql: '7': 5.7 - '8': 8.0 + '8': 8.0 # Located in cloudera.cluster.infrastructure.krb5_common #krb5_realm: CLOUDERA.LOCAL @@ -160,8 +160,8 @@ databases_cm_svcs: name: rman user: rman password: "{{ database_default_password }}" - -# Deprecated in ecs 1.5.0 + + +# Deprecated in ecs 1.5.0 + databases_ecs: ALERTS: host: "{{ database_host }}" diff --git a/roles/deployment/definition/meta/main.yml b/roles/deployment/definition/meta/main.yml index 3892ffe2..a219bdf3 100644 --- a/roles/deployment/definition/meta/main.yml +++ b/roles/deployment/definition/meta/main.yml @@ -1,3 +1,3 @@ --- dependencies: - - cloudera.cluster.infrastructure.krb5_common \ No newline at end of file + - cloudera.cluster.infrastructure.krb5_common diff --git a/roles/deployment/repometa/tasks/prepare-Debian.yml b/roles/deployment/repometa/tasks/prepare-Debian.yml index eea7f6a2..99ecc703 100644 --- a/roles/deployment/repometa/tasks/prepare-Debian.yml +++ b/roles/deployment/repometa/tasks/prepare-Debian.yml @@ -17,4 +17,4 @@ - name: Set OS Distribution for parcel filtering when: cluster_os_distribution is undefined ansible.builtin.set_fact: - cluster_os_distribution: "{{ ansible_distribution_release }}" \ No newline at end of file + cluster_os_distribution: "{{ ansible_distribution_release }}" diff --git a/roles/deployment/repometa/tasks/prepare-RedHat.yml b/roles/deployment/repometa/tasks/prepare-RedHat.yml index 6004baba..01e3d4fa 100644 --- a/roles/deployment/repometa/tasks/prepare-RedHat.yml +++ b/roles/deployment/repometa/tasks/prepare-RedHat.yml @@ -17,4 +17,4 @@ - name: Set OS Distribution for parcel filtering when: cluster_os_distribution is undefined ansible.builtin.set_fact: - cluster_os_distribution: "el{{ ansible_distribution_major_version }}" \ No newline at end of file + cluster_os_distribution: "el{{ ansible_distribution_major_version }}" diff --git a/roles/deployment/repometa/templates/role_mappings/cdh5.j2 b/roles/deployment/repometa/templates/role_mappings/cdh5.j2 index ca563120..5a0334d9 100644 --- a/roles/deployment/repometa/templates/role_mappings/cdh5.j2 +++ b/roles/deployment/repometa/templates/role_mappings/cdh5.j2 @@ -58,7 +58,7 @@ SOLR: - SOLR_SERVER SPARK: - GATEWAY - - SPARK_HISTORY_SERVER + - SPARK_HISTORY_SERVER - SPARK_MASTER - SPARK_WORKER SPARK_ON_YARN: diff --git a/roles/deployment/repometa/templates/role_mappings/spark3.j2 b/roles/deployment/repometa/templates/role_mappings/spark3.j2 index 87b06698..35a1805c 100644 --- a/roles/deployment/repometa/templates/role_mappings/spark3.j2 +++ b/roles/deployment/repometa/templates/role_mappings/spark3.j2 @@ -3,4 +3,4 @@ SPARK3_ON_YARN: - SPARK3_YARN_HISTORY_SERVER LIVY_FOR_SPARK3: - GATEWAY - - LIVY_SERVER_FOR_SPARK3 \ No newline at end of file + - LIVY_SERVER_FOR_SPARK3 diff --git a/roles/deployment/services/kms_ha/defaults/main.yml b/roles/deployment/services/kms_ha/defaults/main.yml index 65ddfc31..86a807c5 100644 --- a/roles/deployment/services/kms_ha/defaults/main.yml +++ b/roles/deployment/services/kms_ha/defaults/main.yml @@ -20,4 +20,4 @@ kms_key_files: - pubring.gpg - secring.gpg kms_user: kms -kms_group: kms \ No newline at end of file +kms_group: kms diff --git a/roles/deployment/services/kts_common/defaults/main.yml b/roles/deployment/services/kts_common/defaults/main.yml index 81c9d1d6..0e0a6a8f 100644 --- a/roles/deployment/services/kts_common/defaults/main.yml +++ b/roles/deployment/services/kts_common/defaults/main.yml @@ -33,4 +33,4 @@ keytrustee_server_gpg_files: keytrustee_server_kbx_files: - pubring.kbx - - pubring.kbx~ \ No newline at end of file + - pubring.kbx~ diff --git a/roles/deployment/services/kts_high_availability/tasks/main.yml b/roles/deployment/services/kts_high_availability/tasks/main.yml index 42d48212..3a5d81fb 100644 --- a/roles/deployment/services/kts_high_availability/tasks/main.yml +++ b/roles/deployment/services/kts_high_availability/tasks/main.yml @@ -88,4 +88,4 @@ keytrustee-orgtool add -n {{ keytrustee_server_org_name }} -c {{ keytrustee_server_org_email }} - --confdir {{ keytrustee_server_conf_dir }} \ No newline at end of file + --confdir {{ keytrustee_server_conf_dir }} diff --git a/roles/deployment/services/mgmt/templates/service.j2 b/roles/deployment/services/mgmt/templates/service.j2 index 6678f3d8..d61e18a2 100644 --- a/roles/deployment/services/mgmt/templates/service.j2 +++ b/roles/deployment/services/mgmt/templates/service.j2 @@ -33,6 +33,6 @@ "config": {{ cm_api.ApiConfigList(merged_configs[role]) }} } {%- endfor -%} - ], + ], "config": {{ cm_api.ApiConfigList(merged_configs['SERVICEWIDE']) }} } diff --git a/roles/deployment/services/wxm/defaults/main.yml b/roles/deployment/services/wxm/defaults/main.yml index 960eefa6..1eec45c5 100644 --- a/roles/deployment/services/wxm/defaults/main.yml +++ b/roles/deployment/services/wxm/defaults/main.yml @@ -1,2 +1,2 @@ altus_key_id: '' -altus_private_key: '' \ No newline at end of file +altus_private_key: '' diff --git a/roles/deployment/services/wxm/tasks/main.yml b/roles/deployment/services/wxm/tasks/main.yml index 3102b82b..4b141105 100644 --- a/roles/deployment/services/wxm/tasks/main.yml +++ b/roles/deployment/services/wxm/tasks/main.yml @@ -30,4 +30,4 @@ - HIVE_ON_TEZ - SPARK_ON_YARN - IMPALA - - WXM \ No newline at end of file + - WXM diff --git a/roles/deployment/services/wxm/templates/add_telemetry.json b/roles/deployment/services/wxm/templates/add_telemetry.json index bd267eb7..f78cb62b 100644 --- a/roles/deployment/services/wxm/templates/add_telemetry.json +++ b/roles/deployment/services/wxm/templates/add_telemetry.json @@ -1,5 +1,5 @@ { - "items" : [ + "items" : [ { "name" : "mgmt-TELEMETRYPUBLISHER", "type" : "TELEMETRYPUBLISHER", @@ -9,4 +9,4 @@ } } ] -} \ No newline at end of file +} diff --git a/roles/deployment/services/wxm/templates/altus_key_config.json b/roles/deployment/services/wxm/templates/altus_key_config.json index ba513f0f..b5ba99f5 100644 --- a/roles/deployment/services/wxm/templates/altus_key_config.json +++ b/roles/deployment/services/wxm/templates/altus_key_config.json @@ -5,4 +5,4 @@ "value": "altus-key-for-wxm" } ] -} \ No newline at end of file +} diff --git a/roles/deployment/services/wxm/templates/wxm_config.json b/roles/deployment/services/wxm/templates/wxm_config.json index 4a18bac8..a1c33ca7 100644 --- a/roles/deployment/services/wxm/templates/wxm_config.json +++ b/roles/deployment/services/wxm/templates/wxm_config.json @@ -5,4 +5,4 @@ "value": "telemetry.upload.job.logs=true\ntelemetry.altus.url={{ wxm_dbus_api_server_url }}" } ] - } \ No newline at end of file + } diff --git a/roles/infrastructure/ca_common/defaults/main.yml b/roles/infrastructure/ca_common/defaults/main.yml index 2abf3cfd..fb642abc 100644 --- a/roles/infrastructure/ca_common/defaults/main.yml +++ b/roles/infrastructure/ca_common/defaults/main.yml @@ -65,4 +65,4 @@ ca_server_attrs_general: OU: PS O: Cloudera, Inc. ST: CA - C: US \ No newline at end of file + C: US diff --git a/roles/infrastructure/ca_common/meta/main.yml b/roles/infrastructure/ca_common/meta/main.yml index b13308b6..016374de 100644 --- a/roles/infrastructure/ca_common/meta/main.yml +++ b/roles/infrastructure/ca_common/meta/main.yml @@ -14,4 +14,4 @@ --- dependencies: - - role: cloudera.cluster.deployment.definition \ No newline at end of file + - role: cloudera.cluster.deployment.definition diff --git a/roles/infrastructure/ca_server/meta/main.yml b/roles/infrastructure/ca_server/meta/main.yml index 85cfa0b8..856162c2 100644 --- a/roles/infrastructure/ca_server/meta/main.yml +++ b/roles/infrastructure/ca_server/meta/main.yml @@ -14,4 +14,4 @@ --- dependencies: - - role: cloudera.cluster.infrastructure.ca_common \ No newline at end of file + - role: cloudera.cluster.infrastructure.ca_common diff --git a/roles/infrastructure/custom_repo/defaults/main.yml b/roles/infrastructure/custom_repo/defaults/main.yml index cdaa027a..a7cc8315 100644 --- a/roles/infrastructure/custom_repo/defaults/main.yml +++ b/roles/infrastructure/custom_repo/defaults/main.yml @@ -19,4 +19,4 @@ repo_tar_local_dir: repo repo_tar_files: "{{ definition.repo_tar_files | default([]) }}" keep_newer: yes -custom_repo_rehost_files: "{{ definition.custom_repo_rehost_files | default([]) }}" \ No newline at end of file +custom_repo_rehost_files: "{{ definition.custom_repo_rehost_files | default([]) }}" diff --git a/roles/infrastructure/haproxy/tasks/main.yml b/roles/infrastructure/haproxy/tasks/main.yml index 19d0648a..4a94427b 100644 --- a/roles/infrastructure/haproxy/tasks/main.yml +++ b/roles/infrastructure/haproxy/tasks/main.yml @@ -25,5 +25,3 @@ name: haproxy enabled: yes state: restarted - - diff --git a/roles/infrastructure/haproxy/templates/haproxy.j2 b/roles/infrastructure/haproxy/templates/haproxy.j2 index 05e5b5bd..166f16ee 100644 --- a/roles/infrastructure/haproxy/templates/haproxy.j2 +++ b/roles/infrastructure/haproxy/templates/haproxy.j2 @@ -135,4 +135,3 @@ backend bk_oozie {% for host in groups['gatewaylb_servers'] %} server oozie{{ host }} {{ host }}:{{ cdh_services | json_query('[?type==`oozie`].oozie_https_port') | first }} ssl ca-file {{ tls.cert_chain }} {% endfor %} - diff --git a/roles/infrastructure/krb5_client/defaults/main.yml b/roles/infrastructure/krb5_client/defaults/main.yml index b3172efb..6a5b59b3 100644 --- a/roles/infrastructure/krb5_client/defaults/main.yml +++ b/roles/infrastructure/krb5_client/defaults/main.yml @@ -1,13 +1,13 @@ --- # Copyright 2023 Cloudera, Inc. All Rights Reserved. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/roles/infrastructure/krb5_client/files/dbus_session.conf b/roles/infrastructure/krb5_client/files/dbus_session.conf index 9032e606..c9a0bff7 100644 --- a/roles/infrastructure/krb5_client/files/dbus_session.conf +++ b/roles/infrastructure/krb5_client/files/dbus_session.conf @@ -3,4 +3,4 @@ 360000 360000 - \ No newline at end of file + diff --git a/roles/infrastructure/krb5_client/handlers/main.yml b/roles/infrastructure/krb5_client/handlers/main.yml index b9bc3586..e2ea5991 100644 --- a/roles/infrastructure/krb5_client/handlers/main.yml +++ b/roles/infrastructure/krb5_client/handlers/main.yml @@ -15,4 +15,4 @@ - name: restart sssd service: name: sssd - state: restarted \ No newline at end of file + state: restarted diff --git a/roles/infrastructure/krb5_client/tasks/pvc_configs.yml b/roles/infrastructure/krb5_client/tasks/pvc_configs.yml index ad897a82..4bcb9746 100644 --- a/roles/infrastructure/krb5_client/tasks/pvc_configs.yml +++ b/roles/infrastructure/krb5_client/tasks/pvc_configs.yml @@ -37,4 +37,4 @@ block: | default_tgs_enctypes= des3-cbc-sha1 aes256-cts-hmac-sha1-96 arcfour-hmac aes128-cts-hmac-sha1-96 des-cbc-md5 default_tkt_enctypes = des3-cbc-sha1 aes256-cts-hmac-sha1-96 arcfour-hmac aes128-cts-hmac-sha1-96 des-cbc-md5 - ignore_errors: true \ No newline at end of file + ignore_errors: true diff --git a/roles/infrastructure/krb5_conf/meta/main.yml b/roles/infrastructure/krb5_conf/meta/main.yml index 474745ea..cb97f8d0 100644 --- a/roles/infrastructure/krb5_conf/meta/main.yml +++ b/roles/infrastructure/krb5_conf/meta/main.yml @@ -14,4 +14,4 @@ --- dependencies: - - role: cloudera.cluster.infrastructure.krb5_common \ No newline at end of file + - role: cloudera.cluster.infrastructure.krb5_common diff --git a/roles/infrastructure/krb5_server/vars/RedHat-7.yml b/roles/infrastructure/krb5_server/vars/RedHat-7.yml index c2f9ba98..bfeea14e 100644 --- a/roles/infrastructure/krb5_server/vars/RedHat-7.yml +++ b/roles/infrastructure/krb5_server/vars/RedHat-7.yml @@ -1,2 +1,2 @@ --- -ipaserver_packages: [ "ipa-server", "libselinux-python" ] \ No newline at end of file +ipaserver_packages: [ "ipa-server", "libselinux-python" ] diff --git a/roles/infrastructure/krb5_server/vars/RedHat-8.yml b/roles/infrastructure/krb5_server/vars/RedHat-8.yml index d49fa517..c791a5bb 100644 --- a/roles/infrastructure/krb5_server/vars/RedHat-8.yml +++ b/roles/infrastructure/krb5_server/vars/RedHat-8.yml @@ -1,2 +1,2 @@ --- -ipaserver_packages: [ "@idm:DL1/server" ] \ No newline at end of file +ipaserver_packages: [ "@idm:DL1/server" ] diff --git a/roles/infrastructure/krb5_server/vars/Ubuntu.yml b/roles/infrastructure/krb5_server/vars/Ubuntu.yml index ac96c558..91668774 100644 --- a/roles/infrastructure/krb5_server/vars/Ubuntu.yml +++ b/roles/infrastructure/krb5_server/vars/Ubuntu.yml @@ -1,2 +1,2 @@ --- -ipaserver_packages: [ "freeipa-server" ] \ No newline at end of file +ipaserver_packages: [ "freeipa-server" ] diff --git a/roles/infrastructure/rdbms/files/utf8-template.sql b/roles/infrastructure/rdbms/files/utf8-template.sql index f95dc7ab..31feb483 100644 --- a/roles/infrastructure/rdbms/files/utf8-template.sql +++ b/roles/infrastructure/rdbms/files/utf8-template.sql @@ -20,4 +20,3 @@ create database template1 with template = template0 encoding = 'UTF8'; update pg_database set datistemplate = TRUE where datname = 'template1'; \c template1 update pg_database set datallowconn = FALSE where datname = 'template0'; - diff --git a/roles/infrastructure/rdbms/tasks/mysql-RedHat.yml b/roles/infrastructure/rdbms/tasks/mysql-RedHat.yml index 4d775fb3..4ad11e57 100644 --- a/roles/infrastructure/rdbms/tasks/mysql-RedHat.yml +++ b/roles/infrastructure/rdbms/tasks/mysql-RedHat.yml @@ -18,7 +18,7 @@ rpm_key: key: https://repo.mysql.com/RPM-GPG-KEY-mysql-2022 state: present - when: + when: - not (skip_rdbms_repo_setup | default(False)) - name: Install MySQL Community repository @@ -26,11 +26,11 @@ name: https://repo.mysql.com/mysql{{ database_version | replace('.','') }}-community-release-el{{ ansible_distribution_major_version }}.rpm update_cache: true lock_timeout: 180 - state: present - when: + state: present + when: - not (skip_rdbms_repo_setup | default(False)) - - + + - name: Install MySQL include_role: - name: ansible-role-mysql \ No newline at end of file + name: ansible-role-mysql diff --git a/roles/infrastructure/rdbms/vars/mysql-RedHat.yml b/roles/infrastructure/rdbms/vars/mysql-RedHat.yml index 22d6eb17..40e536f2 100644 --- a/roles/infrastructure/rdbms/vars/mysql-RedHat.yml +++ b/roles/infrastructure/rdbms/vars/mysql-RedHat.yml @@ -16,4 +16,4 @@ mysql_packages: - mysql - - mysql-server \ No newline at end of file + - mysql-server diff --git a/roles/infrastructure/rdbms/vars/mysql.yml b/roles/infrastructure/rdbms/vars/mysql.yml index 2f3af74b..623d1be3 100644 --- a/roles/infrastructure/rdbms/vars/mysql.yml +++ b/roles/infrastructure/rdbms/vars/mysql.yml @@ -20,4 +20,4 @@ mysql_pid_file: /var/run/mysql/mysql.pid mysql_innodb_large_prefix: 0 mysql_config_include_files: - src: "cloudera.cnf" - force: true \ No newline at end of file + force: true diff --git a/roles/operations/restart_cluster_services/tasks/main.yml b/roles/operations/restart_cluster_services/tasks/main.yml index 72f8b8bf..bbb55100 100644 --- a/roles/operations/restart_cluster_services/tasks/main.yml +++ b/roles/operations/restart_cluster_services/tasks/main.yml @@ -8,4 +8,4 @@ include_tasks: service_restart.yml loop: "{{ services_to_restart }}" loop_control: - loop_var: __service_restart_item \ No newline at end of file + loop_var: __service_restart_item diff --git a/roles/operations/restart_cluster_services/tasks/service_restart.yml b/roles/operations/restart_cluster_services/tasks/service_restart.yml index ba174c19..5d72d36c 100644 --- a/roles/operations/restart_cluster_services/tasks/service_restart.yml +++ b/roles/operations/restart_cluster_services/tasks/service_restart.yml @@ -12,4 +12,4 @@ - name: Wait for restart to acknowledge wait_for: - timeout: 15 \ No newline at end of file + timeout: 15 diff --git a/roles/operations/restart_mgmt_services/tasks/main.yml b/roles/operations/restart_mgmt_services/tasks/main.yml index 3d02e424..cbe3f808 100644 --- a/roles/operations/restart_mgmt_services/tasks/main.yml +++ b/roles/operations/restart_mgmt_services/tasks/main.yml @@ -4,4 +4,4 @@ endpoint: "/cm/service/roleCommands/restart" method: POST body: - items: "{{ services_to_restart }}" \ No newline at end of file + items: "{{ services_to_restart }}" diff --git a/roles/prereqs/jdk/defaults/main.yml b/roles/prereqs/jdk/defaults/main.yml index 138c6c0c..d443f3f5 100644 --- a/roles/prereqs/jdk/defaults/main.yml +++ b/roles/prereqs/jdk/defaults/main.yml @@ -23,5 +23,3 @@ jdk_java_security_paths: - /etc/java-11-openjdk/security - /usr/lib64/jvm/java-1.8.0-openjdk-1.8.0/jre/lib/security jdk_java_security_safe_replace: True - - diff --git a/roles/prereqs/license/defaults/main.yml b/roles/prereqs/license/defaults/main.yml index 28a4a380..7f2e2fe0 100644 --- a/roles/prereqs/license/defaults/main.yml +++ b/roles/prereqs/license/defaults/main.yml @@ -14,4 +14,4 @@ # Path to the license file on the Ansible controller cloudera_manager_license_file: '' -license_local_tmp_path: /tmp/license.txt \ No newline at end of file +license_local_tmp_path: /tmp/license.txt diff --git a/roles/prereqs/mysql_connector/defaults/main.yml b/roles/prereqs/mysql_connector/defaults/main.yml index d9e82ff0..7f9974fb 100644 --- a/roles/prereqs/mysql_connector/defaults/main.yml +++ b/roles/prereqs/mysql_connector/defaults/main.yml @@ -18,4 +18,4 @@ mysql_connector_url: https://cdn.mysql.com//Downloads/Connector-J/mysql-connecto mysql_connector_checksum: md5:5ecd588e13f14de07faa5c67f5caf3f1 mysql_connector_download_dir: "{{ local_temp_dir }}" mysql_connector_extract_dir: "{{ local_temp_dir }}" -mysql_connector_local_path: "{{ mysql_connector_extract_dir }}/mysql-connector-java-5.1.49/mysql-connector-java-5.1.49-bin.jar" \ No newline at end of file +mysql_connector_local_path: "{{ mysql_connector_extract_dir }}/mysql-connector-java-5.1.49/mysql-connector-java-5.1.49-bin.jar" diff --git a/roles/prereqs/mysql_connector/tasks/main.yml b/roles/prereqs/mysql_connector/tasks/main.yml index 5432d750..5df02527 100644 --- a/roles/prereqs/mysql_connector/tasks/main.yml +++ b/roles/prereqs/mysql_connector/tasks/main.yml @@ -79,5 +79,3 @@ - name: Install Mysql packages for python - PyMySQL shell: /usr/local/bin/pip install PyMySQL --force-reinstall --ignore-installed ignore_errors: true - - diff --git a/roles/prereqs/mysql_connector/templates/my_config.h b/roles/prereqs/mysql_connector/templates/my_config.h index f5efd7b1..9f080622 100644 --- a/roles/prereqs/mysql_connector/templates/my_config.h +++ b/roles/prereqs/mysql_connector/templates/my_config.h @@ -1,31 +1,31 @@ -/* Copyright (c) 2009, 2021, Oracle and/or its affiliates. +/* Copyright (c) 2009, 2021, Oracle and/or its affiliates. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, as published by the Free Software Foundation. - + This program is also distributed with certain software (including but not limited to OpenSSL) that is licensed under separate terms, as designated in a particular file or component or in included license documentation. The authors of MySQL hereby grant you an additional permission to link the program and your derivative works with the separately licensed software that they have included with MySQL. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, for more details. - + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - + #ifndef MY_CONFIG_H #define MY_CONFIG_H - + /* * From configure.cmake, in order of appearance */ - + /* Libraries */ #define HAVE_LIBM 1 /* #undef HAVE_LIBNSL */ @@ -35,7 +35,7 @@ #define HAVE_LIBRT 1 /* #undef HAVE_LIBWRAP */ /* #undef HAVE_LIBWRAP_PROTOTYPES */ - + /* Header files */ #define HAVE_ALLOCA_H 1 #define HAVE_ARPA_INET_H 1 @@ -65,7 +65,7 @@ #define HAVE_FNMATCH_H 1 #define HAVE_SYS_UN_H 1 #define HAVE_SASL_SASL_H 1 - + /* Functions */ /* #undef HAVE_ALIGNED_MALLOC */ #define HAVE_BACKTRACE 1 @@ -116,13 +116,13 @@ #define HAVE_NL_LANGINFO 1 /* #undef HAVE_HTONLL */ #define HAVE_EPOLL 1 - + /* WL2373 */ #define HAVE_SYS_TIME_H 1 #define HAVE_SYS_TIMES_H 1 #define HAVE_TIMES 1 #define HAVE_GETTIMEOFDAY 1 - + /* Symbols */ #define HAVE_LRAND48 1 #define GWINSZ_IN_SYS_IOCTL 1 @@ -130,16 +130,16 @@ /* #undef FIONREAD_IN_SYS_FILIO */ #define HAVE_MADV_DONTDUMP 1 #define HAVE_O_TMPFILE - + /* #undef HAVE_KQUEUE */ #define HAVE_SETNS 1 /* #undef HAVE_KQUEUE_TIMERS */ #define HAVE_POSIX_TIMERS 1 - + /* Endianess */ /* #undef WORDS_BIGENDIAN */ #define HAVE_ENDIAN_CONVERSION_MACROS 1 - + /* Type sizes */ #define SIZEOF_VOIDP 8 #define SIZEOF_CHARP 8 @@ -151,10 +151,10 @@ #define HAVE_ULONG 1 #define HAVE_U_INT32_T 1 #define HAVE_TM_GMTOFF 1 - + /* Support for tagging symbols with __attribute__((visibility("hidden"))) */ #define HAVE_VISIBILITY_HIDDEN 1 - + /* Code tests*/ #define HAVE_CLOCK_GETTIME 1 #define HAVE_CLOCK_REALTIME 1 @@ -175,11 +175,11 @@ /* #undef HAVE_PTHREAD_THREADID_NP */ #define HAVE_INTEGER_PTHREAD_SELF 1 #define HAVE_PTHREAD_SETNAME_NP 1 - + /* IPV6 */ /* #undef HAVE_NETINET_IN6_H */ /* #undef HAVE_STRUCT_IN6_ADDR */ - + /* * Platform specific CMake files */ @@ -193,7 +193,7 @@ #define SYSTEM_TYPE "Linux" /* This should mean case insensitive file system */ /* #undef FN_NO_CASE_SENSE */ - + /* * From main CMakeLists.txt */ @@ -211,10 +211,10 @@ /* #undef KERBEROS_LIB_CONFIGURED */ #define SCRAM_LIB_CONFIGURED /* #undef WITH_HYPERGRAPH_OPTIMIZER */ - + /* Lock Order */ /* #undef WITH_LOCK_ORDER */ - + /* Character sets and collations */ #define DEFAULT_MYSQL_HOME "/usr/local/mysql" #define SHAREDIR "/usr/local/mysql/share" @@ -240,8 +240,8 @@ #define USE_NEW_EDITLINE_INTERFACE 1 #define EDITLINE_HAVE_COMPLETION_CHAR 1 /* #undef EDITLINE_HAVE_COMPLETION_INT */ - - + + /* * Libedit */ @@ -252,13 +252,13 @@ /* #undef HAVE_UNVIS */ /* #undef HAVE_GETPW_R_DRAFT */ #define HAVE_GETPW_R_POSIX - + /* * Character sets */ #define MYSQL_DEFAULT_CHARSET_NAME "utf8mb4" #define MYSQL_DEFAULT_COLLATION_NAME "utf8mb4_0900_ai_ci" - + /* * Performance schema */ @@ -280,7 +280,7 @@ /* #undef DISABLE_PSI_METADATA */ /* #undef DISABLE_PSI_MEMORY */ /* #undef DISABLE_PSI_TRANSACTION */ - + /* * MySQL version */ @@ -292,26 +292,26 @@ #define PACKAGE_VERSION "8.0.25" #define VERSION "8.0.25" #define PROTOCOL_VERSION 10 - + /* * CPU info */ #define CPU_LEVEL1_DCACHE_LINESIZE 64 - - + + /* * NDB */ /* #undef HAVE_GETRLIMIT */ /* #undef WITH_NDBCLUSTER_STORAGE_ENGINE */ /* #undef HAVE_PTHREAD_SETSCHEDPARAM */ - + /* * Other */ /* #undef EXTRA_DEBUG */ #define HANDLE_FATAL_SIGNALS 1 - + /* * Hardcoded values needed by libevent/NDB/memcached */ @@ -331,25 +331,25 @@ #define HAVE_SYS_STAT_H 1 #define HAVE_SYS_TYPES_H 1 #define SIZEOF_CHAR 1 - + /* For --secure-file-priv */ #define DEFAULT_SECURE_FILE_PRIV_DIR "NULL" #define HAVE_LIBNUMA 1 - + /* For default value of --early_plugin_load */ /* #undef DEFAULT_EARLY_PLUGIN_LOAD */ - + /* For default value of --partial_revokes */ #define DEFAULT_PARTIAL_REVOKES 0 - + #define SO_EXT ".so" - - + + /* From libmysql/CMakeLists.txt */ #define HAVE_UNIX_DNS_SRV 1 /* #undef HAVE_WIN32_DNS_SRV */ - + /* ARM crc32 support */ /* #undef HAVE_ARMV8_CRC32_INTRINSIC */ - - #endif \ No newline at end of file + + #endif diff --git a/roles/prereqs/postgresql_connector/defaults/main.yml b/roles/prereqs/postgresql_connector/defaults/main.yml index c61d17ed..2733ba6f 100644 --- a/roles/prereqs/postgresql_connector/defaults/main.yml +++ b/roles/prereqs/postgresql_connector/defaults/main.yml @@ -16,4 +16,4 @@ postgresql_connector_url: https://jdbc.postgresql.org/download/postgresql-42.3.3.jar postgresql_connector_checksum: md5:bef0b2e1c6edcd8647c24bed31e1a4ac -install_py3_psycopg2: false \ No newline at end of file +install_py3_psycopg2: false diff --git a/roles/prereqs/pvc_ecs/files/networkmanager.conf b/roles/prereqs/pvc_ecs/files/networkmanager.conf index 13d2a834..16476e25 100644 --- a/roles/prereqs/pvc_ecs/files/networkmanager.conf +++ b/roles/prereqs/pvc_ecs/files/networkmanager.conf @@ -1,2 +1,2 @@ [keyfile] -unmanaged-devices=interface-name:cali*;interface-name:flannel* \ No newline at end of file +unmanaged-devices=interface-name:cali*;interface-name:flannel* diff --git a/roles/prereqs/pvc_ecs/tasks/main.yml b/roles/prereqs/pvc_ecs/tasks/main.yml index 2e269c4e..a9e9e13e 100644 --- a/roles/prereqs/pvc_ecs/tasks/main.yml +++ b/roles/prereqs/pvc_ecs/tasks/main.yml @@ -15,7 +15,7 @@ # limitations under the License. - name: Fail Fast if ECS OS is not RH family - ansible.builtin.fail: + ansible.builtin.fail: msg: ECS is not supported on this OS, should be Centos or RHEL when: ansible_os_family != 'RedHat' @@ -64,24 +64,24 @@ - security loop_control: loop_var: __iptables_flush_item - + ## see https://docs.rke2.io/known_issues - name: Set NetworkManager to ignore any ECS calico & flannel interfaces ansible.builtin.copy: src: networkmanager.conf - dest: /etc/NetworkManager/conf.d/rke2-canal.config + dest: /etc/NetworkManager/conf.d/rke2-canal.config owner: root group: root mode: 0644 - when: + when: - ansible_distribution_major_version|int >= 7 - ansible_facts.services["NetworkManager.service"]['status'] != "not-found" - + - name: Reload NetworkManager daemon ansible.builtin.service: state: restarted daemon_reload: true - name: NetworkManager.service - when: + name: NetworkManager.service + when: - ansible_distribution_major_version|int >= 7 - ansible_facts.services["NetworkManager.service"]['status'] != "not-found" diff --git a/roles/prereqs/user_accounts/meta/main.yml b/roles/prereqs/user_accounts/meta/main.yml index ac7f2739..e99311bd 100644 --- a/roles/prereqs/user_accounts/meta/main.yml +++ b/roles/prereqs/user_accounts/meta/main.yml @@ -15,4 +15,4 @@ --- dependencies: - role: cloudera.cluster.prereqs.local_accounts_common - - role: cloudera.cluster.deployment.definition \ No newline at end of file + - role: cloudera.cluster.deployment.definition diff --git a/roles/prereqs/user_accounts/tasks/main.yml b/roles/prereqs/user_accounts/tasks/main.yml index 45117bed..97724c8b 100644 --- a/roles/prereqs/user_accounts/tasks/main.yml +++ b/roles/prereqs/user_accounts/tasks/main.yml @@ -77,4 +77,3 @@ dest: /var/lib/streamsmsgmgr state: link when: smm_home.stat.exists - diff --git a/roles/prereqs/user_accounts_ecs/tasks/main.yml b/roles/prereqs/user_accounts_ecs/tasks/main.yml index 0f42384b..ffda52a8 100644 --- a/roles/prereqs/user_accounts_ecs/tasks/main.yml +++ b/roles/prereqs/user_accounts_ecs/tasks/main.yml @@ -57,4 +57,3 @@ when: account.when | default(True) when: (not skip_user_group_init) | default(true) - diff --git a/roles/security/tls_clean/meta/main.yml b/roles/security/tls_clean/meta/main.yml index 85cfa0b8..856162c2 100644 --- a/roles/security/tls_clean/meta/main.yml +++ b/roles/security/tls_clean/meta/main.yml @@ -14,4 +14,4 @@ --- dependencies: - - role: cloudera.cluster.infrastructure.ca_common \ No newline at end of file + - role: cloudera.cluster.infrastructure.ca_common diff --git a/roles/security/tls_clean/tasks/main.yml b/roles/security/tls_clean/tasks/main.yml index c9b8883a..f2c96dd7 100644 --- a/roles/security/tls_clean/tasks/main.yml +++ b/roles/security/tls_clean/tasks/main.yml @@ -16,4 +16,4 @@ - name: Delete security artifacts file: name: "{{ base_dir_security }}" - state: absent \ No newline at end of file + state: absent diff --git a/roles/security/tls_generate_csr/meta/main.yml b/roles/security/tls_generate_csr/meta/main.yml index 3b157c54..9f29eddf 100644 --- a/roles/security/tls_generate_csr/meta/main.yml +++ b/roles/security/tls_generate_csr/meta/main.yml @@ -15,4 +15,4 @@ --- dependencies: - role: cloudera.cluster.infrastructure.ca_common - - role: cloudera.cluster.prereqs.local_accounts_common \ No newline at end of file + - role: cloudera.cluster.prereqs.local_accounts_common diff --git a/roles/security/tls_generate_csr/tasks/main.yml b/roles/security/tls_generate_csr/tasks/main.yml index 4ba231a6..ab20f4b5 100644 --- a/roles/security/tls_generate_csr/tasks/main.yml +++ b/roles/security/tls_generate_csr/tasks/main.yml @@ -118,7 +118,7 @@ - name: Set file permissions include_tasks: acls.yml when: "inventory_hostname not in groups['ecs_nodes'] | default([])" - + - name: Set file permissions for ECS include_tasks: acls-ecs.yml when: "inventory_hostname in groups['ecs_nodes'] | default([])" diff --git a/roles/security/tls_install_certs/meta/main.yml b/roles/security/tls_install_certs/meta/main.yml index 85cfa0b8..856162c2 100644 --- a/roles/security/tls_install_certs/meta/main.yml +++ b/roles/security/tls_install_certs/meta/main.yml @@ -14,4 +14,4 @@ --- dependencies: - - role: cloudera.cluster.infrastructure.ca_common \ No newline at end of file + - role: cloudera.cluster.infrastructure.ca_common diff --git a/roles/security/tls_install_certs/tasks/main.yml b/roles/security/tls_install_certs/tasks/main.yml index e44db131..920086eb 100644 --- a/roles/security/tls_install_certs/tasks/main.yml +++ b/roles/security/tls_install_certs/tasks/main.yml @@ -241,4 +241,4 @@ -destkeystore "{{ tls_uber_truststore_path }}" -srcstorepass "{{ tls_truststore_password }}" -deststorepass "{{ tls_truststore_password }}" - -noprompt \ No newline at end of file + -noprompt diff --git a/roles/security/tls_nifi/defaults/main.yml b/roles/security/tls_nifi/defaults/main.yml index dc3f6750..bc1bd8e5 100644 --- a/roles/security/tls_nifi/defaults/main.yml +++ b/roles/security/tls_nifi/defaults/main.yml @@ -23,4 +23,4 @@ nifireg_dir_owner: nifiregistry nifireg_dir_group: nifiregistry cm_auto_host_keystore_name: cm-auto-host_keystore.jks -cm_auto_cluster_truststore_name: cm-auto-in_cluster_truststore.jks \ No newline at end of file +cm_auto_cluster_truststore_name: cm-auto-in_cluster_truststore.jks diff --git a/roles/security/tls_nifi/meta/main.yml b/roles/security/tls_nifi/meta/main.yml index 3018ac96..41a1ed54 100644 --- a/roles/security/tls_nifi/meta/main.yml +++ b/roles/security/tls_nifi/meta/main.yml @@ -15,4 +15,4 @@ --- dependencies: - - role: cloudera.cluster.infrastructure.ca_common \ No newline at end of file + - role: cloudera.cluster.infrastructure.ca_common diff --git a/roles/security/tls_signing/meta/main.yml b/roles/security/tls_signing/meta/main.yml index 85cfa0b8..856162c2 100644 --- a/roles/security/tls_signing/meta/main.yml +++ b/roles/security/tls_signing/meta/main.yml @@ -14,4 +14,4 @@ --- dependencies: - - role: cloudera.cluster.infrastructure.ca_common \ No newline at end of file + - role: cloudera.cluster.infrastructure.ca_common diff --git a/roles/teardown/files/oracle_drop.sql b/roles/teardown/files/oracle_drop.sql index 91b0b9a4..04026d72 100644 --- a/roles/teardown/files/oracle_drop.sql +++ b/roles/teardown/files/oracle_drop.sql @@ -31,6 +31,6 @@ BEGIN DBMS_OUTPUT.put_line('ERROR: DROP ' || cur_rec.object_type || ' ' || cur_rec.object_name ); END; END LOOP; -END; +END; / exit diff --git a/roles/teardown/tasks/teardown_ecs.yml b/roles/teardown/tasks/teardown_ecs.yml index c7de1e23..4d9b22b0 100644 --- a/roles/teardown/tasks/teardown_ecs.yml +++ b/roles/teardown/tasks/teardown_ecs.yml @@ -103,4 +103,4 @@ - name: Delete ECS CLuster include_role: name: cloudera.cluster.operations.delete_cluster - run_once: true \ No newline at end of file + run_once: true diff --git a/roles/verify/inventory/meta/main.yml b/roles/verify/inventory/meta/main.yml index 474745ea..cb97f8d0 100644 --- a/roles/verify/inventory/meta/main.yml +++ b/roles/verify/inventory/meta/main.yml @@ -14,4 +14,4 @@ --- dependencies: - - role: cloudera.cluster.infrastructure.krb5_common \ No newline at end of file + - role: cloudera.cluster.infrastructure.krb5_common diff --git a/roles/verify/inventory/tasks/main.yml b/roles/verify/inventory/tasks/main.yml index 18115579..d17ad5a3 100644 --- a/roles/verify/inventory/tasks/main.yml +++ b/roles/verify/inventory/tasks/main.yml @@ -38,8 +38,8 @@ cluster_hosts: >- {{ groups.cluster | default([]) | union( - (groups.cloudera_manager | default([]) - | union( + (groups.cloudera_manager | default([]) + | union( groups.ecs_nodes | default([]) ) ) diff --git a/roles/verify/parcels_and_roles/tasks/check_cluster.yml b/roles/verify/parcels_and_roles/tasks/check_cluster.yml index 5d5cf57c..4a35381b 100644 --- a/roles/verify/parcels_and_roles/tasks/check_cluster.yml +++ b/roles/verify/parcels_and_roles/tasks/check_cluster.yml @@ -62,7 +62,7 @@ cluster.host_templates | default({}) | json_query('values(@) | map(&keys(@),@)') - | flatten + | flatten | unique }} - set_fact: diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index 02bf2923..6a16e733 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -1,11 +1,11 @@ # Copyright 2024 Cloudera, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 5f964969..d86218dd 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -21,6 +21,7 @@ import json import sys import pytest +import yaml from ansible.module_utils import basic from ansible.module_utils.common.text.converters import to_bytes @@ -36,6 +37,7 @@ AnsibleExitJson, ) + @pytest.fixture(autouse=True) def skip_python(): if sys.version_info < (3, 6): @@ -71,3 +73,14 @@ def prep_args(args=dict()): basic._ANSIBLE_ARGS = to_bytes(args) return prep_args + + +@pytest.fixture +def yaml_args(): + """Prepare module arguments from YAML""" + + def prep_args(args: str = ""): + output = json.dumps({"ANSIBLE_MODULE_ARGS": yaml.safe_load(args)}) + basic._ANSIBLE_ARGS = to_bytes(output) + + return prep_args diff --git a/tests/unit/plugins/modules/cluster/example.yml b/tests/unit/plugins/modules/cluster/example.yml new file mode 100644 index 00000000..2f128db5 --- /dev/null +++ b/tests/unit/plugins/modules/cluster/example.yml @@ -0,0 +1,52 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cluster: + type: + display_name: + state: + cluster_version: + template: + add_repositories: + parcels: {} + tags: {} + contexts: [] + services: + - name: + display_name: + type: + version: + config: {} + role_groups: + - name: + type: + base: false + display_name: + config: {} + tags: {} + host_templates: + - name: + role_groups: [] + hosts: + - name: + config: {} + host_template: + role_groups: [] + roles: + - type: + config: {} + tags: {} + maintenance: false + tls: false + auto_assign: false diff --git a/tests/unit/plugins/modules/cluster/test_cluster.py b/tests/unit/plugins/modules/cluster/test_cluster.py new file mode 100644 index 00000000..b13c3043 --- /dev/null +++ b/tests/unit/plugins/modules/cluster/test_cluster.py @@ -0,0 +1,610 @@ +# -*- coding: utf-8 -*- + +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import logging +import os +import pytest +import yaml + +from ansible_collections.cloudera.cluster.plugins.modules import cluster +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) + +LOG = logging.getLogger(__name__) + + +@pytest.fixture +def conn(): + conn = dict(username=os.getenv("CM_USERNAME"), password=os.getenv("CM_PASSWORD")) + + if os.getenv("CM_HOST", None): + conn.update(host=os.getenv("CM_HOST")) + + if os.getenv("CM_PORT", None): + conn.update(port=os.getenv("CM_PORT")) + + if os.getenv("CM_ENDPOINT", None): + conn.update(url=os.getenv("CM_ENDPOINT")) + + if os.getenv("CM_PROXY", None): + conn.update(proxy=os.getenv("CM_PROXY")) + + return { + **conn, + "verify_tls": "no", + "debug": "no", + } + + +def test_missing_required(conn, module_args): + module_args(conn) + + with pytest.raises(AnsibleFailJson, match="name"): + cluster.main() + + +def test_present_base_minimum(conn, module_args): + conn.update( + name="Example_Base_Minimum", + cluster_version="7", # "1.5.1-b626.p0.42068229", + type="BASE_CLUSTER", + state="present", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + assert e.value.cloudera_manager + + +def test_present_base_hosts(conn, module_args): + conn.update( + name="Example_Base_Hosts", + cluster_version="7", # "1.5.1-b626.p0.42068229", + type="BASE_CLUSTER", + state="present", + hosts=[{"name": "test10-worker-free-01.cldr.internal"}], + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_present_base_hosts_not_found(conn, module_args): + conn.update( + name="Example_Base_Hosts_Not_Found", + cluster_version="7", # "1.5.1-b626.p0.42068229", + type="BASE_CLUSTER", + state="present", + hosts=[{"name": "should.not.find"}], + ) + module_args(conn) + + with pytest.raises( + AnsibleFailJson, match="Did not find the following hosts: should.not.find" + ): + cluster.main() + + +def test_present_base_hosts_in_use(conn, module_args): + conn.update( + name="Example_Base_Hosts_In_Use", + cluster_version="7", # "1.5.1-b626.p0.42068229", + type="BASE_CLUSTER", + state="present", + hosts=[{"name": "test10-worker-02.cldr.internal"}], + ) + module_args(conn) + + with pytest.raises(AnsibleFailJson, match="Invalid host reference!"): + cluster.main() + + +def test_present_base_auto_assign(conn, module_args): + conn.update( + name="Example_Base_Auto_Assign", + cluster_version="7", # "1.5.1-b626.p0.42068229", + type="BASE_CLUSTER", + state="present", + auto_assign=True, + hosts=[{"name": "test10-worker-free-01.cldr.internal"}], + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_present_base_service(conn, module_args): + args = """ + name: Example_Base_Service + cluster_version: 7 + type: BASE_CLUSTER + state: present + services: + - name: ZK-BASE-SERVICE + type: ZOOKEEPER + display_name: ZK_TEST + """ + conn.update(yaml.safe_load(args)) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_present_base_service_config(conn, module_args): + args = """ + name: Example_Base_Service_Config + cluster_version: 7 + type: BASE_CLUSTER + state: present + services: + - name: ZK-BASE-SERVICE-CONFIG + type: ZOOKEEPER + display_name: ZK_TEST + config: + zookeeper_datadir_autocreate: yes + service_config_suppression_server_count_validator: yes + """ + conn.update(yaml.safe_load(args)) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_present_base_service_role_groups(conn, module_args): + args = """ + name: Example_Base_Service_Role_Groups + cluster_version: 7 + type: BASE_CLUSTER + state: present + services: + - name: ZK-BASE-SERVICE-ROLE-GROUPS + type: ZOOKEEPER + display_name: ZK_TEST + role_groups: + - type: SERVER + display_name: Server Base Group + config: + zookeeper_server_java_heapsize: 134217728 # 128MB + - name: NON-BASE-SERVER + type: SERVER + display_name: Server Custom Group + config: + zookeeper_server_java_heapsize: 33554432 # 32MB + """ + conn.update(yaml.safe_load(args)) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_present_base_host_role_group_assignment_base(conn, module_args): + args = """ + name: Example_Base_Host_Role_Group_Assignment + cluster_version: 7 + type: BASE_CLUSTER + state: present + services: + - name: ZK_HOST_ROLE_GROUP_ASSIGNMENT + type: ZOOKEEPER + display_name: ZK_TEST + role_groups: + - type: SERVER + display_name: Server Base Group + config: + zookeeper_server_java_heapsize: 134217728 # 128MB + - name: HOST_ROLE_GROUP_ASSIGNMENT_NON_BASE_SERVER + type: SERVER + display_name: Server Custom Group + config: + zookeeper_server_java_heapsize: 33554432 # 32MB + hosts: + - name: test10-worker-free-02.cldr.internal + role_groups: + - type: SERVER + service: ZK_HOST_ROLE_GROUP_ASSIGNMENT + """ + conn.update(yaml.safe_load(args)) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_present_base_host_role_group_assignment_custom(conn, module_args): + args = """ + name: Example_Base_Host_Role_Group_Assignment + cluster_version: 7 + type: BASE_CLUSTER + state: present + services: + - name: ZK_HOST_ROLE_GROUP_ASSIGNMENT_CUSTOM + type: ZOOKEEPER + display_name: ZK_TEST + role_groups: + - type: SERVER + display_name: Server Base Group + config: + zookeeper_server_java_heapsize: 134217728 # 128MB + - name: ZK_HOST_ROLE_GROUP_ASSIGNMENT_CUSTOM_NON_BASE_SERVER + type: SERVER + display_name: Server Custom Group + config: + zookeeper_server_java_heapsize: 33554432 # 32MB + hosts: + - name: test10-worker-free-03.cldr.internal + role_groups: + - name: ZK_HOST_ROLE_GROUP_ASSIGNMENT_CUSTOM_NON_BASE_SERVER + service: ZK_HOST_ROLE_GROUP_ASSIGNMENT_CUSTOM + """ + conn.update(yaml.safe_load(args)) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_present_base_host_host_template_assignment(conn, module_args): + args = """ + name: Example_Base_Host_Host_Template_Assignment + cluster_version: "7.1.9-1.cdh7.1.9.p0.44702451" # 7 + type: BASE_CLUSTER + state: present + services: + - name: ZK-BASE-SERVICE-ROLE-GROUPS + type: ZOOKEEPER + display_name: ZK_TEST + role_groups: + - name: BASE-SERVER # ignored due to base=True + type: SERVER + base: yes + display_name: Server Base Group + config: + zookeeper_server_java_heapsize: 134217728 # 128MB + - name: NON-BASE-SERVER + type: SERVER + display_name: Server Custom Group + config: + zookeeper_server_java_heapsize: 75497472 # 72MB + hosts: + - name: test10-worker-free-01.cldr.internal + host_template: Example_Template + host_templates: + - name: Example_Template + role_groups: + - NON-BASE-SERVER + parcels: + CDH: "7.1.9-1.cdh7.1.9.p0.44702451" + """ + conn.update(yaml.safe_load(args)) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_present_base_host_role_overrides(conn, module_args): + args = """ + name: Example_Base_Host_Role_Overrides + cluster_version: "7.1.9-1.cdh7.1.9.p0.44702451" # 7 + type: BASE_CLUSTER + state: present + services: + - name: ZK-BASE-SERVICE-ROLE-GROUPS + type: ZOOKEEPER + display_name: ZK_TEST + role_groups: + - name: NON-BASE-SERVER + type: SERVER + display_name: Server Custom Group + config: + zookeeper_server_java_heapsize: 75497472 # 72MB + hosts: + - name: test10-worker-free-02.cldr.internal + roles: + - service: ZK-BASE-SERVICE-ROLE-GROUPS + type: SERVER + config: + zookeeper_server_java_heapsize: 67108864 # 64MB + host_template: Example_Template + host_templates: + - name: Example_Template + role_groups: + - NON-BASE-SERVER + parcels: + CDH: "7.1.9-1.cdh7.1.9.p0.44702451" + """ + conn.update(yaml.safe_load(args)) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_present_basic_cluster(conn, module_args): + args = """ + name: Basic_Cluster + cluster_version: "7.1.9-1.cdh7.1.9.p0.44702451" + type: BASE_CLUSTER + state: present + services: + - name: core-settings-0 + type: CORE_SETTINGS + display_name: CORE_SETTINGS_TEST + - name: zookeeper-0 + type: ZOOKEEPER + display_name: ZK_TEST + config: + zookeeper_datadir_autocreate: yes + - name: hdfs-0 + type: HDFS + display_name: HDFS_TEST + config: + zookeeper_service: zookeeper-0 + core_connector: core-settings-0 + role_groups: + - type: DATANODE + config: + dfs_data_dir_list: /dfs/dn + - type: NAMENODE + config: + dfs_name_dir_list: /dfs/nn + - type: SECONDARYNAMENODE + config: + fs_checkpoint_dir_list: /dfs/snn + - name: yarn-0 + type: YARN + display_name: YARN_TEST + config: + hdfs_service: hdfs-0 + zookeeper_service: zookeeper-0 + role_groups: + - type: RESOURCEMANAGER + config: + yarn_scheduler_maximum_allocation_mb: 4096 + yarn_scheduler_maximum_allocation_vcores: 4 + - type: NODEMANAGER + config: + yarn_nodemanager_resource_memory_mb: 4096 + yarn_nodemanager_resource_cpu_vcores: 4 + yarn_nodemanager_local_dirs: /tmp/nm + yarn_nodemanager_log_dirs: /var/log/nm + - type: GATEWAY + config: + mapred_submit_replication: 3 + mapred_reduce_tasks: 6 + host_templates: + - name: Master1 + role_groups: + - service: HDFS + type: NAMENODE + - service: HDFS + type: SECONDARYNAMENODE + - service: YARN + type: RESOURCEMANAGER + - service: YARN + type: JOBHISTORY + - name: Worker + role_groups: + - service: HDFS + type: DATANODE + - service: YARN + type: NODEMANAGER + - service: ZOOKEEPER + type: SERVER + parcels: + CDH: "7.1.9-1.cdh7.1.9.p0.44702451" + hosts: + - name: test10-worker-free-01.cldr.internal + host_template: Master1 + - name: test10-worker-free-02.cldr.internal + host_template: Worker + - name: test10-worker-free-03.cldr.internal + host_template: Worker + """ + conn.update(yaml.safe_load(args)) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +@pytest.mark.skip(reason="Not yet implemented") +def test_started_base(conn, module_args): + conn.update( + name="PVC-Base", + cluster_version="7.1.9", # "1.5.1-b626.p0.42068229", + state="started", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +@pytest.mark.skip(reason="Not yet implemented") +def test_restarted_base(conn, module_args): + conn.update( + name="PVC-Base", + cluster_version="7.1.9", # "1.5.1-b626.p0.42068229", + state="restarted", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +@pytest.mark.skip(reason="Not yet implemented") +def test_stopped_base(conn, module_args): + conn.update( + name="PVC-Base", + cluster_version="7.1.9", # "1.5.1-b626.p0.42068229", + # type="COMPUTE_CLUSTER", + state="stopped", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + # LOG.info(str(e.value)) + LOG.info(str(e.value.cloudera_manager)) + + +@pytest.mark.skip(reason="Not yet implemented") +def test_absent_base(conn, module_args): + conn.update( + name="Example_Base", + state="absent", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_present_compute_minimum(conn, module_args): + conn.update( + name="Example_Compute", + cluster_version="7.1.9", + contexts=["SDX"], + state="present", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +@pytest.mark.skip(reason="Not yet implemented") +def test_started_compute_minimum(conn, module_args): + conn.update( + name="Example_Compute", + cluster_version="7.1.9", + contexts=["SDX"], + state="started", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_absent_compute(conn, module_args): + conn.update( + name="Example_Compute", + state="absent", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_present_experience_minimum(conn, module_args): + conn.update( + name="Example_Experience", + cluster_version="1.5.3", + type="EXPERIENCE_CLUSTER", + state="present", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_absent_experience(conn, module_args): + conn.update( + name="Example_Experience", + state="absent", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_pytest_cluster_with_template(module_args): + module_args( + { + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), + "port": "7180", + "verify_tls": "no", + "debug": "no", + "cluster_name": "Base_CM_Cluster", + "template": "./files/cluster-template.json", + "add_repositories": "True", + "state": "present", + } + ) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) diff --git a/tests/unit/plugins/modules/cluster_info/test_cluster_info.py b/tests/unit/plugins/modules/cluster_info/test_cluster_info.py new file mode 100644 index 00000000..98ffb8ed --- /dev/null +++ b/tests/unit/plugins/modules/cluster_info/test_cluster_info.py @@ -0,0 +1,90 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type +import os +import logging +import pytest + +from ansible_collections.cloudera.cluster.plugins.modules import cluster_info +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) + +LOG = logging.getLogger(__name__) + + +@pytest.fixture +def conn(): + conn = dict(username=os.getenv("CM_USERNAME"), password=os.getenv("CM_PASSWORD")) + + if os.getenv("CM_HOST", None): + conn.update(host=os.getenv("CM_HOST")) + + if os.getenv("CM_PORT", None): + conn.update(port=os.getenv("CM_PORT")) + + if os.getenv("CM_ENDPOINT", None): + conn.update(url=os.getenv("CM_ENDPOINT")) + + # if os.getenv("CM_PROXY", None): + # conn.update(proxy=os.getenv("CM_PROXY")) + + return { + **conn, + "verify_tls": "no", + "debug": "no", + } + + +def test_cluster_info_base(conn, module_args): + conn.update( + name="Example_Base", + ) + + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster_info.main() + + LOG.debug(str(e.value.clusters)) + + +def test_cluster_info_compute(conn, module_args): + conn.update( + name="Example_Compute", + ) + + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster_info.main() + + LOG.info(str(e.value.clusters)) + + +def test_cluster_info_experience(conn, module_args): + conn.update( + name="Example_Experience", + ) + + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster_info.main() + + LOG.info(str(e.value.clusters)) diff --git a/tests/unit/plugins/modules/cm_cluster/test_cm_cluster.py b/tests/unit/plugins/modules/cm_cluster/test_cm_cluster.py deleted file mode 100644 index 96c358c0..00000000 --- a/tests/unit/plugins/modules/cm_cluster/test_cm_cluster.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import, division, print_function - -__metaclass__ = type - -import os -import logging -import pytest - -from ansible_collections.cloudera.cluster.plugins.modules import cm_cluster -from ansible_collections.cloudera.cluster.tests.unit import AnsibleExitJson, AnsibleFailJson - -LOG = logging.getLogger(__name__) - - -@pytest.fixture() -def conn(): - return { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), - "port": "7180", - "verify_tls": "no", - "debug": "yes", - } - -def test_missing_name_or_template(conn, module_args): - module_args(conn) - - with pytest.raises(AnsibleFailJson, match="name, template") as e: - cm_cluster.main() - -def test_missing_cdh_version(conn, module_args): - module_args({ - **conn, - "name": "Test" - }) - - with pytest.raises(AnsibleFailJson, match="Bad Request") as e: - cm_cluster.main() - - assert "CDH version" in e.value.body['message'] - -def test_absent_not_existing(conn, module_args): - module_args({ - **conn, - "name": "Test", - "state": "absent" - }) - - with pytest.raises(AnsibleExitJson) as e: - cm_cluster.main() - - assert e.value.changed == False - \ No newline at end of file diff --git a/tests/unit/plugins/modules/cm_cluster_info/test_cm_cluster_info.py b/tests/unit/plugins/modules/cm_cluster_info/test_cm_cluster_info.py deleted file mode 100644 index d09ba17b..00000000 --- a/tests/unit/plugins/modules/cm_cluster_info/test_cm_cluster_info.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2024 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import, division, print_function - -__metaclass__ = type -import os -import logging -import pytest - -from ansible_collections.cloudera.cluster.plugins.modules import cm_cluster_info -from ansible_collections.cloudera.cluster.tests.unit import AnsibleExitJson, AnsibleFailJson - -LOG = logging.getLogger(__name__) - -def test_pytest_cm_cluster_info(module_args): - module_args( - { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), - "port": "7180", - "verify_tls": "no", - "cluster_name": "OneNodeCluster", - "debug": "no", - } - ) - - with pytest.raises(AnsibleExitJson) as e: - cm_cluster_info.main() - - LOG.info(str(e.value.cloudera_manager)) diff --git a/tests/unit/plugins/modules/cm_config/test_cm_config.py b/tests/unit/plugins/modules/cm_config/test_cm_config.py index 1dbab465..540db433 100644 --- a/tests/unit/plugins/modules/cm_config/test_cm_config.py +++ b/tests/unit/plugins/modules/cm_config/test_cm_config.py @@ -23,7 +23,10 @@ import pytest from ansible_collections.cloudera.cluster.plugins.modules import cm_config -from ansible_collections.cloudera.cluster.tests.unit import AnsibleExitJson, AnsibleFailJson +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) LOG = logging.getLogger(__name__) @@ -50,12 +53,14 @@ def conn(): "debug": "no", } + def test_missing_parameters(conn, module_args): module_args(conn) with pytest.raises(AnsibleFailJson, match="parameters"): cm_config.main() + def test_set_config(conn, module_args): conn.update( parameters=dict(custom_header_color="PURPLE"), @@ -78,10 +83,7 @@ def test_set_config(conn, module_args): def test_unset_config(conn, module_args): - module_args({ - **conn, - "parameters": dict(custom_header_color=None) - }) + module_args({**conn, "parameters": dict(custom_header_color=None)}) with pytest.raises(AnsibleExitJson) as e: cm_config.main() diff --git a/tests/unit/plugins/modules/cm_config_info/test_cm_config_info.py b/tests/unit/plugins/modules/cm_config_info/test_cm_config_info.py index 5f4a6161..636f47ad 100644 --- a/tests/unit/plugins/modules/cm_config_info/test_cm_config_info.py +++ b/tests/unit/plugins/modules/cm_config_info/test_cm_config_info.py @@ -23,7 +23,10 @@ import pytest from ansible_collections.cloudera.cluster.plugins.modules import cm_config_info -from ansible_collections.cloudera.cluster.tests.unit import AnsibleExitJson, AnsibleFailJson +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) LOG = logging.getLogger(__name__) @@ -31,14 +34,15 @@ @pytest.fixture() def conn(): return { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), "port": "7180", "verify_tls": "no", "debug": "yes", } + def test_get_cluster_config(conn, module_args): module_args(conn) @@ -47,11 +51,9 @@ def test_get_cluster_config(conn, module_args): assert len(e.value.config) > 0 + def test_get_cluster_config_full(conn, module_args): - module_args({ - **conn, - "view": "full" - }) + module_args({**conn, "view": "full"}) with pytest.raises(AnsibleExitJson) as e: cm_config_info.main() diff --git a/tests/unit/plugins/modules/cm_endpoint_info/test_cm_endpoint_info_i.py b/tests/unit/plugins/modules/cm_endpoint_info/test_cm_endpoint_info_i.py index cd2c0350..e6678e8f 100644 --- a/tests/unit/plugins/modules/cm_endpoint_info/test_cm_endpoint_info_i.py +++ b/tests/unit/plugins/modules/cm_endpoint_info/test_cm_endpoint_info_i.py @@ -14,20 +14,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import (absolute_import, division, print_function) +from __future__ import absolute_import, division, print_function + __metaclass__ = type import logging import pytest from ansible_collections.cloudera.cluster.plugins.modules import cm_endpoint_info -from ansible_collections.cloudera.cluster.tests.unit import AnsibleExitJson, AnsibleFailJson +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) from cm_client.rest import RESTClientObject from urllib3.response import HTTPResponse LOG = logging.getLogger(__name__) - + + def test_host_discovery(module_args, monkeypatch): spec = { "username": "testuser", @@ -35,19 +40,17 @@ def test_host_discovery(module_args, monkeypatch): "host": "test.cldr.info", "port": "7180", "verify_tls": "no", - "debug": "yes" + "debug": "yes", } - + def response(): return HTTPResponse() - + monkeypatch.setattr("urllib3.HTTPConnectionPool.urlopen", response) - - + module_args(spec) - + with pytest.raises(AnsibleExitJson) as e: cm_endpoint_info.main() - + assert e.value.endpoint == f"https://{spec['host']}:7183/api/v01" - diff --git a/tests/unit/plugins/modules/cm_license/test_cm_license.py b/tests/unit/plugins/modules/cm_license/test_cm_license.py index 09a5bdb0..ea2dd2a5 100644 --- a/tests/unit/plugins/modules/cm_license/test_cm_license.py +++ b/tests/unit/plugins/modules/cm_license/test_cm_license.py @@ -22,16 +22,20 @@ import pytest from ansible_collections.cloudera.cluster.plugins.modules import cm_license -from ansible_collections.cloudera.cluster.tests.unit import AnsibleExitJson, AnsibleFailJson +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) LOG = logging.getLogger(__name__) + def test_pytest_cm_license(module_args): module_args( { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), "port": "7180", "verify_tls": "no", "debug": "no", diff --git a/tests/unit/plugins/modules/cm_license_info/test_cm_license_info.py b/tests/unit/plugins/modules/cm_license_info/test_cm_license_info.py index 31c74328..4c309e1a 100644 --- a/tests/unit/plugins/modules/cm_license_info/test_cm_license_info.py +++ b/tests/unit/plugins/modules/cm_license_info/test_cm_license_info.py @@ -22,16 +22,20 @@ import pytest from ansible_collections.cloudera.cluster.plugins.modules import cm_license_info -from ansible_collections.cloudera.cluster.tests.unit import AnsibleExitJson, AnsibleFailJson +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) LOG = logging.getLogger(__name__) + def test_pytest_cm_license_info(module_args): module_args( { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), "port": "7180", "verify_tls": "no", "debug": "no", diff --git a/tests/unit/plugins/modules/cm_resource/test_cm_resource_i.py b/tests/unit/plugins/modules/cm_resource/test_cm_resource_i.py index e1b5bd31..f50d853d 100644 --- a/tests/unit/plugins/modules/cm_resource/test_cm_resource_i.py +++ b/tests/unit/plugins/modules/cm_resource/test_cm_resource_i.py @@ -14,7 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import (absolute_import, division, print_function) +from __future__ import absolute_import, division, print_function + __metaclass__ = type import os @@ -22,86 +23,83 @@ import unittest from ansible_collections.cloudera.cluster.plugins.modules import cm_resource -from ansible_collections.cloudera.cluster.tests.unit.plugins.modules.utils import AnsibleExitJson, AnsibleFailJson, ModuleTestCase, setup_module_args +from ansible_collections.cloudera.cluster.tests.unit.plugins.modules.utils import ( + AnsibleExitJson, + AnsibleFailJson, + ModuleTestCase, + setup_module_args, +) -@unittest.skipUnless(os.getenv('CM_USERNAME'), "Cloudera Manager access parameters not set") +@unittest.skipUnless( + os.getenv("CM_USERNAME"), "Cloudera Manager access parameters not set" +) class TestCMResourceIntegration(ModuleTestCase): - - def test_post(self): + def test_post(self): create_module_args = { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "url": os.getenv('CM_ENDPOINT'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "url": os.getenv("CM_ENDPOINT"), "verify_tls": "no", "debug": "yes", "method": "POST", "path": "/users", - "body": { - "items": [ - { - "name": "unit_test", - "password": "UnsecurePassword" - } - ] - } + "body": {"items": [{"name": "unit_test", "password": "UnsecurePassword"}]}, } - + update_module_args = { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - #"host": os.getenv('CM_HOST'), - "url": os.getenv('CM_ENDPOINT'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + # "host": os.getenv('CM_HOST'), + "url": os.getenv("CM_ENDPOINT"), "verify_tls": "no", "debug": "yes", "method": "PUT", "path": "/users/unit_test", - "body": { - "authRoles": [{ "name": "ROLE_LIMITED" }] - } + "body": {"authRoles": [{"name": "ROLE_LIMITED"}]}, } - + delete_module_args = { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - #"host": os.getenv('CM_HOST'), - "url": os.getenv('CM_ENDPOINT'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + # "host": os.getenv('CM_HOST'), + "url": os.getenv("CM_ENDPOINT"), "verify_tls": "no", "debug": "yes", "method": "DELETE", - "path": "/users/unit_test" + "path": "/users/unit_test", } - + # Create setup_module_args(create_module_args) with pytest.raises(AnsibleExitJson) as e: cm_resource.main() - self.assertIsInstance(e.value.args[0]['resources'], list) - + self.assertIsInstance(e.value.args[0]["resources"], list) + # Create fail on duplicate setup_module_args(create_module_args) with pytest.raises(AnsibleFailJson) as e: cm_resource.main() - self.assertEquals(e.value.args[0]['status_code'], 400) - + self.assertEquals(e.value.args[0]["status_code"], 400) + # Update setup_module_args(update_module_args) with pytest.raises(AnsibleExitJson) as e: cm_resource.main() - self.assertIsInstance(e.value.args[0]['resources'], list) - + self.assertIsInstance(e.value.args[0]["resources"], list) + # Delete setup_module_args(delete_module_args) with pytest.raises(AnsibleExitJson) as e: cm_resource.main() - self.assertIsInstance(e.value.args[0]['resources'], list) - + self.assertIsInstance(e.value.args[0]["resources"], list) + # Delete fail on existence setup_module_args(delete_module_args) with pytest.raises(AnsibleFailJson) as e: cm_resource.main() - self.assertEquals(e.value.args[0]['status_code'], 404) - + self.assertEquals(e.value.args[0]["status_code"], 404) + -if __name__ == '__main__': - unittest.main() \ No newline at end of file +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/plugins/modules/cm_resource_info/test_cm_resource_info_i.py b/tests/unit/plugins/modules/cm_resource_info/test_cm_resource_info_i.py index 76f50d39..ae574fc0 100644 --- a/tests/unit/plugins/modules/cm_resource_info/test_cm_resource_info_i.py +++ b/tests/unit/plugins/modules/cm_resource_info/test_cm_resource_info_i.py @@ -14,7 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import (absolute_import, division, print_function) +from __future__ import absolute_import, division, print_function + __metaclass__ = type import os @@ -23,93 +24,108 @@ import unittest from ansible_collections.cloudera.cluster.plugins.modules import cm_resource_info -from ansible_collections.cloudera.cluster.tests.unit.plugins.modules.utils import AnsibleExitJson, AnsibleFailJson, ModuleTestCase, setup_module_args +from ansible_collections.cloudera.cluster.tests.unit.plugins.modules.utils import ( + AnsibleExitJson, + AnsibleFailJson, + ModuleTestCase, + setup_module_args, +) -@unittest.skipUnless(os.getenv('CM_USERNAME'), "Cloudera Manager access parameters not set") +@unittest.skipUnless( + os.getenv("CM_USERNAME"), "Cloudera Manager access parameters not set" +) class TestCMResourceInfoIntegration(ModuleTestCase): - def test_list(self): - setup_module_args({ - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - #"host": os.getenv('CM_HOST'), - "endpoint": os.getenv('CM_ENDPOINT'), - "verify_tls": "no", - "debug": "yes", - "path": "/clusters" - }) - + setup_module_args( + { + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + # "host": os.getenv('CM_HOST'), + "endpoint": os.getenv("CM_ENDPOINT"), + "verify_tls": "no", + "debug": "yes", + "path": "/clusters", + } + ) + with pytest.raises(AnsibleExitJson) as e: cm_resource_info.main() - - self.assertIsInstance(e.value.args[0]['resources'], list) - + + self.assertIsInstance(e.value.args[0]["resources"], list) + def test_item(self): - setup_module_args({ - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), - "verify_tls": "no", - "debug": "yes", - "path": "/cm/license" - }) - + setup_module_args( + { + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), + "verify_tls": "no", + "debug": "yes", + "path": "/cm/license", + } + ) + with pytest.raises(AnsibleExitJson) as e: cm_resource_info.main() - - self.assertIsInstance(e.value.args[0]['resources'], list) + + self.assertIsInstance(e.value.args[0]["resources"], list) def test_invalid_host(self): - setup_module_args({ - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": "nope", - "verify_tls": "no", - "debug": "yes", - "path": "/cm/license" - }) - + setup_module_args( + { + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": "nope", + "verify_tls": "no", + "debug": "yes", + "path": "/cm/license", + } + ) + with pytest.raises(AnsibleFailJson) as e: cm_resource_info.main() - - self.assertRegexpMatches(e.value.args[0]['msg'], "nodename nor servname provided, or not known") - + + self.assertRegexpMatches( + e.value.args[0]["msg"], "nodename nor servname provided, or not known" + ) + def test_invalid_path(self): - setup_module_args({ - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), - "verify_tls": "no", - "debug": "yes", - "path": "/cm/licenseZ" - }) - + setup_module_args( + { + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), + "verify_tls": "no", + "debug": "yes", + "path": "/cm/licenseZ", + } + ) + with pytest.raises(AnsibleFailJson) as e: cm_resource_info.main() - - self.assertRegexpMatches(e.value.args[0]['msg'], "^API error: Not Found$") + + self.assertRegexpMatches(e.value.args[0]["msg"], "^API error: Not Found$") def test_query_params(self): - setup_module_args({ - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), - "verify_tls": "no", - "debug": "yes", - "path": "/tools/echo", - "query": { - "message": "foobarbaz" - }, - "field": "message" - }) - + setup_module_args( + { + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), + "verify_tls": "no", + "debug": "yes", + "path": "/tools/echo", + "query": {"message": "foobarbaz"}, + "field": "message", + } + ) + with pytest.raises(AnsibleExitJson) as e: cm_resource_info.main() - - self.assertRegexpMatches(e.value.args[0]['resources'][0], "^foobarbaz$") - + + self.assertRegexpMatches(e.value.args[0]["resources"][0], "^foobarbaz$") -if __name__ == '__main__': - unittest.main() \ No newline at end of file +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/plugins/modules/cm_service/test_cm_service.py b/tests/unit/plugins/modules/cm_service/test_cm_service.py index 0b175cd7..5614fe61 100644 --- a/tests/unit/plugins/modules/cm_service/test_cm_service.py +++ b/tests/unit/plugins/modules/cm_service/test_cm_service.py @@ -22,21 +22,25 @@ import pytest from ansible_collections.cloudera.cluster.plugins.modules import cm_service -from ansible_collections.cloudera.cluster.tests.unit import AnsibleExitJson, AnsibleFailJson +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) LOG = logging.getLogger(__name__) + def test_pytest_cm_service(module_args): module_args( { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), "port": "7180", "verify_tls": "no", "debug": "yes", "state": "started", - "role" : [ "SERVICEMONITOR" , "HOSTMONITOR", "EVENTSERVER", "ALERTPUBLISHER" ] + "role": ["SERVICEMONITOR", "HOSTMONITOR", "EVENTSERVER", "ALERTPUBLISHER"], } ) diff --git a/tests/unit/plugins/modules/cm_service_info/test_cm_service_info.py b/tests/unit/plugins/modules/cm_service_info/test_cm_service_info.py index d7d7af0b..0561dba6 100644 --- a/tests/unit/plugins/modules/cm_service_info/test_cm_service_info.py +++ b/tests/unit/plugins/modules/cm_service_info/test_cm_service_info.py @@ -14,7 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import (absolute_import, division, print_function) +from __future__ import absolute_import, division, print_function + __metaclass__ = type import os @@ -23,24 +24,33 @@ from ansible_collections.cloudera.cluster.plugins.modules import cm_service_info -from ansible_collections.cloudera.cluster.tests.unit.plugins.modules.utils import AnsibleExitJson, AnsibleFailJson, ModuleTestCase, setup_module_args +from ansible_collections.cloudera.cluster.tests.unit.plugins.modules.utils import ( + AnsibleExitJson, + AnsibleFailJson, + ModuleTestCase, + setup_module_args, +) -@unittest.skipUnless(os.getenv('CM_USERNAME'), "Cloudera Manager access parameters not set") +@unittest.skipUnless( + os.getenv("CM_USERNAME"), "Cloudera Manager access parameters not set" +) class TestCMServiceInfo(ModuleTestCase): - def test_service_info(self): - setup_module_args({ - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), - "port": "7180", - "verify_tls": "no", - "debug": "yes" - }) - + setup_module_args( + { + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), + "port": "7180", + "verify_tls": "no", + "debug": "yes", + } + ) + with pytest.raises(AnsibleExitJson) as e: cm_service_info.main() - -if __name__ == '__main__': - unittest.main() \ No newline at end of file + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/plugins/modules/cm_trial/test_cm_trial_license.py b/tests/unit/plugins/modules/cm_trial/test_cm_trial_license.py index afb1a6ba..a3f10eab 100644 --- a/tests/unit/plugins/modules/cm_trial/test_cm_trial_license.py +++ b/tests/unit/plugins/modules/cm_trial/test_cm_trial_license.py @@ -22,16 +22,20 @@ import pytest from ansible_collections.cloudera.cluster.plugins.modules import cm_trial_license -from ansible_collections.cloudera.cluster.tests.unit import AnsibleExitJson, AnsibleFailJson +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) LOG = logging.getLogger(__name__) + def test_pytest_cm_trial_license(module_args): module_args( { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), "port": "7180", "verify_tls": "no", "debug": "no", diff --git a/tests/unit/plugins/modules/cm_version_info/test_cm_version_info_i.py b/tests/unit/plugins/modules/cm_version_info/test_cm_version_info_i.py index 70114245..c37fdb81 100644 --- a/tests/unit/plugins/modules/cm_version_info/test_cm_version_info_i.py +++ b/tests/unit/plugins/modules/cm_version_info/test_cm_version_info_i.py @@ -14,7 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import (absolute_import, division, print_function) +from __future__ import absolute_import, division, print_function + __metaclass__ = type import os @@ -23,41 +24,56 @@ import unittest from ansible_collections.cloudera.cluster.plugins.modules import cm_version_info -from ansible_collections.cloudera.cluster.tests.unit.plugins.modules.utils import AnsibleExitJson, AnsibleFailJson, ModuleTestCase, setup_module_args +from ansible_collections.cloudera.cluster.tests.unit.plugins.modules.utils import ( + AnsibleExitJson, + AnsibleFailJson, + ModuleTestCase, + setup_module_args, +) -@unittest.skipUnless(os.getenv('CM_USERNAME'), "Cloudera Manager access parameters not set") +@unittest.skipUnless( + os.getenv("CM_USERNAME"), "Cloudera Manager access parameters not set" +) class TestCMVersionIntegration(ModuleTestCase): - def test_host_discovery(self): - setup_module_args({ - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), - "port": "7180", - "verify_tls": "no", - "debug": "yes" - }) - + setup_module_args( + { + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), + "port": "7180", + "verify_tls": "no", + "debug": "yes", + } + ) + with pytest.raises(AnsibleExitJson) as e: cm_version_info.main() - - self.assertEquals(e.value.args[0]['cm']['version'], "7.6.5") - + + self.assertEquals(e.value.args[0]["cm"]["version"], "7.6.5") + def test_direct_endpoint(self): - setup_module_args({ - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "url": "https://" + os.getenv('CM_HOST') + ":" + os.getenv('CM_PORT_TLS') + "/api/" + os.getenv('CM_VERSION'), - "verify_tls": "no", - "debug": "yes" - }) - + setup_module_args( + { + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "url": "https://" + + os.getenv("CM_HOST") + + ":" + + os.getenv("CM_PORT_TLS") + + "/api/" + + os.getenv("CM_VERSION"), + "verify_tls": "no", + "debug": "yes", + } + ) + with pytest.raises(AnsibleExitJson) as e: cm_version_info.main() - - self.assertEquals(e.value.args[0]['cm']['version'], "7.6.5") + + self.assertEquals(e.value.args[0]["cm"]["version"], "7.6.5") -if __name__ == '__main__': - unittest.main() \ No newline at end of file +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/plugins/modules/host/test_host.py b/tests/unit/plugins/modules/host/test_host.py index f5307aaa..3796f1e8 100644 --- a/tests/unit/plugins/modules/host/test_host.py +++ b/tests/unit/plugins/modules/host/test_host.py @@ -22,23 +22,27 @@ import pytest from ansible_collections.cloudera.cluster.plugins.modules import host -from ansible_collections.cloudera.cluster.tests.unit import AnsibleExitJson, AnsibleFailJson +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) LOG = logging.getLogger(__name__) + def test_pytest_add_host_to_cloudera_manager(module_args): module_args( { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), "port": "7180", "verify_tls": "no", "debug": "no", "cluster_hostname": "cloudera.host.example", "rack_id": "/defo", "cluster_host_ip": "10.10.1.1", - "state": "present" + "state": "present", } ) @@ -52,9 +56,9 @@ def test_pytest_add_host_to_cloudera_manager(module_args): def test_pytest_attach_host_to_cluster(module_args): module_args( { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), "port": "7180", "verify_tls": "no", "debug": "no", @@ -62,7 +66,7 @@ def test_pytest_attach_host_to_cluster(module_args): "name": "Cluster_Example", "rack_id": "/defo", "cluster_host_ip": "10.10.1.1", - "state": "attached" + "state": "attached", } ) @@ -76,15 +80,15 @@ def test_pytest_attach_host_to_cluster(module_args): def test_pytest_detach_host_from_cluster(module_args): module_args( { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), "port": "7180", "verify_tls": "no", "debug": "no", "cluster_hostname": "cloudera.host.example", "name": "Cluster_Example", - "state": "detached" + "state": "detached", } ) @@ -92,4 +96,4 @@ def test_pytest_detach_host_from_cluster(module_args): host.main() # LOG.info(str(e.value)) - LOG.info(str(e.value.cloudera_manager)) \ No newline at end of file + LOG.info(str(e.value.cloudera_manager)) diff --git a/tests/unit/plugins/modules/host_info/test_host_info.py b/tests/unit/plugins/modules/host_info/test_host_info.py index e2fe281f..23533b79 100644 --- a/tests/unit/plugins/modules/host_info/test_host_info.py +++ b/tests/unit/plugins/modules/host_info/test_host_info.py @@ -22,16 +22,20 @@ import pytest from ansible_collections.cloudera.cluster.plugins.modules import host_info -from ansible_collections.cloudera.cluster.tests.unit import AnsibleExitJson, AnsibleFailJson +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) LOG = logging.getLogger(__name__) + def test_pytest_hostname_parameter(module_args): module_args( { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), "port": "7180", "verify_tls": "no", "debug": "no", @@ -49,9 +53,9 @@ def test_pytest_hostname_parameter(module_args): def test_pytest_host_id_parameter(module_args): module_args( { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), "port": "7180", "verify_tls": "no", "debug": "no", @@ -69,9 +73,9 @@ def test_pytest_host_id_parameter(module_args): def test_pytest_all_hosts(module_args): module_args( { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), "port": "7180", "verify_tls": "no", "debug": "no", diff --git a/tests/unit/plugins/modules/parcel/test_parcel.py b/tests/unit/plugins/modules/parcel/test_parcel.py index 8f52e20c..8007564b 100644 --- a/tests/unit/plugins/modules/parcel/test_parcel.py +++ b/tests/unit/plugins/modules/parcel/test_parcel.py @@ -17,129 +17,128 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type + import os import logging import pytest from ansible_collections.cloudera.cluster.plugins.modules import parcel -from ansible_collections.cloudera.cluster.tests.unit import AnsibleExitJson, AnsibleFailJson +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) LOG = logging.getLogger(__name__) +@pytest.fixture +def conn(): + conn = dict(username=os.getenv("CM_USERNAME"), password=os.getenv("CM_PASSWORD")) + + if os.getenv("CM_HOST", None): + conn.update(host=os.getenv("CM_HOST")) + + if os.getenv("CM_PORT", None): + conn.update(port=os.getenv("CM_PORT")) + + if os.getenv("CM_ENDPOINT", None): + conn.update(url=os.getenv("CM_ENDPOINT")) + + if os.getenv("CM_PROXY", None): + conn.update(proxy=os.getenv("CM_PROXY")) -def test_pytest_download_parcel(module_args): - module_args( - { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), - "cluster_name": "Example_cluster", - "product": "ECS", - "parcel_version": "1.5.1-b626-ecs-1.5.1-b626.p0.42068229", - "state": "downloaded" - } + return { + **conn, + "verify_tls": "no", + "debug": "no", + } + + +def test_pytest_download_parcel(conn, module_args): + conn.update( + cluster_name=os.getenv("CM_CLUSTER"), + parcel_name=os.getenv("CM_PARCEL_NAME"), + parcel_version=os.getenv("CM_PARCEL_VERSION"), + state="downloaded", ) + module_args(conn) with pytest.raises(AnsibleExitJson) as e: parcel.main() - # LOG.info(str(e.value)) - LOG.info(str(e.value.cloudera_manager)) - -def test_pytest_distribute_parcel(module_args): - module_args( - { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), - "cluster_name": "Example_cluster", - "product": "ECS", - "parcel_version": "1.5.1-b626-ecs-1.5.1-b626.p0.42068229", - "state": "distributed" - } + LOG.info(str(e.value.parcel)) + + +def test_pytest_distribute_parcel(conn, module_args): + conn.update( + cluster_name=os.getenv("CM_CLUSTER"), + parcel_name=os.getenv("CM_PARCEL_NAME"), + parcel_version=os.getenv("CM_PARCEL_VERSION"), + state="distributed", ) + module_args(conn) with pytest.raises(AnsibleExitJson) as e: parcel.main() - # LOG.info(str(e.value)) - LOG.info(str(e.value.cloudera_manager)) - -def test_pytest_activate_parcel(module_args): - module_args( - { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), - "cluster_name": "Example_cluster", - "product": "ECS", - "parcel_version": "1.5.1-b626-ecs-1.5.1-b626.p0.42068229", - "state": "activated" - } + LOG.info(str(e.value.parcel)) + + +def test_pytest_activate_parcel(conn, module_args): + conn.update( + cluster_name=os.getenv("CM_CLUSTER"), + parcel_name=os.getenv("CM_PARCEL_NAME"), + parcel_version=os.getenv("CM_PARCEL_VERSION"), + state="activated", ) + module_args(conn) with pytest.raises(AnsibleExitJson) as e: parcel.main() - # LOG.info(str(e.value)) - LOG.info(str(e.value.cloudera_manager)) - -def test_pytest_remove_parcel(module_args): - module_args( - { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), - "cluster_name": "Example_cluster", - "product": "ECS", - "parcel_version": "1.5.1-b626-ecs-1.5.1-b626.p0.42068229", - "state": "removed" - } + LOG.info(str(e.value.parcel)) + + +def test_pytest_deactivate_parcel(conn, module_args): + conn.update( + cluster_name="Example_Base_Host_Host_Template_Assignment", # os.getenv("CM_CLUSTER"), + parcel_name="SPARK3", + parcel_version="3.3.0.3.3.7180.0-274-1.p0.31212967", + state="distributed", ) + module_args(conn) with pytest.raises(AnsibleExitJson) as e: parcel.main() - # LOG.info(str(e.value)) - LOG.info(str(e.value.cloudera_manager)) - -def test_pytest_undistribute_parcel(module_args): - module_args( - { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), - "cluster_name": "Example_cluster", - "product": "ECS", - "parcel_version": "1.5.1-b626-ecs-1.5.1-b626.p0.42068229", - "state": "undistributed" - } + LOG.info(str(e.value.parcel)) + + +def test_pytest_undistribute_parcel(conn, module_args): + conn.update( + cluster_name=os.getenv("CM_CLUSTER"), + parcel_name=os.getenv("CM_PARCEL_NAME"), + parcel_version=os.getenv("CM_PARCEL_VERSION"), + state="downloaded", ) + module_args(conn) with pytest.raises(AnsibleExitJson) as e: parcel.main() - # LOG.info(str(e.value)) - LOG.info(str(e.value.cloudera_manager)) - - -def test_pytest_deactivate_parcel(module_args): - module_args( - { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), - "cluster_name": "Example_cluster", - "product": "ECS", - "parcel_version": "1.5.1-b626-ecs-1.5.1-b626.p0.42068229", - "state": "deactivated" - } + LOG.info(str(e.value.parcel)) + + +def test_pytest_remove_parcel(conn, module_args): + conn.update( + cluster_name=os.getenv("CM_CLUSTER"), + parcel_name=os.getenv("CM_PARCEL_NAME"), + parcel_version=os.getenv("CM_PARCEL_VERSION"), + state="absent", ) + module_args(conn) with pytest.raises(AnsibleExitJson) as e: parcel.main() - # LOG.info(str(e.value)) - LOG.info(str(e.value.cloudera_manager)) - + LOG.info(str(e.value.parcel)) diff --git a/tests/unit/plugins/modules/parcel_info/test_parcel_info.py b/tests/unit/plugins/modules/parcel_info/test_parcel_info.py index 5ea348bb..9d09e43b 100644 --- a/tests/unit/plugins/modules/parcel_info/test_parcel_info.py +++ b/tests/unit/plugins/modules/parcel_info/test_parcel_info.py @@ -22,16 +22,20 @@ import pytest from ansible_collections.cloudera.cluster.plugins.modules import parcel_info -from ansible_collections.cloudera.cluster.tests.unit import AnsibleExitJson, AnsibleFailJson +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) LOG = logging.getLogger(__name__) + def test_pytest_specific_parcel_info(module_args): module_args( { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), "port": "7180", "verify_tls": "no", "debug": "no", @@ -47,12 +51,13 @@ def test_pytest_specific_parcel_info(module_args): # LOG.info(str(e.value)) LOG.info(str(e.value.cloudera_manager)) + def test_pytest_get_all_parcels(module_args): module_args( { - "username": os.getenv('CM_USERNAME'), - "password": os.getenv('CM_PASSWORD'), - "host": os.getenv('CM_HOST'), + "username": os.getenv("CM_USERNAME"), + "password": os.getenv("CM_PASSWORD"), + "host": os.getenv("CM_HOST"), "port": "7180", "verify_tls": "no", "debug": "no", diff --git a/tests/unit/plugins/modules/service_role_config/test_service_role_config.py b/tests/unit/plugins/modules/service_role_config/test_service_role_config.py index f26e0652..40b47b4d 100644 --- a/tests/unit/plugins/modules/service_role_config/test_service_role_config.py +++ b/tests/unit/plugins/modules/service_role_config/test_service_role_config.py @@ -53,6 +53,7 @@ def conn(): "debug": "no", } + def test_missing_required(conn, module_args): module_args(conn) diff --git a/tests/unit/plugins/modules/service_role_config_group_config/test_service_role_config_group_config.py b/tests/unit/plugins/modules/service_role_config_group_config/test_service_role_config_group_config.py index 0a45e7b1..3ef85033 100644 --- a/tests/unit/plugins/modules/service_role_config_group_config/test_service_role_config_group_config.py +++ b/tests/unit/plugins/modules/service_role_config_group_config/test_service_role_config_group_config.py @@ -59,7 +59,9 @@ def conn(): def test_missing_required(conn, module_args): module_args(conn) - with pytest.raises(AnsibleFailJson, match="cluster, parameters, role_config_group, service"): + with pytest.raises( + AnsibleFailJson, match="cluster, parameters, role_config_group, service" + ): service_role_config_group_config.main() @@ -86,6 +88,7 @@ def test_missing_role_config_group(conn, module_args): with pytest.raises(AnsibleFailJson, match="cluster, parameters, service"): service_role_config_group_config.main() + def test_missing_parameters(conn, module_args): conn.update(parameters={}) module_args(conn) @@ -280,7 +283,7 @@ def test_purge_role_membership(conn, module_args): assert e.value.changed == False assert e.value.role_config_group["name"] == "hdfs-example2" assert not e.value.role_config_group["roles"] - + def test_remove_role_config_group(conn, module_args): conn.update( @@ -341,5 +344,7 @@ def test_remove_role_config_group_invalid_base(conn, module_args): ) module_args(conn) - with pytest.raises(AnsibleFailJson, match="Group 'hdfs-DATANODE-BASE' is a base group"): + with pytest.raises( + AnsibleFailJson, match="Group 'hdfs-DATANODE-BASE' is a base group" + ): service_role_config_group_config.main() diff --git a/tests/unit/plugins/modules/utils.py b/tests/unit/plugins/modules/utils.py index 42905b2d..d737263b 100644 --- a/tests/unit/plugins/modules/utils.py +++ b/tests/unit/plugins/modules/utils.py @@ -14,7 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import (absolute_import, division, print_function) +from __future__ import absolute_import, division, print_function + __metaclass__ = type import json @@ -24,8 +25,9 @@ from ansible.module_utils import basic from ansible.module_utils.common.text.converters import to_bytes + def setup_module_args(args): - args = json.dumps({'ANSIBLE_MODULE_ARGS': args}) + args = json.dumps({"ANSIBLE_MODULE_ARGS": args}) basic._ANSIBLE_ARGS = to_bytes(args) @@ -38,22 +40,23 @@ class AnsibleFailJson(Exception): def exit_json(*args, **kwargs): - if 'changed' not in kwargs: - kwargs['changed'] = False + if "changed" not in kwargs: + kwargs["changed"] = False raise AnsibleExitJson(kwargs) + def fail_json(*args, **kwargs): - kwargs['failed'] = True + kwargs["failed"] = True raise AnsibleFailJson(kwargs) class ModuleTestCase(unittest.TestCase): def setUp(self): - self.mock_module = patch.multiple(basic.AnsibleModule, - exit_json=exit_json, - fail_json=fail_json) + self.mock_module = patch.multiple( + basic.AnsibleModule, exit_json=exit_json, fail_json=fail_json + ) self.mock_module.start() - self.mock_sleep = patch('time.sleep') + self.mock_sleep = patch("time.sleep") self.mock_sleep.start() setup_module_args({}) self.addCleanup(self.mock_module.stop) diff --git a/tests/unit/requirements.txt b/tests/unit/requirements.txt index 8ec82784..ece294ca 100644 --- a/tests/unit/requirements.txt +++ b/tests/unit/requirements.txt @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -cm_client +cm-client