From f0b80954c3bd6a8485ac2131867abec592a567f6 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 27 Sep 2023 18:14:21 -0400 Subject: [PATCH] Add documentation build workflows (#125) * Add initial antsibull-docs files * Add PR docs linting workflow * Add collection namespace and name to doc validation workflow * Add construct_docs workflow using Ansible Community actions * Add publish_docs workflow * Fix location of actions * Update to reflect checkout path * Add always() to antsibull.log artifact upload * Comment out all existing requirements.txt * Remove shared workflow * Remove obsolete Python requirements * Update to use common shared workflows * Add documentation placeholders for all filters * Remove conflicting 'tls' alias * Move ClouderaManagerLookupBase to module_utils/cm_utils.py * Remove erroneous option parameter and add environment variables for username and password * Update docs for top links, Github edits, and extra links * Rewrite README.md to align with content map * Add CONTRIBUTING.md document * Format galaxy.yml for consistency * Update instructions for branching for clarity * Clarify the ansible-galaxy and ansible-builder usage for dependencies * Add additional tags * Update copyright date * Format versions * Clarify Python and collection dependencies and installation processes * Add note about Collection Metadata usage and update callout formatting Signed-off-by: Webster Mudge --- .github/workflows/publish_docs.yml | 36 +++ .github/workflows/validate_pr.yml | 1 - .github/workflows/validate_pr_docs.yml | 20 ++ CONTRIBUTING.md | 82 +++++ LICENSE | 2 +- README.md | 235 +++++++++++--- docs/links.yml | 12 + docsbuild/.gitignore | 6 + docsbuild/antsibull-docs.cfg | 50 +++ docsbuild/build.sh | 29 ++ docsbuild/cloudera.css | 0 docsbuild/conf.py | 64 ++++ docsbuild/requirements.txt | 2 + galaxy.yml | 45 ++- plugins/doc_fragments/cm_options.py | 2 - plugins/filter/append_database_port.yml | 8 + plugins/filter/cluster_service_role_hosts.yml | 8 + plugins/filter/default_database_port.yml | 8 + plugins/filter/extract_custom_role_groups.yml | 8 + plugins/filter/extract_custom_roles.yml | 8 + plugins/filter/extract_parcel_urls.yml | 8 + .../extract_products_from_manifests.yml | 8 + plugins/filter/extract_role_and_group.yml | 8 + plugins/filter/filter_null_configs.yml | 8 + plugins/filter/find_clusters.yml | 8 + plugins/filter/flatten_dict_list.yml | 8 + plugins/filter/format_database_type.yml | 8 + .../filter/get_database_collation_mysql.yml | 8 + .../filter/get_database_encoding_mysql.yml | 8 + plugins/filter/get_major_version.yml | 8 + plugins/filter/get_product_version.yml | 8 + plugins/filter/to_ldap_type_enum.yml | 8 + plugins/lookup/cm_api.py | 131 -------- plugins/lookup/cm_service.py | 13 +- plugins/module_utils/cm_utils.py | 298 ++++++++++++++---- requirements.yml | 12 +- 36 files changed, 898 insertions(+), 278 deletions(-) create mode 100644 .github/workflows/publish_docs.yml create mode 100644 .github/workflows/validate_pr_docs.yml create mode 100644 CONTRIBUTING.md create mode 100644 docs/links.yml create mode 100644 docsbuild/.gitignore create mode 100644 docsbuild/antsibull-docs.cfg create mode 100755 docsbuild/build.sh create mode 100644 docsbuild/cloudera.css create mode 100644 docsbuild/conf.py create mode 100644 docsbuild/requirements.txt create mode 100644 plugins/filter/append_database_port.yml create mode 100644 plugins/filter/cluster_service_role_hosts.yml create mode 100644 plugins/filter/default_database_port.yml create mode 100644 plugins/filter/extract_custom_role_groups.yml create mode 100644 plugins/filter/extract_custom_roles.yml create mode 100644 plugins/filter/extract_parcel_urls.yml create mode 100644 plugins/filter/extract_products_from_manifests.yml create mode 100644 plugins/filter/extract_role_and_group.yml create mode 100644 plugins/filter/filter_null_configs.yml create mode 100644 plugins/filter/find_clusters.yml create mode 100644 plugins/filter/flatten_dict_list.yml create mode 100644 plugins/filter/format_database_type.yml create mode 100644 plugins/filter/get_database_collation_mysql.yml create mode 100644 plugins/filter/get_database_encoding_mysql.yml create mode 100644 plugins/filter/get_major_version.yml create mode 100644 plugins/filter/get_product_version.yml create mode 100644 plugins/filter/to_ldap_type_enum.yml delete mode 100644 plugins/lookup/cm_api.py diff --git a/.github/workflows/publish_docs.yml b/.github/workflows/publish_docs.yml new file mode 100644 index 00000000..b04184ce --- /dev/null +++ b/.github/workflows/publish_docs.yml @@ -0,0 +1,36 @@ +--- + +name: Publish documentation + +on: + push: + branches: + - 'main' + + workflow_dispatch: + +jobs: + build-ansible-docs: + name: Build Ansible Docs + uses: cloudera-labs/github-actions/.github/workflows/construct-ansible-docs.yml@v1 + with: + pages-upload: true + directory-upload: true + antsibull-log-upload: true + + publish-ansible-docs: + name: Publish Ansible Docs + needs: build-ansible-docs + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy Github Pages + id: deployment + uses: actions/deploy-pages@v2 + with: + artifact_name: github-pages diff --git a/.github/workflows/validate_pr.yml b/.github/workflows/validate_pr.yml index a0336cd3..8b5070d8 100644 --- a/.github/workflows/validate_pr.yml +++ b/.github/workflows/validate_pr.yml @@ -20,7 +20,6 @@ on: branches: - 'release/**' - 'devel' - - 'devel-pvc-base' jobs: validate: diff --git a/.github/workflows/validate_pr_docs.yml b/.github/workflows/validate_pr_docs.yml new file mode 100644 index 00000000..f165de9f --- /dev/null +++ b/.github/workflows/validate_pr_docs.yml @@ -0,0 +1,20 @@ +--- + +name: Validate Pull Request documentation + +on: + pull_request: + branches: + - 'release/**' + - 'devel' + + workflow_dispatch: + +jobs: + validate-docs: + name: Validate Ansible Docs + uses: cloudera-labs/github-actions/.github/workflows/lint-ansible-docs.yml@v1 + with: + antsibull-log-upload: true + collection-namespace: cloudera + collection-name: cluster \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..5d849b35 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,82 @@ +# Contributing to cloudera.cluster + +Thank you for considering contributions to the `cloudera.cluster` Ansible collection! + +## Submitting a pull request + +You can start work on issues that are not yet part of a [Milestone](https://github.com/cloudera-labs/cloudera.cluster/milestones) -- anything in our issue tracker that isn't assigned to a Milestone is considered the [backlog](https://github.com/cloudera-labs/cloudera.cluster/issues?q=is%3Aopen+is%3Aissue+no%3Amilestone). + +Before you start working, please announce that you want to do so by commenting on the issue. _([Create an issue](https://github.com/cloudera-labs/cloudera.cluster/issues/new?labels=enhancement) if there isn't one yet, and you can also check out our [Discussions](https://github.com/cloudera-labs/cloudera.cluster/discussions) for ideas.)_ We try to ensure that all active work is assigned to a Milestone in order to keep our backlog accurate. + +**When your work is ready for review, create a branch in your own forked repository from the `devel` branch and submit a pull request against `devel`, referencing your the issue.** + +As a _best practice_, you can prefix your branches with: + +|prefix|Description|Example| +|------|-----------|-------| +|`feature/`|A new feature or changes existing to existing code or documentation|`feature/update-this-modules-params`| +|`fix/`|A non-urgent bug fix|`fix/refactor-module-output-params`| +|`hotfix/`|An urgent bug fix|`hotfix/patch-insecure-module`| + +> [!NOTE] +> :fire_extinguisher: A **hotfix** should branch from `main`. It will then be committed to both the `main` and `devel` branches. + +## Signing your commits + +Note that we require signed commits inline with [Developer Certificate of Origin](https://developercertificate.org/) best-practices for open source collaboration. + +A signed commit is a simple one-liner at the end of your commit message that states that you wrote the patch or otherwise have the right to pass the change into open source. Signing your commits means you agree to: + +``` +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +660 York Street, Suite 102, +San Francisco, CA 94110 USA + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. +``` + +(See [developercertificate.org](https://developercertificate.org/)) + +To agree, make sure to add line at the end of every git commit message, like this: + +``` +Signed-off-by: John Doe +``` + +> [!NOTE] +> :rocket: Add the sign-off automatically when creating the commit via the `-s` flag, e.g. `git commit -s`. + +## Still have questions? Opinions? Comments? + +Come find us on our [Discussions](https://github.com/cloudera-labs/cloudera.cluster/discussions)! diff --git a/LICENSE b/LICENSE index f51c0471..066ee669 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2021 Cloudera, Inc. All rights reserved. + Copyright 2023 Cloudera, Inc. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index f6c105ae..60b52526 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,190 @@ -# Ansible Collection for Cloudera (CDP) Private Cloud +# cloudera.cluster - Cloudera Data Platform (CDP) for Private Cloud and Cloudera Manager (CM) + +[![API documentation](https://github.com/cloudera-labs/cloudera.cluster/actions/workflows/publish_docs.yml/badge.svg?branch=main&event=push)](https://github.com/cloudera-labs/cloudera.cluster/actions/workflows/publish_docs.yml) + +`cloudera.cluster` is an Ansible collection that lets you manage your **[Cloudera Data Platform (CDP)](https://www.cloudera.com/products/cloudera-data-platform.html) Private Cloud** resources and interact with Cloudera Manager for both Private Cloud installations and Public Cloud Data Hub deployments. With this collection, you can: + +* Create and manage [Private Cloud](https://www.cloudera.com/products/cloudera-data-platform.html) deployments and Public Cloud [Data Hubs](https://www.cloudera.com/products/data-hub.html), including: + * Manage services like Impala, NiFi, and Ozone + * Configure Cloudera Manager and `cm_agent`-enabled hosts + +If you have any questions, want to chat about the collection's capabilities and usage, need help using the collection, or just want to stay updated, join us at our [Discussions](https://github.com/cloudera-labs/cloudera.cluster/discussions). + +## Quickstart + +1. [Install the collection](#installation) +2. [Install the requirements](#requirements) +3. [Use the collection](#using-the-collection) + +## API + +See the [API documentation](https://cloudera-labs.github.io/cloudera.cluster/) for details for each plugin and role within the collection. + +## Roadmap + +If you want to see what we are working on or have pending, check out: + +* the [Milestones](https://github.com/cloudera-labs/cloudera.cluster/milestones) and [active issues](https://github.com/cloudera-labs/cloudera.cluster/issues?q=is%3Aissue+is%3Aopen+milestone%3A*) to see our current activity, +* the [issue backlog](https://github.com/cloudera-labs/cloudera.cluster/issues?q=is%3Aopen+is%3Aissue+no%3Amilestone) to see what work is pending or under consideration, and +* read up on the [Ideas](https://github.com/cloudera-labs/cloudera.cluster/discussions/categories/ideas) we have in mind. + +Are we missing something? Let us know by [creating a new issue](https://github.com/cloudera-labs/cloudera.cluster/issues/new) or [posting a new idea](https://github.com/cloudera-labs/cloudera.cluster/discussions/new?category=ideas)! + +## Contribute + +For more information on how to get involved with the `cloudera.cluster` Ansible collection, head over to [CONTRIBUTING.md](CONTRIBUTING.md). + +## Installation + +To install the `cloudera.cluster` collection, you have several options. Please note that we have not yet published this collection to the public Ansible Galaxy server, so you cannot install it via direct namespace, rather you must specify by Git project and (optionally) branch. + +### Option #1: Install from GitHub + +Create or edit your `requirements.yml` file in your project with the +following: + +```yaml +collections: + - name: https://github.com/cloudera-labs/cloudera.cluster.git + type: git + version: main +``` + +And then run in your project: + +```bash +ansible-galaxy collection install -r requirements.yml +``` + +You can also install the collection directly: + +```bash +ansible-galaxy collection install git+https://github.com/cloudera-labs/cloudera.cluster.git@main +``` + +### Option #2: Install the tarball + +Periodically, the collection is packaged into a distribution which you can +install directly: + +```bash +ansible-galaxy collection install +``` + +See [Building the Collection](#building-the-collection) for details on creating a local tarball. ## Requirements -- Python 2.x or 3.x -- [Ansible](http://docs.ansible.com/ansible/intro_installation.html) -- [JMESPath](https://jmespath.org/) +`cloudera.cluster` expects `ansible-core>=2.10,<2.13`. + +> [!WARNING] +> The current `import_template` functionality does not yet work with Ansible version `2.13` and later. -### Dependencies +The collection has the following _required_ dependencies: | Name | Type | Version | +|------|------|---------| | `ansible.posix` | collection | `1.3.0` | | `community.crypto` | collection | `2.2.1` | | `community.general` | collection | `4.5.0` | -### Optional Dependencies +There are a number of _optional_ dependencies for the collection: | Name | Type | Version | +|------|------|---------| | `community.mysql` | collection | `3.1.0` | | `community.postgresql` | collection | `1.6.1` | | `freeipa.ansible_freeipa` | collection | `1.6.2` | | `geerlingguy.postgresql` | role | `2.2.0` | | `geerlingguy.mysql` (patched) | role | `master` | -All dependencies, required and optional, are found in `requirements.yml`. Only the required dependencies are in `galaxy.yml` and thus installed by `ansible-builder` when creating an Environment Execution container. +The collection also requires the following Python libraries to operate its modules: + + * [jmespath](https://jmespath.org/) + * [cm_client](https://cloudera.github.io/cm_api/docs/python-client-swagger/) + +The collection's Python dependencies alone, _not_ the required Python libraries of its collection dependencies, are in `requirements.txt`. + +All collection dependencies, required and optional, can be found in `requirements.yml`; only the _required_ dependencies are in `galaxy.yml`. `ansible-galaxy` will install only the _required_ collection dependencies; you will need to add the _optional_ collection dependencies as needed (see above). + +`ansible-builder` can discover and install all Python dependencies - current collection and dependencies - if you wish to use that application to construct your environment. Otherwise, you will need to read each collection and role dependency and follow its installation instructions. -**Do not use Ansible 2.9.0**. This version has an [issue with templating](https://github.com/ansible/ansible/issues/64745) which causes the playbook execution to fail. Instead, use any 2.8.x version or a later 2.9.x version as these are not affected. +See the [Collection Metadata](https://ansible.readthedocs.io/projects/builder/en/latest/collection_metadata/) section for further details on how to install (and manage) collection dependencies. -## Usage -This Ansible Collection is designed to work hand-in-hand with [Cloudera Deploy](https://github.com/cloudera-labs/cloudera-deploy), which contains reference Playbooks and Example Definitions. +You may wish to use a _virtual environment_ to manage the Python dependencies. + +See the `base` *Execution Environment* configuration in [`cloudera-labs/cldr-runner`](https://github.com/cloudera-labs/cldr-runner) as an example of how you can install the optional dependencies to suit your specific needs. + +## Using the Collection + +This collection is designed to work hand-in-hand with the [`cloudera-deploy` application](https://github.com/cloudera-labs/cloudera-deploy), which uses reference playbooks from the [`cloudera.exe` collection](https://github.com/cloudera-labs/cloudera.exe) and example definitions. Coming releases will decouple these collections further while maintaining backwards compatibility. + +Once installed, reference the collection in your playbooks and roles. + +For example, here we use the +[`cloudera.cluster.cm_resource` module](https://cloudera-labs.github.io/cloudera.cluster/cm_resource_module.html) to patch the Hue service with updated Knox proxy hosts: + +```yaml +- hosts: localhost + connection: local + gather_facts: no + vars: + cm_api: "{{ lookup('ansible.builtin.env', 'CM_API') }}" + user: "{{ lookup('ansible.builtin.env', 'CM_USERNAME') }}" + pwd: "{{ lookup('ansible.builtin.env', 'CM_PASSWORD') }}" + cluster: "my-cluster" + tasks: + - name: Update Hue SSO (Knox Proxies) + cloudera.cluster.cm_resource: + url: "{{ cm_api }}" + username: "{{ user }}" + password: "{{ pwd }}" + path: "v51/clusters/{{ cluster }}/services/hue/config" + method: PUT + parameters: + message: "Patch Knox proxy hosts for Hue (Ansible)" + body: + items: + - name: knox_proxyhosts + value: "{{ ['master1', 'master2', 'master3'] | join(',') }}" +``` + +## Building the Collection + +To create a local collection tarball, run: + +```bash +ansible-galaxy collection build +``` + +## Building the API Documentation + +To create a local copy of the API documentation, first make sure the collection is in your `ANSIBLE_COLLECTIONS_PATHS`. Then run the following: + +```bash +# change into the /docsbuild directory +cd docsbuild + +# install the build requirements (antsibull-docs); you may want to set up a +# dedicated virtual environment +pip install ansible-core https://github.com/cloudera-labs/antsibull-docs/archive/cldr-docsite.tar.gz + +# Install the collection's build dependencies +pip install requirements.txt + +# Then run the build script +./build.sh +``` + +Your local documentation will be found at `docsbuild/build/html`. ## Tested Platforms +Active development is focused on **CDP Private Cloud** deployments and their respective platform compatibility matrices. + +> [!NOTE] +> While the collection's plugins and roles can be used to deploy CDH 5.x and CDH 6.x environments, it is only possible to install a subset of their supported platform components (i.e JDK and database versions) using this tooling. + ### Cloudera Distributions - Cloudera Manager / CDP Private Cloud Base 7.1.x @@ -45,51 +199,20 @@ This Ansible Collection is designed to work hand-in-hand with [Cloudera Deploy]( - Ubuntu 18.04 LTS (Bionic Beaver) - Ubuntu 20.04 LTS (Focal Fossa) -Active development is focused on **CDP Private Cloud Base** (formerly CDP-DC) deployments and their respective platform compatibility matrices. - -> While these roles etc. can be used to deploy CDH 5.x and CDH 6.x environments, it is only possible to install a subset of their supported platform components (i.e JDK and database versions) using this tooling. - -## How do I contribute code? -You need to first sign and return an -[ICLA](icla/Cloudera_ICLA_25APR2018.pdf) -and -[CCLA](icla/Cloudera_CCLA_25APR2018.pdf) -before we can accept and redistribute your contribution. Once these are submitted you are -free to start contributing to cloudera-playbook. Submit these to CLA@cloudera.com. - -### Main steps -* Fork the repo and create a topic branch -* Push commits to your repo -* Create a pull request! - -### Find -We use Github issues to track bugs for this project. Find an issue that you would like to -work on (or file one if you have discovered a new issue!). If no-one is working on it, -assign it to yourself only if you intend to work on it shortly. - -### Fix - -Please write a good, clear commit message, with a short, descriptive title and -a message that is exactly long enough to explain what the problem was, and how it was -fixed. +## Operational Features -## Copyright +> [!WARNING] +> These operational features are deprecated as of version 4.x. If you want to use or build similar features and functions, head over to the [Discussions](https://github.com/cloudera-labs/cloudera.cluster/discussions) to learn more about using the collection to achieve your platform operations needs. -(C) Cloudera, Inc. 2021 All rights reserved. +This collection includes support for: -## License -[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) - -# IMPORTANT NOTICE: - -This release includes support for: - Upgrading Cloudera Manager Server and Cloudera Manager Agents - Upgrading CDH 5 and/or CDH6 to CDP Private Cloud Base - Refreshing the config for running clusters, including adding new services or updating the config of existing services. -These features are potentially very dangerous and can cause damage to running clusters if used incorrectly. If you plan to use these features, please ensure that you test thoroughly on a disposable environment, before running on clusters that are not disposable. +These features are potentially very dangerous and can cause damage to running clusters if used incorrectly. If you plan to use these features, please ensure that you test thoroughly on a disposable environment. -Cloudera recommends that Cloudera Professional Services be engaged before using these features, particularly as none of the automation products are covered under your Cloudera Support agreements. +Cloudera recommends that Cloudera Professional Services be engaged before using these features, particularly as none of these operational features are covered under Cloudera Support agreements. In order to use these capabilities you will need some permutation of the following variables: - `cloudera_runtime_pre_upgrade` (specify the version of the legacy cluster - e.g. 5.16.2) @@ -98,3 +221,21 @@ In order to use these capabilities you will need some permutation of the followi - `activate_runtime_upgrade` (true to do a patch release activation) - `cdh_cdp_upgrade` (true to do a CDH to CDP upgrade) - `upgrade_runtime` (true to upgrade between versions of CDH or CDP) + +## License and Copyright + +Copyright 2023, Cloudera, Inc. + +``` +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` diff --git a/docs/links.yml b/docs/links.yml new file mode 100644 index 00000000..7a8b2442 --- /dev/null +++ b/docs/links.yml @@ -0,0 +1,12 @@ +--- + +edit_on_github: + repository: cloudera-labs/cloudera.cluster + branch: main + path_prefix: '' + +extra_links: + - description: Submit a bug report + url: https://github.com/cloudera-labs/cloudera.cluster/issues/new?labels=bug + - description: Request a feature + url: https://github.com/cloudera-labs/cloudera.cluster/issues/new?labels=enhancement diff --git a/docsbuild/.gitignore b/docsbuild/.gitignore new file mode 100644 index 00000000..12166213 --- /dev/null +++ b/docsbuild/.gitignore @@ -0,0 +1,6 @@ + +# Created with antsibull-docs 2.3.1.post0 + +/temp-rst +/build +/rst \ No newline at end of file diff --git a/docsbuild/antsibull-docs.cfg b/docsbuild/antsibull-docs.cfg new file mode 100644 index 00000000..8f263ca6 --- /dev/null +++ b/docsbuild/antsibull-docs.cfg @@ -0,0 +1,50 @@ + +# Created with antsibull-docs 2.3.1.post0 + +breadcrumbs = true +indexes = true +use_html_blobs = false + +# You can specify ways to convert a collection name (.) to an URL here. +# You can replace either of or by "*" to match all values in that place, +# or use "*" for the collection name to match all collections. In the URL, you can use +# {namespace} and {name} for the two components of the collection name. If you want to use +# "{" or "}" in the URL, write "{{" or "}}" instead. Basically these are Python format +# strings (https://docs.python.org/3.8/library/string.html#formatstrings). +collection_url = { + * = "https://github.com/cloudera-labs/{namespace}.{name}" +} + +# The same wildcard rules and formatting rules as for collection_url apply. +collection_install = { + * = "ansible-galaxy collection install git+https://github.com/cloudera-labs/{namespace}.{name}.git" +} + +logging_cfg = { + version = 1.0 + outputs = { + logfile = { + output = twiggy.outputs.FileOutput + args = [ + ./antsibull.log + ] + } + stderr = { + output = twiggy.outputs.StreamOutput + format = twiggy.formats.shell_format + } + } + emitters = { + # Log everything a user might be interested in to a log file. + all = { + # DEBUG is the most verbose level + level = DEBUG + output_name = logfile + } + # Log problems to stderr so they can be reported and taken care of. + problems = { + level = WARNING + output_name = stderr + } + } +} \ No newline at end of file diff --git a/docsbuild/build.sh b/docsbuild/build.sh new file mode 100755 index 00000000..9b4d18b4 --- /dev/null +++ b/docsbuild/build.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Created with antsibull-docs 2.3.1.post0 + +set -e + +pushd "$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +trap "{ popd; }" EXIT + +# Create collection documentation into temporary directory +rm -rf temp-rst +mkdir -p temp-rst +chmod og-w temp-rst # antsibull-docs wants that directory only readable by itself +antsibull-docs \ + --config-file antsibull-docs.cfg \ + collection \ + --use-current \ + --squash-hierarchy \ + --dest-dir temp-rst \ + cloudera.cluster + +# Copy collection documentation into source directory +rsync -cprv --delete-after temp-rst/ rst/ + +# Build Sphinx site +sphinx-build -M html rst build -c . -W --keep-going + +# Copy Cloudera CSS overrides into source directory +cp cloudera.css build/html/_static/css diff --git a/docsbuild/cloudera.css b/docsbuild/cloudera.css new file mode 100644 index 00000000..e69de29b diff --git a/docsbuild/conf.py b/docsbuild/conf.py new file mode 100644 index 00000000..21534fa8 --- /dev/null +++ b/docsbuild/conf.py @@ -0,0 +1,64 @@ + +# Created with antsibull-docs 2.3.1.post0 + +# This file only contains a selection of the most common options. For a full list see the +# documentation: +# http://www.sphinx-doc.org/en/master/config + +project = 'cloudera.cluster' +copyright = 'Cloudera, Inc.' + +title = 'Cloudera Labs' +html_short_title = 'Cloudera Labs' + +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx_antsibull_ext'] + +pygments_style = 'ansible' + +highlight_language = 'YAML+Jinja' + +html_theme = 'sphinx_ansible_theme' +html_show_sphinx = False + +display_version = False + +html_use_smartypants = True +html_use_modindex = False +html_use_index = False +html_copy_source = False + +# See https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#confval-intersphinx_mapping for the syntax +intersphinx_mapping = { + 'python': ('https://docs.python.org/2/', (None, '../python2.inv')), + 'python3': ('https://docs.python.org/3/', (None, '../python3.inv')), + 'jinja2': ('http://jinja.palletsprojects.com/', (None, '../jinja2.inv')), + 'ansible_devel': ('https://docs.ansible.com/ansible/devel/', (None, '../ansible_devel.inv')), + # If you want references to resolve to a released Ansible version (say, `5`), uncomment and replace X by this version: + # 'ansibleX': ('https://docs.ansible.com/ansible/X/', (None, '../ansibleX.inv')), +} + +default_role = 'any' + +nitpicky = True + +html_css_files = [ + 'css/cloudera.css', +] + +html_last_updated_fmt = '%b %d, %Y' + +html_theme_options = { + 'vcs_pageview_mode': 'edit', + 'documentation_home_url': 'https://github.com/cloudera-labs/', + 'topbar_links': { + 'Cloudera Data Platform (CDP)': 'https://www.cloudera.com/products/cloudera-data-platform.html', + 'Documentation': 'https://docs.cloudera.com/', + 'Downloads': 'https://www.cloudera.com/downloads.html', + 'Training': 'https://www.cloudera.com/about/training.html', + 'Certification': 'https://www.cloudera.com/about/training/certification.html', + }, +} + +html_content = { + 'display_github': 'True', +} diff --git a/docsbuild/requirements.txt b/docsbuild/requirements.txt new file mode 100644 index 00000000..d4a6fbc4 --- /dev/null +++ b/docsbuild/requirements.txt @@ -0,0 +1,2 @@ + +# Created with antsibull-docs 2.3.1.post0 diff --git a/galaxy.yml b/galaxy.yml index 593849aa..c8d1cd20 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -14,22 +14,37 @@ # See the License for the specific language governing permissions and # limitations under the License. -namespace: cloudera -name: cluster -version: 4.0.0-rc1 +namespace: cloudera +name: cluster +version: 4.0.0-rc1 +readme: README.md +authors: [] -authors: [] -readme: README.md -description: Cloudera assets for managing Cloudera clusters -license_file: 'LICENSE' -tags: [] +description: > + A set of roles, modules, and other plugins for interacting with the services + and endpoints provided by the Cloudera Data Platform (CDP) for Private Cloud + and Cloudera Manager (CM). +license_file: LICENSE +tags: +- cloudera +- cdp +- private cloud +- data services +- cloudera manager +- cm dependencies: - 'ansible.posix': '1.3.0' - 'community.crypto': '2.2.1' - 'community.general': '4.5.0' + 'ansible.posix': '1.3.0' + 'community.crypto': '2.2.1' + 'community.general': '4.5.0' -repository: https://github.com/cloudera-labs/cloudera.cluster -documentation: https://github.com/cloudera-labs/cloudera.cluster -homepage: https://github.com/cloudera-labs/cloudera.cluster -issues: https://github.com/cloudera-labs/cloudera.cluster/issues +repository: https://github.com/cloudera-labs/cloudera.cluster +homepage: https://github.com/cloudera-labs/cloudera.cluster +issues: https://github.com/cloudera-labs/cloudera.cluster/issues +documentation: https://cloudera-labs.github.io/cloudera.cluster + +build_ignore: +- '.*' +- docs +- docsrc +- site diff --git a/plugins/doc_fragments/cm_options.py b/plugins/doc_fragments/cm_options.py index 179ba85b..7ca17a18 100644 --- a/plugins/doc_fragments/cm_options.py +++ b/plugins/doc_fragments/cm_options.py @@ -55,8 +55,6 @@ class ModuleDocFragment(object): type: bool required: False default: True - aliases: - - tls username: description: - Username for access to the CM API endpoint. diff --git a/plugins/filter/append_database_port.yml b/plugins/filter/append_database_port.yml new file mode 100644 index 00000000..bb1b5b19 --- /dev/null +++ b/plugins/filter/append_database_port.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: append_database_port + short_description: append_database_port + description: append_database_port + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/cluster_service_role_hosts.yml b/plugins/filter/cluster_service_role_hosts.yml new file mode 100644 index 00000000..69d91379 --- /dev/null +++ b/plugins/filter/cluster_service_role_hosts.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: cluster_service_role_hosts + short_description: cluster_service_role_hosts + description: cluster_service_role_hosts + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/default_database_port.yml b/plugins/filter/default_database_port.yml new file mode 100644 index 00000000..ee07c9ea --- /dev/null +++ b/plugins/filter/default_database_port.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: default_database_port + short_description: default_database_port + description: default_database_port + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/extract_custom_role_groups.yml b/plugins/filter/extract_custom_role_groups.yml new file mode 100644 index 00000000..e5c0f4aa --- /dev/null +++ b/plugins/filter/extract_custom_role_groups.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: extract_custom_role_groups + short_description: extract_custom_role_groups + description: extract_custom_role_groups + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/extract_custom_roles.yml b/plugins/filter/extract_custom_roles.yml new file mode 100644 index 00000000..bb6512a8 --- /dev/null +++ b/plugins/filter/extract_custom_roles.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: extract_custom_roles + short_description: extract_custom_roles + description: extract_custom_roles + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/extract_parcel_urls.yml b/plugins/filter/extract_parcel_urls.yml new file mode 100644 index 00000000..5abdc4db --- /dev/null +++ b/plugins/filter/extract_parcel_urls.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: extract_parcel_urls + short_description: extract_parcel_urls + description: extract_parcel_urls + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/extract_products_from_manifests.yml b/plugins/filter/extract_products_from_manifests.yml new file mode 100644 index 00000000..1d05f48a --- /dev/null +++ b/plugins/filter/extract_products_from_manifests.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: extract_products_from_manifests + short_description: extract_products_from_manifests + description: extract_products_from_manifests + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/extract_role_and_group.yml b/plugins/filter/extract_role_and_group.yml new file mode 100644 index 00000000..d172969b --- /dev/null +++ b/plugins/filter/extract_role_and_group.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: extract_role_and_group + short_description: extract_role_and_group + description: extract_role_and_group + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/filter_null_configs.yml b/plugins/filter/filter_null_configs.yml new file mode 100644 index 00000000..a8d4be3a --- /dev/null +++ b/plugins/filter/filter_null_configs.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: fill_null_configs + short_description: fill_null_configs + description: fill_null_configs + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/find_clusters.yml b/plugins/filter/find_clusters.yml new file mode 100644 index 00000000..99a0be9a --- /dev/null +++ b/plugins/filter/find_clusters.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: find_clusters + short_description: find_clusters + description: find_clusters + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/flatten_dict_list.yml b/plugins/filter/flatten_dict_list.yml new file mode 100644 index 00000000..39b6fe33 --- /dev/null +++ b/plugins/filter/flatten_dict_list.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: flatten_dict_list + short_description: flatten_dict_list + description: flatten_dict_list + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/format_database_type.yml b/plugins/filter/format_database_type.yml new file mode 100644 index 00000000..1cc588a5 --- /dev/null +++ b/plugins/filter/format_database_type.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: format_database_type + short_description: format_database_type + description: format_database_type + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/get_database_collation_mysql.yml b/plugins/filter/get_database_collation_mysql.yml new file mode 100644 index 00000000..ce199519 --- /dev/null +++ b/plugins/filter/get_database_collation_mysql.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: get_database_collation_mysql + short_description: get_database_collation_mysql + description: get_database_collation_mysql + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/get_database_encoding_mysql.yml b/plugins/filter/get_database_encoding_mysql.yml new file mode 100644 index 00000000..c20f2969 --- /dev/null +++ b/plugins/filter/get_database_encoding_mysql.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: get_database_encoding_mysql + short_description: get_database_encoding_mysql + description: get_database_encoding_mysql + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/get_major_version.yml b/plugins/filter/get_major_version.yml new file mode 100644 index 00000000..ea2a5cb3 --- /dev/null +++ b/plugins/filter/get_major_version.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: get_major_version + short_description: get_major_version + description: get_major_version + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/get_product_version.yml b/plugins/filter/get_product_version.yml new file mode 100644 index 00000000..c084b1aa --- /dev/null +++ b/plugins/filter/get_product_version.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: get_product_version + short_description: get_product_version + description: get_product_version + +EXAMPLES: + +RETURN: diff --git a/plugins/filter/to_ldap_type_enum.yml b/plugins/filter/to_ldap_type_enum.yml new file mode 100644 index 00000000..acfc3895 --- /dev/null +++ b/plugins/filter/to_ldap_type_enum.yml @@ -0,0 +1,8 @@ +DOCUMENTATION: + name: to_ldap_type_enum + short_description: to_ldap_type_enum + description: to_ldap_type_enum + +EXAMPLES: + +RETURN: diff --git a/plugins/lookup/cm_api.py b/plugins/lookup/cm_api.py deleted file mode 100644 index 680d36aa..00000000 --- a/plugins/lookup/cm_api.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import io -import json -import logging - -from urllib.error import HTTPError -from urllib.parse import urljoin -from urllib3 import disable_warnings -from urllib3.exceptions import InsecureRequestWarning, MaxRetryError, HTTPError -from urllib3.util import Url - -from ansible.errors import AnsibleError -from ansible.module_utils.common.text.converters import to_text, to_native -from ansible.plugins.lookup import LookupBase -from ansible.utils.display import Display - -from cm_client import ApiClient, Configuration -from cm_client.rest import ApiException, RESTClientObject - -display = Display() - -""" -A common Ansible Lookup plugin for API access to Cloudera Manager. -""" -class ClouderaManagerLookupBase(LookupBase): - def initialize_client(self): - # Set up core CM API client parameters - config = Configuration() - config.username = self.get_option('username') - config.password = self.get_option('password') - config.verify_ssl = self.get_option('verify_tls') - config.debug = self.get_option('debug') - - # Configure logging - _log_format = '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s' - if self.get_option('debug'): - self._setup_logger(logging.DEBUG, _log_format) - self.logger.debug("CM API agent: %s", self.get_option('agent_header')) - else: - self._setup_logger(logging.ERROR, _log_format) - - if self.get_option('verify_tls') is False: - disable_warnings(InsecureRequestWarning) - - # If provided a CM API endpoint URL, use it directly - if self.get_option('endpoint'): - config.host = self.get_option('endpoint') - # Otherwise, run discovery on missing parts - else: - config.host = self._discover_endpoint(config) - - self.api_client = ApiClient() - - def _setup_logger(self, log_level, log_format): - """Configures the logging of the HTTP activity""" - self.logger = logging.getLogger('urllib3') - self.logger.setLevel(log_level) - - def _get_auth_headers(self, config): - """Constructs a Basic Auth header dictionary from the Configuration. - This dictionary can be used directly with the API client's REST client.""" - headers = dict() - auth = config.auth_settings().get('basic') - headers[auth['key']] = auth['value'] - return headers - - def _discover_endpoint(self, config): - """Discovers the scheme and version of a potential Cloudara Manager host""" - # Get the authentication headers and REST client - headers = self._get_auth_headers(config) - rest = RESTClientObject() - - # Resolve redirects to establish HTTP scheme and port - pre_rendered = Url(scheme="https" if self.get_option('force_tls') else "http", host=self.get_option('host'), port=self.get_option('port')) - rendered = rest.pool_manager.request('GET', pre_rendered.url, headers=headers.copy()) - rendered_url = rendered.geturl() - - # Discover API version if not set - if not self.version: - pre_versioned = urljoin(rendered_url, "/api/version") - versioned = rest.pool_manager.request('GET', pre_versioned, headers=headers) - self.version = versioned.data.decode('utf-8') - - # Construct the discovered API endpoint - return urljoin(rendered_url, "/api/" + self.version) - - def get(self, path, query=None, field='items', body=None): - """Wrapper to GET a CM API endpoint path directly.""" - path_params = [] - header_params = {} - header_params['Accept'] = self.api_client.select_header_accept(['application/json']) - header_params['Content-Type'] = self.api_client.select_header_content_type(['application/json']) - - try: - results =self.api_client.call_api( - path, "GET", path_params, query, - header_params, auth_settings=['basic'], - _preload_content=False) - - if 200 >= results[1] <= 299: - data = json.loads(results[0].data.decode('utf-8')) - if field in data: - data = data[field] - return data if type(data) is list else [data] - else: - raise AnsibleError("Error interacting with CM resource. Status code: %s" % to_text(results[1])) - except ApiException as ae: - body = ae.body.decode('utf-8') - if body != "": - body = json.loads(body) - raise AnsibleError("API error: %s; Status code: %s" % (ae.reason, ae.status), obj=body, orig_exc=ae) - except MaxRetryError as maxe: - raise AnsibleError("Request error: %s" % to_text(maxe.reason)) - except HTTPError as he: - raise AnsibleError("HTTP request error", orig_exc=he) diff --git a/plugins/lookup/cm_service.py b/plugins/lookup/cm_service.py index 3d26c18a..1ba7edfd 100644 --- a/plugins/lookup/cm_service.py +++ b/plugins/lookup/cm_service.py @@ -30,8 +30,6 @@ description: - A C(service) or list of services to lookup within the CDP Datahub cluster. required: True - sample: - - KUDU cluster: description: Name of the Datahub cluster to query. type: string @@ -44,11 +42,14 @@ description: Username for accessing the Cloudera Manager API. type: string required: True + env: + - name: CM_USERNAME password: description: Password for accessing the Cloudera Manager API. type: string required: True - no_log: True + env: + - name: CM_PASSWORD endpoint: description: API endpoint of Cloudera Manager. type: string @@ -93,11 +94,7 @@ - Requires C(cm_client). ''' -from ansible_collections.cloudera.cluster.plugins.lookup.cm_api import ClouderaManagerLookupBase - -from ansible.utils.display import Display - -display = Display() +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ClouderaManagerLookupBase class LookupModule(ClouderaManagerLookupBase): def run(self, terms, variables=None, **kwargs): diff --git a/plugins/module_utils/cm_utils.py b/plugins/module_utils/cm_utils.py index 4c72e099..82e23e01 100644 --- a/plugins/module_utils/cm_utils.py +++ b/plugins/module_utils/cm_utils.py @@ -16,7 +16,7 @@ # limitations under the License. """ -A common Ansible Module for shared functions for Cloudera Manager +A common Ansible Module and Lookup for shared functions for Cloudera Manager """ import io @@ -30,6 +30,10 @@ from urllib.parse import urljoin from ansible.module_utils.basic import AnsibleModule +from ansible.errors import AnsibleError +from ansible.module_utils.common.text.converters import to_text +from ansible.plugins.lookup import LookupBase +from ansible.utils.display import Display from cm_client import ApiClient, Configuration from cm_client.rest import ApiException, RESTClientObject @@ -37,15 +41,16 @@ __credits__ = ["frisch@cloudera.com"] -__maintainer__ = [ - "wmudge@cloudera.com" -] +__maintainer__ = ["wmudge@cloudera.com"] + +display = Display() class ClouderaManagerModule(object): @classmethod def handle_process(cls, f): """Wrapper to handle log capture and common HTTP errors""" + @wraps(f) def _impl(self, *args, **kwargs): try: @@ -56,31 +61,34 @@ def _impl(self, *args, **kwargs): self.log_lines.append(self.log_out.splitlines()) return result except ApiException as ae: - body = ae.body.decode('utf-8') + body = ae.body.decode("utf-8") if body != "": body = json.loads(body) - self.module.fail_json(msg="API error: " + str(ae.reason), - status_code=ae.status, body=body) + self.module.fail_json( + msg="API error: " + str(ae.reason), status_code=ae.status, body=body + ) except MaxRetryError as maxe: self.module.fail_json(msg="Request error: " + str(maxe.reason)) except HTTPError as he: self.module.fail_json(msg="HTTP request: " + str(he)) + return _impl - + """A base Cloudera Manager (CM) module class""" + def __init__(self, module): # Set common parameters self.module = module - self.url = self._get_param('url', None) - self.force_tls = self._get_param('force_tls') - self.host = self._get_param('host') - self.port = self._get_param('port') - self.version = self._get_param('version') - self.username = self._get_param('username') - self.password = self._get_param('password') - self.verify_tls = self._get_param('verify_tls') - self.debug = self._get_param('debug') - self.agent_header = self._get_param('agent_header') + self.url = self._get_param("url", None) + self.force_tls = self._get_param("force_tls") + self.host = self._get_param("host") + self.port = self._get_param("port") + self.version = self._get_param("version") + self.username = self._get_param("username") + self.password = self._get_param("password") + self.verify_tls = self._get_param("verify_tls") + self.debug = self._get_param("debug") + self.agent_header = self._get_param("agent_header") # Initialize common return values self.log_out = None @@ -93,9 +101,11 @@ def __init__(self, module): config.password = self.password config.verify_ssl = self.verify_tls config.debug = self.debug - + # Configure logging - _log_format = '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s' + _log_format = ( + "%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s" + ) if self.debug: self._setup_logger(logging.DEBUG, _log_format) self.logger.debug("CM API agent: %s", self.agent_header) @@ -104,16 +114,16 @@ def __init__(self, module): if self.verify_tls is False: disable_warnings(InsecureRequestWarning) - + def _get_param(self, param, default=None): """Fetches an Ansible input parameter if it exists, else returns optional default or None""" if self.module is not None: return self.module.params[param] if param in self.module.params else default return default - + def _setup_logger(self, log_level, log_format): """Configures the logging of the HTTP activity""" - self.logger = logging.getLogger('urllib3') + self.logger = logging.getLogger("urllib3") self.logger.setLevel(log_level) self.__log_capture = io.StringIO() @@ -124,35 +134,35 @@ def _setup_logger(self, log_level, log_format): handler.setFormatter(formatter) self.logger.addHandler(handler) - + def _get_log(self): """Retrieves the contents of the captured log""" contents = self.__log_capture.getvalue() self.__log_capture.truncate(0) return contents - + def _initialize_client(self): """Configures and creates the API client""" config = Configuration() - + # If provided a CML endpoint URL, use it directly if self.url: config.host = self.url # Otherwise, run discovery on missing parts else: config.host = self._discover_endpoint(config) - + # Create and set the API Client self.api_client = ApiClient() - + def get_auth_headers(self, config): """Constructs a Basic Auth header dictionary from the Configuration. This dictionary can be used directly with the API client's REST client.""" headers = dict() - auth = config.auth_settings().get('basic') - headers[auth['key']] = auth['value'] + auth = config.auth_settings().get("basic") + headers[auth["key"]] = auth["value"] return headers - + def _discover_endpoint(self, config): """Discovers the scheme and version of a potential Cloudara Manager host""" # Get the authentication headers and REST client @@ -160,77 +170,231 @@ def _discover_endpoint(self, config): rest = RESTClientObject() # Resolve redirects to establish HTTP scheme and port - pre_rendered = Url(scheme="https" if self.force_tls else "http", host=self.host, port=self.port) - rendered = rest.pool_manager.request('GET', pre_rendered.url, headers=headers.copy()) + pre_rendered = Url( + scheme="https" if self.force_tls else "http", host=self.host, port=self.port + ) + rendered = rest.pool_manager.request( + "GET", pre_rendered.url, headers=headers.copy() + ) rendered_url = rendered.geturl() - + # Discover API version if not set if not self.version: pre_versioned = urljoin(rendered_url, "/api/version") - versioned = rest.pool_manager.request('GET', pre_versioned, headers=headers) - self.version = versioned.data.decode('utf-8') - + versioned = rest.pool_manager.request("GET", pre_versioned, headers=headers) + self.version = versioned.data.decode("utf-8") + # Construct the discovered API endpoint return urljoin(rendered_url, "/api/" + self.version) - + def set_session_cookie(self): """Utility to cache the session cookie for intra-module operations.""" if not self.api_client.last_response: api_instance = ClouderaManagerResourceApi(self.api_client) api_instance.get_version() - self.api_client.cookie = self.api_client.last_response.getheader('Set-Cookie') + self.api_client.cookie = self.api_client.last_response.getheader("Set-Cookie") - def call_api(self, path, method, query=None, field='items', body=None): + def call_api(self, path, method, query=None, field="items", body=None): """Wrapper to call a CM API endpoint path directly.""" path_params = [] header_params = {} - header_params['Accept'] = self.api_client.select_header_accept(['application/json']) - header_params['Content-Type'] = self.api_client.select_header_content_type(['application/json']) - - results =self.api_client.call_api(path, method, path_params, query, - header_params, body, auth_settings=['basic'], - _preload_content=False) - + header_params["Accept"] = self.api_client.select_header_accept( + ["application/json"] + ) + header_params["Content-Type"] = self.api_client.select_header_content_type( + ["application/json"] + ) + + results = self.api_client.call_api( + path, + method, + path_params, + query, + header_params, + body, + auth_settings=["basic"], + _preload_content=False, + ) + if 200 >= results[1] <= 299: - data = json.loads(results[0].data.decode('utf-8')) + data = json.loads(results[0].data.decode("utf-8")) if field in data: data = data[field] return data if type(data) is list else [data] else: - self.module.fail_json(msg="Error interacting with CM resource", status_code=results[1]) - + self.module.fail_json( + msg="Error interacting with CM resource", status_code=results[1] + ) @staticmethod def ansible_module_discovery(argument_spec={}, required_together=[], **kwargs): """INTERNAL: Creates the Ansible module argument spec and dependencies for CM API endpoint discovery. - Typically, modules will use the ansible_module method to include direct API endpoint URL support.""" + Typically, modules will use the ansible_module method to include direct API endpoint URL support. + """ return AnsibleModule( argument_spec=dict( **argument_spec, - host=dict(type='str', aliases=['hostname']), - port=dict(type='int', default=7180), - version=dict(type='str'), - force_tls=dict(type='bool', default=False), - verify_tls=dict(required=False, type='bool', default=True, aliases=['tls']), - username=dict(required=True, type='str'), - password=dict(required=True, type='str', no_log=True), - debug=dict(required=False, type='bool', default=False, aliases=['debug_endpoints']), - agent_header=dict(required=False, type='str', default='ClouderaFoundry') + host=dict(type="str", aliases=["hostname"]), + port=dict(type="int", default=7180), + version=dict(type="str"), + force_tls=dict(type="bool", default=False), + verify_tls=dict( + required=False, type="bool", default=True, aliases=["tls"] + ), + username=dict(required=True, type="str"), + password=dict(required=True, type="str", no_log=True), + debug=dict( + required=False, + type="bool", + default=False, + aliases=["debug_endpoints"], + ), + agent_header=dict( + required=False, type="str", default="ClouderaFoundry" + ), ), - required_together=required_together + [['username', 'password']], - **kwargs + required_together=required_together + [["username", "password"]], + **kwargs, ) @staticmethod - def ansible_module(argument_spec={}, mutually_exclusive=[], required_one_of=[], required_together=[], **kwargs): + def ansible_module( + argument_spec={}, + mutually_exclusive=[], + required_one_of=[], + required_together=[], + **kwargs + ): """Creates the base Ansible module argument spec and dependencies, including discovery and direct endpoint URL support.""" return ClouderaManagerModule.ansible_module_discovery( argument_spec=dict( **argument_spec, - url=dict(type='str', aliases=['endpoint', 'cm_endpoint_url']), + url=dict(type="str", aliases=["endpoint", "cm_endpoint_url"]), ), - mutually_exclusive=mutually_exclusive + [['url', 'host']], - required_one_of=required_one_of + [['url', 'host']], + mutually_exclusive=mutually_exclusive + [["url", "host"]], + required_one_of=required_one_of + [["url", "host"]], required_together=required_together, - **kwargs + **kwargs, + ) + + +""" +A common Ansible Lookup plugin for API access to Cloudera Manager. +""" + + +class ClouderaManagerLookupBase(LookupBase): + def initialize_client(self): + # Set up core CM API client parameters + config = Configuration() + config.username = self.get_option("username") + config.password = self.get_option("password") + config.verify_ssl = self.get_option("verify_tls") + config.debug = self.get_option("debug") + + # Configure logging + _log_format = ( + "%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s" + ) + if self.get_option("debug"): + self._setup_logger(logging.DEBUG, _log_format) + self.logger.debug("CM API agent: %s", self.get_option("agent_header")) + else: + self._setup_logger(logging.ERROR, _log_format) + + if self.get_option("verify_tls") is False: + disable_warnings(InsecureRequestWarning) + + # If provided a CM API endpoint URL, use it directly + if self.get_option("endpoint"): + config.host = self.get_option("endpoint") + # Otherwise, run discovery on missing parts + else: + config.host = self._discover_endpoint(config) + + self.api_client = ApiClient() + + def _setup_logger(self, log_level, log_format): + """Configures the logging of the HTTP activity""" + self.logger = logging.getLogger("urllib3") + self.logger.setLevel(log_level) + + def _get_auth_headers(self, config): + """Constructs a Basic Auth header dictionary from the Configuration. + This dictionary can be used directly with the API client's REST client.""" + headers = dict() + auth = config.auth_settings().get("basic") + headers[auth["key"]] = auth["value"] + return headers + + def _discover_endpoint(self, config): + """Discovers the scheme and version of a potential Cloudara Manager host""" + # Get the authentication headers and REST client + headers = self._get_auth_headers(config) + rest = RESTClientObject() + + # Resolve redirects to establish HTTP scheme and port + pre_rendered = Url( + scheme="https" if self.get_option("force_tls") else "http", + host=self.get_option("host"), + port=self.get_option("port"), + ) + rendered = rest.pool_manager.request( + "GET", pre_rendered.url, headers=headers.copy() + ) + rendered_url = rendered.geturl() + + # Discover API version if not set + if not self.version: + pre_versioned = urljoin(rendered_url, "/api/version") + versioned = rest.pool_manager.request("GET", pre_versioned, headers=headers) + self.version = versioned.data.decode("utf-8") + + # Construct the discovered API endpoint + return urljoin(rendered_url, "/api/" + self.version) + + def get(self, path, query=None, field="items", body=None): + """Wrapper to GET a CM API endpoint path directly.""" + path_params = [] + header_params = {} + header_params["Accept"] = self.api_client.select_header_accept( + ["application/json"] + ) + header_params["Content-Type"] = self.api_client.select_header_content_type( + ["application/json"] ) + + try: + results = self.api_client.call_api( + path, + "GET", + path_params, + query, + header_params, + auth_settings=["basic"], + _preload_content=False, + ) + + if 200 >= results[1] <= 299: + data = json.loads(results[0].data.decode("utf-8")) + if field in data: + data = data[field] + return data if type(data) is list else [data] + else: + raise AnsibleError( + "Error interacting with CM resource. Status code: %s" + % to_text(results[1]) + ) + except ApiException as ae: + body = ae.body.decode("utf-8") + if body != "": + body = json.loads(body) + raise AnsibleError( + "API error: %s; Status code: %s" % (ae.reason, ae.status), + obj=body, + orig_exc=ae, + ) + except MaxRetryError as maxe: + raise AnsibleError("Request error: %s" % to_text(maxe.reason)) + except HTTPError as he: + raise AnsibleError("HTTP request error", orig_exc=he) diff --git a/requirements.yml b/requirements.yml index c3609a75..b17c2933 100644 --- a/requirements.yml +++ b/requirements.yml @@ -24,14 +24,14 @@ roles: collections: - name: ansible.posix - version: "1.3.0" + version: 1.3.0 - name: community.crypto - version: "2.2.1" + version: 2.2.1 - name: community.general - version: "4.5.0" + version: 4.5.0 - name: community.mysql - version: "3.1.0" + version: 3.1.0 - name: community.postgresql - version: "1.6.1" + version: 1.6.1 - name: freeipa.ansible_freeipa - version: "1.6.2" + version: 1.6.2