cluster.yml

---

# Copyright 2021 Cloudera, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This Playbook is specifically for Deploying Cloudera Clusters
# Edit with extreme caution
# This plays are imported from a separate playbook so that Ansible tags are intuitively propagated from main.yml

# STARTBLOCK # Init run
- name: Init Cluster Deployment Tasks
  hosts: localhost
  tags: always
  gather_facts: no
  tasks:
    # TODO: Move to dependent roles with delegation if necessary
    - name: Create local temp directories
      file: "{{ __dir }}"
      loop:
        - path: "{{ local_temp_dir }}/csrs"
          state: directory
        - path: "{{ local_temp_dir }}/certs"
          state: directory
      loop_control:
        loop_var: __dir

- name: Check Inventory Connectivity
  hosts: all
  tags: always
  gather_facts: no
  tasks:
    - name: Check connectivity to Inventory
      ansible.builtin.wait_for_connection:

    - name: Gather facts from connected inventory
      setup:

# ENDBLOCK # Init run
# STARTBLOCK # Verify Inventory and Definition
# Moved from tasks/group_hosts and tasks/group_hosts_by_tls
- name: Group hosts using host template information
  hosts: all
  gather_facts: no
  roles:
    - cloudera.cluster.deployment.groupby
  tags:
    - always

- name: Verify inventory [verify_inventory]
  hosts: localhost
  gather_facts: no
  roles:
    - cloudera.cluster.verify.inventory
  tags:
    - verify
    - verify_inventory
    - default_cluster
    - full_cluster

- name: Verify definition [verify_definition]
  hosts: cloudera_manager
  gather_facts: no
  roles:
    - cloudera.cluster.verify.definition
  tags:
    - verify
    - verify_definition
    - default_cluster
    - full_cluster

# Moved before parcel verification to allow rehosting
- name: Install custom parcel repository
  hosts: custom_repo
  become: yes
  roles:
    - cloudera.cluster.infrastructure.custom_repo
  tags:
    - custom_repo
    - default_cluster
    - full_cluster

# Moved from verify_parcels to reduce duplication
- name: Verify definition [verify_parcels_and_roles]
  hosts: cloudera_manager
  gather_facts: no
  roles:
    - cloudera.cluster.verify.parcels_and_roles
  tags:
    - verify
    - verify_parcels
    - default_cluster
    - full_cluster

# ENDBLOCK # Verify Inventory and Definition
# STARTBLOCK # Prepare Nodes

- name: Apply OS pre-requisite configurations
  hosts: cloudera_manager, cluster, ca_server
  become: yes
  roles:
    - cloudera.cluster.prereqs.os
  tags:
    - os
    - default_cluster
    - full_cluster

- name: Create local user accounts
  hosts: cloudera_manager, cluster, tls
  become: yes
  gather_facts: no
  roles:
    - cloudera.cluster.prereqs.user_accounts
  tags:
    - users
    - default_cluster
    - full_cluster

- name: Install JDK
  hosts: cloudera_manager, cluster, tls
  become: yes
  roles:
    - cloudera.cluster.prereqs.jdk
  tags:
    - jdk
    - default_cluster
    - full_cluster

- name: Install MySQL Connector
  hosts: cloudera_manager, cluster
  gather_facts: no
  become: yes
  roles:
    - role: cloudera.cluster.prereqs.mysql_connector
      when: database_type == 'mysql' or database_type == 'mariadb'
  tags:
    - mysql_connector
    - full_cluster

- name: Install Oracle Connector
  hosts: cloudera_manager, cluster
  gather_facts: no
  become: yes
  roles:
    - role: cloudera.cluster.prereqs.oracle_connector
      when: database_type == 'oracle'
  tags:
    - oracle_connector
    - full_cluster

- name: Install PostgreSQL Connector
  hosts: cloudera_manager, cluster
  gather_facts: no
  become: yes
  roles:
    - role: cloudera.cluster.prereqs.postgresql_connector
      when: database_type == 'postgresql'
  tags:
    - postgresql_connector
    - full_cluster

# ENDBLOCK # Prepare Nodes
# STARTBLOCK # Create Cluster Service Infrastructure

- name: Install KDC
  hosts: krb5_server
  become: yes
  roles:
    - role: cloudera.cluster.infrastructure.krb5_server
  tags:
    - security
    - kerberos
    - tls
    - full_cluster

- name: Setup KRB5 clients
  hosts: cloudera_manager, cluster
  become: yes
  roles:
    - role: cloudera.cluster.infrastructure.krb5_client
      when: "'krb5_server' in groups"
  tags:
    - security
    - kerberos
    - tls
    - full_cluster

- name: Install CA server
  hosts: ca_server
  become: yes
  roles:
    - cloudera.cluster.infrastructure.ca_server
  tags:
    - security
    - tls
    - full_cluster

- name: Install HAProxy
  hosts: haproxy
  become: yes
  roles:
    - cloudera.cluster.infrastructure.haproxy
  tags:
    - ha
    - full_cluster

# ENDBLOCK # Create Cluster Service Infrastructure
# STARTBLOCK # Prepare TLS

- name: Build TLS keystores and truststores
  hosts: tls
  become: yes
  gather_facts: no
  roles:
    - cloudera.cluster.security.tls_generate_csr
    - cloudera.cluster.security.tls_signing
    - cloudera.cluster.security.tls_install_certs
  vars:
    local_certs_dir: "{{ local_temp_dir }}/certs"
    local_csrs_dir: "{{ local_temp_dir }}/csrs"
  tags:
    - tls
    - full_cluster

- name: Delete temp directories
  hosts: localhost
  become: no
  gather_facts: no
  tasks:
    - file:
        path: "{{ [local_temp_dir, dir] | path_join }}"
        state: absent
      loop:
        - csrs
        - certs
      loop_control:
        loop_var: dir
  tags:
    - always

# ENDBLOCK # Prepare TLS
# STARTBLOCK # Install Cluster Service Infrastructure II
- name: Install RDBMS
  hosts: db_server
  become: yes
  roles:
    - cloudera.cluster.infrastructure.rdbms
  tags:
    - database
    - default_cluster
    - full_cluster
# ENDBLOCK # Install Cluster Service Infrastructure II
# STARTBLOCK # NiFi TLS

- name: Setup symlinks for NiFi TLS keystore and truststore
  hosts: cluster
  become: yes
  gather_facts: no
  roles:
    - role: cloudera.cluster.security.tls_nifi
      when: >
        (tls | default(False)
        or manual_tls_cert_distribution | default(False))
        and not (autotls | default(False))

# ENDBLOCK # NiFi TLS
# STARTBLOCK # Install Cloudera Manager

- name: Install Cloudera Manager daemons
  hosts: cloudera_manager, cluster
  become: yes
  any_errors_fatal: true
  roles:
    - role: cloudera.cluster.cloudera_manager.daemons
  tags:
    - cm
    - default_cluster
    - full_cluster

- name: Install Cloudera Manager server
  hosts: cloudera_manager
  become: yes
  roles:
    - role: cloudera.cluster.cloudera_manager.server
  tags:
    - cm
    - default_cluster
    - full_cluster

- name: Install Cloudera Manager License
  hosts: cloudera_manager
  become: yes
  roles:
    - role: cloudera.cluster.cloudera_manager.license
  tags:
    - cm
    - license
    - default_cluster
    - full_cluster

- name: Install Cloudera Manager agents
  hosts: cloudera_manager, cluster
  become: yes
  any_errors_fatal: true
  roles:
    - role: cloudera.cluster.cloudera_manager.agent
  tags:
    - cm
    - default_cluster
    - full_cluster

- name: Configure Cloudera Manager server for TLS
  hosts: cloudera_manager
  become: yes
  gather_facts: no
  roles:
    - role: cloudera.cluster.cloudera_manager.server_tls
      when: tls | default(False) or manual_tls_cert_distribution | default(False)
  tags:
    - tls
    - cm
    - full_cluster

- name: Configure Cloudera Manager agents
  hosts: cloudera_manager, cluster
  become: yes
  any_errors_fatal: true
  roles:
    - cloudera.cluster.cloudera_manager.agent_config
  tags:
    - cm
    - default_cluster
    - full_cluster

- name: Configure Cloudera Manager server
  hosts: localhost
  gather_facts: no
  roles:
    - cloudera.cluster.cloudera_manager.config
  tags:
    - cm
    - default_cluster
    - full_cluster

- name: Configure Cloudera Manager auth and accounts
  hosts: cloudera_manager
  gather_facts: no
  roles:
    - role: cloudera.cluster.cloudera_manager.external_auth
    - role: cloudera.cluster.cloudera_manager.external_account
  tags:
    - cm
    - full_cluster

- name: Configure Cloudera Manager Password
  hosts: cloudera_manager
  gather_facts: no
  roles:
    - role: cloudera.cluster.cloudera_manager.admin_password.set
  tags:
    - cm
    - default_cluster
    - full_cluster

# ENDBLOCK # Install Cloudera Manager
# STARTBLOCK # Cloudera Manager Password

- name: Check Cloudera Manager admin password
  hosts: cloudera_manager, cluster
  gather_facts: no
  roles:
    - cloudera.cluster.cloudera_manager.api_client
  tags:
    - cm
    - default_cluster
    - full_cluster

# ENDBLOCK # Cloudera Manager Password
# STARTBLOCK # Prepare Security
# Insert AutoTLS with tag
- name: Enable Auto-TLS
  hosts: cloudera_manager
  gather_facts: no
  roles:
    - cloudera.cluster.cloudera_manager.autotls
  tags:
    - autotls
    - never

- name: Install pre-requisite packages for Kerberos
  hosts: cloudera_manager, cluster
  become: yes
  roles:
    - role: cloudera.cluster.prereqs.kerberos
      when: krb5_kdc_host is defined or 'krb5_server' in groups
  tags:
    - kerberos
    - prereqs
    - full_cluster

- name: Configure Cloudera Manager server for Kerberos
  hosts: cloudera_manager
  gather_facts: no
  roles:
    - role: cloudera.cluster.cloudera_manager.kerberos
      when: krb5_kdc_host is defined or 'krb5_server' in groups
  tags:
    - kerberos
    - full_cluster

# ENDBLOCK # Prepare Security
# STARTBLOCK # Install Cluster

- name: Restart Cloudera Manager Agents
  hosts: cloudera_manager, cluster
  gather_facts: no
  become: yes
  roles:
    - role: cloudera.cluster.operations.restart_agents
  tags:
    - never
    - restart_agents

- name: Ensure that the agents are heartbeating
  hosts: cloudera_manager, cluster
  gather_facts: yes
  any_errors_fatal: yes
  roles:
    - role: cloudera.cluster.cloudera_manager.wait_for_heartbeat
      when: cloudera_manager_agent_wait_for_heartbeat | default(True)
  tags:
    - heartbeat
    - default_cluster
    - full_cluster

- name: Deploy Cloudera Management Service
  hosts: localhost
  gather_facts: no
  roles:
    - cloudera.cluster.deployment.services.mgmt
  tags:
    - mgmt
    - default_cluster
    - full_cluster

- name: Preload parcels from custom repo to Cloudera Manager
  hosts: cloudera_manager
  become: yes
  gather_facts: no
  roles:
    - role: cloudera.cluster.cloudera_manager.preload_parcels
      when: "'custom_repo' in groups"
  tags:
    - preload_parcels
    - default_cluster
    - full_cluster

- name: Deploy clusters
  hosts: cloudera_manager
  gather_facts: no
  roles:
    - cloudera.cluster.deployment.cluster
  tags:
    - cluster
    - default_cluster
    - full_cluster

# ENDBLOCK # Install Cluster
# STARTBLOCK # Setup HDFS Encryption

- name: Setup KTS HA
  hosts: localhost
  become: yes
  gather_facts: no
  roles:
    - role: cloudera.cluster.deployment.services.kts_high_availability
      when:
        - "'kts_active' in groups"
        - "'kts_passive' in groups"
  tags:
    - kts
    - full_cluster

- name: Handle KMS services
  hosts: localhost
  gather_facts: no
  become: yes
  roles:
    - role: cloudera.cluster.deployment.services.kms
      when: "'kms_servers' in groups"
    - role: cloudera.cluster.deployment.services.kms_ha
      when: "'kms_servers' in groups"
  tags:
    - kms
    - full_cluster

- name: Handle KMS services
  hosts: cloudera_manager
  gather_facts: no
  become: no
  roles:
    - role: cloudera.cluster.operations.refresh_ranger_kms_repo
      when: "'kms_servers' in groups"
  tags:
    - kms
    - full_cluster

- name: Restart and re-deploy stale client configs
  hosts: localhost
  gather_facts: no
  roles:
    - role: cloudera.cluster.operations.restart_stale
      when: "'kms_servers' in groups"
  vars:
    client_config_timeout: "{{ restart_client_config_timeout | default(300) }}"
  tags:
    - kms
    - restart_stale
    - full_cluster

# ENDBLOCK # Setup HDFS Encryption
# STARTBLOCK # Teardown
# Teardown CA

- name: Teardown CA server
  hosts: ca_server
  gather_facts: no
  become: yes
  tasks:
    - ansible.builtin.include_role:
        name: cloudera.cluster.infrastructure.ca_certs
        tasks_from: clean.yml
  tags:
    - teardown_ca
    - teardown_all
    - never

- name: Teardown security artifact directories
  hosts: tls
  gather_facts: no
  become: yes
  roles:
    - role: cloudera.cluster.security.tls_clean
      when: "'tls' in groups"
  tags:
    - teardown_tls
    - teardown_all
    - never

# Teardown Cluster

- name: Teardown
  hosts: all
  gather_facts: no
  become: yes
  any_errors_fatal: true
  roles:
    - cloudera.cluster.teardown
  tags:
    - teardown_cluster
    - teardown_all
    - never

# ENDBLOCK # Teardown
# End run
###