provision-kafka

#!/usr/bin/env ansible-playbook
# (c) 2016 DataNexus Inc.  All Rights Reserved.
# Licensed software not for distribution
#
# main routine for provisioning kafka
---
# note that the tags are not currently used, but are present for future integration into
# the datanexus infrastructure components  
- name: KAFKA OVERLAY | preparing {{ cloud }} layer
  tags:
    - confluent
  hosts: localhost
  connection: local
  # this vars list needs to contain every port across every application and all variable dependencies
  vars_files:
    - "{{ tenant_config_path }}/config/site.yml"
    - "{{ tenant_config_path }}/config/applications/kafka.yml"
    - roles/apache/defaults/main.yml
    - roles/confluent/defaults/main.yml
    - "{{ tenant_config_path }}/config/applications/confluent.yml"
    - roles/controlcenter/defaults/main.yml
    - roles/cruisecontrol/defaults/main.yml
    - roles/zookeeper/defaults/main.yml
    - roles/kafka/defaults/main.yml
    - roles/kafkarest/defaults/main.yml
    - roles/ksql/defaults/main.yml
    - roles/registry/defaults/main.yml
    - roles/connect/defaults/main.yml
    - roles/replicator/defaults/main.yml
    - "{{ tenant_config_path }}/config/{{ project }}.yml"
  gather_facts: yes
  tasks:
    - block:
      
      - include_role:
          name: aws
          tasks_from: discover-vpc
          
      - include_role:
          name: aws
          tasks_from: create-securitygroup
      
      - include_role:
          name: aws
          tasks_from: apply-securitygroup
          
      when: cloud == 'aws'
    
    - block:
      
      - include_role:
          name: azure
          tasks_from: discover-resourcegroup
          
      - include_role:
          name: azure
          tasks_from: modify-existing-securitygroup
          
      when: cloud == 'azure'

# complete preflight for all host groups
- name: KAFKA OVERLAY | completing preflight OS configuration
  hosts: zookeeper:kafka_broker:registry:cruisecontrol:controlcenter:kafka_connect:rest_proxy:kafka_ksql:kafka_replicator
  tags:
    - confluent
  vars_files:
    - roles/confluent/defaults/main.yml
    - roles/apache/defaults/main.yml
    - "{{ tenant_config_path }}/config/site.yml"
    - "{{ tenant_config_path }}/config/os/linux.yml"
    - "{{ tenant_config_path }}/config/applications/confluent.yml"
    - "{{ tenant_config_path }}/config/applications/apache.yml"
  vars:
      # this is semi clever; application gets set based on each host group
      application: "{{ group_names | first }}"
  gather_facts: yes
  tasks:
    - include_role:
        name: preflight

- name: KAFKA OVERLAY | installing base components across all nodes
  hosts: zookeeper:kafka_broker:kafka_connect
  tags:
    - confluent
  vars_files:
    - "{{ tenant_config_path }}/config/site.yml"
    - "{{ tenant_config_path }}/config/applications/confluent.yml"
    - "{{ tenant_config_path }}/config/applications/apache.yml"
  gather_facts: yes
  tasks:
    
    - import_role:
        name: apache
      when:
        - apache_kafka
        - not confluent_kafka
        
    - block:

      # here we want to configure platform or community based on confluent.yml
      - set_fact:
          confluent_distribution: "{{ (confluent_community == true) | ternary('confluent-community','confluent-platform') }}"

      - import_role:
          name: confluent

      when:
        - confluent_kafka
        - not apache_kafka

- name: KAFKA OVERLAY | installing confluent components across all nodes
  hosts: registry:controlcenter:rest_proxy:kafka_ksql:kafka_replicator
  tags:
    - confluent
  vars_files:
    - "{{ tenant_config_path }}/config/site.yml"
    - "{{ tenant_config_path }}/config/applications/confluent.yml"
  gather_facts: yes
  tasks:

    - set_fact:
        confluent_distribution: "{{ (confluent_community == true) | ternary('confluent-community','confluent-platform') }}"

    - import_role:
        name: confluent

- name: KAFKA OVERLAY | configuring zookeeper
  hosts: zookeeper
  tags:
    - confluent
  vars_files:
    - roles/apache/defaults/main.yml
    - roles/confluent/defaults/main.yml
    - "{{ tenant_config_path }}/config/site.yml"
    - "{{ tenant_config_path }}/config/applications/zookeeper.yml"
    - "{{ tenant_config_path }}/config/applications/confluent.yml"
    - "{{ tenant_config_path }}/config/applications/apache.yml"
  gather_facts: yes
  tasks:
    # we always build the zookeeper nodes off the admin plane internal IP
    - name: KAFKA OVERLAY (ZOOKEEPER) | building eth0 zookeeper host group
      add_host: hostname="{{ hostvars[item].ansible_eth0.ipv4.address }}" groupname=zookeeper_private
      with_items: "{{ groups['zookeeper'] }}"
      when:
        - "'zookeeper' in groups | default([])"

    - import_role:
        name: zookeeper
      
# this is necessary before we build host groups from the ansible server
- name: KAFKA OVERLAY | discovering broker facts
  tags:
    - confluent
  hosts: kafka_broker
  tasks:
    - setup:

# this is necessary before we build host groups from the ansible server
- name: KAFKA OVERLAY | discovering control center broker facts
  tags:
    - confluent
  hosts: controlcenter_broker
  tasks:
    - setup:

- name: KAFKA OVERLAY | building kafka public internal host group
  tags:
    - confluent
  hosts: localhost
  gather_facts: no
  tasks:
    # eth0 is the data plane in a single interface system
    - name: KAFKA OVERLAY | building eth0 broker host group
      add_host: hostname="{{ hostvars[item].ansible_eth0.ipv4.address }}" groupname=kafka_public
      with_items: "{{ groups['kafka_broker'] }}"
      when:
        - "'kafka_broker' in groups | default([])"
        - hostvars[item].ansible_eth1 is not defined

    # eth1 is the dataplane in a multi interface system
    - name: KAFKA OVERLAY | building eth1 broker host group
      add_host: hostname="{{ hostvars[item].ansible_eth1.ipv4.address }}" groupname=kafka_public
      with_items: "{{ groups['kafka_broker'] }}"
      when:
        - "'kafka_broker' in groups | default([])"
        - hostvars[item].ansible_eth1 is defined

    # this is primarily for controlcenter monitoring of secondary clusters
    # eth0 is the data plane in a single interface system
    - name: KAFKA OVERLAY | building eth0 control center broker host group
      add_host: hostname="{{ hostvars[item].ansible_eth0.ipv4.address }}" groupname=controlcenter_broker_public
      with_items: "{{ groups['controlcenter_broker'] }}"
      when:
        - "'controlcenter_broker' in groups | default([])"
        - hostvars[item].ansible_eth1 is not defined

    # eth1 is the dataplane in a multi interface system
    - name: KAFKA OVERLAY | building eth1 control center broker host group
      add_host: hostname="{{ hostvars[item].ansible_eth1.ipv4.address }}" groupname=controlcenter_broker_public
      with_items: "{{ groups['controlcenter_broker'] }}"
      when:
        - "'controlcenter_broker' in groups | default([])"
        - hostvars[item].ansible_eth1 is defined

- name: KAFKA OVERLAY | configuring kafka
  hosts: kafka_broker
  tags:
    - confluent
  vars_files:
    - roles/apache/defaults/main.yml
    - roles/confluent/defaults/main.yml
    - roles/zookeeper/defaults/main.yml
    - roles/cruisecontrol/defaults/main.yml
    - "{{ tenant_config_path }}/config/site.yml"
    - "{{ tenant_config_path }}/config/applications/apache.yml"
    - "{{ tenant_config_path }}/config/applications/confluent.yml"
    - "{{ tenant_config_path }}/config/applications/kafka.yml"
    - "{{ tenant_config_path }}/config/os/linux.yml"
  tasks:
    - import_role:
        name: kafka

- name: KAFKA OVERLAY | configuring confluent schema registry
  hosts: registry
  tags:
    - confluent
  vars_files:
    - roles/apache/defaults/main.yml
    - roles/confluent/defaults/main.yml
    - roles/zookeeper/defaults/main.yml
    - "{{ tenant_config_path }}/config/site.yml"
    - "{{ tenant_config_path }}/config/applications/confluent.yml"
    - "{{ tenant_config_path }}/config/applications/registry.yml"
  tasks:
    - import_role:
        name: registry

- name: KAFKA OVERLAY | building kafka ksql public internal host group
  tags:
    - confluent
  hosts: localhost
  gather_facts: no
  tasks:
    # eth0 is the data plane in a single interface system
    - name: KAFKA OVERLAY | building eth0 ksql host group
      add_host: hostname="{{ hostvars[item].ansible_eth0.ipv4.address }}" groupname=kafka_ksql_public
      with_items: "{{ groups['kafka_ksql'] }}"
      when:
        - "'kafka_ksql' in groups | default([])"
        - hostvars[item].ansible_eth1 is not defined

    # eth1 is the dataplane in a multi interface system
    - name: KAFKA OVERLAY | building eth1 ksql host group
      add_host: hostname="{{ hostvars[item].ansible_eth1.ipv4.address }}" groupname=kafka_ksql_public
      with_items: "{{ groups['kafka_ksql'] }}"
      when:
        - "'kafka_ksql' in groups | default([])"
        - hostvars[item].ansible_eth1 is defined

- name: KAFKA OVERLAY | building kafka connect public internal host group
  tags:
    - confluent
  hosts: localhost
  gather_facts: no
  tasks:
    # eth0 is the data plane in a single interface system
    - name: KAFKA OVERLAY | building eth0 connect host group
      add_host: hostname="{{ hostvars[item].ansible_eth0.ipv4.address }}" groupname=kafka_connect_public
      with_items: "{{ groups['kafka_connect'] }}"
      when:
        - "'kafka_connect' in groups | default([])"
        - hostvars[item].ansible_eth1 is not defined

    # eth1 is the dataplane in a multi interface system
    - name: KAFKA OVERLAY | building eth1 connect host group
      add_host: hostname="{{ hostvars[item].ansible_eth1.ipv4.address }}" groupname=kafka_connect_public
      with_items: "{{ groups['kafka_connect'] }}"
      when:
        - "'kafka_connect' in groups | default([])"
        - hostvars[item].ansible_eth1 is defined

- name: CRUISE CONTROL OVERLAY | configuring linkedin cruise control
  hosts: cruisecontrol
  tags:
    - confluent
  vars_files:
    - roles/confluent/defaults/main.yml
    - roles/apache/defaults/main.yml
    - roles/kafka/defaults/main.yml
    - roles/zookeeper/defaults/main.yml
    - "{{ tenant_config_path }}/config/site.yml"
    - "{{ tenant_config_path }}/config/applications/cruisecontrol.yml"
    - "{{ tenant_config_path }}/config/applications/confluent.yml"
    - "{{ tenant_config_path }}/config/applications/apache.yml"
  vars:
      # this is semi clever; application gets set based on each host group
      application: "{{ group_names | first }}"
  gather_facts: yes
  tasks:
    - include_role:
        name: cruisecontrol

- name: KAFKA OVERLAY | configuring confluent controlcenter
  hosts: controlcenter
  tags:
    - confluent
  vars_files:
    - "{{ tenant_config_path }}/config/site.yml"
    - roles/apache/defaults/main.yml
    - roles/confluent/defaults/main.yml
    - roles/zookeeper/defaults/main.yml
    - roles/kafka/defaults/main.yml
    - roles/connect/defaults/main.yml
    - roles/ksql/defaults/main.yml
    - roles/registry/defaults/main.yml
    - "{{ tenant_config_path }}/config/applications/controlcenter.yml"
    - "{{ tenant_config_path }}/config/os/linux.yml"
    - "{{ tenant_config_path }}/config/applications/confluent.yml"
  gather_facts: yes
  tasks:
    - import_role:
        name: controlcenter

- name: KAFKA OVERLAY | configuring kafka connect
  hosts: kafka_connect
  tags:
    - confluent
  vars_files:
    - roles/apache/defaults/main.yml
    - roles/confluent/defaults/main.yml
    - roles/kafka/defaults/main.yml
    - "{{ tenant_config_path }}/config/site.yml"
    - "{{ tenant_config_path }}/config/applications/apache.yml"
    - "{{ tenant_config_path }}/config/applications/confluent.yml"
    - "{{ tenant_config_path }}/config/applications/connect.yml"
  gather_facts: yes
  tasks:
    - import_role:
        name: connect

- name: KAFKA OVERLAY | configuring confluent kafka ReST proxy
  hosts: rest_proxy
  tags:
    - confluent
  vars_files:
    - roles/apache/defaults/main.yml
    - roles/confluent/defaults/main.yml
    - roles/zookeeper/defaults/main.yml
    - roles/kafka/defaults/main.yml
    - roles/registry/defaults/main.yml
    - "{{ tenant_config_path }}/config/site.yml"
    - "{{ tenant_config_path }}/config/applications/confluent.yml"
    - "{{ tenant_config_path }}/config/applications/kafkarest.yml"
  gather_facts: yes
  tasks:
    - import_role:
        name: kafkarest

- name: KAFKA OVERLAY | configuring confluent kafka ksql
  hosts: kafka_ksql
  tags:
    - confluent
  vars_files:
    - roles/apache/defaults/main.yml
    - roles/confluent/defaults/main.yml
    - roles/kafka/defaults/main.yml
    - "{{ tenant_config_path }}/config/site.yml"
    - "{{ tenant_config_path }}/config/applications/confluent.yml"
    - "{{ tenant_config_path }}/config/applications/ksql.yml"
  gather_facts: yes
  tasks:
    - import_role:
        name: ksql

- name: KAFKA OVERLAY | completing postflight actions
  hosts: zookeeper:kafka_broker:registry:kafka_connect:rest_proxy:kafka_ksql:kafka_replicator
  tags:
    - confluent
  vars_files:
    - "{{ tenant_config_path }}/config/site.yml"
    - roles/apache/defaults/main.yml
    - roles/confluent/defaults/main.yml
    - roles/kafka/defaults/main.yml
    - "{{ tenant_config_path }}/config/applications/zookeeper.yml"
    - roles/zookeeper/defaults/main.yml
    - "{{ tenant_config_path }}/config/applications/apache.yml"
    - "{{ tenant_config_path }}/config/applications/confluent.yml"
    - "{{ tenant_config_path }}/config/applications/kafka.yml"
  vars:
      # this is semi clever; application gets set based on each host group
      application: "{{ group_names | first }}"
  gather_facts: no
  tasks:
    - include_role:
        name: postflight