[WIP] Simplify dns management

playbooks/common/openshift-cluster/upgrades/v3_7/master_config_upgrade.yml

@@ -1,7 +1,7 @@
 ---
 - modify_yaml:
     dest: "{{ openshift.common.config_base}}/master/master-config.yaml"
-    yaml_key: 'admissionConfig.pluginConfig'
+    yaml_key: 'admissionConfig.pluginConfig."openshift.io/ImagePolicyConfig"'
     yaml_value: "{{ openshift.master.admission_plugin_config }}"
   when: "'admission_plugin_config' in openshift.master"
 

playbooks/common/openshift-glusterfs/config.yml

@@ -1,46 +1,40 @@
 ---
-- name: GlusterFS Install Checkpoint Start
-  hosts: oo_all_hosts
-  gather_facts: false
-  tasks:
-  - name: Set GlusterFS install 'In Progress'
-    set_stats:
-      data:
-        installer_phase_glusterfs: "In Progress"
-      aggregate: false
-
 - name: Open firewall ports for GlusterFS nodes
   hosts: glusterfs
-  tasks:
-  - include_role:
-      name: openshift_storage_glusterfs
-      tasks_from: firewall.yml
+  vars:
+    os_firewall_allow:
+    - service: glusterfs_sshd
+      port: "2222/tcp"
+    - service: glusterfs_daemon
+      port: "24007/tcp"
+    - service: glusterfs_management
+      port: "24008/tcp"
+    - service: glusterfs_bricks
+      port: "49152-49251/tcp"
+  roles:
+  - role: os_firewall
     when:
     - openshift_storage_glusterfs_is_native | default(True) | bool
 
 - name: Open firewall ports for GlusterFS registry nodes
   hosts: glusterfs_registry
-  tasks:
-  - include_role:
-      name: openshift_storage_glusterfs
-      tasks_from: firewall.yml
+  vars:
+    os_firewall_allow:
+    - service: glusterfs_sshd
+      port: "2222/tcp"
+    - service: glusterfs_daemon
+      port: "24007/tcp"
+    - service: glusterfs_management
+      port: "24008/tcp"
+    - service: glusterfs_bricks
+      port: "49152-49251/tcp"
+  roles:
+  - role: os_firewall
     when:
     - openshift_storage_glusterfs_registry_is_native | default(True) | bool
 
 - name: Configure GlusterFS
   hosts: oo_first_master
-  tasks:
-  - name: setup glusterfs
-    include_role:
-      name: openshift_storage_glusterfs
+  roles:
+  - role: openshift_storage_glusterfs
     when: groups.oo_glusterfs_to_config | default([]) | count > 0
-
-- name: GlusterFS Install Checkpoint End
-  hosts: oo_all_hosts
-  gather_facts: false
-  tasks:
-  - name: Set GlusterFS install 'Complete'
-    set_stats:
-      data:
-        installer_phase_glusterfs: "Complete"
-      aggregate: false

playbooks/common/openshift-node/configure_nodes.yml

@@ -13,5 +13,6 @@
   roles:
   - role: os_firewall
   - role: openshift_node
+  - role: openshift_node_dnsmasq
   - role: tuned
   - role: nickhammond.logrotate

roles/openshift_node/meta/main.yml

@@ -18,4 +18,3 @@ dependencies:
 - role: openshift_clock
 - role: openshift_docker
 - role: openshift_cloud_provider
-- role: openshift_node_dnsmasq

roles/openshift_node/templates/node.service.j2

@@ -14,10 +14,12 @@ After=dnsmasq.service
 Type=notify
 EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node
 Environment=GOTRACEBACK=crash
-ExecStartPre=/usr/bin/cp /etc/origin/node/node-dnsmasq.conf /etc/dnsmasq.d/
+{% if not openshift.common.version_gte_3_7 %}
+ExecStartPre=-/usr/bin/cp /etc/origin/node/node-dnsmasq.conf /etc/dnsmasq.d/
 ExecStartPre=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:/in-addr.arpa/127.0.0.1,/{{ openshift.common.dns_domain }}/127.0.0.1
-ExecStopPost=/usr/bin/rm /etc/dnsmasq.d/node-dnsmasq.conf
+ExecStopPost=-/usr/bin/rm /etc/dnsmasq.d/node-dnsmasq.conf
 ExecStopPost=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:
+{% endif %}
 ExecStart=/usr/bin/openshift start node --config=${CONFIG_FILE} $OPTIONS
 LimitNOFILE=65536
 LimitCORE=infinity

roles/openshift_node/templates/node.yaml.v1.j2

@@ -1,9 +1,5 @@
 allowDisabledDocker: false
 apiVersion: v1
-{% if openshift.common.version_gte_3_6 %}
-dnsBindAddress: 127.0.0.1:53
-dnsRecursiveResolvConf: /etc/origin/node/resolv.conf
-{% endif %}
 dnsDomain: {{ openshift.common.dns_domain }}
 {% if 'dns_ip' in openshift.node %}
 dnsIP: {{ openshift.node.dns_ip }}

roles/openshift_node/templates/openshift.docker.node.service

@@ -20,8 +20,12 @@ After=dnsmasq.service
 EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node
 EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node-dep
 ExecStartPre=-/usr/bin/docker rm -f {{ openshift.common.service_type }}-node
-ExecStartPre=/usr/bin/cp /etc/origin/node/node-dnsmasq.conf /etc/dnsmasq.d/
+{% if not openshift.common.version_gte_3_7 %}
+ExecStartPre=-/usr/bin/cp /etc/origin/node/node-dnsmasq.conf /etc/dnsmasq.d/
 ExecStartPre=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:/in-addr.arpa/127.0.0.1,/{{ openshift.common.dns_domain }}/127.0.0.1
+ExecStopPost=-/usr/bin/rm /etc/dnsmasq.d/node-dnsmasq.conf
+ExecStopPost=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:
+{% endif %}
 ExecStart=/usr/bin/docker run --name {{ openshift.common.service_type }}-node \
   --rm --privileged --net=host --pid=host --env-file=/etc/sysconfig/{{ openshift.common.service_type }}-node \
   -v /:/rootfs:ro,rslave -e CONFIG_FILE=${CONFIG_FILE} -e OPTIONS=${OPTIONS} \
@@ -41,8 +45,6 @@ ExecStart=/usr/bin/docker run --name {{ openshift.common.service_type }}-node \
   {{ openshift.node.node_image }}:${IMAGE_VERSION}
 ExecStartPost=/usr/bin/sleep 10
 ExecStop=/usr/bin/docker stop {{ openshift.common.service_type }}-node
-ExecStopPost=/usr/bin/rm /etc/dnsmasq.d/node-dnsmasq.conf
-ExecStopPost=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:
 SyslogIdentifier={{ openshift.common.service_type }}-node
 Restart=always
 RestartSec=5s

roles/openshift_node_dnsmasq/files/networkmanager/99-origin-dns.sh

@@ -9,13 +9,15 @@
 # a pod would fail.
 #
 # To use this,
-# - If this host is also a master, reconfigure master dnsConfig to listen on
-#   8053 to avoid conflicts on port 53 and open port 8053 in the firewall
 # - Drop this script in /etc/NetworkManager/dispatcher.d/
 # - systemctl restart NetworkManager
 # - Configure node-config.yaml to set dnsIP: to the ip address of this
 #   node
 #
+# dnsmasq will bind to all interfaces except lo by default
+# If you want to bind to specific interfaces set OPENSHIFT_NODE_DNSMASQ_INTERFACES
+# to the comma separated list of interfaces you wish to bind to
+#
 # Test it:
 # host kubernetes.default.svc.cluster.local
 # host google.com
@@ -25,21 +27,16 @@
 
 cd /etc/sysconfig/network-scripts
 . ./network-functions
+. /etc/sysconfig/atomic-openshift-node
 
 [ -f ../network ] && . ../network
 
 if [[ $2 =~ ^(up|dhcp4-change|dhcp6-change)$ ]]; then
-  # If the origin-upstream-dns config file changed we need to restart
   NEEDS_RESTART=0
-  UPSTREAM_DNS='/etc/dnsmasq.d/origin-upstream-dns.conf'
-  # We'll regenerate the dnsmasq origin config in a temp file first
-  UPSTREAM_DNS_TMP=`mktemp`
-  UPSTREAM_DNS_TMP_SORTED=`mktemp`
-  CURRENT_UPSTREAM_DNS_SORTED=`mktemp`
   NEW_RESOLV_CONF=`mktemp`
-  NEW_NODE_RESOLV_CONF=`mktemp`
-
-
+  if [ ! -f /etc/origin/node/resolv.conf ]; then
+    cp /etc/resolv.conf /etc/origin/node/resolv.conf
+  fi
   ######################################################################
   # couldn't find an existing method to determine if the interface owns the
   # default route
@@ -49,52 +46,26 @@ if [[ $2 =~ ^(up|dhcp4-change|dhcp6-change)$ ]]; then
   if [[ ${DEVICE_IFACE} == ${def_route_int} ]]; then
     if [ ! -f /etc/dnsmasq.d/origin-dns.conf ]; then
       cat << EOF > /etc/dnsmasq.d/origin-dns.conf
-no-resolv
 domain-needed
-server=/cluster.local/172.30.0.1
-server=/30.172.in-addr.arpa/172.30.0.1
 enable-dbus
+bind-interfaces
+dns-loop-detect
+resolv-file=/etc/origin/node/resolv.conf
+except-interface=lo
 EOF
+      if [ ! -z $OPENSHIFT_NODE_DNSMASQ_INTERFACES ]; then
+        interfaces=$(echo $OPENSHIFT_NODE_DNSMASQ_INTERFACES | tr "," "\n")
+        echo "${interfaces[@]}"
+        for i in $interfaces; do
+          echo "interface=${i}" >> /etc/dnsmasq.d/origin-dns.conf
+        done
+      fi
+
       # New config file, must restart
       NEEDS_RESTART=1
     fi
 
-    # If network manager doesn't know about the nameservers then the best
-    # we can do is grab them from /etc/resolv.conf but only if we've got no
-    # watermark
-    if ! grep -q '99-origin-dns.sh' /etc/resolv.conf; then
-      if [[ -z "${IP4_NAMESERVERS}" || "${IP4_NAMESERVERS}" == "${def_route_ip}" ]]; then
-            IP4_NAMESERVERS=`grep '^nameserver ' /etc/resolv.conf | awk '{ print $2 }'`
-      fi
-      ######################################################################
-      # Write out default nameservers for /etc/dnsmasq.d/origin-upstream-dns.conf
-      # and /etc/origin/node/resolv.conf in their respective formats
-      for ns in ${IP4_NAMESERVERS}; do
-        if [[ ! -z $ns ]]; then
-          echo "server=${ns}" >> $UPSTREAM_DNS_TMP
-          echo "nameserver ${ns}" >> $NEW_NODE_RESOLV_CONF
-        fi
-      done
-      # Sort it in case DNS servers arrived in a different order
-      sort $UPSTREAM_DNS_TMP > $UPSTREAM_DNS_TMP_SORTED
-      sort $UPSTREAM_DNS > $CURRENT_UPSTREAM_DNS_SORTED
-      # Compare to the current config file (sorted)
-      NEW_DNS_SUM=`md5sum ${UPSTREAM_DNS_TMP_SORTED} | awk '{print $1}'`
-      CURRENT_DNS_SUM=`md5sum ${CURRENT_UPSTREAM_DNS_SORTED} | awk '{print $1}'`
-      if [ "${NEW_DNS_SUM}" != "${CURRENT_DNS_SUM}" ]; then
-        # DNS has changed, copy the temp file to the proper location (-Z
-        # sets default selinux context) and set the restart flag
-        cp -Z $UPSTREAM_DNS_TMP $UPSTREAM_DNS
-        NEEDS_RESTART=1
-      fi
-      # compare /etc/origin/node/resolv.conf checksum and replace it if different
-      NEW_NODE_RESOLV_CONF_MD5=`md5sum ${NEW_NODE_RESOLV_CONF}`
-      OLD_NODE_RESOLV_CONF_MD5=`md5sum /etc/origin/node/resolv.conf`
-      if [ "${NEW_NODE_RESOLV_CONF_MD5}" != "${OLD_NODE_RESOLV_CONF_MD5}" ]; then
-        cp -Z $NEW_NODE_RESOLV_CONF /etc/origin/node/resolv.conf
-      fi
-    fi
-
+    # dnsmasq not running, needs a restart
     if ! `systemctl -q is-active dnsmasq.service`; then
       NEEDS_RESTART=1
     fi
@@ -122,5 +93,5 @@ EOF
   fi
 
   # Clean up after yourself
-  rm -f $UPSTREAM_DNS_TMP $UPSTREAM_DNS_TMP_SORTED $CURRENT_UPSTREAM_DNS_SORTED $NEW_RESOLV_CONF
+  rm -f $NEW_RESOLV_CONF
 fi

roles/openshift_node_dnsmasq/handlers/main.yml

@@ -9,3 +9,12 @@
   systemd:
     name: dnsmasq
     state: restarted
+
+- name: restart node
+  systemd:
+    name: "{{ openshift.common.service_type }}-node"
+    state: restarted
+  register: l_openshift_node_restart_node_result
+  until: not l_openshift_node_restart_node_result | failed
+  retries: 3
+  delay: 30

roles/openshift_node_dnsmasq/meta/main.yml

@@ -13,3 +13,4 @@ galaxy_info:
   - cloud
 dependencies:
 - role: openshift_node_facts
+- role: lib_utils

roles/openshift_node_dnsmasq/tasks/main.yml

@@ -14,33 +14,15 @@
   package: name=dnsmasq state=installed
   when: not openshift.common.is_atomic | bool
 
-- name: ensure origin/node directory exists
+- name: Remove older dnsmasq configuration files
   file:
-    state: directory
     path: "{{ item }}"
-    owner: root
-    group: root
-    mode: '0700'
+    state: absent
   with_items:
-  - /etc/origin
-  - /etc/origin/node
-
-# this file is copied to /etc/dnsmasq.d/ when the node starts and is removed
-# when the node stops. A dbus-message is sent to dnsmasq to add the same entries
-# so that dnsmasq doesn't need to be restarted. Once we can use dnsmasq 2.77 or
-# newer we can use --server-file option to update the servers dynamically and
-# reload them by sending dnsmasq a SIGHUP. We write the file in case someone else
-# triggers a restart of dnsmasq but not a node restart.
-- name: Install node-dnsmasq.conf
-  template:
-    src: node-dnsmasq.conf.j2
-    dest: /etc/origin/node/node-dnsmasq.conf
-
-- name: Install dnsmasq configuration
-  template:
-    src: origin-dns.conf.j2
-    dest: /etc/dnsmasq.d/origin-dns.conf
-  notify: restart dnsmasq
+    - "/etc/dnsmasq.d/origin-upstream-dns.conf"
+    - "/etc/dnsmasq.d/node-dnsmasq.conf"
+    - "/etc/dnsmasq.d/origin-dns.conf"
+  when: openshift.common.version_gte_3_7 | bool
 
 - name: Deploy additional dnsmasq.conf
   template:
@@ -65,3 +47,13 @@
 # Relies on ansible in order to configure static config
 - include: ./no-network-manager.yml
   when: not network_manager_active | bool
+
+- name: Reconfigure the node for dnsmasq
+  yedit:
+    state: present
+    src: /etc/origin/node/node-config.yaml
+    edits:
+      - key: "dnsBindAddress"
+        value: "127.0.0.1:53"
+      - key: "dnsRecursiveResolvConf"
+        value: "/etc/origin/node/resolv.conf"

roles/openshift_node_dnsmasq/tasks/network-manager.yml

@@ -1,9 +1,31 @@
 ---
+
+- name: Configured dnsmasq interfaces
+  lineinfile:
+    path: /etc/sysconfig/atomic-openshift-node
+    line: "OPENSHIFT_NODE_DNSMASQ_INTERFACES={{ openshift_node_dnsmasq_interfaces | split(',') | join (',') }}"
+  notify: restart NetworkManager
+  when: openshift_node_dnsmasq_interfaces is defined
+
 - name: Install network manager dispatch script
   copy:
     src: networkmanager/99-origin-dns.sh
     dest: /etc/NetworkManager/dispatcher.d/
     mode: 0755
   notify: restart NetworkManager
 
+- name: Ensure dnsmasq can read /etc/origin/node/resolv.conf
+  file:
+    path: "{{ item }}"
+    mode: "o+x"
+  with_items:
+  - "/etc/origin"
+  - "/etc/origin/node"
+
+- name: Install node-dnsmasq.conf on < 3.7
+  template:
+    dest: /etc/origin/node/node-dnsmasq.conf
+    src: node-dnsmasq.conf.j2
+  when: not openshift.common.version_gte_3_7 | bool
+
 - meta: flush_handlers