Allow individual config_db per host via ztp.json

partition/roles/dhcp/defaults/main.yaml

@@ -19,7 +19,7 @@ dhcp_global_options: []
 # examples:
 # - default-url = "http://{{ ansible_host }}/onie-installer"
 # - ztp_provisioning_script_url code 239 = text
-# - ztp_provisioning_script_url "http://{{ ansible_host }}/ztp.sh"
+# - ztp_provisioning_script_url "http://{{ ansible_host }}/user.sh"
 
 dhcp_global_deny_list: []
 # examples:

partition/roles/sonic/README.md

@@ -18,6 +18,7 @@ It depends on the `switch_facts` module from `ansible-common`, so make sure modu
 | sonic_ip_masquerade                                        |           | Enable ip masquerading on eth0.                                                                                      |
 | sonic_breakouts                                            |           | The breakout configuration for ports, e.g. `dict('Ethernet0'='4x25G')`                                               |
 | sonic_config_action                                        |           | Either `load` or `reload`. In the latter case all services will be restarted. If not given, defaults to `load`       |
+| sonic_render_config_db_template                            |           | When `true` the `metal.yaml.j2` template will be rendered into `/etc/sonic/config_db.json`                           |
 | sonic_ports                                                |           | Configuration for ports (mtu, fec, have highest precedence). These ports will be up by default.                      |
 | sonic_ports.name                                           |           | The port name.                                                                                                       |
 | sonic_ports.speed                                          |           | Speed of the port.                                                                                                   |

partition/roles/sonic/defaults/main.yaml

@@ -5,6 +5,7 @@ sonic_nameservers: []
 sonic_ip_masquerade: false
 sonic_timezone: Europe/Berlin
 sonic_config_action: load
+sonic_render_config_db_template: true
 
 ## Physical settings
 sonic_ports: []

partition/roles/sonic/tasks/config_db.yaml

@@ -0,0 +1,79 @@
+---
+- name: Check mandatory variables on non-empty sonic_ports are set
+  assert:
+    fail_msg: "default port configuration is necessary on non-empty sonic_ports"
+    quiet: yes
+    that:
+      - sonic_ports_default_speed
+      - sonic_ports_default_mtu
+  when: sonic_ports
+
+- name: Check mandatory variables on non-empty sonic_portchannels are set
+  assert:
+    fail_msg: "default configuration is necessary on non-empty sonic_portchannels"
+    quiet: yes
+    that:
+      - sonic_portchannels_default_mtu
+  when: sonic_portchannels
+
+- name: Populate sonic_ports_dict
+  set_fact:
+    sonic_ports_dict: "{{ sonic_ports_dict|default({}) | combine( {item.name: item} ) }}"
+  loop: "{{ sonic_ports }}"
+
+# Dependencies are returned by config.
+- name: Configure breakouts
+  command: "config interface breakout --yes {{ item.key }} '{{ item.value }}'"
+  register: breakout_result
+  changed_when: "'Breakout process got successfully completed.' in breakout_result.stdout"
+  failed_when: "breakout_result.rc != 0 or 'Dependecies Exist. No further action will be taken' in breakout_result.stdout"
+  with_dict: "{{ sonic_breakouts }}"
+  when: sonic_breakouts is defined
+
+- name: Delete deprecated metal.yaml
+  ansible.builtin.file:
+    path: "/etc/sonic/metal.yaml"
+    state: absent
+
+- name: Get running configuration
+  ansible.builtin.command: show runningconfiguration all
+  register: sonic_running_cfg_result
+  changed_when: false
+
+- name: Parse running configuration
+  ansible.builtin.set_fact:
+    sonic_running_cfg: "{{ sonic_running_cfg_result.stdout | from_json }}"
+
+- name: Extract running configuration for breakouts and ports
+  ansible.builtin.set_fact:
+    sonic_running_cfg_breakouts: "{{ sonic_running_cfg | community.general.json_query('BREAKOUT_CFG') }}"
+    sonic_running_cfg_hwsku: "{{ sonic_running_cfg | community.general.json_query('DEVICE_METADATA.localhost.hwsku') }}"
+    sonic_running_cfg_mac: "{{ sonic_running_cfg | community.general.json_query('DEVICE_METADATA.localhost.mac') }}"
+    sonic_running_cfg_platform: "{{ sonic_running_cfg | community.general.json_query('DEVICE_METADATA.localhost.platform') }}"
+    sonic_running_cfg_ports: "{{ sonic_running_cfg | community.general.json_query('PORT')  }}"
+
+- name: Fail if running configuration doesn't contain required information
+  ansible.builtin.assert:
+    that:
+      - sonic_running_cfg_hwsku
+      - sonic_running_cfg_mac
+      - sonic_running_cfg_platform
+      - sonic_running_cfg_ports
+    fail_msg: The running configuration is incomplete because it does not contain 'PORT' or complete 'DEVICE_METADATA'.
+
+- name: Fail if running configuration doesn't contain breakout configuration
+  ansible.builtin.assert:
+    that:
+      - sonic_running_cfg_breakouts
+    fail_msg: The running configuration is incomplete because it does not contain 'BREAKOUT_CFG'.
+  when: sonic_breakouts is defined
+
+- name: Render config_db
+  set_fact:
+    config_db: "{{ lookup('template', 'metal.yaml.j2') }}"
+
+- name: Save config_db as JSON file
+  copy:
+    content: "{{ config_db | from_yaml | to_nice_json }}"
+    dest: /etc/sonic/config_db.json
+  notify: "config {{ sonic_config_action }}"

partition/roles/sonic/tasks/main.yaml

@@ -15,28 +15,6 @@
       - sonic_nameservers is defined
       - metal_stack_switch_os_is_sonic
 
-- name: Check mandatory variables on non-empty sonic_ports are set
-  assert:
-    fail_msg: "default port configuration is necessary on non-empty sonic_ports"
-    quiet: yes
-    that:
-      - sonic_ports_default_speed
-      - sonic_ports_default_mtu
-  when: sonic_ports
-
-- name: Check mandatory variables on non-empty sonic_portchannels are set
-  assert:
-    fail_msg: "default configuration is necessary on non-empty sonic_portchannels"
-    quiet: yes
-    that:
-      - sonic_portchannels_default_mtu
-  when: sonic_portchannels
-
-- name: Populate sonic_ports_dict
-  set_fact:
-    sonic_ports_dict: "{{ sonic_ports_dict|default({}) | combine( {item.name: item} ) }}"
-  loop: "{{ sonic_ports }}"
-
 - name: render resolv.conf
   template:
     src: resolv.conf.j2
@@ -58,62 +36,9 @@
     value: "1"
   when: sonic_ip_masquerade
 
-# Dependencies are returned by config.
-- name: Configure breakouts
-  command: "config interface breakout --yes {{ item.key }} '{{ item.value }}'"
-  register: breakout_result
-  changed_when: "'Breakout process got successfully completed.' in breakout_result.stdout"
-  failed_when: "breakout_result.rc != 0 or 'Dependecies Exist. No further action will be taken' in breakout_result.stdout"
-  with_dict: "{{ sonic_breakouts }}"
-  when: sonic_breakouts is defined
-
-- name: Delete deprecated metal.yaml
-  ansible.builtin.file:
-    path: "/etc/sonic/metal.yaml"
-    state: absent
-
-- name: Get running configuration
-  ansible.builtin.command: show runningconfiguration all
-  register: sonic_running_cfg_result
-  changed_when: false
-
-- name: Parse running configuration
-  ansible.builtin.set_fact:
-    sonic_running_cfg: "{{ sonic_running_cfg_result.stdout | from_json }}"
-
-- name: Extract running configuration for breakouts and ports
-  ansible.builtin.set_fact:
-    sonic_running_cfg_breakouts: "{{ sonic_running_cfg | community.general.json_query('BREAKOUT_CFG') }}"
-    sonic_running_cfg_hwsku: "{{ sonic_running_cfg | community.general.json_query('DEVICE_METADATA.localhost.hwsku') }}"
-    sonic_running_cfg_mac: "{{ sonic_running_cfg | community.general.json_query('DEVICE_METADATA.localhost.mac') }}"
-    sonic_running_cfg_platform: "{{ sonic_running_cfg | community.general.json_query('DEVICE_METADATA.localhost.platform') }}"
-    sonic_running_cfg_ports: "{{ sonic_running_cfg | community.general.json_query('PORT')  }}"
-
-- name: Fail if running configuration doesn't contain required information
-  ansible.builtin.assert:
-    that:
-      - sonic_running_cfg_hwsku
-      - sonic_running_cfg_mac
-      - sonic_running_cfg_platform
-      - sonic_running_cfg_ports
-    fail_msg: The running configuration is incomplete because it does not contain 'PORT' or complete 'DEVICE_METADATA'.
-
-- name: Fail if running configuration doesn't contain breakout configuration
-  ansible.builtin.assert:
-    that:
-      - sonic_running_cfg_breakouts
-    fail_msg: The running configuration is incomplete because it does not contain 'BREAKOUT_CFG'.
-  when: sonic_breakouts is defined
-
-- name: Render config_db
-  set_fact:
-    config_db: "{{ lookup('template', 'metal.yaml.j2') }}"
-
-- name: Save config_db as JSON file
-  copy:
-    content: "{{ config_db | from_yaml | to_nice_json }}"
-    dest: /etc/sonic/config_db.json
-  notify: "config {{ sonic_config_action }}"
+- name: Render and save config_db
+  import_tasks: config_db.yaml
+  when: sonic_render_config_db_template
 
 - name: Set NTP timezone
   timezone:

partition/roles/sonic/templates/frr.conf.j2

@@ -4,6 +4,7 @@ hostname {{ inventory_hostname }}
 !
 service integrated-vtysh-config
 !
+agentx
 log syslog {{ sonic_frr_syslog_level }}
 {% if sonic_frr_debug_options is defined %}
 {% for option in sonic_frr_debug_options %}

partition/roles/sonic/test/data/exit/frr.conf

@@ -3,6 +3,7 @@ hostname exit01
 !
 service integrated-vtysh-config
 !
+agentx
 log syslog informational
 !
 vrf VrfMpls
@@ -105,4 +106,4 @@ route-map LOOPBACKS permit 10
 ip route 0.0.0.0/0 10.1.2.1
 !
 line vty
-!
+!

partition/roles/sonic/test/data/l2_leaf/frr.conf

@@ -3,6 +3,7 @@ hostname l2leaf01
 !
 service integrated-vtysh-config
 !
+agentx
 log syslog informational
 !
 vrf Vrf46
@@ -62,4 +63,4 @@ route-map LOOPBACKS permit 10
   match interface Loopback0
 !
 line vty
-!
+!

partition/roles/sonic/test/data/mgmtleaf/frr.conf

@@ -3,6 +3,7 @@ hostname r01mgmtleaf
 !
 service integrated-vtysh-config
 !
+agentx
 log syslog informational
 !
 interface Ethernet120
@@ -31,4 +32,4 @@ route-map DENY_MGMT deny 10
 route-map DENY_MGMT permit 20
 !
 line vty
-!
+!

partition/roles/sonic/test/data/sonic-vs/frr.conf

@@ -3,6 +3,7 @@ hostname sonic-vs
 !
 service integrated-vtysh-config
 !
+agentx
 log syslog informational
 !
 interface Ethernet0
@@ -26,4 +27,4 @@ route-map DENY_MGMT deny 10
 route-map DENY_MGMT permit 20
 !
 line vty
-!
+!

partition/roles/sonic/test/data/spine/frr.conf

@@ -3,6 +3,7 @@ hostname spine01
 !
 service integrated-vtysh-config
 !
+agentx
 log syslog informational
 !
 interface Ethernet120
@@ -35,4 +36,4 @@ route-map LOOPBACKS permit 10
   match interface Loopback0
 !
 line vty
-!
+!

partition/roles/ztp/README.md

@@ -2,24 +2,11 @@
 
 Configures a server for providing zero-touch-provisioning scripts for switches.
 
-## Variables
-
-| Name                 | Mandatory | Description                                                 |
-| -------------------- | --------- | ----------------------------------------------------------- |
-| ztp_nginx_image_name | yes       | the docker image to use to serve ztp scripts.               |
-| ztp_nginx_image_tag  | yes       | the tag of the docker image to use to serve ztp scripts.    |
-| ztp_host_dir_path    |           | the path to serve ztp scripts from.                         |
-| ztp_listen_address   |           | the address used to serve ztp requests                      |
-| ztp_port             |           | the port to serve ztp scripts on.                           |
-| ztp_authorized_keys  | yes       | the authorized keys that should be installed by ztp.        |
-| ztp_admin_user       |           | the user for which the authorized keys will be provisioned. |
-| ztp_additional_files |           | puts additional files into serve directory.                 |
-
 ## Provisioning SONiC Switches via ztp.json
 
 On SONiC switches it is possible to describe the ZTP procedure in a file called `ztp.json`.
 It contains all steps that should be performed during ZTP along with some additional options.
-We use `ztp.json` to trigger a restart of the BGP service after the initial switch provisioning.
+For example, host-specific download paths for the `config_db.json` or any additional files or scripts can be provided in the `ztp.json`.
 To use the `ztp.json` file, add a DHCP option with code 67 to the DHCP server that serves the file.
 For example, add a section like the following to `/etc/dhcp/dhcpd.conf`:
 
@@ -34,3 +21,51 @@ host leaf01 {
 ```
 
 For more information on the `ztp.json` format refer to the [documentation](https://github.com/sonic-net/SONiC/blob/master/doc/ztp/ztp.md).
+
+Note that each switch that uses the `ztp.json` file needs an individual `config_db.json`, that it can download at `http://{{ ztp_listen_address }}:{{ ztp_port }}/<hostname>_config_db.json`.
+For example, if the switch's hostname is `r01leaf02`, there should be a file called `r01leaf02_config_db.json` located in `{{ ztp_host_dir_path }}/config/`.
+The configs can be added to the `ztp_additional_files` variable, e.g.
+
+```yaml
+ztp_additional_files:
+  - name: r01leaf02_config_db.json
+    data: "{{ lookup('file', 'path/to/r01leaf02_config_db.json)' | string }}" # using `string` to keep the formatting
+  - name: r02leaf01_config_db.json
+    data: ...
+```
+
+When a SONiC switch is deployed via `ztp.json` and configured by the `sonic` role afterwards, make sure to leave the `sonic_ports`, `sonic_portchannels` and `sonic_breakouts` variables empty and set `sonic_render_config_db_template` to false.
+Otherwise the `sonic` role will override the `config_db.json` provided by the `ztp.json`.
+The result of this may not be intended and, in the worst case, the switch will reach a broken state from which it only can be restored by a factory reset.
+Of course it is also possible to load only a minimal `config_db.json` via ZTP and allow the `sonic` role to render its template based on the `sonic_ports`, `sonic_portchannels` and `sonic_breakouts` variables.
+Both approaches have their pros and cons.
+
+### Pros and Cons of Loading a Static config_db.json via ZTP
+
+The main advantage of loading the `config_db.json` once via ZTP and disabling template rendering by the `sonic` role is a better stability and the ability to configure the switch exactly as needed without relying on the complex templating logic in the `sonic` role.
+As mentioned above, the problem with loading a new config each time the `sonic` role is run is that even seemingly small changes might break the system (swss crash).
+On the other hand, with a ZTP-only approach, since ZTP only runs during initial setup of the switch, the only way of changing the config is by resetting the switch to activate ZTP.
+So the desicion of whether to use the `sonic` role's dynamic config or a static ZTP-only config comes down to questions like:
+
+- how often will the config need to change?
+- do all ports on the switch look more or less the same or are there ports that require some specific configuration?
+
+In the latter case the templating might run into certain edge cases, where the resulting config breaks the system.
+Then you should consider using only a static config.
+
+> For the time being it is up to the user which provisioning procedure they prefer.
+> In the future we hope to come up with a single solution that is both flexible and reliable.
+
+## Variables
+
+| Name                    | Mandatory | Description                                                 |
+| ----------------------- | --------- | ----------------------------------------------------------- |
+| ztp_nginx_image_name    | yes       | the docker image to use to serve ztp scripts.               |
+| ztp_nginx_image_tag     | yes       | the tag of the docker image to use to serve ztp scripts.    |
+| ztp_host_dir_path       |           | the path to serve ztp scripts from.                         |
+| ztp_listen_address      |           | the address used to serve ztp requests                      |
+| ztp_port                |           | the port to serve ztp scripts on.                           |
+| ztp_authorized_keys     | yes       | the authorized keys that should be installed by ztp.        |
+| ztp_admin_user          |           | the user for which the authorized keys will be provisioned. |
+| ztp_additional_files    |           | puts additional files into serve directory.                 |
+| ztp_provisioning_script |           | shell script to be executed as a last step in the ztp.json  |

partition/roles/ztp/files/frr.conf

@@ -0,0 +1,9 @@
+frr defaults datacenter
+password zebra
+enable password zebra
+!
+log syslog informational
+log facility local4
+!
+agentx
+!