[telemetry] Rotate streaming telemetry secrets.

dockers/docker-sonic-telemetry/Dockerfile.j2

@@ -26,7 +26,10 @@ RUN apt-get clean -y      && \
     apt-get autoremove -y && \
     rm -rf /debs
 
-COPY ["start.sh", "telemetry.sh", "dialout.sh", "/usr/bin/"]
+# Install the python `inotify` package
+RUN pip3 install inotify
+
+COPY ["start.sh", "telemetry.sh", "dialout.sh", "certificate_rollover_checker", "/usr/bin/"]
 COPY ["telemetry_vars.j2", "/usr/share/sonic/templates/"]
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]

dockers/docker-sonic-telemetry/certificate_rotation_checker

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+
+"""
+certificate_rotation_checker
+
+This script will be leveraged to periodically check whether the certificate and private key
+files of streaming telemetry were rotated by dSMS service or not. The streaming telemetry
+server process will be restarted if the certificate and private key are rotated by dSMS service
+and then updated by the acms agent running in ACMS container.
+"""
+
+import os
+import subprocess
+import sys
+import syslog
+import time
+
+import inotify.adapters
+
+from swsscommon import swsscommon
+
+MAX_RETRY_TIMES = 10
+CERTIFICATE_CHECKING_INTERVAL_SECS = 3600
+
+CREDENTIALS_DIR_PATH = "/etc/sonic/credentials/"
+
+
+def get_command_result(command):
+    """Executes the command and returns the exiting code and resulting output.
+
+    Args:
+        command: A string contains the command to be executed.
+
+    Returns:
+        An integer indicates the exiting code.
+        A string which contains the output of command.
+    """
+    command_stdout = ""
+    command_stderr = ""
+
+    try:
+        proc_instance = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                                         shell=True, universal_newlines=True)
+        command_stdout, command_stderr = proc_instance.communicate()
+    except (OSError, ValueError) as err:
+        syslog.syslog(syslog.LOG_ERR, "Failed to execute the command '{}'. Error: '{}'"
+                      .format(command, err))
+        return 2, command_stderr
+
+    return proc_instance.returncode, command_stdout.strip()
+
+
+def check_telemetry_server_running():
+    """Checkes whether telemetry server process is running.
+
+    Args:
+        None.
+
+    Returns:
+        None.
+    """
+    processes_status_cmd = "supervisorctl status"
+    retry_times = 0
+    is_running = False
+
+    while retry_times <= MAX_RETRY_TIMES:
+        retry_times += 1
+        exit_code, command_stdout = get_command_result(processes_status_cmd)
+        if exit_code != 3:
+            syslog.syslog(syslog.LOG_INFO,
+                          "Failed to get the processes running status in telemetry container and retry after 60 seconds ...")
+            time.sleep(60)
+        else:
+            for line in command_stdout.splitlines():
+                if "telemetry" in line and "RUNNING" in line:
+                    is_running = True
+                    break
+            if is_running:
+                syslog.syslog(syslog.LOG_INFO,
+                              "Telemetry server process is running after certificate and private key were rotated!")
+                break
+
+    if not is_running:
+        syslog.syslog(syslog.LOG_ERR,
+                      "Telemetry server process is not running after certificate and private key were rotated and exiting ...")
+        sys.exit(1)
+
+
+def restart_telemetry_server():
+    """Restarts the telemetry server process by Supervisord and then checks
+    it is actually running.
+
+    Args:
+        None
+
+    Returns:
+        None
+    """
+    restart_telemetry_server_cmd = "supervisorctl restart telemetry"
+    retry_times = 0
+
+    while retry_times <= MAX_RETRY_TIMES:
+        retry_times += 1
+        exit_code, command_stdout = get_command_result(restart_telemetry_server_cmd)
+        if exit_code != 0:
+            syslog.syslog(syslog.LOG_INFO,
+                          "Failed to restart telemetry server process and retry after 60 seconds ...")
+            time.sleep(60)
+        else:
+            break
+
+    if retry_times > MAX_RETRY_TIMES:
+        syslog.syslog(syslog.LOG_ERR,
+                      "Failed to restart telemetry server process after trying '{}' times and exiting ..."
+                      .format(MAX_RETRY_TIMES))
+        sys.exit(2)
+
+    check_telemetry_server_running()
+
+
+def check_certificate_rotated(certificate_file_name, private_key_file_name):
+    """Leverages the 'inotify' module to monitor the file system events under the
+    directory which stores the SONiC credentials and restarts telemetry server
+    process if certificate and private key were rotated.
+
+
+    Args:
+        certificate_file_name: A string indicates the telemetry certificate file name.
+        private_key_file_name: A string indicates the telemetry private key file name.
+
+    Returns:
+        None.
+    """
+    certificate_file_rotated = False
+    private_key_file_rotated = False
+
+    inotify_instance = inotify.adapters.Inotify()
+    inotify_instance.add_watch(CREDENTIALS_DIR_PATH)
+    for event in inotify_instance.event_gen(yield_nones=False):
+        header, event_type, monitoring_path, file_name = event
+        if (file_name == certificate_file_name
+                and ("IN_CREATE" in event_type or "IN_MOVED_TO" in event_type)):
+            certificate_file_rotated = True
+        if (file_name == private_key_file_name
+                and ("IN_CREATE" in event_type or "IN_MOVED_TO" in event_type)):
+            private_key_file_rotated = True
+
+        if certificate_file_rotated and private_key_file_rotated:
+            certificate_file_rotated = False
+            private_key_file_rotated = False
+            syslog.syslog(syslog.LOG_INFO,
+                          "Certificate and private key were rotated and restarting telemetry server process ...")
+            restart_telemetry_server()
+
+            # Wait for specified seconds and then do the next round checking
+            syslog.syslog(syslog.LOG_INFO,
+                          "Sleeping '{}' seconds before doing the next round certifcate rotation checking ..."
+                          .format(CERTIFICATE_CHECKING_INTERVAL_SECS))
+            time.sleep(CERTIFICATE_CHECKING_INTERVAL_SECS)
+
+
+def certificate_rotated_checker():
+    """Checks rotation of certificate and key files and restart streaming telemetry server if necessary.
+
+    Leverages 'inotify' module to check whether the certificate and private key files of
+    streaming telemetry were already rotated by dSMS service and updated by acms agent running
+    in ACMS container. The streaming telemetry server process will be restarted if they were rotated.
+
+    Args:
+        None
+
+    Returns:
+        None
+    """
+    certificate_file_path = ""
+    private_key_file_path = ""
+    certificate_file_name = ""
+    private_key_file_name = ""
+
+    config_db = swsscommon.DBConnector("CONFIG_DB", 0)
+    telemetry_table = swsscommon.Table(config_db, "TELEMETRY")
+    telemetry_table_keys = telemetry_table.getKeys()
+    if "certs" in telemetry_table_keys:
+        certs_info = dict(telemetry_table.get("certs")[1])
+        if "server_crt" in certs_info and "server_key" in certs_info:
+            certificate_file_path = certs_info["server_crt"]
+            private_key_file_path = certs_info["server_key"]
+            syslog.syslog(syslog.LOG_INFO, "Path of certificate file is '{}'".format(certificate_file_path))
+            syslog.syslog(syslog.LOG_INFO, "Path of key file is '{}'".format(private_key_file_path))
+        else:
+            syslog.syslog(syslog.LOG_ERR,
+                          "Failed to retrieve the path of certificate and key file from 'TELEMETRY' table!")
+            sys.exit(3)
+    else:
+        syslog.syslog(syslog.LOG_ERR,
+                      "Failed to retrieve the certificate and key information from 'TELEMETRY' table!")
+        sys.exit(4)
+
+    while True:
+        if not os.path.exists(certificate_file_path) or not os.path.exists(private_key_file_path):
+            syslog.syslog(syslog.LOG_ERR,
+                          "Certificate or key file did not exist on device and sleep '{}' seconds to check again ..."
+                          .format(CERTIFICATE_CHECKING_INTERVAL_SECS))
+            time.sleep(CERTIFICATE_CHECKING_INTERVAL_SECS)
+        else:
+            break
+
+    certificate_file_name = certificate_file_path.strip().split("/")[-1]
+    private_key_file_name = private_key_file_path.strip().split("/")[-1]
+    syslog.syslog(syslog.LOG_INFO, "cer_file_name: {}, key_file_name: {}".format(certificate_file_name, private_key_file_name))
+    if not certificate_file_name or not private_key_file_name:
+        syslog.syslog(syslog.LOG_ERR,
+                      "Failed to retrieve the file name of certificate or private key!")
+        sys.exit(5)
+
+    check_certificate_rotated(certificate_file_name, private_key_file_name)
+
+
+def main():
+    certificate_rotated_checker()
+
+
+if __name__ == "__main__":
+    main()
+    sys.exit(0)

dockers/docker-sonic-telemetry/critical_processes

@@ -1,2 +1,3 @@
 program:telemetry
 program:dialout
+program:certificate_rollover_checker

dockers/docker-sonic-telemetry/supervisord.conf

@@ -58,3 +58,13 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for=telemetry:running
+
+[program:certificate_rollover_checker]
+command=/usr/bin/certificate_rollover_checker
+priority=5
+autostart=false
+autorestart=false
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=start:exited

dockers/docker-sonic-telemetry/telemetry.sh

@@ -19,35 +19,47 @@ CERTS=$(echo $TELEMETRY_VARS | jq -r '.certs')
 TELEMETRY_ARGS=" -logtostderr"
 export CVL_SCHEMA_PATH=/usr/sbin/schema
 
-if [ -n "$CERTS" ]; then
-    SERVER_CRT=$(echo $CERTS | jq -r '.server_crt')
-    SERVER_KEY=$(echo $CERTS | jq -r '.server_key')
-    if [ -z $SERVER_CRT  ] || [ -z $SERVER_KEY  ]; then
-        TELEMETRY_ARGS+=" --insecure"
-    else
-        TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY "
-    fi
+while true
+do
+    if [ -n "$CERTS" ]; then
+        SERVER_CRT=$(echo $CERTS | jq -r '.server_crt')
+        SERVER_KEY=$(echo $CERTS | jq -r '.server_key')
+        CA_CRT=$(echo $CERTS | jq -r '.ca_crt')
 
-    CA_CRT=$(echo $CERTS | jq -r '.ca_crt')
-    if [ ! -z $CA_CRT ]; then
-        TELEMETRY_ARGS+=" --ca_crt $CA_CRT"
-    fi
-elif [ -n "$X509" ]; then
-    SERVER_CRT=$(echo $X509 | jq -r '.server_crt')
-    SERVER_KEY=$(echo $X509 | jq -r '.server_key')
-    if [ -z $SERVER_CRT  ] || [ -z $SERVER_KEY  ]; then
-        TELEMETRY_ARGS+=" --insecure"
+        logger "Trying to retrieve server certificate, key and Root CA certificate ..."
+        logger "The file path of server certificate in CONFIG_DB is: $SERVER_CRT"
+        logger "The file path of server provate key in CONFIG_DB is: $SERVER_KEY"
+        logger "The file path of Root CA certificate in CONFIG_DB is: $CA_CRT"
+
+        if [[ -f $SERVER_CRT && -f $SERVER_KEY && -f $CA_CRT ]]; then
+            logger "Succeeded in retrieving server certificate, key and Root CA certificate."
+            TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY --ca_crt $CA_CRT"
+            break
+        else
+            logger "Failed to retrieve server certificate, key or Root CA certificate!"
+        fi
+    elif [ -n "$X509" ]; then
+        SERVER_CRT=$(echo $X509 | jq -r '.server_crt')
+        SERVER_KEY=$(echo $X509 | jq -r '.server_key')
+        if [ -z $SERVER_CRT  ] || [ -z $SERVER_KEY  ]; then
+            TELEMETRY_ARGS+=" --insecure"
+        else
+            TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY "
+        fi
+
+        CA_CRT=$(echo $X509 | jq -r '.ca_crt')
+        if [ ! -z $CA_CRT ]; then
+            TELEMETRY_ARGS+=" --ca_crt $CA_CRT"
+        fi
+        break
     else
-        TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY "
+        TELEMETRY_ARGS+=" --noTLS"
+        break
     fi
 
-    CA_CRT=$(echo $X509 | jq -r '.ca_crt')
-    if [ ! -z $CA_CRT ]; then
-        TELEMETRY_ARGS+=" --ca_crt $CA_CRT"
-    fi
-else
-    TELEMETRY_ARGS+=" --noTLS"
-fi
+    logger "Sleeping 3600 seconds and checks the existence of secrets again ..."
+    sleep 3600
+done
 
 # If no configuration entry exists for TELEMETRY, create one default port
 if [ -z "$GNMI" ]; then

src/sonic-config-engine/minigraph.py

@@ -1633,9 +1633,9 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw
             'log_level': '2'
         },
         'certs': {
-            'server_crt': '/etc/sonic/telemetry/streamingtelemetryserver.cer',
-            'server_key': '/etc/sonic/telemetry/streamingtelemetryserver.key',
-            'ca_crt': '/etc/sonic/telemetry/dsmsroot.cer'
+            'server_crt': '/etc/sonic/credentials/streamingtelemetryserver.cer',
+            'server_key': '/etc/sonic/credentials/streamingtelemetryserver.key',
+            'ca_crt': '/etc/sonic/credentials/dsmsroot.cer'
         }
     }
     results['RESTAPI'] = {