From a921b8e87a5f450058f906659b62030add613b23 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 20 Dec 2021 16:28:33 -0800 Subject: [PATCH 01/21] [telemetry] Roll over streaming telemetry secrets by ACMS. Signed-off-by: Yong Zhao --- dockers/docker-sonic-telemetry/Dockerfile.j2 | 2 +- .../certificate_rollover_checker | 139 ++++++++++++++++++ .../docker-sonic-telemetry/critical_processes | 1 + .../docker-sonic-telemetry/supervisord.conf | 13 +- dockers/docker-sonic-telemetry/telemetry.sh | 84 +++++++---- src/sonic-config-engine/minigraph.py | 3 + 6 files changed, 211 insertions(+), 31 deletions(-) create mode 100755 dockers/docker-sonic-telemetry/certificate_rollover_checker diff --git a/dockers/docker-sonic-telemetry/Dockerfile.j2 b/dockers/docker-sonic-telemetry/Dockerfile.j2 index 88ff94318208..3f8d92f10411 100644 --- a/dockers/docker-sonic-telemetry/Dockerfile.j2 +++ b/dockers/docker-sonic-telemetry/Dockerfile.j2 @@ -26,7 +26,7 @@ RUN apt-get clean -y && \ apt-get autoremove -y && \ rm -rf /debs -COPY ["start.sh", "telemetry.sh", "dialout.sh", "/usr/bin/"] +COPY ["start.sh", "telemetry.sh", "dialout.sh", "certificate_rollover_checker", "/usr/bin/"] COPY ["telemetry_vars.j2", "/usr/share/sonic/templates/"] COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] COPY ["files/supervisor-proc-exit-listener", "/usr/bin"] diff --git a/dockers/docker-sonic-telemetry/certificate_rollover_checker b/dockers/docker-sonic-telemetry/certificate_rollover_checker new file mode 100755 index 000000000000..beeb88f5ff7f --- /dev/null +++ b/dockers/docker-sonic-telemetry/certificate_rollover_checker @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 + +""" +certificate_rollover_checker + +This script will be leveraged to periodically check whether the certificate and private key +files of streaming telemetry were rolled over by dSMS service or not. The streaming telemetry +container will be restarted if the certificate and private key are rolled over by dSMS service +and then updated by ACMS agent running in ACMS container. +""" + +import os +import signal +import sys +import syslog +import time + +from swsscommon import swsscommon + +CERTIFICATE_CHECKING_INTERVAL_SECS = 3600 + + +def get_file_last_mod_time(file_path): + """Gets the last modification time of a specific file. + + Args: + file_path: A string represents the file path. + + Returns: + last_mod_time: A float number in seconds represents the last moditification time of file + since epoch. + """ + last_mod_time = 0.0 + + try: + last_mod_time = os.path.getmtime(file_path) + except OSError as error: + syslog.syslog(syslog.LOG_ERR, + "Could not get last modification time of the file and error message is '{}'.".format(error)) + sys.exit(1) + + return last_mod_time + + +def restart_streaming_telemetry(): + """Restarts the streaming telemetry container by terminating the root process. + + Args: + None + + Returns: + None + """ + root_process_pid = os.getppid() + syslog.syslog(syslog.LOG_INFO, + "Restarting streaming telemetry service by terminating the process with pid: '{}'".format(root_process_pid)) + os.kill(root_process_pid, signal.SIGTERM) + sys.exit(0) + + +def certificate_rollover_check(): + """Checks certificate and key files and restart streaming telemetry contianer if necessary. + + Checks the last modification time of certificate and private key files of streaming telemetry + to see whether they were already rolled over by dSMS service and updated by ACMS agent running + in ACMS container. The streaming telemetry container will be restarted if they were rolled over. + + Args: + None + + Returns: + None + """ + certificate_path = "" + private_key_path = "" + certificate_last_mod_time = 0 + private_key_last_mod_time = 0 + + config_db = swsscommon.DBConnector("CONFIG_DB", 0) + telemetry_table = swsscommon.Table(config_db, "TELEMETRY") + telemetry_table_keys = telemetry_table.getKeys() + if "certs" in telemetry_table_keys: + certs_info = dict(telemetry_table.get("certs")[1]) + if "server_crt_acms" in certs_info and "server_key_acms" in certs_info: + certificate_path = certs_info["server_crt_acms"] + private_key_path = certs_info["server_key_acms"] + syslog.syslog(syslog.LOG_INFO, "Path of certificate file is '{}'".format(certificate_path)) + syslog.syslog(syslog.LOG_INFO, "Path of key file is '{}'".format(private_key_path)) + else: + syslog.syslog(syslog.LOG_ERR, + "Failed to retrieve the path of certificate and key file from 'TELEMETRY' table!") + sys.exit(2) + else: + syslog.syslog(syslog.LOG_ERR, + "Failed to retrieve the certificate information from 'TELEMETRY' table!") + sys.exit(3) + + while True: + if not os.path.exists(certificate_path) or not os.path.exists(private_key_path): + syslog.syslog(syslog.LOG_ERR, + "Certificate or key file did not exist on device and sleep '{}' seconds to check again {} ...".format(CERTIFICATE_CHECKING_INTERVAL_SECS)) + time.sleep(CERTIFICATE_CHECKING_INTERVAL_SECS) + else: + break + + certificate_last_mod_time = get_file_last_mod_time(certificate_path) + private_key_last_mod_time = get_file_last_mod_time(private_key_path) + + while True: + certificate_mod_time = get_file_last_mod_time(certificate_path) + private_key_mod_time = get_file_last_mod_time(private_key_path) + syslog.syslog(syslog.LOG_INFO, + "Last modification time of certificate file is: '{}'".format(time.ctime(certificate_last_mod_time))) + syslog.syslog(syslog.LOG_INFO, + "Last modification time of key file is: '{}'".format(time.ctime(private_key_last_mod_time))) + + if (certificate_mod_time > certificate_last_mod_time + or private_key_mod_time > private_key_last_mod_time): + syslog.syslog(syslog.LOG_INFO, + "Last modification time of certificate file is changed to '{}': ".format(time.ctime(certificate_mod_time))) + syslog.syslog(syslog.LOG_INFO, + "Last modification time of key file is changed to '{}': ".format(time.ctime(private_key_mod_time))) + syslog.syslog(syslog.LOG_INFO, + "Secrets were rolled over and restarting streaming telemetry service ...") + restart_streaming_telemetry() + + # Wait for specified seconds and then do the next round checking + syslog.syslog(syslog.LOG_INFO, + "Sleeping '{}' seconds before doing the next round rollover checking ...".format(CERTIFICATE_CHECKING_INTERVAL_SECS)) + time.sleep(CERTIFICATE_CHECKING_INTERVAL_SECS) + + +def main(): + certificate_rollover_check() + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/dockers/docker-sonic-telemetry/critical_processes b/dockers/docker-sonic-telemetry/critical_processes index 612a94d9edac..810415d53fc5 100644 --- a/dockers/docker-sonic-telemetry/critical_processes +++ b/dockers/docker-sonic-telemetry/critical_processes @@ -1,2 +1,3 @@ program:telemetry program:dialout +program:certificate_rollover_checker diff --git a/dockers/docker-sonic-telemetry/supervisord.conf b/dockers/docker-sonic-telemetry/supervisord.conf index 5ab0e3ca59da..53962aaceb96 100644 --- a/dockers/docker-sonic-telemetry/supervisord.conf +++ b/dockers/docker-sonic-telemetry/supervisord.conf @@ -10,14 +10,13 @@ autorestart=unexpected startretries=0 exitcodes=0,3 events=PROCESS_STATE -buffer_size=1024 +buffer_size=50 [eventlistener:supervisor-proc-exit-listener] command=/usr/bin/supervisor-proc-exit-listener --container-name telemetry events=PROCESS_STATE_EXITED,PROCESS_STATE_RUNNING autostart=true autorestart=false -buffer_size=1024 [program:rsyslogd] command=/usr/sbin/rsyslogd -n -iNONE @@ -58,3 +57,13 @@ stdout_logfile=syslog stderr_logfile=syslog dependent_startup=true dependent_startup_wait_for=telemetry:running + +[program:certificate_rollover_checker] +command=/usr/bin/certificate_rollover_checker +priority=5 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog +dependent_startup=true +dependent_startup_wait_for=start:exited diff --git a/dockers/docker-sonic-telemetry/telemetry.sh b/dockers/docker-sonic-telemetry/telemetry.sh index 1f92657e3b8f..38e7fe3d24ff 100755 --- a/dockers/docker-sonic-telemetry/telemetry.sh +++ b/dockers/docker-sonic-telemetry/telemetry.sh @@ -19,39 +19,67 @@ CERTS=$(echo $TELEMETRY_VARS | jq -r '.certs') TELEMETRY_ARGS=" -logtostderr" export CVL_SCHEMA_PATH=/usr/sbin/schema -if [ -n "$CERTS" ]; then - SERVER_CRT=$(echo $CERTS | jq -r '.server_crt') - SERVER_KEY=$(echo $CERTS | jq -r '.server_key') - if [ -z $SERVER_CRT ] || [ -z $SERVER_KEY ]; then - TELEMETRY_ARGS+=" --insecure" - else - TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY " - fi +while true +do + if [ -n "$CERTS" ]; then + SERVER_CRT=$(echo $CERTS | jq -r '.server_crt') + SERVER_KEY=$(echo $CERTS | jq -r '.server_key') + CA_CRT=$(echo $CERTS | jq -r '.ca_crt') + + logger "Trying to retrieve server certificate, key and Root CA certificate managed by HwProxy ..." + logger "The file path of server certificate in CONFIG_DB is: $SERVER_CRT" + logger "The file path of server provate key in CONFIG_DB is: $SERVER_KEY" + logger "The file path of Root CA certificate in CONFIG_DB is: $CA_CRT" - CA_CRT=$(echo $CERTS | jq -r '.ca_crt') - if [ ! -z $CA_CRT ]; then - TELEMETRY_ARGS+=" --ca_crt $CA_CRT" - fi -elif [ -n "$X509" ]; then - SERVER_CRT=$(echo $X509 | jq -r '.server_crt') - SERVER_KEY=$(echo $X509 | jq -r '.server_key') - if [ -z $SERVER_CRT ] || [ -z $SERVER_KEY ]; then - TELEMETRY_ARGS+=" --insecure" - else - TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY " - fi + if [[ -f $SERVER_CRT && -f $SERVER_KEY && -f $CA_CRT ]]; then + logger "Succeeded in retrieving server certificate, key and Root CA certificate from HwProxy." + TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY --ca_crt $CA_CRT" + break + else + logger "Failed to retrieve server certificate, key or Root CA certificate from HwProxy!" + fi - CA_CRT=$(echo $X509 | jq -r '.ca_crt') - if [ ! -z $CA_CRT ]; then - TELEMETRY_ARGS+=" --ca_crt $CA_CRT" - fi -else - TELEMETRY_ARGS+=" --noTLS" -fi + SERVER_CRT_ACMS=$(echo $CERTS | jq -r '.server_crt_acms') + SERVER_KEY_ACMS=$(echo $CERTS | jq -r '.server_key_acms') + CA_CRT_ACMS=$(echo $CERTS | jq -r '.ca_crt_acms') + + logger "Trying to retrieve server certificate, key and Root CA certificate managed by ACMS ..." + logger "The file path of server certificate in CONFIG_DB is: $SERVER_CRT_ACMS" + logger "The file path of server private key in CONFIG_DB is: $SERVER_KEY_ACMS" + logger "The file path of Root CA certificate in CONFIG_DB is: $CA_CRT_ACMS" + + if [[ -f $SERVER_CRT_ACMS && -f $SERVER_KEY_ACMS && -f $CA_CRT_ACMS ]]; then + logger "Succeeded in retrieving the certificate, key and Root CA certificate from ACMS." + continue + else + logger "Failed to retrieve server certificate, key or Root CA certificate from ACMS!" + fi + elif [ -n "$X509" ]; then + SERVER_CRT=$(echo $X509 | jq -r '.server_crt') + SERVER_KEY=$(echo $X509 | jq -r '.server_key') + if [ -z $SERVER_CRT ] || [ -z $SERVER_KEY ]; then + TELEMETRY_ARGS+=" --insecure" + else + TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY " + fi + + CA_CRT=$(echo $X509 | jq -r '.ca_crt') + if [ ! -z $CA_CRT ]; then + TELEMETRY_ARGS+=" --ca_crt $CA_CRT" + fi + break + else + TELEMETRY_ARGS+=" --noTLS" + break + fi + + logger "Sleeping 3600 seconds and checks the existence of secrets again ..." + sleep 3600 +done # If no configuration entry exists for TELEMETRY, create one default port if [ -z "$GNMI" ]; then - PORT=8080 + PORT=8080 else PORT=$(echo $GNMI | jq -r '.port') fi diff --git a/src/sonic-config-engine/minigraph.py b/src/sonic-config-engine/minigraph.py index 8f8c90fab584..44331ad1cb1c 100644 --- a/src/sonic-config-engine/minigraph.py +++ b/src/sonic-config-engine/minigraph.py @@ -1636,6 +1636,9 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw 'server_crt': '/etc/sonic/telemetry/streamingtelemetryserver.cer', 'server_key': '/etc/sonic/telemetry/streamingtelemetryserver.key', 'ca_crt': '/etc/sonic/telemetry/dsmsroot.cer' + 'server_crt_acms': '/etc/sonic/credentials/streamingtelemetryserver.cer', + 'server_key_acms': '/etc/sonic/credentials/streamingtelemetryserver.key', + 'ca_crt_acms': '/etc/sonic/credentials/dsmsroot.cer' } } results['RESTAPI'] = { From 7da4005b182aaa82293c161f7447e3ed4f00a487 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 20 Dec 2021 16:37:18 -0800 Subject: [PATCH 02/21] [telemetry] Fix the indent in the script `telemetry.sh`. Signed-off-by: Yong Zhao --- dockers/docker-sonic-telemetry/telemetry.sh | 60 ++++++++++----------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/dockers/docker-sonic-telemetry/telemetry.sh b/dockers/docker-sonic-telemetry/telemetry.sh index 38e7fe3d24ff..6430528b771f 100755 --- a/dockers/docker-sonic-telemetry/telemetry.sh +++ b/dockers/docker-sonic-telemetry/telemetry.sh @@ -25,52 +25,52 @@ do SERVER_CRT=$(echo $CERTS | jq -r '.server_crt') SERVER_KEY=$(echo $CERTS | jq -r '.server_key') CA_CRT=$(echo $CERTS | jq -r '.ca_crt') - + logger "Trying to retrieve server certificate, key and Root CA certificate managed by HwProxy ..." logger "The file path of server certificate in CONFIG_DB is: $SERVER_CRT" logger "The file path of server provate key in CONFIG_DB is: $SERVER_KEY" logger "The file path of Root CA certificate in CONFIG_DB is: $CA_CRT" - if [[ -f $SERVER_CRT && -f $SERVER_KEY && -f $CA_CRT ]]; then - logger "Succeeded in retrieving server certificate, key and Root CA certificate from HwProxy." - TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY --ca_crt $CA_CRT" - break + if [[ -f $SERVER_CRT && -f $SERVER_KEY && -f $CA_CRT ]]; then + logger "Succeeded in retrieving server certificate, key and Root CA certificate from HwProxy." + TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY --ca_crt $CA_CRT" + break else - logger "Failed to retrieve server certificate, key or Root CA certificate from HwProxy!" + logger "Failed to retrieve server certificate, key or Root CA certificate from HwProxy!" fi - SERVER_CRT_ACMS=$(echo $CERTS | jq -r '.server_crt_acms') - SERVER_KEY_ACMS=$(echo $CERTS | jq -r '.server_key_acms') - CA_CRT_ACMS=$(echo $CERTS | jq -r '.ca_crt_acms') - - logger "Trying to retrieve server certificate, key and Root CA certificate managed by ACMS ..." - logger "The file path of server certificate in CONFIG_DB is: $SERVER_CRT_ACMS" - logger "The file path of server private key in CONFIG_DB is: $SERVER_KEY_ACMS" - logger "The file path of Root CA certificate in CONFIG_DB is: $CA_CRT_ACMS" - - if [[ -f $SERVER_CRT_ACMS && -f $SERVER_KEY_ACMS && -f $CA_CRT_ACMS ]]; then - logger "Succeeded in retrieving the certificate, key and Root CA certificate from ACMS." - continue - else - logger "Failed to retrieve server certificate, key or Root CA certificate from ACMS!" + SERVER_CRT_ACMS=$(echo $CERTS | jq -r '.server_crt_acms') + SERVER_KEY_ACMS=$(echo $CERTS | jq -r '.server_key_acms') + CA_CRT_ACMS=$(echo $CERTS | jq -r '.ca_crt_acms') + + logger "Trying to retrieve server certificate, key and Root CA certificate managed by ACMS ..." + logger "The file path of server certificate in CONFIG_DB is: $SERVER_CRT_ACMS" + logger "The file path of server private key in CONFIG_DB is: $SERVER_KEY_ACMS" + logger "The file path of Root CA certificate in CONFIG_DB is: $CA_CRT_ACMS" + + if [[ -f $SERVER_CRT_ACMS && -f $SERVER_KEY_ACMS && -f $CA_CRT_ACMS ]]; then + logger "Succeeded in retrieving the certificate, key and Root CA certificate from ACMS." + continue + else + logger "Failed to retrieve server certificate, key or Root CA certificate from ACMS!" fi elif [ -n "$X509" ]; then - SERVER_CRT=$(echo $X509 | jq -r '.server_crt') - SERVER_KEY=$(echo $X509 | jq -r '.server_key') - if [ -z $SERVER_CRT ] || [ -z $SERVER_KEY ]; then - TELEMETRY_ARGS+=" --insecure" + SERVER_CRT=$(echo $X509 | jq -r '.server_crt') + SERVER_KEY=$(echo $X509 | jq -r '.server_key') + if [ -z $SERVER_CRT ] || [ -z $SERVER_KEY ]; then + TELEMETRY_ARGS+=" --insecure" else - TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY " + TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY " fi - CA_CRT=$(echo $X509 | jq -r '.ca_crt') - if [ ! -z $CA_CRT ]; then - TELEMETRY_ARGS+=" --ca_crt $CA_CRT" + CA_CRT=$(echo $X509 | jq -r '.ca_crt') + if [ ! -z $CA_CRT ]; then + TELEMETRY_ARGS+=" --ca_crt $CA_CRT" fi break else - TELEMETRY_ARGS+=" --noTLS" - break + TELEMETRY_ARGS+=" --noTLS" + break fi logger "Sleeping 3600 seconds and checks the existence of secrets again ..." From ede36b97667861385b40d9add430510ef424a1a8 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 20 Dec 2021 16:44:28 -0800 Subject: [PATCH 03/21] [telemetry] Fix the indent issue in the script `telemetry.sh`. Signed-off-by: Yong Zhao --- dockers/docker-sonic-telemetry/telemetry.sh | 66 ++++++++++----------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/dockers/docker-sonic-telemetry/telemetry.sh b/dockers/docker-sonic-telemetry/telemetry.sh index 6430528b771f..47800c73e105 100755 --- a/dockers/docker-sonic-telemetry/telemetry.sh +++ b/dockers/docker-sonic-telemetry/telemetry.sh @@ -32,54 +32,54 @@ do logger "The file path of Root CA certificate in CONFIG_DB is: $CA_CRT" if [[ -f $SERVER_CRT && -f $SERVER_KEY && -f $CA_CRT ]]; then - logger "Succeeded in retrieving server certificate, key and Root CA certificate from HwProxy." - TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY --ca_crt $CA_CRT" - break - else - logger "Failed to retrieve server certificate, key or Root CA certificate from HwProxy!" - fi + logger "Succeeded in retrieving server certificate, key and Root CA certificate from HwProxy." + TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY --ca_crt $CA_CRT" + break + else + logger "Failed to retrieve server certificate, key or Root CA certificate from HwProxy!" + fi SERVER_CRT_ACMS=$(echo $CERTS | jq -r '.server_crt_acms') - SERVER_KEY_ACMS=$(echo $CERTS | jq -r '.server_key_acms') - CA_CRT_ACMS=$(echo $CERTS | jq -r '.ca_crt_acms') + SERVER_KEY_ACMS=$(echo $CERTS | jq -r '.server_key_acms') + CA_CRT_ACMS=$(echo $CERTS | jq -r '.ca_crt_acms') logger "Trying to retrieve server certificate, key and Root CA certificate managed by ACMS ..." - logger "The file path of server certificate in CONFIG_DB is: $SERVER_CRT_ACMS" - logger "The file path of server private key in CONFIG_DB is: $SERVER_KEY_ACMS" - logger "The file path of Root CA certificate in CONFIG_DB is: $CA_CRT_ACMS" + logger "The file path of server certificate in CONFIG_DB is: $SERVER_CRT_ACMS" + logger "The file path of server private key in CONFIG_DB is: $SERVER_KEY_ACMS" + logger "The file path of Root CA certificate in CONFIG_DB is: $CA_CRT_ACMS" if [[ -f $SERVER_CRT_ACMS && -f $SERVER_KEY_ACMS && -f $CA_CRT_ACMS ]]; then - logger "Succeeded in retrieving the certificate, key and Root CA certificate from ACMS." - continue + logger "Succeeded in retrieving the certificate, key and Root CA certificate from ACMS." + continue else - logger "Failed to retrieve server certificate, key or Root CA certificate from ACMS!" - fi + logger "Failed to retrieve server certificate, key or Root CA certificate from ACMS!" + fi elif [ -n "$X509" ]; then - SERVER_CRT=$(echo $X509 | jq -r '.server_crt') - SERVER_KEY=$(echo $X509 | jq -r '.server_key') + SERVER_CRT=$(echo $X509 | jq -r '.server_crt') + SERVER_KEY=$(echo $X509 | jq -r '.server_key') if [ -z $SERVER_CRT ] || [ -z $SERVER_KEY ]; then - TELEMETRY_ARGS+=" --insecure" - else - TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY " - fi + TELEMETRY_ARGS+=" --insecure" + else + TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY " + fi CA_CRT=$(echo $X509 | jq -r '.ca_crt') - if [ ! -z $CA_CRT ]; then - TELEMETRY_ARGS+=" --ca_crt $CA_CRT" - fi - break - else - TELEMETRY_ARGS+=" --noTLS" - break - fi - - logger "Sleeping 3600 seconds and checks the existence of secrets again ..." - sleep 3600 + if [ ! -z $CA_CRT ]; then + TELEMETRY_ARGS+=" --ca_crt $CA_CRT" + fi + break + else + TELEMETRY_ARGS+=" --noTLS" + break + fi + + logger "Sleeping 3600 seconds and checks the existence of secrets again ..." + sleep 3600 done # If no configuration entry exists for TELEMETRY, create one default port if [ -z "$GNMI" ]; then - PORT=8080 + PORT=8080 else PORT=$(echo $GNMI | jq -r '.port') fi From 7c3c7d304976d2c9b339b5fb2883c1b60be0c857 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 20 Dec 2021 16:46:58 -0800 Subject: [PATCH 04/21] [telemetry] Add the field of `buffer_size=1024` in supervisord configuration file. Signed-off-by: Yong Zhao --- dockers/docker-sonic-telemetry/supervisord.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dockers/docker-sonic-telemetry/supervisord.conf b/dockers/docker-sonic-telemetry/supervisord.conf index 53962aaceb96..aeffc201280f 100644 --- a/dockers/docker-sonic-telemetry/supervisord.conf +++ b/dockers/docker-sonic-telemetry/supervisord.conf @@ -10,13 +10,14 @@ autorestart=unexpected startretries=0 exitcodes=0,3 events=PROCESS_STATE -buffer_size=50 +buffer_size=1024 [eventlistener:supervisor-proc-exit-listener] command=/usr/bin/supervisor-proc-exit-listener --container-name telemetry events=PROCESS_STATE_EXITED,PROCESS_STATE_RUNNING autostart=true autorestart=false +buffer_size=1024 [program:rsyslogd] command=/usr/sbin/rsyslogd -n -iNONE From 90d9d03b25d650635920ab9cf2019998c69b9b08 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 20 Dec 2021 16:50:31 -0800 Subject: [PATCH 05/21] [telemetry] Fix the indent issue. Signed-off-by: Yong Zhao --- dockers/docker-sonic-telemetry/telemetry.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dockers/docker-sonic-telemetry/telemetry.sh b/dockers/docker-sonic-telemetry/telemetry.sh index 47800c73e105..a5d6c072a7f1 100755 --- a/dockers/docker-sonic-telemetry/telemetry.sh +++ b/dockers/docker-sonic-telemetry/telemetry.sh @@ -54,10 +54,10 @@ do else logger "Failed to retrieve server certificate, key or Root CA certificate from ACMS!" fi - elif [ -n "$X509" ]; then + elif [ -n "$X509" ]; then SERVER_CRT=$(echo $X509 | jq -r '.server_crt') SERVER_KEY=$(echo $X509 | jq -r '.server_key') - if [ -z $SERVER_CRT ] || [ -z $SERVER_KEY ]; then + if [ -z $SERVER_CRT ] || [ -z $SERVER_KEY ]; then TELEMETRY_ARGS+=" --insecure" else TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY " From 0bfcec90780a9913e02092fe75ee1b7bc5c813e3 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 20 Dec 2021 17:26:48 -0800 Subject: [PATCH 06/21] [telemetry] Fix a syntax error. Signed-off-by: Yong Zhao --- .../docker-sonic-telemetry/certificate_rollover_checker | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dockers/docker-sonic-telemetry/certificate_rollover_checker b/dockers/docker-sonic-telemetry/certificate_rollover_checker index beeb88f5ff7f..393263e6839f 100755 --- a/dockers/docker-sonic-telemetry/certificate_rollover_checker +++ b/dockers/docker-sonic-telemetry/certificate_rollover_checker @@ -21,9 +21,7 @@ CERTIFICATE_CHECKING_INTERVAL_SECS = 3600 def get_file_last_mod_time(file_path): - """Gets the last modification time of a specific file. - - Args: + """Gets the last modification time of a specific file. Args: file_path: A string represents the file path. Returns: @@ -55,7 +53,6 @@ def restart_streaming_telemetry(): syslog.syslog(syslog.LOG_INFO, "Restarting streaming telemetry service by terminating the process with pid: '{}'".format(root_process_pid)) os.kill(root_process_pid, signal.SIGTERM) - sys.exit(0) def certificate_rollover_check(): @@ -98,7 +95,7 @@ def certificate_rollover_check(): while True: if not os.path.exists(certificate_path) or not os.path.exists(private_key_path): syslog.syslog(syslog.LOG_ERR, - "Certificate or key file did not exist on device and sleep '{}' seconds to check again {} ...".format(CERTIFICATE_CHECKING_INTERVAL_SECS)) + "Certificate or key file did not exist on device and sleep '{}' seconds to check again ...".format(CERTIFICATE_CHECKING_INTERVAL_SECS)) time.sleep(CERTIFICATE_CHECKING_INTERVAL_SECS) else: break From 9db5b04a66998cbb06865873aa92ccd375b352fe Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 20 Dec 2021 17:58:50 -0800 Subject: [PATCH 07/21] [telemetry] Fix a syntax error. Signed-off-by: Yong Zhao --- src/sonic-config-engine/minigraph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-config-engine/minigraph.py b/src/sonic-config-engine/minigraph.py index 44331ad1cb1c..21a91b380563 100644 --- a/src/sonic-config-engine/minigraph.py +++ b/src/sonic-config-engine/minigraph.py @@ -1635,7 +1635,7 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw 'certs': { 'server_crt': '/etc/sonic/telemetry/streamingtelemetryserver.cer', 'server_key': '/etc/sonic/telemetry/streamingtelemetryserver.key', - 'ca_crt': '/etc/sonic/telemetry/dsmsroot.cer' + 'ca_crt': '/etc/sonic/telemetry/dsmsroot.cer', 'server_crt_acms': '/etc/sonic/credentials/streamingtelemetryserver.cer', 'server_key_acms': '/etc/sonic/credentials/streamingtelemetryserver.key', 'ca_crt_acms': '/etc/sonic/credentials/dsmsroot.cer' From b1844b75c01634a9d4bde2548abd05967472f301 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 6 Jan 2022 12:59:04 -0800 Subject: [PATCH 08/21] [telemetry] Install `inotify` module. Signed-off-by: Yong Zhao --- files/build_templates/sonic_debian_extension.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 6cd852e6e8ea..7e9c0f8c2853 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -127,6 +127,9 @@ sudo rm -rf $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY3_WHEEL_NAME # Install Python module for psutil sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install psutil +# Install Python module for inotify +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install inotify + # Install Python module for ipaddr sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install ipaddr From e28f64d307367d5895ebcda1a783b2c84974b77a Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Fri, 7 Jan 2022 12:04:24 -0800 Subject: [PATCH 09/21] [telemetry] Install the inotify. Signed-off-by: Yong Zhao --- files/build_templates/sonic_debian_extension.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 7e9c0f8c2853..453481d1ca56 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -130,6 +130,7 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install psutil # Install Python module for inotify sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install inotify + # Install Python module for ipaddr sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install ipaddr From 536fc4cbc0f5afb69b7abec6a2d0c18a106ad1e8 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Sun, 9 Jan 2022 17:01:43 -0800 Subject: [PATCH 10/21] [telemetry] Install the `inotify` in telemetry container. Signed-off-by: Yong Zhao --- dockers/docker-sonic-telemetry/Dockerfile.j2 | 3 +++ files/build_templates/sonic_debian_extension.j2 | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dockers/docker-sonic-telemetry/Dockerfile.j2 b/dockers/docker-sonic-telemetry/Dockerfile.j2 index 3f8d92f10411..0cdfa226a677 100644 --- a/dockers/docker-sonic-telemetry/Dockerfile.j2 +++ b/dockers/docker-sonic-telemetry/Dockerfile.j2 @@ -26,6 +26,9 @@ RUN apt-get clean -y && \ apt-get autoremove -y && \ rm -rf /debs +# Install the python `inotify` package +RUN pip3 install inotify + COPY ["start.sh", "telemetry.sh", "dialout.sh", "certificate_rollover_checker", "/usr/bin/"] COPY ["telemetry_vars.j2", "/usr/share/sonic/templates/"] COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 453481d1ca56..6cd852e6e8ea 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -127,10 +127,6 @@ sudo rm -rf $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY3_WHEEL_NAME # Install Python module for psutil sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install psutil -# Install Python module for inotify -sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install inotify - - # Install Python module for ipaddr sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install ipaddr From 17c4d60a90bfe1afe0ca90e5d48e3b52a56ff312 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 10 Jan 2022 11:49:38 -0800 Subject: [PATCH 11/21] [telemetry] Use `inotify` module to monitor the rotation of certificate and private key. Signed-off-by: Yong Zhao --- .../certificate_rollover_checker | 136 ----------- .../certificate_rotation_checker | 226 ++++++++++++++++++ dockers/docker-sonic-telemetry/telemetry.sh | 22 +- src/sonic-config-engine/minigraph.py | 9 +- 4 files changed, 232 insertions(+), 161 deletions(-) delete mode 100755 dockers/docker-sonic-telemetry/certificate_rollover_checker create mode 100755 dockers/docker-sonic-telemetry/certificate_rotation_checker diff --git a/dockers/docker-sonic-telemetry/certificate_rollover_checker b/dockers/docker-sonic-telemetry/certificate_rollover_checker deleted file mode 100755 index 393263e6839f..000000000000 --- a/dockers/docker-sonic-telemetry/certificate_rollover_checker +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python3 - -""" -certificate_rollover_checker - -This script will be leveraged to periodically check whether the certificate and private key -files of streaming telemetry were rolled over by dSMS service or not. The streaming telemetry -container will be restarted if the certificate and private key are rolled over by dSMS service -and then updated by ACMS agent running in ACMS container. -""" - -import os -import signal -import sys -import syslog -import time - -from swsscommon import swsscommon - -CERTIFICATE_CHECKING_INTERVAL_SECS = 3600 - - -def get_file_last_mod_time(file_path): - """Gets the last modification time of a specific file. Args: - file_path: A string represents the file path. - - Returns: - last_mod_time: A float number in seconds represents the last moditification time of file - since epoch. - """ - last_mod_time = 0.0 - - try: - last_mod_time = os.path.getmtime(file_path) - except OSError as error: - syslog.syslog(syslog.LOG_ERR, - "Could not get last modification time of the file and error message is '{}'.".format(error)) - sys.exit(1) - - return last_mod_time - - -def restart_streaming_telemetry(): - """Restarts the streaming telemetry container by terminating the root process. - - Args: - None - - Returns: - None - """ - root_process_pid = os.getppid() - syslog.syslog(syslog.LOG_INFO, - "Restarting streaming telemetry service by terminating the process with pid: '{}'".format(root_process_pid)) - os.kill(root_process_pid, signal.SIGTERM) - - -def certificate_rollover_check(): - """Checks certificate and key files and restart streaming telemetry contianer if necessary. - - Checks the last modification time of certificate and private key files of streaming telemetry - to see whether they were already rolled over by dSMS service and updated by ACMS agent running - in ACMS container. The streaming telemetry container will be restarted if they were rolled over. - - Args: - None - - Returns: - None - """ - certificate_path = "" - private_key_path = "" - certificate_last_mod_time = 0 - private_key_last_mod_time = 0 - - config_db = swsscommon.DBConnector("CONFIG_DB", 0) - telemetry_table = swsscommon.Table(config_db, "TELEMETRY") - telemetry_table_keys = telemetry_table.getKeys() - if "certs" in telemetry_table_keys: - certs_info = dict(telemetry_table.get("certs")[1]) - if "server_crt_acms" in certs_info and "server_key_acms" in certs_info: - certificate_path = certs_info["server_crt_acms"] - private_key_path = certs_info["server_key_acms"] - syslog.syslog(syslog.LOG_INFO, "Path of certificate file is '{}'".format(certificate_path)) - syslog.syslog(syslog.LOG_INFO, "Path of key file is '{}'".format(private_key_path)) - else: - syslog.syslog(syslog.LOG_ERR, - "Failed to retrieve the path of certificate and key file from 'TELEMETRY' table!") - sys.exit(2) - else: - syslog.syslog(syslog.LOG_ERR, - "Failed to retrieve the certificate information from 'TELEMETRY' table!") - sys.exit(3) - - while True: - if not os.path.exists(certificate_path) or not os.path.exists(private_key_path): - syslog.syslog(syslog.LOG_ERR, - "Certificate or key file did not exist on device and sleep '{}' seconds to check again ...".format(CERTIFICATE_CHECKING_INTERVAL_SECS)) - time.sleep(CERTIFICATE_CHECKING_INTERVAL_SECS) - else: - break - - certificate_last_mod_time = get_file_last_mod_time(certificate_path) - private_key_last_mod_time = get_file_last_mod_time(private_key_path) - - while True: - certificate_mod_time = get_file_last_mod_time(certificate_path) - private_key_mod_time = get_file_last_mod_time(private_key_path) - syslog.syslog(syslog.LOG_INFO, - "Last modification time of certificate file is: '{}'".format(time.ctime(certificate_last_mod_time))) - syslog.syslog(syslog.LOG_INFO, - "Last modification time of key file is: '{}'".format(time.ctime(private_key_last_mod_time))) - - if (certificate_mod_time > certificate_last_mod_time - or private_key_mod_time > private_key_last_mod_time): - syslog.syslog(syslog.LOG_INFO, - "Last modification time of certificate file is changed to '{}': ".format(time.ctime(certificate_mod_time))) - syslog.syslog(syslog.LOG_INFO, - "Last modification time of key file is changed to '{}': ".format(time.ctime(private_key_mod_time))) - syslog.syslog(syslog.LOG_INFO, - "Secrets were rolled over and restarting streaming telemetry service ...") - restart_streaming_telemetry() - - # Wait for specified seconds and then do the next round checking - syslog.syslog(syslog.LOG_INFO, - "Sleeping '{}' seconds before doing the next round rollover checking ...".format(CERTIFICATE_CHECKING_INTERVAL_SECS)) - time.sleep(CERTIFICATE_CHECKING_INTERVAL_SECS) - - -def main(): - certificate_rollover_check() - - -if __name__ == "__main__": - main() - sys.exit(0) diff --git a/dockers/docker-sonic-telemetry/certificate_rotation_checker b/dockers/docker-sonic-telemetry/certificate_rotation_checker new file mode 100755 index 000000000000..246ec6bb3bcc --- /dev/null +++ b/dockers/docker-sonic-telemetry/certificate_rotation_checker @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 + +""" +certificate_rotation_checker + +This script will be leveraged to periodically check whether the certificate and private key +files of streaming telemetry were rotated by dSMS service or not. The streaming telemetry +server process will be restarted if the certificate and private key are rotated by dSMS service +and then updated by the acms agent running in ACMS container. +""" + +import os +import signal +import subprocess +import sys +import syslog +import time + +import inotify.adapters + +from swsscommon import swsscommon + +MAX_RETRY_TIMES = 10 +CERTIFICATE_CHECKING_INTERVAL_SECS = 3600 + +CREDENTIALS_DIR_PATH = "/etc/sonic/credentials/" + + +def get_command_result(command): + """Executes the command and returns the exiting code and resulting output. + + Args: + command: A string contains the command to be executed. + + Returns: + An integer indicates the exiting code. + A string which contains the output of command. + """ + command_stdout = "" + command_stderr = "" + + try: + proc_instance = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True, universal_newlines=True) + command_stdout, command_stderr = proc_instance.communicate() + except (OSError, ValueError) as err: + syslog.syslog(syslog.LOG_ERR, "Failed to execute the command '{}'. Error: '{}'" + .format(command, err)) + return 2, command_stderr + + return proc_instance.returncode, command_stdout.strip() + + +def check_telemetry_server_running(): + """Checkes whether telemetry server process is running. + + Args: + None. + + Returns: + None. + """ + processes_status_cmd = "supervisorctl status" + retry_times = 0 + is_running = False + + while retry_times <= MAX_RETRY_TIMES: + retry_times += 1 + exit_code, command_stdout = get_command_result(processes_status_cmd) + if exit_code != 3: + syslog.syslog(syslog.LOG_INFO, + "Failed to get the processes running status in telemetry container and retry after 60 seconds ...") + time.sleep(60) + else: + for line in command_stdout.splitlines(): + if "telemetry" in line and "RUNNING" in line: + is_running = True + break + if is_running: + syslog.syslog(syslog.LOG_INFO, + "Telemetry server process is running after certificate and private key were rotated!") + break + + if not is_running: + syslog.syslog(syslog.LOG_ERR, + "Telemetry server process is not running after certificate and private key were rotated and exiting ...") + sys.exit(1) + + +def restart_telemetry_server(): + """Restarts the telemetry server process by Supervisord and then checks + it is actually running. + + Args: + None + + Returns: + None + """ + restart_telemetry_server_cmd = "supervisorctl restart telemetry" + retry_times = 0 + + while retry_times <= MAX_RETRY_TIMES: + retry_times += 1 + exit_code, command_stdout = get_command_result(restart_telemetry_server_cmd) + if exit_code != 0: + syslog.syslog(syslog.LOG_INFO, + "Failed to restart telemetry server process and retry after 60 seconds ...") + time.sleep(60) + else: + break + + if retry_times > MAX_RETRY_TIMES: + syslog.syslog(syslog.LOG_ERR, + "Failed to restart telemetry server process after trying '{}' times and exiting ..." + .format(MAX_RETRY_TIMES)) + sys.exit(2) + + check_telemetry_server_running() + + +def check_certificate_rotated(certificate_file_name, private_key_file_name): + """Leverages the 'inotify' module to monitor the file system events under the + directory which stores the SONiC credentials and restarts telemetry server + process if certificate and private key were rotated. + + + Args: + certificate_file_name: A string indicates the telemetry certificate file name. + private_key_file_name: A string indicates the telemetry private key file name. + + Returns: + None. + """ + certificate_file_rotated = False + private_key_file_rotated = False + + inotify_instance = inotify.adapters.Inotify() + inotify_instance.add_watch(CREDENTIALS_DIR_PATH) + for event in inotify_instance.event_gen(yield_nones=False): + header, event_type, monitoring_path, file_name = event + if (file_name == certificate_file_name + and ("IN_CREATE" in event_type or "IN_MOVED_TO" in event_type)): + certificate_file_rotated = True + if (file_name == private_key_file_name + and ("IN_CREATE" in event_type or "IN_MOVED_TO" in event_type)): + private_key_file_rotated = True + + if certificate_file_rotated and private_key_file_rotated: + certificate_file_rotated = False + private_key_file_rotated = False + syslog.syslog(syslog.LOG_INFO, + "Certificate and private key were rotated and restarting telemetry server process ...") + restart_telemetry_server() + + # Wait for specified seconds and then do the next round checking + syslog.syslog(syslog.LOG_INFO, + "Sleeping '{}' seconds before doing the next round certifcate rotation checking ..." + .format(CERTIFICATE_CHECKING_INTERVAL_SECS)) + time.sleep(CERTIFICATE_CHECKING_INTERVAL_SECS) + + +def certificate_rotated_checker(): + """Checks rotation of certificate and key files and restart streaming telemetry server if necessary. + + Leverages 'inotify' module to check whether the certificate and private key files of + streaming telemetry were already rotated by dSMS service and updated by acms agent running + in ACMS container. The streaming telemetry server process will be restarted if they were rotated. + + Args: + None + + Returns: + None + """ + certificate_file_path = "" + private_key_file_path = "" + certificate_file_name = "" + private_key_file_name = "" + + config_db = swsscommon.DBConnector("CONFIG_DB", 0) + telemetry_table = swsscommon.Table(config_db, "TELEMETRY") + telemetry_table_keys = telemetry_table.getKeys() + if "certs" in telemetry_table_keys: + certs_info = dict(telemetry_table.get("certs")[1]) + if "server_crt" in certs_info and "server_key" in certs_info: + certificate_file_path = certs_info["server_crt"] + private_key_file_path = certs_info["server_key"] + syslog.syslog(syslog.LOG_INFO, "Path of certificate file is '{}'".format(certificate_file_path)) + syslog.syslog(syslog.LOG_INFO, "Path of key file is '{}'".format(private_key_file_path)) + else: + syslog.syslog(syslog.LOG_ERR, + "Failed to retrieve the path of certificate and key file from 'TELEMETRY' table!") + sys.exit(3) + else: + syslog.syslog(syslog.LOG_ERR, + "Failed to retrieve the certificate and key information from 'TELEMETRY' table!") + sys.exit(4) + + while True: + if not os.path.exists(certificate_file_path) or not os.path.exists(private_key_file_path): + syslog.syslog(syslog.LOG_ERR, + "Certificate or key file did not exist on device and sleep '{}' seconds to check again ..." + .format(CERTIFICATE_CHECKING_INTERVAL_SECS)) + time.sleep(CERTIFICATE_CHECKING_INTERVAL_SECS) + else: + break + + certificate_file_name = certificate_file_path.strip().split("/")[-1] + private_key_file_name = private_key_file_path.strip().split("/")[-1] + syslog.syslog(syslog.LOG_INFO, "cer_file_name: {}, key_file_name: {}".format(certificate_file_name, private_key_file_name)) + if not certificate_file_name or not private_key_file_name: + syslog.syslog(syslog.LOG_ERR, + "Failed to retrieve the file name of certificate or private key!") + sys.exit(5) + + check_certificate_rotated(certificate_file_name, private_key_file_name) + + +def main(): + certificate_rotated_checker() + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/dockers/docker-sonic-telemetry/telemetry.sh b/dockers/docker-sonic-telemetry/telemetry.sh index a5d6c072a7f1..a61284c72dc8 100755 --- a/dockers/docker-sonic-telemetry/telemetry.sh +++ b/dockers/docker-sonic-telemetry/telemetry.sh @@ -26,33 +26,17 @@ do SERVER_KEY=$(echo $CERTS | jq -r '.server_key') CA_CRT=$(echo $CERTS | jq -r '.ca_crt') - logger "Trying to retrieve server certificate, key and Root CA certificate managed by HwProxy ..." + logger "Trying to retrieve server certificate, key and Root CA certificate ..." logger "The file path of server certificate in CONFIG_DB is: $SERVER_CRT" logger "The file path of server provate key in CONFIG_DB is: $SERVER_KEY" logger "The file path of Root CA certificate in CONFIG_DB is: $CA_CRT" if [[ -f $SERVER_CRT && -f $SERVER_KEY && -f $CA_CRT ]]; then - logger "Succeeded in retrieving server certificate, key and Root CA certificate from HwProxy." + logger "Succeeded in retrieving server certificate, key and Root CA certificate." TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY --ca_crt $CA_CRT" break else - logger "Failed to retrieve server certificate, key or Root CA certificate from HwProxy!" - fi - - SERVER_CRT_ACMS=$(echo $CERTS | jq -r '.server_crt_acms') - SERVER_KEY_ACMS=$(echo $CERTS | jq -r '.server_key_acms') - CA_CRT_ACMS=$(echo $CERTS | jq -r '.ca_crt_acms') - - logger "Trying to retrieve server certificate, key and Root CA certificate managed by ACMS ..." - logger "The file path of server certificate in CONFIG_DB is: $SERVER_CRT_ACMS" - logger "The file path of server private key in CONFIG_DB is: $SERVER_KEY_ACMS" - logger "The file path of Root CA certificate in CONFIG_DB is: $CA_CRT_ACMS" - - if [[ -f $SERVER_CRT_ACMS && -f $SERVER_KEY_ACMS && -f $CA_CRT_ACMS ]]; then - logger "Succeeded in retrieving the certificate, key and Root CA certificate from ACMS." - continue - else - logger "Failed to retrieve server certificate, key or Root CA certificate from ACMS!" + logger "Failed to retrieve server certificate, key or Root CA certificate!" fi elif [ -n "$X509" ]; then SERVER_CRT=$(echo $X509 | jq -r '.server_crt') diff --git a/src/sonic-config-engine/minigraph.py b/src/sonic-config-engine/minigraph.py index 21a91b380563..97c83b06616d 100644 --- a/src/sonic-config-engine/minigraph.py +++ b/src/sonic-config-engine/minigraph.py @@ -1633,12 +1633,9 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw 'log_level': '2' }, 'certs': { - 'server_crt': '/etc/sonic/telemetry/streamingtelemetryserver.cer', - 'server_key': '/etc/sonic/telemetry/streamingtelemetryserver.key', - 'ca_crt': '/etc/sonic/telemetry/dsmsroot.cer', - 'server_crt_acms': '/etc/sonic/credentials/streamingtelemetryserver.cer', - 'server_key_acms': '/etc/sonic/credentials/streamingtelemetryserver.key', - 'ca_crt_acms': '/etc/sonic/credentials/dsmsroot.cer' + 'server_crt': '/etc/sonic/credentials/streamingtelemetryserver.cer', + 'server_key': '/etc/sonic/credentials/streamingtelemetryserver.key', + 'ca_crt': '/etc/sonic/credentials/dsmsroot.cer' } } results['RESTAPI'] = { From 59dfe9b7dcb09fc02748e22e36cc45384feffeed Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 10 Jan 2022 12:21:57 -0800 Subject: [PATCH 12/21] [telemetry] Remove the unused import from script. Signed-off-by: Yong Zhao --- dockers/docker-sonic-telemetry/certificate_rotation_checker | 1 - 1 file changed, 1 deletion(-) diff --git a/dockers/docker-sonic-telemetry/certificate_rotation_checker b/dockers/docker-sonic-telemetry/certificate_rotation_checker index 246ec6bb3bcc..9d73f9184787 100755 --- a/dockers/docker-sonic-telemetry/certificate_rotation_checker +++ b/dockers/docker-sonic-telemetry/certificate_rotation_checker @@ -10,7 +10,6 @@ and then updated by the acms agent running in ACMS container. """ import os -import signal import subprocess import sys import syslog From ce9f7135cde67586d008ae00573597eeb9640ef1 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 12 Jan 2022 13:47:34 -0800 Subject: [PATCH 13/21] [telemetry] Use the rotation of certificate file as an indicator. Signed-off-by: Yong Zhao --- .../certificate_rotation_checker | 57 +++++++------------ 1 file changed, 19 insertions(+), 38 deletions(-) mode change 100755 => 100644 dockers/docker-sonic-telemetry/certificate_rotation_checker diff --git a/dockers/docker-sonic-telemetry/certificate_rotation_checker b/dockers/docker-sonic-telemetry/certificate_rotation_checker old mode 100755 new mode 100644 index 9d73f9184787..bacfc30556b4 --- a/dockers/docker-sonic-telemetry/certificate_rotation_checker +++ b/dockers/docker-sonic-telemetry/certificate_rotation_checker @@ -3,13 +3,13 @@ """ certificate_rotation_checker -This script will be leveraged to periodically check whether the certificate and private key -files of streaming telemetry were rotated by dSMS service or not. The streaming telemetry -server process will be restarted if the certificate and private key are rotated by dSMS service -and then updated by the acms agent running in ACMS container. +This script will be leveraged to periodically check whether the certificate file +of streaming telemetry was rotated or not. The streaming telemetry server process +will be restarted if the certificate file was rotated. """ import os +import signal import subprocess import sys import syslog @@ -118,21 +118,19 @@ def restart_telemetry_server(): check_telemetry_server_running() -def check_certificate_rotated(certificate_file_name, private_key_file_name): +def check_certificate_rotated(certificate_file_name): """Leverages the 'inotify' module to monitor the file system events under the directory which stores the SONiC credentials and restarts telemetry server - process if certificate and private key were rotated. + process if its certificate was rotated. Args: certificate_file_name: A string indicates the telemetry certificate file name. - private_key_file_name: A string indicates the telemetry private key file name. Returns: None. """ certificate_file_rotated = False - private_key_file_rotated = False inotify_instance = inotify.adapters.Inotify() inotify_instance.add_watch(CREDENTIALS_DIR_PATH) @@ -141,30 +139,19 @@ def check_certificate_rotated(certificate_file_name, private_key_file_name): if (file_name == certificate_file_name and ("IN_CREATE" in event_type or "IN_MOVED_TO" in event_type)): certificate_file_rotated = True - if (file_name == private_key_file_name - and ("IN_CREATE" in event_type or "IN_MOVED_TO" in event_type)): - private_key_file_rotated = True - if certificate_file_rotated and private_key_file_rotated: + if certificate_file_rotated: certificate_file_rotated = False - private_key_file_rotated = False syslog.syslog(syslog.LOG_INFO, - "Certificate and private key were rotated and restarting telemetry server process ...") + "Certificate was rotated and restarting telemetry server process ...") restart_telemetry_server() - # Wait for specified seconds and then do the next round checking - syslog.syslog(syslog.LOG_INFO, - "Sleeping '{}' seconds before doing the next round certifcate rotation checking ..." - .format(CERTIFICATE_CHECKING_INTERVAL_SECS)) - time.sleep(CERTIFICATE_CHECKING_INTERVAL_SECS) - def certificate_rotated_checker(): - """Checks rotation of certificate and key files and restart streaming telemetry server if necessary. + """Checks rotation of certificate file and restart streaming telemetry server if necessary. - Leverages 'inotify' module to check whether the certificate and private key files of - streaming telemetry were already rotated by dSMS service and updated by acms agent running - in ACMS container. The streaming telemetry server process will be restarted if they were rotated. + Leverages 'inotify' module to check whether the certificate file of streaming telemetry was + already rotated or not. The streaming telemetry server process will be restarted if it was rotated. Args: None @@ -173,47 +160,41 @@ def certificate_rotated_checker(): None """ certificate_file_path = "" - private_key_file_path = "" certificate_file_name = "" - private_key_file_name = "" config_db = swsscommon.DBConnector("CONFIG_DB", 0) telemetry_table = swsscommon.Table(config_db, "TELEMETRY") telemetry_table_keys = telemetry_table.getKeys() if "certs" in telemetry_table_keys: certs_info = dict(telemetry_table.get("certs")[1]) - if "server_crt" in certs_info and "server_key" in certs_info: + if "server_crt" in certs_info: certificate_file_path = certs_info["server_crt"] - private_key_file_path = certs_info["server_key"] syslog.syslog(syslog.LOG_INFO, "Path of certificate file is '{}'".format(certificate_file_path)) - syslog.syslog(syslog.LOG_INFO, "Path of key file is '{}'".format(private_key_file_path)) else: syslog.syslog(syslog.LOG_ERR, - "Failed to retrieve the path of certificate and key file from 'TELEMETRY' table!") + "Failed to retrieve the path of certificate file from 'TELEMETRY' table!") sys.exit(3) else: syslog.syslog(syslog.LOG_ERR, - "Failed to retrieve the certificate and key information from 'TELEMETRY' table!") + "Failed to retrieve the certificate information from 'TELEMETRY' table!") sys.exit(4) while True: - if not os.path.exists(certificate_file_path) or not os.path.exists(private_key_file_path): + if not os.path.exists(certificate_file_path): syslog.syslog(syslog.LOG_ERR, - "Certificate or key file did not exist on device and sleep '{}' seconds to check again ..." + "Certificate file did not exist on device and sleep '{}' seconds to check again ..." .format(CERTIFICATE_CHECKING_INTERVAL_SECS)) time.sleep(CERTIFICATE_CHECKING_INTERVAL_SECS) else: break certificate_file_name = certificate_file_path.strip().split("/")[-1] - private_key_file_name = private_key_file_path.strip().split("/")[-1] - syslog.syslog(syslog.LOG_INFO, "cer_file_name: {}, key_file_name: {}".format(certificate_file_name, private_key_file_name)) - if not certificate_file_name or not private_key_file_name: + if not certificate_file_name: syslog.syslog(syslog.LOG_ERR, - "Failed to retrieve the file name of certificate or private key!") + "Failed to retrieve the file name of certificate!") sys.exit(5) - check_certificate_rotated(certificate_file_name, private_key_file_name) + check_certificate_rotated(certificate_file_name) def main(): From 68bd0b65224e9f34a2ed2b70abe1b646698ade93 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 12 Jan 2022 13:49:02 -0800 Subject: [PATCH 14/21] [telemetry] Change the script to be executable. Signed-off-by: Yong Zhao --- dockers/docker-sonic-telemetry/certificate_rotation_checker | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 dockers/docker-sonic-telemetry/certificate_rotation_checker diff --git a/dockers/docker-sonic-telemetry/certificate_rotation_checker b/dockers/docker-sonic-telemetry/certificate_rotation_checker old mode 100644 new mode 100755 From 6ad0ef5d3a234b376f1b9822df0779cbb40c52fa Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 12 Jan 2022 14:02:03 -0800 Subject: [PATCH 15/21] [telemetry] Remove the unused import. Signed-off-by: Yong Zhao --- dockers/docker-sonic-telemetry/certificate_rotation_checker | 1 - 1 file changed, 1 deletion(-) diff --git a/dockers/docker-sonic-telemetry/certificate_rotation_checker b/dockers/docker-sonic-telemetry/certificate_rotation_checker index bacfc30556b4..38f8a3d8d8d0 100755 --- a/dockers/docker-sonic-telemetry/certificate_rotation_checker +++ b/dockers/docker-sonic-telemetry/certificate_rotation_checker @@ -9,7 +9,6 @@ will be restarted if the certificate file was rotated. """ import os -import signal import subprocess import sys import syslog From 6ce4e4d2f1a885f7f4e86b858c2b4ee476d3fd31 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 12 Jan 2022 14:41:45 -0800 Subject: [PATCH 16/21] [telemetry] Change the script name. Signed-off-by: Yong Zhao --- dockers/docker-sonic-telemetry/Dockerfile.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dockers/docker-sonic-telemetry/Dockerfile.j2 b/dockers/docker-sonic-telemetry/Dockerfile.j2 index 0cdfa226a677..8cd79203a704 100644 --- a/dockers/docker-sonic-telemetry/Dockerfile.j2 +++ b/dockers/docker-sonic-telemetry/Dockerfile.j2 @@ -29,7 +29,8 @@ RUN apt-get clean -y && \ # Install the python `inotify` package RUN pip3 install inotify -COPY ["start.sh", "telemetry.sh", "dialout.sh", "certificate_rollover_checker", "/usr/bin/"] +COPY ["start.sh", "telemetry.sh", "dialout.sh", "/usr/bin/"] +COPY ["certificate_rotation_checker", "/usr/bin/"] COPY ["telemetry_vars.j2", "/usr/share/sonic/templates/"] COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] COPY ["files/supervisor-proc-exit-listener", "/usr/bin"] From 6512a6f50122595654eda92ee75df8f607c351b5 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 17 Jan 2022 17:35:29 -0800 Subject: [PATCH 17/21] [telemetry] Checks whether both certificate and private key files existed on device and log an error message if certificate file was deleted accidentally. Signed-off-by: Yong Zhao --- .../certificate_rotation_checker | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dockers/docker-sonic-telemetry/certificate_rotation_checker b/dockers/docker-sonic-telemetry/certificate_rotation_checker index 38f8a3d8d8d0..48829454adea 100755 --- a/dockers/docker-sonic-telemetry/certificate_rotation_checker +++ b/dockers/docker-sonic-telemetry/certificate_rotation_checker @@ -139,6 +139,9 @@ def check_certificate_rotated(certificate_file_name): and ("IN_CREATE" in event_type or "IN_MOVED_TO" in event_type)): certificate_file_rotated = True + if (file_name == certificate_file_name and "IN_DELETE" in event_type): + syslog.syslog(syslog.LOG_ERR, "Certificate file on device was deleted!") + if certificate_file_rotated: certificate_file_rotated = False syslog.syslog(syslog.LOG_INFO, @@ -159,6 +162,7 @@ def certificate_rotated_checker(): None """ certificate_file_path = "" + private_key_file_path = "" certificate_file_name = "" config_db = swsscommon.DBConnector("CONFIG_DB", 0) @@ -166,9 +170,11 @@ def certificate_rotated_checker(): telemetry_table_keys = telemetry_table.getKeys() if "certs" in telemetry_table_keys: certs_info = dict(telemetry_table.get("certs")[1]) - if "server_crt" in certs_info: + if "server_crt" in certs_info and "server_key" in certs_info: certificate_file_path = certs_info["server_crt"] + private_key_file_path = certs_info["server_key"] syslog.syslog(syslog.LOG_INFO, "Path of certificate file is '{}'".format(certificate_file_path)) + syslog.syslog(syslog.LOG_INFO, "Path of private key file is '{}'".format(private_key_file_path)) else: syslog.syslog(syslog.LOG_ERR, "Failed to retrieve the path of certificate file from 'TELEMETRY' table!") @@ -179,9 +185,9 @@ def certificate_rotated_checker(): sys.exit(4) while True: - if not os.path.exists(certificate_file_path): + if not os.path.exists(certificate_file_path) or not os.path.exists(private_key_file_path): syslog.syslog(syslog.LOG_ERR, - "Certificate file did not exist on device and sleep '{}' seconds to check again ..." + "Certificate or private key file did not exist on device and sleep '{}' seconds to check again ..." .format(CERTIFICATE_CHECKING_INTERVAL_SECS)) time.sleep(CERTIFICATE_CHECKING_INTERVAL_SECS) else: From c6e2fddbdc6412e402fa3d974787254073e81ef2 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Tue, 25 Jan 2022 14:22:06 -0800 Subject: [PATCH 18/21] [telemetry] Reload telemetry server configuration by sending a `SIGHUP` signal. Signed-off-by: Yong Zhao --- .../certificate_rotation_checker | 107 +++++++++--------- 1 file changed, 55 insertions(+), 52 deletions(-) diff --git a/dockers/docker-sonic-telemetry/certificate_rotation_checker b/dockers/docker-sonic-telemetry/certificate_rotation_checker index 48829454adea..4ece188d8670 100755 --- a/dockers/docker-sonic-telemetry/certificate_rotation_checker +++ b/dockers/docker-sonic-telemetry/certificate_rotation_checker @@ -3,12 +3,13 @@ """ certificate_rotation_checker -This script will be leveraged to periodically check whether the certificate file -of streaming telemetry was rotated or not. The streaming telemetry server process -will be restarted if the certificate file was rotated. +This script will be leveraged to periodically check whether the certificate file +of streaming telemetry was rotated or not. The configuration of streaming telemetry +server process will be reloaded if the certificate file was rotated. """ import os +import signal import subprocess import sys import syslog @@ -49,78 +50,78 @@ def get_command_result(command): return proc_instance.returncode, command_stdout.strip() -def check_telemetry_server_running(): - """Checkes whether telemetry server process is running. +def get_telemetry_server_info(): + """Gets telemetry server process information. Args: None. Returns: - None. + If telemetry server process is running, returns True and process id; + Otherwise returns False and -1. """ processes_status_cmd = "supervisorctl status" retry_times = 0 - is_running = False while retry_times <= MAX_RETRY_TIMES: retry_times += 1 exit_code, command_stdout = get_command_result(processes_status_cmd) if exit_code != 3: syslog.syslog(syslog.LOG_INFO, - "Failed to get the processes running status in telemetry container and retry after 60 seconds ...") + "Failed to get the telemetry server process information and retry after 60 seconds ...") time.sleep(60) else: for line in command_stdout.splitlines(): if "telemetry" in line and "RUNNING" in line: - is_running = True - break - if is_running: - syslog.syslog(syslog.LOG_INFO, - "Telemetry server process is running after certificate and private key were rotated!") - break + return True, line.split()[3].strip(",") - if not is_running: - syslog.syslog(syslog.LOG_ERR, - "Telemetry server process is not running after certificate and private key were rotated and exiting ...") - sys.exit(1) + return False, -1 -def restart_telemetry_server(): - """Restarts the telemetry server process by Supervisord and then checks - it is actually running. +def reload_telemetry_server_configuration(): + """Reloads the telemetry server configuration by sending signal 'SIGHUP' + to telemetry server process and checks it is actually running after doing the reload. Args: None Returns: - None + Returns True if the configuration was reloaded successfully; Otherwise, return False. """ - restart_telemetry_server_cmd = "supervisorctl restart telemetry" - retry_times = 0 + telemetry_server_pid = -1 + is_running = False - while retry_times <= MAX_RETRY_TIMES: - retry_times += 1 - exit_code, command_stdout = get_command_result(restart_telemetry_server_cmd) - if exit_code != 0: - syslog.syslog(syslog.LOG_INFO, - "Failed to restart telemetry server process and retry after 60 seconds ...") - time.sleep(60) - else: - break + is_running, telemetry_server_pid = get_telemetry_server_info() + if not is_running: + syslog.syslog(syslog.LOG_ERR, + "Telemetry server process is not running before reloading configuration!") + return False + + syslog.syslog(syslog.LOG_INFO, + "Telemetry server process is running with PID: {}".format(telemetry_server_pid)) + syslog.syslog(syslog.LOG_INFO, "Sending 'SIGHUP' signal to telemetry server process ...") + + os.kill(int(telemetry_server_pid), signal.SIGHUP) + + syslog.syslog(syslog.LOG_INFO, "'SIGHUP' signal was sent out.") + + # Wait for 120 seconds to check whether telemetry server process comes back + time.sleep(120) - if retry_times > MAX_RETRY_TIMES: + is_running, telemetry_server_pid = get_telemetry_server_info() + if not is_running: syslog.syslog(syslog.LOG_ERR, - "Failed to restart telemetry server process after trying '{}' times and exiting ..." - .format(MAX_RETRY_TIMES)) - sys.exit(2) + "Telemetry server process is not running after reloading configuration!") + return False - check_telemetry_server_running() + syslog.syslog(syslog.LOG_INFO, "Telemetry server process is running after reloading configuration!") + return True def check_certificate_rotated(certificate_file_name): """Leverages the 'inotify' module to monitor the file system events under the - directory which stores the SONiC credentials and restarts telemetry server - process if its certificate was rotated. + directory which stores the SONiC credentials and reloads telemetry server + configuration if its certificate was rotated. Args: @@ -139,21 +140,23 @@ def check_certificate_rotated(certificate_file_name): and ("IN_CREATE" in event_type or "IN_MOVED_TO" in event_type)): certificate_file_rotated = True - if (file_name == certificate_file_name and "IN_DELETE" in event_type): - syslog.syslog(syslog.LOG_ERR, "Certificate file on device was deleted!") - if certificate_file_rotated: certificate_file_rotated = False syslog.syslog(syslog.LOG_INFO, - "Certificate was rotated and restarting telemetry server process ...") - restart_telemetry_server() + "Certificate was rotated and reloading telemetry server configuration ...") + + if not reload_telemetry_server_configuration(): + syslog.syslog(syslog.LOG_ERR, + "Failed to reload the telemetry server configuration!") + + syslog.syslog(syslog.LOG_INFO, "Telemetry server configuration was reloaded successfully!") def certificate_rotated_checker(): - """Checks rotation of certificate file and restart streaming telemetry server if necessary. + """Checks rotation of certificate file and then reloads streaming telemetry server configuration. Leverages 'inotify' module to check whether the certificate file of streaming telemetry was - already rotated or not. The streaming telemetry server process will be restarted if it was rotated. + rotated or not. The configuration of telemetry server process will be reloaded if it was rotated. Args: None @@ -177,17 +180,17 @@ def certificate_rotated_checker(): syslog.syslog(syslog.LOG_INFO, "Path of private key file is '{}'".format(private_key_file_path)) else: syslog.syslog(syslog.LOG_ERR, - "Failed to retrieve the path of certificate file from 'TELEMETRY' table!") - sys.exit(3) + "Failed to retrieve the path of certificate or private key file from 'TELEMETRY' table!") + sys.exit(1) else: syslog.syslog(syslog.LOG_ERR, "Failed to retrieve the certificate information from 'TELEMETRY' table!") - sys.exit(4) + sys.exit(2) while True: if not os.path.exists(certificate_file_path) or not os.path.exists(private_key_file_path): syslog.syslog(syslog.LOG_ERR, - "Certificate or private key file did not exist on device and sleep '{}' seconds to check again ..." + "Certificate or private key file did not exist on device and checks again after '{}' seconds ..." .format(CERTIFICATE_CHECKING_INTERVAL_SECS)) time.sleep(CERTIFICATE_CHECKING_INTERVAL_SECS) else: @@ -197,7 +200,7 @@ def certificate_rotated_checker(): if not certificate_file_name: syslog.syslog(syslog.LOG_ERR, "Failed to retrieve the file name of certificate!") - sys.exit(5) + sys.exit(3) check_certificate_rotated(certificate_file_name) From 3dbb79db9243513a14ac70295dd2d3ebb7b41aa4 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 26 Jan 2022 10:46:37 -0800 Subject: [PATCH 19/21] [telemetry] Fix the undefinied fields issue in telemetry server starting script. Signed-off-by: Yong Zhao --- dockers/docker-sonic-telemetry/telemetry.sh | 52 +++++++++++---------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/dockers/docker-sonic-telemetry/telemetry.sh b/dockers/docker-sonic-telemetry/telemetry.sh index a61284c72dc8..f7e245f56f6e 100755 --- a/dockers/docker-sonic-telemetry/telemetry.sh +++ b/dockers/docker-sonic-telemetry/telemetry.sh @@ -41,17 +41,20 @@ do elif [ -n "$X509" ]; then SERVER_CRT=$(echo $X509 | jq -r '.server_crt') SERVER_KEY=$(echo $X509 | jq -r '.server_key') - if [ -z $SERVER_CRT ] || [ -z $SERVER_KEY ]; then - TELEMETRY_ARGS+=" --insecure" - else - TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY " - fi - CA_CRT=$(echo $X509 | jq -r '.ca_crt') - if [ ! -z $CA_CRT ]; then - TELEMETRY_ARGS+=" --ca_crt $CA_CRT" + + logger "Trying to retrieve server certificate, key and Root CA certificate ..." + logger "The file path of server certificate in CONFIG_DB is: $SERVER_CRT" + logger "The file path of server provate key in CONFIG_DB is: $SERVER_KEY" + logger "The file path of Root CA certificate in CONFIG_DB is: $CA_CRT" + + if [[ -f $SERVER_CRT && -f $SERVER_KEY && -f $CA_CRT ]]; then + logger "Succeeded in retrieving server certificate, key and Root CA certificate." + TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY --ca_crt $CA_CRT" + break + else + logger "Failed to retrieve server certificate, key or Root CA certificate!" fi - break else TELEMETRY_ARGS+=" --noTLS" break @@ -61,24 +64,25 @@ do sleep 3600 done -# If no configuration entry exists for TELEMETRY, create one default port -if [ -z "$GNMI" ]; then - PORT=8080 -else +if [ -n "$GNMI" ]; then PORT=$(echo $GNMI | jq -r '.port') -fi -TELEMETRY_ARGS+=" --port $PORT" + if [ ! -z $PORT ] || [ $PORT != "null" ]; then + TELEMETRY_ARGS+=" --port $PORT" + else + TELEMETRY_ARGS+=" --port 8080" + fi -CLIENT_AUTH=$(echo $GNMI | jq -r '.client_auth') -if [ -z $CLIENT_AUTH ] || [ $CLIENT_AUTH == "false" ]; then - TELEMETRY_ARGS+=" --allow_no_client_auth" -fi + LOG_LEVEL=$(echo $GNMI | jq -r '.log_level') + if [ ! -z $LOG_LEVEL ] || [ $LOG_LEVEL != "null" ]; then + TELEMETRY_ARGS+=" -v=$LOG_LEVEL" + else + TELEMETRY_ARGS+=" -v=2" + fi -LOG_LEVEL=$(echo $GNMI | jq -r '.log_level') -if [ ! -z $LOG_LEVEL ]; then - TELEMETRY_ARGS+=" -v=$LOG_LEVEL" -else - TELEMETRY_ARGS+=" -v=2" + CLIENT_AUTH=$(echo $GNMI | jq -r '.client_auth') + if [ -z $CLIENT_AUTH ] || [ $CLIENT_AUTH == "null" ] || [ $CLIENT_AUTH == "false" ]; then + TELEMETRY_ARGS+=" --allow_no_client_auth" + fi fi exec /usr/sbin/telemetry ${TELEMETRY_ARGS} From 088170488d5b4e77e51497df07ea85065e8eada8 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 26 Jan 2022 10:55:12 -0800 Subject: [PATCH 20/21] [telemetry] Fix the indentation. Signed-off-by: Yong Zhao --- dockers/docker-sonic-telemetry/telemetry.sh | 32 ++++++++++----------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/dockers/docker-sonic-telemetry/telemetry.sh b/dockers/docker-sonic-telemetry/telemetry.sh index f7e245f56f6e..59dccf635311 100755 --- a/dockers/docker-sonic-telemetry/telemetry.sh +++ b/dockers/docker-sonic-telemetry/telemetry.sh @@ -65,24 +65,24 @@ do done if [ -n "$GNMI" ]; then - PORT=$(echo $GNMI | jq -r '.port') - if [ ! -z $PORT ] || [ $PORT != "null" ]; then - TELEMETRY_ARGS+=" --port $PORT" - else - TELEMETRY_ARGS+=" --port 8080" - fi + PORT=$(echo $GNMI | jq -r '.port') + if [ ! -z $PORT ] || [ $PORT != "null" ]; then + TELEMETRY_ARGS+=" --port $PORT" + else + TELEMETRY_ARGS+=" --port 8080" + fi - LOG_LEVEL=$(echo $GNMI | jq -r '.log_level') - if [ ! -z $LOG_LEVEL ] || [ $LOG_LEVEL != "null" ]; then - TELEMETRY_ARGS+=" -v=$LOG_LEVEL" - else - TELEMETRY_ARGS+=" -v=2" - fi + LOG_LEVEL=$(echo $GNMI | jq -r '.log_level') + if [ ! -z $LOG_LEVEL ] || [ $LOG_LEVEL != "null" ]; then + TELEMETRY_ARGS+=" -v=$LOG_LEVEL" + else + TELEMETRY_ARGS+=" -v=2" + fi - CLIENT_AUTH=$(echo $GNMI | jq -r '.client_auth') - if [ -z $CLIENT_AUTH ] || [ $CLIENT_AUTH == "null" ] || [ $CLIENT_AUTH == "false" ]; then - TELEMETRY_ARGS+=" --allow_no_client_auth" - fi + CLIENT_AUTH=$(echo $GNMI | jq -r '.client_auth') + if [ -z $CLIENT_AUTH ] || [ $CLIENT_AUTH == "null" ] || [ $CLIENT_AUTH == "false" ]; then + TELEMETRY_ARGS+=" --allow_no_client_auth" + fi fi exec /usr/sbin/telemetry ${TELEMETRY_ARGS} From 41377a190583878db0379b07d842271e98a2e4a0 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 26 Jan 2022 10:56:46 -0800 Subject: [PATCH 21/21] [telemetry] Fix the indentation issue. Signed-off-by: Yong Zhao --- dockers/docker-sonic-telemetry/telemetry.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockers/docker-sonic-telemetry/telemetry.sh b/dockers/docker-sonic-telemetry/telemetry.sh index 59dccf635311..4fb2a2773fd2 100755 --- a/dockers/docker-sonic-telemetry/telemetry.sh +++ b/dockers/docker-sonic-telemetry/telemetry.sh @@ -43,7 +43,7 @@ do SERVER_KEY=$(echo $X509 | jq -r '.server_key') CA_CRT=$(echo $X509 | jq -r '.ca_crt') - logger "Trying to retrieve server certificate, key and Root CA certificate ..." + logger "Trying to retrieve server certificate, key and Root CA certificate ..." logger "The file path of server certificate in CONFIG_DB is: $SERVER_CRT" logger "The file path of server provate key in CONFIG_DB is: $SERVER_KEY" logger "The file path of Root CA certificate in CONFIG_DB is: $CA_CRT"