[sonic-utilities] Build and install as a Python wheel package

build_debian.sh

@@ -317,6 +317,10 @@ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y in
     cron                    \
     haveged
 
+# Install "wheel" package so that we can install .whl packages and not
+# encounter a "error: invalid command 'bdist_wheel'" error
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install wheel
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install wheel
 
 if [[ $CONFIGURED_ARCH == amd64 ]]; then
 ## Pre-install the fundamental packages for amd64 (x86)
@@ -396,7 +400,7 @@ sudo mkdir -p $FILESYSTEM_ROOT/var/core
 
 # Config sysctl
 sudo augtool --autosave "
-set /files/etc/sysctl.conf/kernel.core_pattern '|/usr/bin/coredump-compress %e %t %p %P'
+set /files/etc/sysctl.conf/kernel.core_pattern '|/usr/local/bin/coredump-compress %e %t %p %P'
 set /files/etc/sysctl.conf/kernel.softlockup_panic 1
 set /files/etc/sysctl.conf/kernel.panic 10
 set /files/etc/sysctl.conf/vm.panic_on_oom 2

dockers/docker-fpm-frr/base_image_files/TSC

@@ -2,4 +2,4 @@
 
 docker exec -i bgp /usr/bin/TSC
 
-/usr/bin/portstat -p 5
+portstat -p 5

files/build_templates/docker_image_ctl.j2

@@ -125,10 +125,10 @@ function postStartAction()
         $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1"
     fi
 
-    if [[ -x /usr/bin/db_migrator.py ]]; then
+    if [[ -x /usr/local/bin/db_migrator.py ]]; then
         # Migrate the DB to the latest schema version if needed
         if [ -z "$DEV" ]; then
-            /usr/bin/db_migrator.py -o migrate
+            /usr/local/bin/db_migrator.py -o migrate
         fi
     fi
     # Add redis UDS to the redis group and give read/write access to the group

files/build_templates/sonic_debian_extension.j2

@@ -169,27 +169,29 @@ sudo cp {{platform_common_py2_wheel_path}} $FILESYSTEM_ROOT/$PLATFORM_COMMON_PY2
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $PLATFORM_COMMON_PY2_WHEEL_NAME
 sudo rm -rf $FILESYSTEM_ROOT/$PLATFORM_COMMON_PY2_WHEEL_NAME
 
+# Install Debian packages and their dependencies which are needed by sonic-utilities
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f python-m2crypto
+
 # Install built Python Click package (and its dependencies via 'apt-get -y install -f')
 # Do this before installing sonic-utilities so that it doesn't attempt to install
 # an older version as part of its dependencies
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python-click*_all.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
 
-# Install python pexpect used by sonic-utilities consutil
-# using pip install instead to get a more recent version than is available through debian
-sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install pexpect
-
-# Install tabulate >= 0.8.1 via pip in order to support multi-line row output for sonic-utilities
-sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install tabulate==0.8.2
-
-# Install packages to support Dynamic Port Breakout config command for sonic-utilities
-sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install xmltodict==0.12.0
-sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install jsondiff==1.2.0
+# Install SONiC Utilities Python 2 package
+SONIC_UTILITIES_PY2_WHEEL_NAME=$(basename {{sonic_utilities_py2_wheel_path}})
+sudo cp {{sonic_utilities_py2_wheel_path}} $FILESYSTEM_ROOT/$SONIC_UTILITIES_PY2_WHEEL_NAME
+sudo LANG=C chroot $FILESYSTEM_ROOT pip install $SONIC_UTILITIES_PY2_WHEEL_NAME
+sudo rm -rf $FILESYSTEM_ROOT/$SONIC_UTILITIES_PY2_WHEEL_NAME
 
-# Install SONiC Utilities (and its dependencies via 'apt-get -y install -f')
-sudo dpkg --root=$FILESYSTEM_ROOT -i $python_debs_path/python-sonic-utilities_*.deb || \
+# Install sonic-utilities data files (and any dependencies via 'apt-get -y install -f')
+sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-utilities-data_*.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
 
+# sonic-utilities-data installs bash-completion as a dependency. However, it is disabled by default
+# in bash.bashrc, so we copy a version of the file with it enabled here.
+sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/
+
 {% if enable_ztp == "y" %}
 # Install ZTP (and its dependencies via 'apt-get -y install -f')
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-ztp_*.deb || \
@@ -202,10 +204,6 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-host-service_*.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
 {% endif %}
 
-# SONiC utilities installs bash-completion as a dependency. However, it is disabled by default
-# in bash.bashrc, so we copy a version of the file with it enabled here.
-sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/
-
 # Install SONiC Device Data  (and its dependencies via 'apt-get -y install -f')
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-device-data_*.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f

files/image_config/config-setup/config-setup

@@ -188,9 +188,9 @@ load_config()
         return $?
     fi
 
-    if [[ -x /usr/bin/db_migrator.py ]]; then
+    if [[ -x /usr/local/bin/db_migrator.py ]]; then
         # Migrate the DB to the latest schema version if needed
-        /usr/bin/db_migrator.py -o migrate
+        /usr/local/bin/db_migrator.py -o migrate
     fi
 
     sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1"

files/image_config/fstrim/fstrim.service

@@ -3,5 +3,5 @@ Description=Discard unused blocks
 
 [Service]
 Type=oneshot
-ExecStartPre=/usr/bin/log_ssd_health
+ExecStartPre=/usr/local/bin/log_ssd_health
 ExecStart=/sbin/fstrim -av

files/image_config/monit/conf.d/sonic-host

@@ -27,6 +27,6 @@ check process rsyslog with pidfile /var/run/rsyslogd.pid
 # Hence for any discrepancy, there will be log messages for "ERR" level
 # from both route_check.py & monit.
 #
-check program routeCheck with path "/usr/bin/route_check.py"
+check program routeCheck with path "/usr/local/bin/route_check.py"
     every 5 cycles
     if status != 0 then alert

files/image_config/sudoers/sudoers

@@ -19,26 +19,25 @@ Defaults        lecture_file = /etc/sudoers.lecture
 
 # Cmnd alias specification
 # Note: bcmcmd is dangerous for users in read only netgroups because it may operate ASIC
-Cmnd_Alias      READ_ONLY_CMDS = /sbin/brctl show, \
-                                 /usr/bin/decode-syseeprom, \
-                                 /usr/bin/docker images *, \
+Cmnd_Alias      READ_ONLY_CMDS = /bin/cat /var/log/syslog*, \
+                                 /sbin/brctl show, \
                                  /usr/bin/docker exec snmp cat /etc/snmp/snmpd.conf, \
                                  /usr/bin/docker exec bgp cat /etc/quagga/bgpd.conf, \
                                  /usr/bin/docker exec * ps aux, \
+                                 /usr/bin/docker images *, \
                                  /usr/bin/docker ps*, \
-                                 /usr/bin/generate_dump, \
                                  /usr/bin/lldpctl, \
-                                 /usr/bin/lldpshow, \
-                                 /usr/bin/psuutil *, \
                                  /usr/bin/sensors, \
-                                 /usr/bin/sonic-installer list, \
-                                 /usr/bin/sfputil show *, \
-                                 /usr/bin/teamshow, \
+                                 /usr/bin/tail -F /var/log/syslog, \
                                  /usr/bin/vtysh -c show *, \
-                                 /bin/cat /var/log/syslog*, \
-                                 /usr/bin/tail -F /var/log/syslog
-
-Cmnd_Alias      PASSWD_CMDS = /usr/bin/config tacacs passkey *, \
+                                 /usr/local/bin/decode-syseeprom, \
+                                 /usr/local/bin/generate_dump, \
+                                 /usr/local/bin/lldpshow, \
+                                 /usr/local/bin/psuutil *, \
+                                 /usr/local/bin/sonic-installer list, \
+                                 /usr/local/bin/sfputil show *
+
+Cmnd_Alias      PASSWD_CMDS = /usr/local/bin/config tacacs passkey *, \
                               /usr/sbin/chpasswd *
 
 # User privilege specification

files/image_config/warmboot-finalizer/finalize-warmboot.sh

@@ -6,7 +6,7 @@ VERBOSE=no
 COMP_LIST="orchagent neighsyncd bgp natsyncd"
 EXP_STATE="reconciled"
 
-ASSISTANT_SCRIPT="/usr/bin/neighbor_advertiser"
+ASSISTANT_SCRIPT="/usr/local/bin/neighbor_advertiser"
 
 
 function debug()

files/image_config/watchdog-control/watchdog-control.sh

@@ -1,7 +1,7 @@
 #! /bin/bash
 
 VERBOSE=no
-WATCHDOG_UTIL="/usr/bin/watchdogutil"
+WATCHDOG_UTIL="/usr/local/bin/watchdogutil"
 
 function debug()
 {

platform/broadcom/sonic-platform-modules-dell/common/fw-updater

@@ -16,7 +16,7 @@ HOST_GRUB_CFG = HOST_GRUB_DIR + '/grub/grub.cfg'
 HOST_GRUB_ENV = HOST_GRUB_DIR + '/grub/grubenv'
 HOST_GRUB_BOOT_DIR = '--boot-directory=' + HOST_GRUB_DIR
 HOST_PLATFORM_INFO = HOST_GRUB_DIR + '/platform'
-dell_reload_tool = '/usr/bin/reboot'
+dell_reload_tool = '/usr/local/bin/reboot'
 
 
 

platform/mellanox/mlnx-onie-fw-update.sh

@@ -94,7 +94,7 @@ system_reboot() {
     sleep 5s
 
     # Use SONiC reboot scenario
-    /usr/bin/reboot
+    /usr/local/bin/reboot
 }
 
 terminate_handler() {

platform/p4/docker-sonic-p4.mk

@@ -11,7 +11,7 @@ $(DOCKER_SONIC_P4)_DEPENDS += $(SWSS) \
                               $(LIBTEAMDCTL) \
                               $(LIBTEAM_UTILS) \
                               $(SONIC_DEVICE_DATA) \
-                              $(SONIC_UTILS) \
+                              $(SONIC_UTILITIES_PY2) \
                               $(IPROUTE2)
 
 # ifeq ($(ROUTING_STACK), quagga)

platform/vs/docker-sonic-vs.mk

@@ -10,9 +10,8 @@ $(DOCKER_SONIC_VS)_DEPENDS += $(SWSS) \
                               $(SONIC_DEVICE_DATA) \
                               $(LIBYANG) \
                               $(LIBYANG_CPP) \
-                              $(LIBYANG_PY2)
-
-$(DOCKER_SONIC_VS)_PYTHON_DEBS += $(SONIC_UTILS)
+                              $(LIBYANG_PY2) \
+                              $(SONIC_UTILITIES_DATA)
 
 # swsssdk is a dependency of sonic-py-common
 # TODO: sonic-py-common should depend on swsscommon instead
@@ -21,7 +20,8 @@ $(DOCKER_SONIC_VS)_PYTHON_WHEELS += $(SWSSSDK_PY2) \
                                     $(SONIC_PY_COMMON_PY2) \
                                     $(SONIC_PY_COMMON_PY3) \
                                     $(SONIC_YANG_MODELS_PY3) \
-                                    $(SONIC_YANG_MGMT_PY)
+                                    $(SONIC_YANG_MGMT_PY) \
+                                    $(SONIC_UTILITIES_PY2)
 
 ifeq ($(INSTALL_DEBUG_TOOLS), y)
 $(DOCKER_SONIC_VS)_DEPENDS += $(SWSS_DBG) \

platform/vs/docker-sonic-vs/Dockerfile.j2

@@ -46,7 +46,8 @@ RUN apt-get install -y net-tools \
                        conntrack \
                        iptables \
                        python3-pip \
-                       jq
+                       jq \
+                       python-m2crypto
 
 # install redis-server
 RUN curl -o redis-tools_6.0.6-1~bpo10+1_amd64.deb "https://sonicstorage.blob.core.windows.net/packages/redis/redis-tools_6.0.6-1~bpo10+1_amd64.deb?sv=2015-04-05&sr=b&sig=73zbmjkf3pi%2Bn0R8Hy7CWT2EUvOAyzM5aLYJWCLySGM%3D&se=2030-09-06T19%3A44%3A59Z&sp=r"
@@ -56,6 +57,8 @@ RUN rm redis-tools_6.0.6-1~bpo10+1_amd64.deb redis-server_6.0.6-1~bpo10+1_amd64.
 
 RUN pip install setuptools
 RUN pip3 install setuptools
+RUN pip install wheel
+RUN pip3 install wheel
 RUN pip install py2_ipaddress
 RUN pip install six
 RUN pip install pyroute2==0.5.3 netifaces==0.10.7
@@ -102,11 +105,7 @@ COPY python-wheels/{{ whl }}  python-wheels/
 # install PKGs after copying all PKGs to avoid dependency failure
 # use py3 to find python3 package, which is forced by wheel as of now
 {%- for whl in docker_sonic_vs_whls.split(' ') %}
-{%- if 'py3' in whl %}
-RUN pip3 install python-wheels/{{ whl }}
-{% else -%}
-RUN pip install python-wheels/{{ whl }}
-{%- endif %}
+RUN pip{% if 'py3' in whl %}3{% endif %} install python-wheels/{{ whl }}
 {%- endfor %}
 {% endif %}
 

rules/sonic-utilities-data.dep

@@ -0,0 +1,9 @@
+SPATH       := $($(SONIC_UTILITIES_DATA)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/sonic-utilities-data.mk rules/sonic-utilities-data.dep   
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(SPATH))
+
+$(SONIC_UTILITIES_DATA)_CACHE_MODE  := GIT_CONTENT_SHA
+$(SONIC_UTILITIES_DATA)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_UTILITIES_DATA)_DEP_FILES   := $(DEP_FILES)
+

rules/sonic-utilities-data.mk

@@ -0,0 +1,5 @@
+# SONiC command line utilities data package 
+
+SONIC_UTILITIES_DATA = sonic-utilities-data_1.0-1_all.deb
+$(SONIC_UTILITIES_DATA)_SRC_PATH = $(SRC_PATH)/sonic-utilities/sonic-utilities-data
+SONIC_DPKG_DEBS += $(SONIC_UTILITIES_DATA)

rules/sonic-utilities.dep

@@ -1,12 +1,10 @@
-
-SPATH       := $($(SONIC_UTILS)_SRC_PATH)
+SPATH       := $($(SONIC_UTILITIES_PY2)_SRC_PATH)
 DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/sonic-utilities.mk rules/sonic-utilities.dep   
 DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
 SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
 
-$(SONIC_UTILS)_CACHE_MODE  := GIT_CONTENT_SHA 
-$(SONIC_UTILS)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
-$(SONIC_UTILS)_DEP_FILES   := $(DEP_FILES)
-$(SONIC_UTILS)_SMDEP_FILES := $(SMDEP_FILES)
-$(SONIC_UTILS)_SMDEP_PATHS := $(SPATH)
-
+$(SONIC_UTILITIES_PY2)_CACHE_MODE  := GIT_CONTENT_SHA
+$(SONIC_UTILITIES_PY2)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_UTILITIES_PY2)_DEP_FILES   := $(DEP_FILES)
+$(SONIC_UTILITIES_PY2)_SMDEP_FILES := $(SMDEP_FILES)
+$(SONIC_UTILITIES_PY2)_SMDEP_PATHS := $(SPATH)

rules/sonic-utilities.mk

@@ -6,19 +6,18 @@
 # added here also. However, the current build system assumes all runtime
 # dependencies are .deb packages.
 #
-# TODO: Create a way to specify both .deb and .whl runtime dependencies
-#       then add the aforementioned runtime dependencies here.
-#
 
-SONIC_UTILS = python-sonic-utilities_1.2-1_all.deb
-$(SONIC_UTILS)_SRC_PATH = $(SRC_PATH)/sonic-utilities
-$(SONIC_UTILS)_DEBS_DEPENDS = $(LIBYANG) \
-                              $(LIBYANG_CPP) \
-                              $(LIBYANG_PY2) \
-                              $(LIBYANG_PY3)
-$(SONIC_UTILS)_WHEEL_DEPENDS = $(SONIC_PY_COMMON_PY2) \
-                               $(SONIC_PY_COMMON_PY3) \
-                               $(SONIC_CONFIG_ENGINE) \
-                               $(SONIC_YANG_MGMT_PY) \
-                               $(SONIC_YANG_MODELS_PY3)
-SONIC_PYTHON_STDEB_DEBS += $(SONIC_UTILS)
+SONIC_UTILITIES_PY2 = sonic_utilities-1.2-py2-none-any.whl
+$(SONIC_UTILITIES_PY2)_SRC_PATH = $(SRC_PATH)/sonic-utilities
+$(SONIC_UTILITIES_PY2)_PYTHON_VERSION = 2
+$(SONIC_UTILITIES_PY2)_DEPENDS += $(SONIC_PY_COMMON_PY2) \
+                                  $(SONIC_PY_COMMON_PY3) \
+			          $(SWSSSDK_PY2) \
+                                  $(SONIC_CONFIG_ENGINE) \
+                                  $(SONIC_YANG_MGMT_PY) \
+                                  $(SONIC_YANG_MODELS_PY3)
+$(SONIC_UTILITIES_PY2)_DEBS_DEPENDS = $(LIBYANG) \
+                                      $(LIBYANG_CPP) \
+                                      $(LIBYANG_PY2) \
+                                      $(LIBYANG_PY3)
+SONIC_PYTHON_WHEELS += $(SONIC_UTILITIES_PY2)

slave.mk

@@ -802,12 +802,13 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
                 $(LIBPAM_TACPLUS) \
                 $(LIBNSS_TACPLUS) \
                 $(MONIT) \
-                $(PYTHON_SWSSCOMMON)) \
+                $(PYTHON_SWSSCOMMON) \
+                $(SONIC_UTILITIES_DATA)) \
         $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \
         $$(addprefix $(FILES_PATH)/,$$($$*_FILES)) \
         $(if $(findstring y,$(ENABLE_ZTP)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_ZTP))) \
         $(if $(findstring y,$(INCLUDE_HOST_SERVICE)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_HOST_SERVICE))) \
-        $(addprefix $(PYTHON_DEBS_PATH)/,$(SONIC_UTILS)) \
+        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_UTILITIES_PY2)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY2)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY3)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE)) \
@@ -858,6 +859,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export multi_instance="false"
 	export python_swss_debs="$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$($(LIBSWSSCOMMON)_RDEPENDS))"
 	export python_swss_debs+=" $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(LIBSWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON_SWSSCOMMON))"
+	export sonic_utilities_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_UTILITIES_PY2))"
 
 	$(foreach docker, $($*_DOCKERS),\
 		export docker_image="$(docker)"