[202205] Non recursive automake and Debian packaging changes (#700) (#…

…772)

Cherry-pick of #700

Switch to using non-recursive automake
This can help increase build parallelization. On my dev VM, this decreased the compilation time from 2:55 to about 2:28 (Bullseye, Python 2 disabled, -j12). Time improvements may be more noticeable on systems with more CPU cores.

Update debian packaging
Update to using debhelper compat 12, remove some overrides that are no longer needed, use automatically-generated dbgsym packages, and disable the static build.

Always enable all hardening
Make sure hardened build flags are always used.

Re-add libboost1.71-dev as a dependency, since the Buster slave containers have that installed instead of just libboost-dev

Makefile.am

@@ -1,3 +1,24 @@
-SUBDIRS = common pyext sonic-db-cli tests
+lib_LTLIBRARIES =
+bin_PROGRAMS =
+
+if DEBUG
+DBGFLAGS = -ggdb -DDEBUG
+else
+DBGFLAGS = -g -DNDEBUG
+endif
+
+SWIG_SOURCES = pyext/swsscommon.i
+SWIG_FLAG = -Wall -c++ -python -keyword
+if ARCH64
+SWIG_FLAG += -DSWIGWORDSIZE64
+endif
+
+CLEANFILES =
+
+include common/Makefile.am
+include pyext/Makefile.am
+include sonic-db-cli/Makefile.am
+include tests/Makefile.am
+
 
 ACLOCAL_AMFLAGS = -I m4

common/Makefile.am

@@ -1,81 +1,71 @@
-INCLUDES = -I $(top_srcdir)
 
-lib_LTLIBRARIES = libswsscommon.la
+lib_LTLIBRARIES += common/libswsscommon.la
 
-EXTRA_DIST = \
-    consumer_state_table_pops.lua \
-    consumer_table_pops.lua \
-    producer_state_table_apply_view.lua \
-    table_dump.lua \
-    redis_multi.lua \
-    portcounter.lua \
-    fdb_flush.lua \
-    fdb_flush.v2.lua
+dist_swss_DATA = \
+    common/consumer_state_table_pops.lua \
+    common/consumer_table_pops.lua \
+    common/producer_state_table_apply_view.lua \
+    common/table_dump.lua \
+    common/redis_multi.lua \
+    common/portcounter.lua \
+    common/fdb_flush.lua \
+    common/fdb_flush.v2.lua
 
-EXTRA_CONF_DIST = database_config.json
+dist_swsscommon_DATA= common/database_config.json
 
 swssdir = $(datadir)/swss
 swsscommondir = /var/run/redis/sonic-db
 
-dist_swss_DATA = $(EXTRA_DIST)
-dist_swsscommon_DATA = $(EXTRA_CONF_DIST)
+bin_PROGRAMS += common/swssloglevel
 
-bin_PROGRAMS = swssloglevel
+common_libswsscommon_la_SOURCES = \
+    common/logger.cpp                \
+    common/redisreply.cpp            \
+    common/configdb.cpp              \
+    common/dbconnector.cpp           \
+    common/dbinterface.cpp           \
+    common/sonicv2connector.cpp      \
+    common/table.cpp                 \
+    common/json.cpp                  \
+    common/producertable.cpp         \
+    common/producerstatetable.cpp    \
+    common/rediscommand.cpp          \
+    common/redistran.cpp             \
+    common/redisselect.cpp           \
+    common/select.cpp                \
+    common/selectableevent.cpp       \
+    common/selectabletimer.cpp       \
+    common/consumertable.cpp         \
+    common/consumertablebase.cpp     \
+    common/consumerstatetable.cpp    \
+    common/ipaddress.cpp             \
+    common/ipprefix.cpp              \
+    common/ipaddresses.cpp           \
+    common/macaddress.cpp            \
+    common/netdispatcher.cpp         \
+    common/netlink.cpp               \
+    common/nfnetlink.cpp             \
+    common/notificationconsumer.cpp  \
+    common/notificationproducer.cpp  \
+    common/linkcache.cpp             \
+    common/portmap.cpp               \
+    common/pubsub.cpp                \
+    common/tokenize.cpp              \
+    common/exec.cpp                  \
+    common/saiaclschema.cpp          \
+    common/subscriberstatetable.cpp  \
+    common/timestamp.cpp             \
+    common/warm_restart.cpp          \
+    common/luatable.cpp              \
+    common/countertable.cpp          \
+    common/redisutility.cpp
 
-if DEBUG
-DBGFLAGS = -ggdb -DDEBUG
-else
-DBGFLAGS = -g -DNDEBUG
-endif
+common_libswsscommon_la_CXXFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(LIBNL_CFLAGS) $(CODE_COVERAGE_CXXFLAGS)
+common_libswsscommon_la_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(LIBNL_CPPFLAGS) $(CODE_COVERAGE_CPPFLAGS)
+common_libswsscommon_la_LIBADD = -lpthread $(LIBNL_LIBS) $(CODE_COVERAGE_LIBS)
 
-libswsscommon_la_SOURCES = \
-    logger.cpp                \
-    redisreply.cpp            \
-    configdb.cpp              \
-    dbconnector.cpp           \
-    dbinterface.cpp           \
-    sonicv2connector.cpp      \
-    table.cpp                 \
-    json.cpp                  \
-    producertable.cpp         \
-    producerstatetable.cpp    \
-    rediscommand.cpp          \
-    redistran.cpp             \
-    redisselect.cpp           \
-    select.cpp                \
-    selectableevent.cpp       \
-    selectabletimer.cpp       \
-    consumertable.cpp         \
-    consumertablebase.cpp     \
-    consumerstatetable.cpp    \
-    ipaddress.cpp             \
-    ipprefix.cpp              \
-    ipaddresses.cpp           \
-    macaddress.cpp            \
-    netdispatcher.cpp         \
-    netlink.cpp               \
-    nfnetlink.cpp             \
-    notificationconsumer.cpp  \
-    notificationproducer.cpp  \
-    linkcache.cpp             \
-    portmap.cpp               \
-    pubsub.cpp                \
-    tokenize.cpp              \
-    exec.cpp                  \
-    saiaclschema.cpp          \
-    subscriberstatetable.cpp  \
-    timestamp.cpp             \
-    warm_restart.cpp          \
-    luatable.cpp              \
-    countertable.cpp          \
-    redisutility.cpp
+common_swssloglevel_SOURCES = common/loglevel.cpp
 
-libswsscommon_la_CXXFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(LIBNL_CFLAGS) $(CODE_COVERAGE_CXXFLAGS)
-libswsscommon_la_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(LIBNL_CPPFLAGS) $(CODE_COVERAGE_CPPFLAGS)
-libswsscommon_la_LIBADD = -lpthread $(LIBNL_LIBS) $(CODE_COVERAGE_LIBS)
-
-swssloglevel_SOURCES = loglevel.cpp
-
-swssloglevel_CXXFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CODE_COVERAGE_CXXFLAGS)
-swssloglevel_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CODE_COVERAGE_CPPFLAGS)
-swssloglevel_LDADD = libswsscommon.la $(CODE_COVERAGE_LIBS)
+common_swssloglevel_CXXFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CODE_COVERAGE_CXXFLAGS)
+common_swssloglevel_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CODE_COVERAGE_CPPFLAGS)
+common_swssloglevel_LDADD = common/libswsscommon.la $(CODE_COVERAGE_LIBS)

configure.ac

@@ -3,7 +3,7 @@ AC_CONFIG_SRCDIR([])
 AC_CONFIG_AUX_DIR(config)
 AM_CONFIG_HEADER(config.h)
 AC_CONFIG_MACRO_DIR([m4])
-AM_INIT_AUTOMAKE([foreign])
+AM_INIT_AUTOMAKE([foreign subdir-objects])
 AC_LANG_C
 AC_LANG([C++])
 AC_PROG_CC
@@ -94,12 +94,6 @@ AC_SUBST(CFLAGS_COMMON)
 
 AC_CONFIG_FILES([
     Makefile
-    common/Makefile
-    pyext/Makefile
-    pyext/py2/Makefile
-    pyext/py3/Makefile
-    sonic-db-cli/Makefile
-    tests/Makefile
 ])
 
 AC_OUTPUT

debian/compat

@@ -1 +1 @@
-10
+12

debian/control

@@ -2,7 +2,7 @@ Source: sonic
 Maintainer: Shuotian Cheng <shuche@microsoft.com>
 Section: net
 Priority: optional
-Build-Depends: dh-exec (>=0.3), debhelper (>= 9), autotools-dev, libboost-dev | libboost1.71-dev | libboost1.74-dev
+Build-Depends: dh-exec (>=0.3), debhelper (>= 12), autotools-dev, libboost-dev | libboost1.71-dev
 Standards-Version: 1.0.0
 
 Package: libswsscommon
@@ -11,15 +11,9 @@ Depends: ${shlibs:Depends}, ${misc:Pre-Depends}
 Section: libs
 Description: This package contains Switch State Service common library.
 
-Package: libswsscommon-dbg
-Architecture: any
-Depends: libswsscommon (= ${binary:Version}), ${misc:Depends}
-Section: debug
-Description: debugging symbols for libswsscommon library.
-
 Package: libswsscommon-dev
 Architecture: any
-Depends: libswsscommon (= ${binary:Version}), libboost-dev | libboost1.71-dev | libboost1.74-dev
+Depends: libswsscommon (= ${binary:Version}), libboost-dev | libboost1.71-dev
 Section: libdevel
 Description: This package contains development files for Switch State Service.
 

debian/rules

@@ -8,18 +8,18 @@ DPKG_EXPORT_BUILDFLAGS = 1
 include /usr/share/dpkg/default.mk
 
 # see FEATURE AREAS in dpkg-buildflags(1)
-#export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+export DEB_BUILD_MAINT_OPTIONS = hardening=+all
 
 # see ENVIRONMENT in dpkg-buildflags(1)
 # package maintainers to append CFLAGS
 #export DEB_CFLAGS_MAINT_APPEND  = -Wall -pedantic
 # package maintainers to append LDFLAGS
 #export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed
 
-DEBIAN_DIST_CODENAME := $(shell lsb_release -sc | sed -e 's/\#/ /g')
-
 DOPACKAGES = $(shell dh_listpackages)
 
+CONFIGURE_ARGS = --disable-static
+
 # For Debian jessie, stretch, and buster, and Ubuntu bionic and focal, build
 # Python 2 bindings. This is controlled by the build profile being used.
 ifneq (,$(filter python-swsscommon,$(DOPACKAGES)))
@@ -28,19 +28,12 @@ else
 CONFIGURE_ARGS += --disable-python2
 endif
 
-# main packaging script based on dh7 syntax
+# main packaging script
 %:
-	dh $@  --with autoreconf
+	dh $@
 
 # dh_make generated override targets
 # This is example for Cmake (See https://bugs.debian.org/641051 )
 override_dh_auto_configure:
 	dh_auto_configure -- $(CONFIGURE_ARGS) $(DEB_CONFIGURE_EXTRA_FLAGS)
 #	-DCMAKE_LIBRARY_PATH=$(DEB_HOST_MULTIARCH)
-
-override_dh_clean:
-	dh_clean
-	rm -f pyext/swsscommon.py pyext/swsscommon_wrap.cpp
-
-override_dh_strip:
-	dh_strip -plibswsscommon --dbg-package=libswsscommon-dbg

pyext/Makefile.am

@@ -1,6 +1,4 @@
-SUBDIRS = py3
 if PYTHON2
-SUBDIRS += py2
+include pyext/py2/Makefile.am
 endif
-
-ACLOCAL_AMFLAGS = -I m4
+include pyext/py3/Makefile.am

pyext/py2/Makefile.am

@@ -1,19 +1,12 @@
-SWIG_SOURCES = ../swsscommon.i
+pkgpython_PYTHON = pyext/py2/swsscommon.py pyext/py2/__init__.py
+pkgpyexec_LTLIBRARIES = pyext/py2/_swsscommon.la
 
-pkgpython_PYTHON = swsscommon.py __init__.py
-pkgpyexec_LTLIBRARIES = _swsscommon.la
+pyext_py2__swsscommon_la_SOURCES = pyext/py2/swsscommon_wrap.cpp
+pyext_py2__swsscommon_la_CPPFLAGS = -std=c++11 -Icommon -I/usr/include/python$(PYTHON_VERSION)
+pyext_py2__swsscommon_la_LDFLAGS = -module
+pyext_py2__swsscommon_la_LIBADD = common/libswsscommon.la -lpython$(PYTHON_VERSION)
 
-_swsscommon_la_SOURCES = swsscommon_wrap.cpp
-_swsscommon_la_CPPFLAGS = -std=c++11 -I../../common -I/usr/include/python$(PYTHON_VERSION)
-_swsscommon_la_LDFLAGS = -module
-_swsscommon_la_LIBADD = ../../common/libswsscommon.la -lpython$(PYTHON_VERSION)
+pyext/py2/swsscommon_wrap.cpp: $(SWIG_SOURCES)
+	$(SWIG) $(SWIG_FLAG) -Icommon -o $@ $<
 
-SWIG_FLAG = -Wall -c++ -python -keyword
-if ARCH64
-SWIG_FLAG += -DSWIGWORDSIZE64
-endif
-
-swsscommon_wrap.cpp: $(SWIG_SOURCES)
-	$(SWIG) $(SWIG_FLAG) -I../../common -o $@ $<
-
-CLEANFILES = swsscommon_wrap.cpp
+CLEANFILES += pyext/py2/swsscommon_wrap.cpp pyext/py2/swsscommon.py

pyext/py3/Makefile.am

@@ -1,19 +1,12 @@
-SWIG_SOURCES = ../swsscommon.i
+pkgpython3_PYTHON = pyext/py3/swsscommon.py pyext/py3/__init__.py
+pkgpy3exec_LTLIBRARIES = pyext/py3/_swsscommon.la
 
-pkgpython3_PYTHON = swsscommon.py __init__.py
-pkgpy3exec_LTLIBRARIES = _swsscommon.la
+pyext_py3__swsscommon_la_SOURCES = pyext/py3/swsscommon_wrap.cpp
+pyext_py3__swsscommon_la_CPPFLAGS = -std=c++11 -Icommon -I/usr/include/python$(PYTHON3_VERSION)
+pyext_py3__swsscommon_la_LDFLAGS = -module
+pyext_py3__swsscommon_la_LIBADD = common/libswsscommon.la $(PYTHON3_BLDLIBRARY)
 
-_swsscommon_la_SOURCES = swsscommon_wrap.cpp
-_swsscommon_la_CPPFLAGS = -std=c++11 -I../../common -I/usr/include/python$(PYTHON3_VERSION)
-_swsscommon_la_LDFLAGS = -module
-_swsscommon_la_LIBADD = ../../common/libswsscommon.la $(PYTHON3_BLDLIBRARY)
+pyext/py3/swsscommon_wrap.cpp: $(SWIG_SOURCES)
+	$(SWIG) $(SWIG_FLAG) -Icommon -o $@ $<
 
-SWIG_FLAG = -Wall -c++ -python -keyword
-if ARCH64
-SWIG_FLAG += -DSWIGWORDSIZE64
-endif
-
-swsscommon_wrap.cpp: $(SWIG_SOURCES)
-	$(SWIG) $(SWIG_FLAG) -I../../common -o $@ $<
-
-CLEANFILES = swsscommon_wrap.cpp
+CLEANFILES += pyext/py3/swsscommon_wrap.cpp pyext/py3/swsscommon.py

sonic-db-cli/Makefile.am

@@ -1,19 +1,11 @@
-INCLUDES = -I $(top_srcdir)
+lib_LTLIBRARIES += sonic-db-cli/libsonicdbcli.la
+sonic_db_cli_libsonicdbcli_la_SOURCES = sonic-db-cli/sonic-db-cli.cpp
+sonic_db_cli_libsonicdbcli_la_CFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON)
+sonic_db_cli_libsonicdbcli_la_CPPFLAGS = $(DBGFLAGS) $(AM_CPPFLAGS) $(CFLAGS_COMMON)
+sonic_db_cli_libsonicdbcli_la_LIBADD = common/libswsscommon.la -lpthread
 
-if DEBUG
-DBGFLAGS = -ggdb -DDEBUG
-else
-DBGFLAGS = -g -DNDEBUG
-endif
-
-lib_LTLIBRARIES = libsonicdbcli.la
-libsonicdbcli_la_SOURCES = sonic-db-cli.cpp
-libsonicdbcli_la_CFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON)
-libsonicdbcli_la_CPPFLAGS = $(DBGFLAGS) $(AM_CPPFLAGS) $(CFLAGS_COMMON)
-libsonicdbcli_la_LDFLAGS = -L$(top_srcdir)/common -lswsscommon -lpthread
-
-bin_PROGRAMS = sonic-db-cli
-sonic_db_cli_SOURCES = sonic-db-cli.cpp main.cpp
-sonic_db_cli_CFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON)
-sonic_db_cli_CPPFLAGS = $(DBGFLAGS) $(AM_CPPFLAGS) $(CFLAGS_COMMON)
-sonic_db_cli_LDFLAGS = -L$(top_srcdir)/common -lswsscommon -lpthread
+bin_PROGRAMS += sonic-db-cli/sonic-db-cli
+sonic_db_cli_sonic_db_cli_SOURCES = sonic-db-cli/sonic-db-cli.cpp sonic-db-cli/main.cpp
+sonic_db_cli_sonic_db_cli_CFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON)
+sonic_db_cli_sonic_db_cli_CPPFLAGS = $(DBGFLAGS) $(AM_CPPFLAGS) $(CFLAGS_COMMON)
+sonic_db_cli_sonic_db_cli_LDADD = common/libswsscommon.la -lpthread