Integrated hw-mgmt automation script into SONiC

platform/mellanox/hw-management.mk

@@ -24,3 +24,84 @@ MLNX_HW_MANAGEMENT = hw-management_1.mlnx.$(MLNX_HW_MANAGEMENT_VERSION)_$(CONFIG
 $(MLNX_HW_MANAGEMENT)_SRC_PATH = $(PLATFORM_PATH)/hw-management
 $(MLNX_HW_MANAGEMENT)_DEPENDS += $(LINUX_HEADERS) $(LINUX_HEADERS_COMMON)
 SONIC_MAKE_DEBS += $(MLNX_HW_MANAGEMENT)
+
+TEMP_HW_MGMT_DIR = /tmp/hw_mgmt
+PTCH_DIR = $(TEMP_HW_MGMT_DIR)/patch_dir/
+NON_UP_PTCH_DIR = $(TEMP_HW_MGMT_DIR)/non_up_patch_dir/
+PTCH_LIST  = $(TEMP_HW_MGMT_DIR)/series
+KCFG_LIST = $(TEMP_HW_MGMT_DIR)/kconfig
+HWMGMT_NONUP_LIST = $(BUILD_WORKDIR)/$($(MLNX_HW_MANAGEMENT)_SRC_PATH)/hwmgmt_nonup_patches
+USER_OUTFILE = $(BUILD_WORKDIR)/integrate-mlnx-hw-mgmt_jenkins.out
+TMPFILE = /tmp/intg-hw-mgmt.out
+
+integrate-mlnx-hw-mgmt:
+	$(FLUSH_LOG)
+	rm -rf $(TEMP_HW_MGMT_DIR) $(TMPFILE)
+	mkdir -p $(PTCH_DIR) $(NON_UP_PTCH_DIR)
+	touch $(PTCH_LIST) $(KCFG_LIST)
+
+	# clean up existing untracked files
+	pushd $(BUILD_WORKDIR); git stash; git clean -f -- platform/mellanox/; popd
+	pushd $(BUILD_WORKDIR)/src/sonic-linux-kernel; git stash; git clean -f -- patch/; popd
+
+	echo "#### Integrate HW-MGMT ${MLNX_HW_MANAGEMENT_VERSION} Kernel Patches into SONiC" > ${USER_OUTFILE}
+	pushd $(BUILD_WORKDIR)/$(PLATFORM_PATH) $(LOG_SIMPLE)
+
+	# Run tests
+	pushd integration-scripts/tests; pytest-3 -v; popd
+
+	# Pre-processing before runing hw_mgmt script
+	integration-scripts/hwmgmt_kernel_patches.py pre \
+							--config_inclusion $(KCFG_LIST) \
+							--build_root $(BUILD_WORKDIR) $(LOG_SIMPLE)
+
+	$(BUILD_WORKDIR)/$($(MLNX_HW_MANAGEMENT)_SRC_PATH)/hw-mgmt/recipes-kernel/linux/deploy_kernel_patches.py \
+							--dst_accepted_folder $(PTCH_DIR) \
+							--dst_candidate_folder $(NON_UP_PTCH_DIR) \
+							--series_file $(PTCH_LIST) \
+							--config_file $(KCFG_LIST) \
+							--kernel_version $(KERNEL_VERSION) \
+							--os_type sonic $(LOG_SIMPLE)
+
+	# Post-processing
+	integration-scripts/hwmgmt_kernel_patches.py post \
+							--patches $(PTCH_DIR) \
+							--non_up_patches $(NON_UP_PTCH_DIR) \
+							--config_inclusion $(KCFG_LIST) \
+							--series $(PTCH_LIST) \
+							--current_non_up_patches $(HWMGMT_NONUP_LIST) \
+							--build_root $(BUILD_WORKDIR) $(LOG_SIMPLE)
+
+	# Commit the changes in linux kernel and and log the diff
+	pushd $(BUILD_WORKDIR)/src/sonic-linux-kernel
+	git add -- patch/; git commit -m "Intgerate HW-MGMT ${MLNX_HW_MANAGEMENT_VERSION} Changes";
+
+	echo -en "\n###-> series file changes in sonic-linux-kernel <-###\n" >> ${USER_OUTFILE}
+	git diff --no-color  HEAD~1 HEAD -- patch/series >> ${USER_OUTFILE}
+
+	echo -en "\n###-> kconfig-inclusion file changes in sonic-linux-kernel <-###\n" >> ${USER_OUTFILE}
+	git diff --no-color  HEAD~1 HEAD -- patch/kconfig-inclusions >> ${USER_OUTFILE}
+
+	echo -en '\n###-> Summary of files updated in sonic-linux-kernel <-###\n' >> ${USER_OUTFILE}
+	git diff --no-color  HEAD~1 HEAD --stat --output=${TMPFILE}
+	cat ${TMPFILE} | tee -a ${USER_OUTFILE}
+	popd
+
+	# Commit the changes in buildimage and log the diff
+	pushd $(BUILD_WORKDIR)
+	git add -- $(PLATFORM_PATH); git commit -m "Intgerate HW-MGMT ${MLNX_HW_MANAGEMENT_VERSION} Changes";
+
+	echo -en '\n###-> Non Upstream series.patch changes <-###\n' >> ${USER_OUTFILE}
+	git diff --no-color  HEAD~1 HEAD -- $(PLATFORM_PATH)/non-upstream-patches/series.patch >> ${USER_OUTFILE}
+
+	echo -en '\n###-> Non Upstream patch list file <-###\n' >> ${USER_OUTFILE}
+	git diff --no-color  HEAD~1 HEAD -- $($(MLNX_HW_MANAGEMENT)_SRC_PATH)/hwmgmt_nonup_patches >> ${USER_OUTFILE}
+
+	echo -en '\n###-> Summary of buildimage changes <-###\n' >> ${USER_OUTFILE}
+	git diff --no-color  HEAD~1 HEAD --stat --output=${TMPFILE} -- $(PLATFORM_PATH)
+	cat ${TMPFILE} | tee -a ${USER_OUTFILE}
+	popd
+
+	popd $(LOG_SIMPLE)
+
+SONIC_PHONY_TARGETS += integrate-mlnx-hw-mgmt

platform/mellanox/hw-management/hwmgmt_nonup_patches

@@ -0,0 +1,63 @@
+# Current non-upstream patch list, should be updated by integrate_kernel_patches.py script
+0099-mlxsw-core_hwmon-Fix-variable-names-for-hwmon-attrib.patch
+0100-mlxsw-core_thermal-Rename-labels-according-to-naming.patch
+0101-mlxsw-core_thermal-Remove-obsolete-API-for-query-res.patch
+0102-mlxsw-reg-Add-mgpir_-prefix-to-MGPIR-fields-comments.patch
+0103-mlxsw-core-Remove-unnecessary-asserts.patch
+0104-mlxsw-reg-Extend-MTMP-register-with-new-slot-number-.patch
+0105-mlxsw-reg-Extend-MTBR-register-with-new-slot-number-.patch
+0106-mlxsw-reg-Extend-MCIA-register-with-new-slot-number-.patch
+0107-mlxsw-reg-Extend-MCION-register-with-new-slot-number.patch
+0108-mlxsw-reg-Extend-PMMP-register-with-new-slot-number-.patc
+0109-mlxsw-reg-Extend-MGPIR-register-with-new-slot-fields.patch
+0110-mlxsw-core_env-Pass-slot-index-during-PMAOS-register.patch
+0111-mlxsw-reg-Add-new-field-to-Management-General-Periph.patch
+0112-mlxsw-core-Extend-interfaces-for-cable-info-access-w.patch
+0113-mlxsw-core-Extend-port-module-data-structures-for-li.patch
+0114-mlxsw-core-Move-port-module-events-enablement-to-a-s.patch
+0115-mlxsw-core_hwmon-Split-gearbox-initialization.patch
+0116-mlxsw-core_hwmon-Extend-internal-structures-to-suppo.patch
+0117-mlxsw-core_hwmon-Introduce-slot-parameter-in-hwmon-i.patch
+0118-mlxsw-core_hwmon-Extend-hwmon-device-with-gearbox-ma.patch
+0119-mlxsw-core_thermal-Extend-internal-structures-to-sup.patch
+0120-mlxsw-core_thermal-Split-gearbox-initialization.patch
+0121-mlxsw-core_thermal-Extend-thermal-area-with-gearbox-.patch
+0122-mlxsw-core_thermal-Add-line-card-id-prefix-to-line-c.patch
+0123-mlxsw-core_thermal-Use-exact-name-of-cooling-devices.patch
+0124-mlxsw-core_thermal-Use-common-define-for-thermal-zon.patch
+0125-devlink-add-support-to-create-line-card-and-expose-t.patch
+0126-devlink-implement-line-card-provisioning.patch
+0127-devlink-implement-line-card-active-state.patch
+0128-devlink-add-port-to-line-card-relationship-set.patch
+0129-devlink-introduce-linecard-info-get-message.patch
+0130-devlink-introduce-linecard-info-get-message.patch
+0131-mlxsw-reg-Add-Ports-Mapping-event-Configuration-Regi.patch
+0132-mlxsw-reg-Add-Management-DownStream-Device-Query-Reg.patch
+0133-mlxsw-reg-Add-Management-DownStream-Device-Control-R.patch
+0134-mlxsw-reg-Add-Management-Binary-Code-Transfer-Regist.patch
+0135-mlxsw-core_linecards-Add-line-card-objects-and-imple.patch
+0136-mlxsw-core_linecards-Implement-line-card-activation-.patch
+0137-mlxsw-core-Extend-driver-ops-by-remove-selected-port.patch
+0138-mlxsw-spectrum-Add-port-to-linecard-mapping.patch
+0139-mlxsw-reg-Introduce-Management-Temperature-Extended-.patch
+0140-mlxsw-core-Add-APIs-for-thermal-sensor-mapping.patch
+0141-mlxsw-reg-Add-Management-DownStream-Device-Tunneling.patch
+0142-mlxsw-core_linecards-Probe-devices-for-provisioned-l.patch
+0143-mlxsw-core_linecards-Expose-device-FW-version-over-d.patch
+0144-mlxsw-core-Introduce-flash-update-components.patch
+0145-mlxfw-Get-the-PSID-value-using-op-instead-of-passing.patch
+0146-mlxsw-core_linecards-Implement-line-card-device-flas.patch
+0147-mlxsw-core_linecards-Introduce-ops-for-linecards-sta.patch
+0148-mlxsw-core-Add-interfaces-for-line-card-initializati.patch
+0149-mlxsw-core_thermal-Add-interfaces-for-line-card-init.patch
+0150-mlxsw-core_hwmon-Add-interfaces-for-line-card-initia.patch
+0151-mlxsw-minimal-Prepare-driver-for-modular-system-supp.patch
+0152-mlxsw-core-Extend-bus-init-function-with-event-handl.patch
+0153-mlxsw-i2c-Add-support-for-system-events-handling.patch
+0154-mlxsw-core-Export-line-card-API.patch
+0155-mlxsw-minimal-Add-system-event-handler.patch
+0156-mlxsw-minimal-Add-interfaces-for-line-card-initializ.patch
+0163-platform-mellanox-Introduce-support-for-rack-manager.patch
+0176-platform-mellanox-fix-reset_pwr_converter_fail-attri.patch
+0177-Documentation-ABI-fix-description-of-fix-reset_pwr_c.patch
+0178-platform-mellanox-Introduce-support-for-next-generat.patch

platform/mellanox/integration-scripts/helper.py

@@ -0,0 +1,173 @@
+import os
+import glob
+import re
+
+MARK_ID = "###->"
+MLNX_KFG_MARKER = "mellanox"
+HW_MGMT_MARKER = "mellanox_hw_mgmt"
+SLK_PATCH_LOC = "src/sonic-linux-kernel/patch/"
+SLK_KCONFIG = SLK_PATCH_LOC + "kconfig-inclusions"
+SLK_SERIES = SLK_PATCH_LOC + "series"
+NON_UP_PATCH_DIR = "platform/mellanox/non-upstream-patches/"
+NON_UP_PATCH_LOC = NON_UP_PATCH_DIR + "patches"
+NON_UP_PATCH_DIFF = NON_UP_PATCH_DIR + "series.patch"
+KCFG_HDR_RE = "\[(.*)\]"
+ # kconfig_inclusion headers to consider
+HDRS = ["common", "amd64"]
+
+class FileHandler:
+
+    @staticmethod
+    def write_lines(path, lines, raw=False):
+        # Create the dir if it doesn't exist already
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        with open(path, 'w') as f:
+            for line in lines:
+                if raw:
+                    f.write(f"{line}")
+                else:
+                    f.write(f"{line}\n")
+
+    @staticmethod
+    def read_raw(path):
+        # Read the data line by line into a list
+        data = []
+        with open(path) as im:
+            data = im.readlines()
+        return data
+
+    @staticmethod
+    def read_strip(path, as_string=False):
+        # Read the data line by line into a list and strip whitelines
+        data = FileHandler.read_raw(path)
+        data = [d.strip() for d in data]
+        if as_string:
+            return "\n".join(data)
+        return data
+
+    @staticmethod
+    def read_strip_minimal(path, as_string=False, ignore_start_with="#"):
+        # Read the data line by line into a list, strip spaces and ignore comments
+        data = FileHandler.read_raw(path)
+        filtered_data = []
+        for l in data:
+            l = l.strip()
+            if l and not l.startswith(ignore_start_with):
+                filtered_data.append(l)
+        if as_string:
+            return "\n".join(filtered_data)
+        return filtered_data
+
+    @staticmethod
+    def read_dir(path, ext="*") -> list:
+        return [os.path.basename(f) for f in glob.glob(os.path.join(path, ext))]
+
+    @staticmethod
+    def find_marker_indices(lines: list, marker=None) -> tuple:
+        i_start = -1
+        i_end = len(lines)
+        # print("TEST", marker, lines)
+        if marker:
+            for index, line in enumerate(lines):
+                # assumes one unique marker per file
+                # if multiple marker sections are present, reads the first one
+                if line.strip().startswith(MARK_ID):
+                    if marker+"-start" in line:
+                        i_start = index
+                    elif marker+"-end" in line:
+                        i_end = index
+        # print(i_start, i_end)             
+        return (i_start, i_end)
+
+    @staticmethod
+    def read_kconfig_inclusion(path, marker=MLNX_KFG_MARKER):
+        lines = FileHandler.read_strip(path)
+        if not marker:
+            return lines
+        i_start, i_end = FileHandler.find_marker_indices(lines, marker)
+
+        if i_start < 0 or i_end >= len(lines):
+            print("-> WARNING No Marker Found")
+            return []
+
+        return lines[i_start+1:i_end]
+
+    @staticmethod
+    def write_lines_marker(path, writable_opts: list, marker=None):
+        # if marker is none, just write the opts into the file,
+        # otherwise write the data only b/w the marker
+        curr_data = FileHandler.read_raw(path)
+        i_start, i_end = FileHandler.find_marker_indices(curr_data, marker)
+        newline_writ_opts = [opt + "\n" for opt in writable_opts]
+        if i_start < 0 or i_end >= len(curr_data):
+            print("-> WARNING No Marker Found, writing data at the end of file")
+            curr_data.extend(["\n"])
+            curr_data.extend(newline_writ_opts)
+        else:
+            curr_data = curr_data[0:i_start+1] + newline_writ_opts + curr_data[i_end:]
+
+        print("-> INFO Written the following opts: \n{}".format("".join(FileHandler.read_raw(path))))
+        FileHandler.write_lines(path, curr_data, True)
+
+    @staticmethod
+    def read_kconfig_parser(path) -> dict:
+        # kconfig_inclusion output formatted to {"no_parent", "common":[,], "amd64": [,], "arm64": [,]}
+        lines = FileHandler.read_strip_minimal(path)
+        ret = dict({"no_parent":[]})
+        curr_hdr = ""
+        for line in lines:
+            match = re.search(KCFG_HDR_RE, line)
+            if match:
+                curr_hdr = match.group(1)
+                ret[curr_hdr] = []
+            elif curr_hdr in ret:
+                ret[curr_hdr].append(line)
+            else:
+                ret["no_parent"].append(line)
+        return ret
+
+
+class KCFG:
+
+    @staticmethod
+    def parse_opt_str(opt: str) -> tuple:
+        if not opt.startswith("CONFIG"):
+            print("-> DEBUG: Malformed kconfig opt, {}".format(opt))
+            return ()
+
+        tmp = opt.split("=")
+        if len(tmp) != 2:
+            print("-> DEBUG: Malformed kconfig opt, {}".format(opt))
+            return ()
+
+        if not tmp[0].startswith("CONFIG"):
+            print("-> DEBUG: Malformed kconfig opt, {}".format(opt))
+            return ()
+        return (tmp[0], tmp[1])
+
+    @staticmethod
+    def parse_opts_strs(kcfg_sec: list) -> list(tuple()):
+        opts = [] # list of tuples (CONFIG_*, "m|y|n")
+        for kcfg in kcfg_sec:
+            tmp = KCFG.parse_opt_str(kcfg)
+            if tmp: 
+                opts.append(tmp)
+        return opts
+
+    @staticmethod
+    def get_writable_opts(opts):
+        lines = []
+        for opt in opts:
+            lines.append("{}={}".format(opt[0].upper(), opt[1]))
+        return lines
+
+
+class Action():
+    def __init__(self, args):
+        self.args = args
+
+    def perform(self):
+        pass
+
+    def write_user_out(self):
+        pass