From 6798a357f307bc4225a72d54c795bd21172b0595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Fri, 3 Jan 2025 17:48:50 +0800 Subject: [PATCH 1/4] fix: major_hold_scene dmesg get MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add： version check task --- .../check/tasks/obproxy/version/old_version.yaml | 9 +++++++++ .../check/tasks/observer/version/old_version.yaml | 14 ++++++++++++++ plugins/rca/major_hold.py | 8 ++++---- 3 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 plugins/check/tasks/obproxy/version/old_version.yaml diff --git a/plugins/check/tasks/obproxy/version/old_version.yaml b/plugins/check/tasks/obproxy/version/old_version.yaml new file mode 100644 index 00000000..31864823 --- /dev/null +++ b/plugins/check/tasks/obproxy/version/old_version.yaml @@ -0,0 +1,9 @@ +info: 'Check obproxy version . Some versions of obproxy are not recommended' +task: + - steps: + - type: ssh + ssh: "export LD_LIBRARY_PATH=#{remote_home_path}/lib && #{remote_home_path}/bin/obproxy --version 2>&1 | grep \"obproxy (\" | awk '{print $3}'" + result: + set_value: obproxy_version + verify: '[[ ! "${obproxy_version}" == "4.0"* ]] && [[ ! "${obproxy_version}" == "4.1"* ]] && [[ ! "${obproxy_version}" == "4.2"* ]] && [[ ! "${obproxy_version}" == "4.3.0"* ]] && [[ ! "${obproxy_version}" == "3"* ]] && [[ ! "${obproxy_version}" == "4.3.1"* ]]' + err_msg: '#{obproxy_version} is not recommended, please upgrade to the bproxy' \ No newline at end of file diff --git a/plugins/check/tasks/observer/version/old_version.yaml b/plugins/check/tasks/observer/version/old_version.yaml index bcbc9b59..72135a38 100644 --- a/plugins/check/tasks/observer/version/old_version.yaml +++ b/plugins/check/tasks/observer/version/old_version.yaml @@ -7,6 +7,20 @@ task: set_value: observer_version verify: '[[ ! "#{observer_version}" == *"3.1."* ]] || [[ ! "#{observer_version}" == *"CE"* ]]' err_msg: 'Starting from June 30, 2024, OceanBase Database Community Edition V3.1.x will no longer receive any bug fixes or version updates.Please upgrade as soon as possible' + - type: ssh + ssh: "echo \"#{observer_version}\" | grep -oP '(?<=\()[^ ]+ \K[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+'" + result: + set_value: observer_version + verify: '[[ ! "${observer_version}" == "4.0"* ]] && [[ ! "${observer_version}" == "4.1"* ]] && [[ ! "${observer_version}" == "4.2.2"* ]] && [[ ! "${observer_version}" == "4.3.0"* ]] && [[ ! "${observer_version}" == "4.3.1"* ]]' + err_msg: '#{observer_version} is not recommended, please upgrade to the obcluster' + - type: ssh + ssh: "echo \"#{observer_version}\" | awk -F. '{print $NF}'" + result: + set_value: observer_version_end_str + verify: '[[ "$observer_version" == "4.2.1"* && "$observer_version_end_str" -ge 5 ]] || [[ ! "$observer_version" == "4.2.1"* ]]' + err_msg: '#{observer_version} is not recommended, please upgrade to the obcluster' + + diff --git a/plugins/rca/major_hold.py b/plugins/rca/major_hold.py index 5f60fa45..fd4b85a2 100644 --- a/plugins/rca/major_hold.py +++ b/plugins/rca/major_hold.py @@ -62,7 +62,7 @@ def execute(self): CDB_OB_MAJOR_COMPACTION_err_tenant_ids.append(str(data.get('TENANT_ID'))) self.record.add_record("CDB_OB_MAJOR_COMPACTION have IS_ERROR='YES',the tenant_ids are {0}".format(err_tenant_ids)) err_tenant_ids.extend(CDB_OB_MAJOR_COMPACTION_err_tenant_ids) - + self.record.add_record("check on CDB_OB_MAJOR_COMPACTION IS_ERROR is 'YES'.\n sql:{0}".format(sql), result=err_tenant_ids) except Exception as e: self.stdio.warn("MajorHoldScene execute CDB_OB_MAJOR_COMPACTION panic: {0}".format(e)) raise RCAExecuteException("MajorHoldScene execute CDB_OB_MAJOR_COMPACTION panic: {0}".format(e)) @@ -206,9 +206,9 @@ def execute(self): # all node execute for observer_node in self.observer_nodes: ssh_client = observer_node["ssher"] - ssh_client.exec_cmd("dmesg -T > /tmp/dmesg_{0}.log".format(observer_node.get_name())) - ssh_client.download("/tmp/dmesg_{0}.log".format(observer_node.get_name()), self.local_path + "/dmesg_log") - tenant_record.add_record("download /tmp/dmesg_{0}.log to {1}".format(observer_node.get_name(), self.local_path + "/dmesg_log")) + ssh_client.exec_cmd("dmesg -T > /tmp/dmesg_{0}.log".format(ssh_client.get_name())) + ssh_client.download("/tmp/dmesg_{0}.log".format(ssh_client.get_name()), self.local_path + "/dmesg_log") + tenant_record.add_record("download /tmp/dmesg_{0}.log to {1}".format(ssh_client.get_name(), self.local_path + "/dmesg_log")) except Exception as e: self.stdio.warn("MajorHoldScene execute 6 get dmesg exception: {0}".format(e)) tenant_record.add_suggest("send the {0} to the oceanbase community".format(self.local_path)) From c813fa40e044c3b6a15a8cdd780992669bad94fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 6 Jan 2025 17:25:09 +0800 Subject: [PATCH 2/4] rca disconnection support env 'from' 'to' --- plugins/rca/disconnection.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/plugins/rca/disconnection.py b/plugins/rca/disconnection.py index f8cccfce..86030f3f 100644 --- a/plugins/rca/disconnection.py +++ b/plugins/rca/disconnection.py @@ -12,7 +12,7 @@ """ @time: 2024/03/11 -@file: disconnection_scene.py +@file: disconnection.py @desc: """ import re @@ -68,9 +68,20 @@ def __execute_obproxy_one_node(self, node): self.gather_log.set_parameters("nodes_list", [node]) self.gather_log.set_parameters("target", "obproxy") self.gather_log.set_parameters("scope", "obproxy_diagnosis") + # get log time + log_time_tag = None if self.input_parameters.get("since") is not None: since = self.input_parameters.get("since") self.gather_log.set_parameters("since", since) + log_time_tag = "since: {0}".format(since) + if self.input_parameters.get("from") is not None and self.input_parameters.get("to") is not None: + from_time = self.input_parameters.get("from") + to_time = self.input_parameters.get("to") + self.gather_log.set_parameters("from", from_time) + self.gather_log.set_parameters("to", to_time) + log_time_tag = "from {0}, to {1}".format(from_time, to_time) + if log_time_tag is not None: + self.record.add_record("on node: {1} get log_time:{0}".format(log_time_tag, node.get("ip"))) self.work_path = self.store_dir logs_name = self.gather_log.execute() if len(logs_name) == 0: From f87ea4479ddd73ea310e9f961a625524370c4bcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 6 Jan 2025 20:23:42 +0800 Subject: [PATCH 3/4] rca disconnection support env 'from' 'to' --- plugins/rca/major_hold.py | 5 +++-- src/handler/gather/step/base.py | 26 ++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/plugins/rca/major_hold.py b/plugins/rca/major_hold.py index fd4b85a2..66a8b124 100644 --- a/plugins/rca/major_hold.py +++ b/plugins/rca/major_hold.py @@ -52,6 +52,7 @@ def execute(self): err_tenant_ids = [] self.record.add_record("check major task is error or not") try: + sql = 'select * from oceanbase.CDB_OB_MAJOR_COMPACTION where IS_ERROR="YES";' COMPACTING_data = self.ob_connector.execute_sql_return_cursor_dictionary('select * from oceanbase.CDB_OB_MAJOR_COMPACTION where IS_ERROR="YES";').fetchall() if len(COMPACTING_data) == 0: self.record.add_record("CDB_OB_MAJOR_COMPACTION is not exist IS_ERROR='YES'") @@ -62,7 +63,7 @@ def execute(self): CDB_OB_MAJOR_COMPACTION_err_tenant_ids.append(str(data.get('TENANT_ID'))) self.record.add_record("CDB_OB_MAJOR_COMPACTION have IS_ERROR='YES',the tenant_ids are {0}".format(err_tenant_ids)) err_tenant_ids.extend(CDB_OB_MAJOR_COMPACTION_err_tenant_ids) - self.record.add_record("check on CDB_OB_MAJOR_COMPACTION IS_ERROR is 'YES'.\n sql:{0}".format(sql), result=err_tenant_ids) + self.record.add_record("check on CDB_OB_MAJOR_COMPACTION IS_ERROR is 'YES'.\n sql:{0}".format(sql)) except Exception as e: self.stdio.warn("MajorHoldScene execute CDB_OB_MAJOR_COMPACTION panic: {0}".format(e)) raise RCAExecuteException("MajorHoldScene execute CDB_OB_MAJOR_COMPACTION panic: {0}".format(e)) @@ -108,7 +109,7 @@ def execute(self): # execute record need more for err_tenant_id in err_tenant_ids: - tenant_record = RCA_ResultRecord() + tenant_record = RCA_ResultRecord(stdio=self.stdio) first_record_records = self.record.records.copy() tenant_record.records.extend(first_record_records) self.stdio.verbose("tenant_id is {0}".format(err_tenant_id)) diff --git a/src/handler/gather/step/base.py b/src/handler/gather/step/base.py index 8b1d8e7a..f2ae2998 100644 --- a/src/handler/gather/step/base.py +++ b/src/handler/gather/step/base.py @@ -16,6 +16,7 @@ @desc: """ from src.common.ssh_client.ssh import SshClient +from src.common.tool import Util from src.handler.gather.gather_component_log import GatherComponentLogHandler from src.common.stdio import SafeStdio from src.handler.gather.step.ssh import SshHandler @@ -39,6 +40,7 @@ def __init__(self, context, step, node, cluster, report_path, task_variable_dict self.report_path = report_path self.env = env self.node_number = node_number + self.options = self.context.options def execute(self): self.stdio.verbose("step: {0}".format(self.step)) @@ -69,7 +71,17 @@ def execute(self): elif self.step["type"] == "log" and (skip_type != "ssh"): if self.node.get("host_type") and self.node.get("host_type") == "OBSERVER": handler = GatherComponentLogHandler() - handler.init(self.context, target="observer", grep=self.step.get("grep"), nodes=[self.node], store_dir=self.report_path, is_scene=True) + handler.init( + self.context, + target="observer", + grep=self.step.get("grep"), + nodes=[self.node], + store_dir=self.report_path, + from_option=Util.get_option(self.options, 'from'), + to_option=Util.get_option(self.options, 'to'), + since=Util.get_option(self.options, 'since'), + is_scene=True, + ) handler.handle() else: self.stdio.verbose("node host_type is {0} not OBSERVER, skipping gather log".format(self.node.get("host_type"))) @@ -78,7 +90,17 @@ def execute(self): self.context.set_variable('filter_nodes_list', [self.node]) self.context.set_variable('gather_grep', self.step.get("grep")) handler = GatherComponentLogHandler() - handler.init(self.context, target="obproxy", grep=self.step.get("grep"), nodes=[self.node], store_dir=self.report_path, is_scene=True) + handler.init( + self.context, + target="obproxy", + grep=self.step.get("grep"), + nodes=[self.node], + store_dir=self.report_path, + from_option=Util.get_option(self.options, 'from'), + to_option=Util.get_option(self.options, 'to'), + since=Util.get_option(self.options, 'since'), + is_scene=True, + ) handler.handle() else: self.stdio.verbose("node host_type is {0} not OBPROXY, skipping gather log".format(self.node.get("host_type"))) From 4879ab65c981cfbe86bb49c2eaf5c0b121ce4f11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Tue, 7 Jan 2025 20:47:17 +0800 Subject: [PATCH 4/4] rca disconnection support env 'from' 'to' --- plugins/rca/major_hold.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/rca/major_hold.py b/plugins/rca/major_hold.py index 9274b378..3149e96f 100644 --- a/plugins/rca/major_hold.py +++ b/plugins/rca/major_hold.py @@ -53,7 +53,7 @@ def execute(self): self.record.add_record("check major task is error or not") try: sql = 'select * from oceanbase.CDB_OB_MAJOR_COMPACTION where IS_ERROR="YES";' - COMPACTING_data = self.ob_connector.execute_sql_return_cursor_dictionary('select * from oceanbase.CDB_OB_MAJOR_COMPACTION where IS_ERROR="YES";').fetchall() + COMPACTING_data = self.ob_connector.execute_sql_return_cursor_dictionary(sql).fetchall() if len(COMPACTING_data) == 0: self.record.add_record("CDB_OB_MAJOR_COMPACTION is not exist IS_ERROR='YES'") else: