From 8cbb843f693d2809f2718afdc08877cce41e16ed Mon Sep 17 00:00:00 2001 From: Nandakumar Vadivelu Date: Wed, 13 Nov 2024 18:18:53 +0530 Subject: [PATCH 1/2] HDDS-11695. SCM follower should not log NotLeaderException during Pipeline Report processing. --- .../scm/pipeline/PipelineReportHandler.java | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java index dcc7bdb3fba..f27cefb49ff 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.safemode.SafeModeManager; import org.apache.hadoop.hdds.scm.server @@ -91,35 +92,45 @@ public void onMessage(PipelineReportFromDatanode pipelineReportFromDatanode, for (PipelineReport report : pipelineReport.getPipelineReportList()) { try { processPipelineReport(report, dn, publisher); - } catch (NotLeaderException ex) { - // Avoid NotLeaderException logging which happens when processing - // pipeline report on followers. } catch (PipelineNotFoundException e) { - LOGGER.error("Could not find pipeline {}", report.getPipelineID()); + handlePipelineNotFoundException(report, dn, publisher); } catch (IOException | TimeoutException e) { - LOGGER.error("Could not process pipeline report={} from dn={}.", - report, dn, e); + // Ignore NotLeaderException logging which happens when processing + // pipeline report on followers. + if (!isNotLeaderException(e)) { + LOGGER.error("Could not process pipeline report={} from dn={}.", + report, dn, e); + } } } } - protected void processPipelineReport(PipelineReport report, - DatanodeDetails dn, EventPublisher publisher) - throws IOException, TimeoutException { - PipelineID pipelineID = PipelineID.getFromProtobuf(report.getPipelineID()); - Pipeline pipeline; - try { - pipeline = pipelineManager.getPipeline(pipelineID); - } catch (PipelineNotFoundException e) { - if (scmContext.isLeader()) { - LOGGER.info("Reported pipeline {} is not found", pipelineID); - SCMCommand< ? > command = new ClosePipelineCommand(pipelineID); + private void handlePipelineNotFoundException(final PipelineReport report, + final DatanodeDetails dn, final EventPublisher publisher) { + final PipelineID pipelineID = PipelineID.getFromProtobuf(report.getPipelineID()); + LOGGER.error("Pipeline {}, reported by datanode {} is not found.", pipelineID, dn); + if (scmContext.isLeader()) { + try { + final SCMCommand command = new ClosePipelineCommand(pipelineID); command.setTerm(scmContext.getTermOfLeader()); publisher.fireEvent(SCMEvents.DATANODE_COMMAND, new CommandForDatanode<>(dn.getUuid(), command)); + } catch (NotLeaderException ex) { + // Do nothing if the leader has changed. } - return; } + } + + private static boolean isNotLeaderException(final Exception e) { + return e instanceof SCMException && ((SCMException) e).getResult().equals( + SCMException.ResultCodes.SCM_NOT_LEADER); + } + + protected void processPipelineReport(PipelineReport report, + DatanodeDetails dn, EventPublisher publisher) + throws IOException, TimeoutException { + final PipelineID pipelineID = PipelineID.getFromProtobuf(report.getPipelineID()); + final Pipeline pipeline = pipelineManager.getPipeline(pipelineID); setReportedDatanode(pipeline, dn); setPipelineLeaderId(report, pipeline, dn); From 4177dbf3132198b11d026b21380c5a4df4b6c73f Mon Sep 17 00:00:00 2001 From: Nandakumar Vadivelu Date: Thu, 14 Nov 2024 16:49:53 +0530 Subject: [PATCH 2/2] Review comment addressed. Co-authored-by: Chung En Lee --- .../apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java index f27cefb49ff..b931f122a97 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java @@ -108,7 +108,7 @@ public void onMessage(PipelineReportFromDatanode pipelineReportFromDatanode, private void handlePipelineNotFoundException(final PipelineReport report, final DatanodeDetails dn, final EventPublisher publisher) { final PipelineID pipelineID = PipelineID.getFromProtobuf(report.getPipelineID()); - LOGGER.error("Pipeline {}, reported by datanode {} is not found.", pipelineID, dn); + LOGGER.info("Pipeline {}, reported by datanode {} is not found.", pipelineID, dn); if (scmContext.isLeader()) { try { final SCMCommand command = new ClosePipelineCommand(pipelineID);