From 91569b85d885539adab2f350c4e932002d476dfb Mon Sep 17 00:00:00 2001 From: Arjun Date: Mon, 10 Apr 2023 15:59:40 -0700 Subject: [PATCH] change some deprecated code and fix minor codestyle --- .../java/org/apache/gobblin/hive/HiveSerDeWrapper.java | 7 ++++--- .../gobblin/yarn/HelixInstancePurgerWithMetrics.java | 4 ++-- .../main/java/org/apache/gobblin/yarn/YarnService.java | 10 +++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveSerDeWrapper.java b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveSerDeWrapper.java index fea2e99d1dd..d5c3319d054 100644 --- a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveSerDeWrapper.java +++ b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveSerDeWrapper.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; import org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat; import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; +import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.avro.AvroSerDe; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; @@ -88,7 +89,7 @@ public String toString() { } } - private Optional serDe = Optional.absent(); + private Optional serDe = Optional.absent(); private final String serDeClassName; private final String inputFormatClassName; private final String outputFormatClassName; @@ -107,10 +108,10 @@ private HiveSerDeWrapper(String serDeClassName, String inputFormatClassName, Str * Get the {@link SerDe} instance associated with this {@link HiveSerDeWrapper}. * This method performs lazy initialization. */ - public SerDe getSerDe() throws IOException { + public AbstractSerDe getSerDe() throws IOException { if (!this.serDe.isPresent()) { try { - this.serDe = Optional.of(SerDe.class.cast(Class.forName(this.serDeClassName).newInstance())); + this.serDe = Optional.of(AbstractSerDe.class.cast(Class.forName(this.serDeClassName).newInstance())); } catch (Throwable t) { throw new IOException("Failed to instantiate SerDe " + this.serDeClassName, t); } diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/HelixInstancePurgerWithMetrics.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/HelixInstancePurgerWithMetrics.java index efb6644b489..9ac0ccad700 100644 --- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/HelixInstancePurgerWithMetrics.java +++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/HelixInstancePurgerWithMetrics.java @@ -42,9 +42,9 @@ public class HelixInstancePurgerWithMetrics { /** - * Blocking call for purging all offline helix instances. Provides boiler plate code for providing periodic updates + * Blocking call for purging all offline helix instances. Provides boilerplate code for providing periodic updates * and sending a GTE if it's an unexpected amount of time. - * + *

* All previous helix instances should be purged on startup. Gobblin task runners are stateless from helix * perspective because all important state is persisted separately in Workunit State Store or Watermark store. */ diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java index 7955c791abe..5b6610f8147 100644 --- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java +++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java @@ -333,7 +333,7 @@ public void handleContainerReleaseRequest(ContainerReleaseRequest containerRelea // Record that this container was explicitly released so that a new one is not spawned to replace it // Put the container id in the releasedContainerCache before releasing it so that handleContainerCompletion() // can check for the container id and skip spawning a replacement container. - // Note that this is best effort since these are asynchronous operations and a container may abort concurrently + // Note that this is the best effort since these are asynchronous operations and a container may abort concurrently // with the release call. So in some cases a replacement container may have already been spawned before // the container is put into the black list. this.releasedContainerCache.put(container.getId(), ""); @@ -453,7 +453,7 @@ private EventSubmitter buildEventSubmitter() { /** * Request an allocation of containers. If numTargetContainers is larger than the max of current and expected number * of containers then additional containers are requested. - * + *

* If numTargetContainers is less than the current number of allocated containers then release free containers. * Shrinking is relative to the number of currently allocated containers since it takes time for containers * to be allocated and assigned work and we want to avoid releasing a container prematurely before it is assigned @@ -709,7 +709,7 @@ private boolean shouldStickToTheSameNode(int containerExitStatus) { * Handle the completion of a container. A new container will be requested to replace the one * that just exited. Depending on the exit status and if container host affinity is enabled, * the new container may or may not try to be started on the same node. - * + *

* A container completes in either of the following conditions: 1) some error happens in the * container and caused the container to exit, 2) the container gets killed due to some reason, * for example, if it runs over the allowed amount of virtual or physical memory, 3) the gets @@ -860,11 +860,11 @@ private ImmutableMap.Builder buildContainerStatusEventMetadata(C * Get the number of matching container requests for the specified resource memory and cores. * Due to YARN-1902 and YARN-660, this API is not 100% accurate. {@link AMRMClientCallbackHandler#onContainersAllocated(List)} * contains logic for best effort clean up of requests, and the resource tend to match the allocated container. So in practice the count is pretty accurate. - * + *

* This API call gets the count of container requests for containers that are > resource if there is no request with the exact same resource * The RM can return containers that are larger (because of normalization etc). * Container may be larger by memory or cpu (e.g. container (1000M, 3cpu) can fit request (1000M, 1cpu) or request (500M, 3cpu). - * + *

* Thankfully since each helix tag / resource has a different priority, matching requests for one helix tag / resource * have complete isolation from another helix tag / resource */