[GOBBLIN-1914] Configurable Gobblin Application Master class for Yarn (…

…#3781) * [GOBBLIN-1914] Configurable Gobblin Application Master class for Yarn * Configurable logs
apache · Sep 21, 2023 · 1ca3192 · 1ca3192
1 parent fc73db8
commit 1ca3192
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 6 deletions.
diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
@@ -801,23 +801,34 @@ private void addJobConfPackage(String jobConfPackagePath, Path destDir, Map<Stri
 
   @VisibleForTesting
   protected String buildApplicationMasterCommand(String applicationId, int memoryMbs) {
-    String appMasterClassName = GobblinApplicationMaster.class.getSimpleName();
+    Class appMasterClass;
+    try {
+       String appMasterClassName = ConfigUtils.getString(
+          config, GobblinYarnConfigurationKeys.APP_MASTER_CLASS, GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_CLASS);
+       appMasterClass = Class.forName(appMasterClassName);
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException(e);
+    }
+
+    String logFileName = ConfigUtils.getString(config,
+        GobblinYarnConfigurationKeys.APP_MASTER_LOG_FILE_NAME, appMasterClass.getSimpleName());
+
     return new StringBuilder()
         .append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java")
         .append(" -Xmx").append((int) (memoryMbs * this.jvmMemoryXmxRatio) - this.jvmMemoryOverheadMbs).append("M")
         .append(" -D").append(GobblinYarnConfigurationKeys.JVM_USER_TIMEZONE_CONFIG).append("=").append(this.containerTimezone)
         .append(" -D").append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_DIR_NAME).append("=").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR)
-        .append(" -D").append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_FILE_NAME).append("=").append(appMasterClassName).append(".").append(ApplicationConstants.STDOUT)
+        .append(" -D").append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_FILE_NAME).append("=").append(logFileName).append(".").append(ApplicationConstants.STDOUT)
         .append(" ").append(JvmUtils.formatJvmArguments(this.appMasterJvmArgs))
-        .append(" ").append(GobblinApplicationMaster.class.getName())
+        .append(" ").append(appMasterClass.getName())
         .append(" --").append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME)
         .append(" ").append(this.applicationName)
         .append(" --").append(GobblinClusterConfigurationKeys.APPLICATION_ID_OPTION_NAME)
         .append(" ").append(applicationId)
         .append(" 1>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator).append(
-            appMasterClassName).append(".").append(ApplicationConstants.STDOUT)
+            logFileName).append(".").append(ApplicationConstants.STDOUT)
         .append(" 2>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator).append(
-            appMasterClassName).append(".").append(ApplicationConstants.STDERR)
+            logFileName).append(".").append(ApplicationConstants.STDERR)
         .toString();
   }
 
@@ -1071,4 +1082,4 @@ public void run() {
 
     gobblinYarnAppLauncher.launch();
   }
-}
+}
diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
@@ -30,6 +30,9 @@ public class GobblinYarnConfigurationKeys {
   public static final String GOBBLIN_YARN_PREFIX = "gobblin.yarn.";
 
   // General Gobblin Yarn application configuration properties.
+  public static final String APP_MASTER_CLASS = GOBBLIN_YARN_PREFIX + "app.master.class";
+  public static final String DEFAULT_APP_MASTER_CLASS = GobblinApplicationMaster.class.getName();
+  public static final String APP_MASTER_LOG_FILE_NAME = GOBBLIN_YARN_PREFIX + "app.master.log.file.name";
   public static final String APPLICATION_NAME_KEY = GOBBLIN_YARN_PREFIX + "app.name";
   public static final String APP_QUEUE_KEY = GOBBLIN_YARN_PREFIX + "app.queue";
   public static final String APP_REPORT_INTERVAL_MINUTES_KEY = GOBBLIN_YARN_PREFIX + "app.report.interval.minutes";

diff --git a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java
@@ -229,6 +229,9 @@ public void testBuildApplicationMasterCommand() {
 
     // 41 is from 64 * 0.8 - 10
     Assert.assertTrue(command.contains("-Xmx41"));
+    Assert.assertTrue(command.contains("org.apache.gobblin.yarn.GobblinApplicationMaster"));
+    Assert.assertTrue(command.contains("GobblinApplicationMaster.stdout"));
+    Assert.assertTrue(command.contains("GobblinApplicationMaster.stderr"));
   }
 
   @Test