From 53f686cca43a1640885db12b836d563f3152d618 Mon Sep 17 00:00:00 2001 From: harshbafna Date: Wed, 13 May 2020 13:07:55 +0530 Subject: [PATCH 1/2] fixed code for build failure in python virtual environment --- .../serve/metrics/MetricCollector.java | 26 +++++++++--- .../pytorch/serve/util/ConfigManagerTest.java | 40 ++++--------------- 2 files changed, 28 insertions(+), 38 deletions(-) diff --git a/frontend/server/src/main/java/org/pytorch/serve/metrics/MetricCollector.java b/frontend/server/src/main/java/org/pytorch/serve/metrics/MetricCollector.java index 80d7396b2c..1c8b2310e3 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/metrics/MetricCollector.java +++ b/frontend/server/src/main/java/org/pytorch/serve/metrics/MetricCollector.java @@ -7,8 +7,10 @@ import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; import org.pytorch.serve.util.ConfigManager; import org.pytorch.serve.wlm.ModelManager; @@ -40,23 +42,37 @@ public void run() { String pythonEnv; if ((pythonPath == null || pythonPath.isEmpty()) && (!workingDir.getAbsolutePath().contains("site-package"))) { - pythonEnv = "PYTHONPATH=" + workingDir.getAbsolutePath(); + pythonEnv = workingDir.getAbsolutePath(); } else { - pythonEnv = "PYTHONPATH=" + pythonPath; + pythonEnv = pythonPath; if (!workingDir.getAbsolutePath().contains("site-package")) { pythonEnv += File.pathSeparatorChar + workingDir.getAbsolutePath(); // NOPMD } } // sbin added for macs for python sysctl pythonpath + HashMap environment = new HashMap<>(System.getenv()); + environment.put("PYTHONPATH", pythonEnv); + StringBuilder path = new StringBuilder(); - path.append("PATH=").append(System.getenv("PATH")); + path.append(System.getenv("PATH")); String osName = System.getProperty("os.name"); if (osName.startsWith("Mac OS X")) { path.append(File.pathSeparatorChar).append("/sbin/"); } - String[] env = {pythonEnv, path.toString()}; - final Process p = Runtime.getRuntime().exec(args, env, workingDir); + environment.put("PATH", path.toString()); + ArrayList envList = new ArrayList<>(); + Pattern blackList = configManager.getBlacklistPattern(); + + for (Map.Entry entry : environment.entrySet()) { + if (!blackList.matcher(entry.getKey()).matches()) { + envList.add(entry.getKey() + '=' + entry.getValue()); + } + } + + final Process p = + Runtime.getRuntime() + .exec(args, envList.toArray(new String[0]), workingDir); // NOPMD ModelManager modelManager = ModelManager.getInstance(); Map workerMap = modelManager.getWorkers(); try (OutputStream os = p.getOutputStream()) { diff --git a/frontend/server/src/test/java/org/pytorch/serve/util/ConfigManagerTest.java b/frontend/server/src/test/java/org/pytorch/serve/util/ConfigManagerTest.java index 9d78aa2a2f..bd9fb82f68 100644 --- a/frontend/server/src/test/java/org/pytorch/serve/util/ConfigManagerTest.java +++ b/frontend/server/src/test/java/org/pytorch/serve/util/ConfigManagerTest.java @@ -6,7 +6,6 @@ import java.lang.reflect.Field; import java.security.GeneralSecurityException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import org.junit.Assert; @@ -38,39 +37,15 @@ private Metric createMetric(String metricName, String requestId) { } @SuppressWarnings("unchecked") - private void modifyEnv(String key, String val) - throws ClassNotFoundException, NoSuchFieldException, IllegalAccessException { - try { - Class processEnvironmentClass = Class.forName("java.lang.ProcessEnvironment"); - Field theEnvironmentField = processEnvironmentClass.getDeclaredField("theEnvironment"); - theEnvironmentField.setAccessible(true); - Map env = (Map) theEnvironmentField.get(null); - env.put(key, val); - Field theCIEField = - processEnvironmentClass.getDeclaredField("theCaseInsensitiveEnvironment"); - theCIEField.setAccessible(true); - Map cienv = (Map) theCIEField.get(null); - cienv.put(key, val); - } catch (NoSuchFieldException e) { - Class[] classes = Collections.class.getDeclaredClasses(); - Map env = System.getenv(); - for (Class cl : classes) { - if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) { - Field field = cl.getDeclaredField("m"); - field.setAccessible(true); - Object obj = field.get(env); - Map map = (Map) obj; - map.clear(); - map.put(key, val); - } - } - } + private void modifyEnv(String key, String val) throws ReflectiveOperationException { + Map env = System.getenv(); + Field field = env.getClass().getDeclaredField("m"); + field.setAccessible(true); + ((Map) field.get(env)).put(key, val); } @Test - public void test() - throws IOException, GeneralSecurityException, IllegalAccessException, - NoSuchFieldException, ClassNotFoundException { + public void test() throws IOException, GeneralSecurityException, ReflectiveOperationException { modifyEnv("TS_DEFAULT_RESPONSE_TIMEOUT", "130"); ConfigManager.Arguments args = new ConfigManager.Arguments(); args.setModels(new String[] {"noop_v0.1"}); @@ -104,8 +79,7 @@ public void test() } @Test - public void testNoEnvVars() - throws IllegalAccessException, NoSuchFieldException, ClassNotFoundException { + public void testNoEnvVars() throws ReflectiveOperationException { System.setProperty("tsConfigFile", "src/test/resources/config_test_env.properties"); modifyEnv("TS_DEFAULT_RESPONSE_TIMEOUT", "130"); ConfigManager.Arguments args = new ConfigManager.Arguments(); From 41b1b58fa37aa6536b79fc6e4cd879dd843c9f36 Mon Sep 17 00:00:00 2001 From: harshbafna Date: Fri, 22 May 2020 10:08:53 +0530 Subject: [PATCH 2/2] added exception missed on resolving conflicts --- .../src/test/java/org/pytorch/serve/util/ConfigManagerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/server/src/test/java/org/pytorch/serve/util/ConfigManagerTest.java b/frontend/server/src/test/java/org/pytorch/serve/util/ConfigManagerTest.java index ecec62347b..fc5759a9b6 100644 --- a/frontend/server/src/test/java/org/pytorch/serve/util/ConfigManagerTest.java +++ b/frontend/server/src/test/java/org/pytorch/serve/util/ConfigManagerTest.java @@ -79,7 +79,7 @@ public void test() throws IOException, GeneralSecurityException, ReflectiveOpera } @Test - public void testNoEnvVars() throws ReflectiveOperationException { + public void testNoEnvVars() throws ReflectiveOperationException, IOException { System.setProperty("tsConfigFile", "src/test/resources/config_test_env.properties"); modifyEnv("TS_DEFAULT_RESPONSE_TIMEOUT", "130"); ConfigManager.Arguments args = new ConfigManager.Arguments();