Improvement: Conda environments symlink into the Python tempdir (pala…

…ntir#57)

core/src/main/scala/org/apache/spark/api/conda/CondaEnvironmentManager.scala

@@ -130,10 +130,10 @@ final class CondaEnvironmentManager(condaBinaryPath: String,
     require(condaChannelUrls.nonEmpty, "Can't have an empty list of conda channel URLs")
     val name = "conda-env"
 
-    // must link in /tmp to reduce path length in case baseDir is very long...
+    // must link in a temp directory to reduce path length in case baseDir is very long...
     // If baseDir path is too long, this breaks conda's 220 character limit for binary replacement.
     // Don't even try to use java.io.tmpdir - yarn sets this to a very long path
-    val linkedBaseDir = Utils.createTempDir("/tmp", "conda").toPath.resolve("real")
+    val linkedBaseDir = Utils.createTempDir(getTempDir, "conda").toPath.resolve("real")
     logInfo(s"Creating symlink $linkedBaseDir -> $baseDir")
     Files.createSymbolicLink(linkedBaseDir, Paths.get(baseDir))
 
@@ -173,10 +173,10 @@ final class CondaEnvironmentManager(condaBinaryPath: String,
         "via spark.conda.bootstrapPackageUrlsUserInfo.")
     val name = "conda-env"
 
-    // must link in /tmp to reduce path length in case baseDir is very long...
+    // must link in a temp directory to reduce path length in case baseDir is very long...
     // If baseDir path is too long, this breaks conda's 220 character limit for binary replacement.
     // Don't even try to use java.io.tmpdir - yarn sets this to a very long path
-    val linkedBaseDir = Utils.createTempDir("/tmp", "conda").toPath.resolve("real")
+    val linkedBaseDir = Utils.createTempDir(getTempDir, "conda").toPath.resolve("real")
     logInfo(s"Creating symlink $linkedBaseDir -> $baseDir")
     Files.createSymbolicLink(linkedBaseDir, Paths.get(baseDir))
 
@@ -310,6 +310,13 @@ final class CondaEnvironmentManager(condaBinaryPath: String,
     }
     out.toString
   }
+
+  def getTempDir: String = {
+    sys.env.get("TMPDIR")
+      .orElse(sys.env.get("TEMP"))
+      .orElse(sys.env.get("TMP"))
+      .getOrElse("/tmp")
+  }
 }
 
 object CondaEnvironmentManager extends Logging {

python/pyspark/sql/types.py

@@ -767,7 +767,7 @@ def __init__(self, row, schema, when_sorted, when_unsorted):
             "Detected potential mismatch between schema and named arguments in row: {0} and {1}."
             "\nAs work-around, use positional instead of named arguments and ensure that the order "
             "of values matches the schema.\nE.g. for schema ['a','b'] change Row(b=2,a=1) to "
-            "Row(1,2).\nIf that's not possible, please file support ticket with Palantir."
+            "Row(1,2). If that's not possible, please file a support ticket with Palantir."
             .format(schema.simpleString(), row))
         self.row = row
         self.schema = schema