[SPARK-6186] [EC2] Make Tachyon version configurable in EC2 deploymen…

…t script

This PR comes from Tachyon community to solve the issue:
https://tachyon.atlassian.net/browse/TACHYON-11

An accompanying PR is in mesos/spark-ec2:
mesos/spark-ec2#101

Author: cheng chang <myairia@gmail.com>

Closes apache#4901 from uronce-cc/master and squashes the following commits:

313aa36 [cheng chang] minor re-wording
fd2a48e [cheng chang] Remove Tachyon when deploying through git hash
1d53c5c [cheng chang] add default value to --tachyon-version
6f8887e [cheng chang] make tachyon version configurable

ec2/deploy.generic/root/spark-ec2/ec2-variables.sh

@@ -26,9 +26,10 @@ export SPARK_LOCAL_DIRS="{{spark_local_dirs}}"
 export MODULES="{{modules}}"
 export SPARK_VERSION="{{spark_version}}"
 export SHARK_VERSION="{{shark_version}}"
+export TACHYON_VERSION="{{tachyon_version}}"
 export HADOOP_MAJOR_VERSION="{{hadoop_major_version}}"
 export SWAP_MB="{{swap}}"
 export SPARK_WORKER_INSTANCES="{{spark_worker_instances}}"
 export SPARK_MASTER_OPTS="{{spark_master_opts}}"
 export AWS_ACCESS_KEY_ID="{{aws_access_key_id}}"
-export AWS_SECRET_ACCESS_KEY="{{aws_secret_access_key}}"
+export AWS_SECRET_ACCESS_KEY="{{aws_secret_access_key}}"

ec2/spark_ec2.py

@@ -62,6 +62,16 @@
     "1.2.1",
 ])
 
+SPARK_TACHYON_MAP = {
+    "1.0.0": "0.4.1",
+    "1.0.1": "0.4.1",
+    "1.0.2": "0.4.1",
+    "1.1.0": "0.5.0",
+    "1.1.1": "0.5.0",
+    "1.2.0": "0.5.0",
+    "1.2.1": "0.5.0",
+}
+
 DEFAULT_SPARK_VERSION = SPARK_EC2_VERSION
 DEFAULT_SPARK_GITHUB_REPO = "https://github.com/apache/spark"
 
@@ -370,6 +380,10 @@ def get_validate_spark_version(version, repo):
 }
 
 
+def get_tachyon_version(spark_version):
+    return SPARK_TACHYON_MAP.get(spark_version, "")
+
+
 # Attempt to resolve an appropriate AMI given the architecture and region of the request.
 def get_spark_ami(opts):
     if opts.instance_type in EC2_INSTANCE_TYPES:
@@ -919,9 +933,13 @@ def deploy_files(conn, root_dir, opts, master_nodes, slave_nodes, modules):
     if "." in opts.spark_version:
         # Pre-built Spark deploy
         spark_v = get_validate_spark_version(opts.spark_version, opts.spark_git_repo)
+        tachyon_v = get_tachyon_version(spark_v)
     else:
         # Spark-only custom deploy
         spark_v = "%s|%s" % (opts.spark_git_repo, opts.spark_version)
+        tachyon_v = ""
+        print "Deploying Spark via git hash; Tachyon won't be set up"
+        modules = filter(lambda x: x != "tachyon", modules)
 
     template_vars = {
         "master_list": '\n'.join([i.public_dns_name for i in master_nodes]),
@@ -934,6 +952,7 @@ def deploy_files(conn, root_dir, opts, master_nodes, slave_nodes, modules):
         "swap": str(opts.swap),
         "modules": '\n'.join(modules),
         "spark_version": spark_v,
+        "tachyon_version": tachyon_v,
         "hadoop_major_version": opts.hadoop_major_version,
         "spark_worker_instances": "%d" % opts.worker_instances,
         "spark_master_opts": opts.master_opts