forked from bigdatagenomics/adam
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[ADAM-1847] Update ADAM scripts to support self-contained pip install.
Resolves bigdatagenomics#1847. Cribs heavily from PySpark's script flow for supporting a full, self-contained pip install-able Spark by finding the JARs and bin scripts and packaging them up as packages which are deployed to pip. We then needed to modify the bin scripts to find the pip installed JARs.
- Loading branch information
Showing
6 changed files
with
221 additions
and
51 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| #!/usr/bin/env python | ||
|
|
||
| # | ||
| # Licensed to Big Data Genomics (BDG) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The BDG licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
|
|
||
| # This script is copied from Apache Spark, with minor modifications. | ||
| # | ||
| # This script attempt to determine the correct setting for ADAM_HOME given | ||
| # that ADAM may have been installed on the system with pip. | ||
|
|
||
| from __future__ import print_function | ||
| import os | ||
| import sys | ||
|
|
||
|
|
||
| def _find_adam_home(): | ||
| """Find the ADAM_HOME.""" | ||
| # If the enviroment has ADAM_HOME set trust it. | ||
| if "ADAM_HOME" in os.environ: | ||
| return os.environ["ADAM_HOME"] | ||
|
|
||
| def is_adam_home(path): | ||
| """Takes a path and returns true if the provided path could be a reasonable ADAM_HOME""" | ||
| return (os.path.isfile(os.path.join(path, "bin/adam-submit")) and | ||
| (os.path.isdir(os.path.join(path, "jars")) or | ||
| os.path.isdir(os.path.join(path, "assembly")))) | ||
|
|
||
| paths = ["../", os.path.dirname(os.path.realpath(__file__))] | ||
|
|
||
| # Add the path of the ADAM module if it exists | ||
| if sys.version < "3": | ||
| import imp | ||
| try: | ||
| module_home = imp.find_module("bdgenomics.adam")[1] | ||
| paths.append(module_home) | ||
| # If we are installed in edit mode also look two dirs up | ||
| paths.append(os.path.join(module_home, "../../")) | ||
| except ImportError: | ||
| # Not pip installed no worries | ||
| pass | ||
| else: | ||
| from importlib.util import find_spec | ||
| try: | ||
| module_home = os.path.dirname(find_spec("bdgenomics.adam").origin) | ||
| paths.append(module_home) | ||
| # If we are installed in edit mode also look two dirs up | ||
| paths.append(os.path.join(module_home, "../../")) | ||
| except ImportError: | ||
| # Not pip installed no worries | ||
| pass | ||
|
|
||
| # Normalize the paths | ||
| paths = [os.path.abspath(p) for p in paths] | ||
|
|
||
| try: | ||
| return next(path for path in paths if is_spark_home(path)) | ||
| except StopIteration: | ||
| print("Could not find valid ADAM_HOME while searching {0}".format(paths), file=sys.stderr) | ||
| exit(-1) | ||
|
|
||
| if __name__ == "__main__": | ||
| print(_find_adam_home()) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| #!/usr/bin/env bash | ||
| # | ||
| # Licensed to Big Data Genomics (BDG) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The BDG licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
|
|
||
| set -e | ||
|
|
||
| # Attempts to find a proper value for ADAM_HOME. Should be included using "source" directive. | ||
|
|
||
| FIND_ADAM_HOME_PYTHON_SCRIPT="$(cd "$(dirname "$0")"; pwd)/find_adam_home.py" | ||
|
|
||
| # Short cirtuit if the user already has this set. | ||
| if [ ! -z "${ADAM_HOME}" ]; then | ||
| exit 0 | ||
| elif [ ! -f "$FIND_ADAM_HOME_PYTHON_SCRIPT" ]; then | ||
| # If we are not in the same directory as find_adam_home.py we are not pip installed so we don't | ||
| # need to search the different Python directories for a Adam installation. | ||
| # Note only that, if the user has pip installed adam but is directly calling pyadam or | ||
| # adam-submit in another directory we want to use that version of adam rather than the | ||
| # pip installed version of adam. | ||
| export ADAM_HOME="$(cd "$(dirname "$0")"/..; pwd)" | ||
| else | ||
| # We are pip installed, use the Python script to resolve a reasonable ADAM_HOME | ||
| # Default to standard python interpreter unless told otherwise | ||
| if [[ -z "$PYSPARK_DRIVER_PYTHON" ]]; then | ||
| PYSPARK_DRIVER_PYTHON="${PYSPARK_PYTHON:-"python"}" | ||
| fi | ||
| export ADAM_HOME=$($PYSPARK_DRIVER_PYTHON "$FIND_ADAM_HOME_PYTHON_SCRIPT") | ||
| fi |
This file was deleted.
Oops, something went wrong.