From 4d2010a852dafc7bc2a8dc846e5c5f7fa1cfc0ad Mon Sep 17 00:00:00 2001 From: Devin Petersohn Date: Thu, 29 Nov 2018 11:05:24 -0800 Subject: [PATCH] Ship Modin with Ray. (#3109) --- .gitignore | 3 +++ .travis.yml | 3 +++ .travis/install-dependencies.sh | 8 ++++---- .travis/test-wheels.sh | 4 ++-- python/ray/__init__.py | 3 +++ python/ray/test/test_modin.py | 12 ++++++++++++ thirdparty/scripts/build_modin.sh | 29 +++++++++++++++++++++++++++++ thirdparty/scripts/setup.sh | 5 +++++ 8 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 python/ray/test/test_modin.py create mode 100755 thirdparty/scripts/build_modin.sh diff --git a/.gitignore b/.gitignore index f8130b3a2f85..91189b6f9c41 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ /src/ray/object_manager/format/*_generated.h /src/ray/raylet/format/*_generated.h +# Modin source files +/python/ray/modin + # Redis temporary files *dump.rdb diff --git a/.travis.yml b/.travis.yml index debf450738a7..b48089d52ecd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -171,6 +171,9 @@ script: # ray temp file tests - python -m pytest -v test/tempfile_test.py + # modin test files + - python python/ray/test/test_modin.py + deploy: - provider: s3 access_key_id: AKIAJ2L7XDUSZVTXI5QA diff --git a/.travis/install-dependencies.sh b/.travis/install-dependencies.sh index 293c1b8b6b04..0fb597d4686f 100755 --- a/.travis/install-dependencies.sh +++ b/.travis/install-dependencies.sh @@ -24,7 +24,7 @@ if [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "linux" ]]; then wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" - pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.22 requests \ + pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \ feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then sudo apt-get update @@ -33,7 +33,7 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" - pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.22 requests \ + pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \ feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then # check that brew is installed @@ -50,7 +50,7 @@ elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" - pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.22 requests \ + pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \ feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then # check that brew is installed @@ -67,7 +67,7 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" - pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.22 requests \ + pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \ feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout elif [[ "$LINT" == "1" ]]; then sudo apt-get update diff --git a/.travis/test-wheels.sh b/.travis/test-wheels.sh index 1765135ec9be..f7870ea52d49 100755 --- a/.travis/test-wheels.sh +++ b/.travis/test-wheels.sh @@ -59,7 +59,7 @@ if [[ "$platform" == "linux" ]]; then if [[ "$NUMBER_OF_WHEELS" != "5" ]]; then echo "Wrong number of wheels found." ls -l $ROOT_DIR/../.whl/ - exit 1 + exit 2 fi elif [[ "$platform" == "macosx" ]]; then @@ -94,5 +94,5 @@ elif [[ "$platform" == "macosx" ]]; then done else echo "Unrecognized environment." - exit 1 + exit 3 fi diff --git a/python/ray/__init__.py b/python/ray/__init__.py index a507cdd2e7a2..6d4e0ba9a6b9 100644 --- a/python/ray/__init__.py +++ b/python/ray/__init__.py @@ -46,6 +46,9 @@ e.args += (helpful_message, ) raise +modin_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "modin") +sys.path.insert(0, modin_path) + from ray.raylet import ObjectID, _config # noqa: E402 from ray.profiling import profile # noqa: E402 from ray.worker import (error_info, init, connect, disconnect, get, put, wait, diff --git a/python/ray/test/test_modin.py b/python/ray/test/test_modin.py new file mode 100644 index 000000000000..83c11895ec7b --- /dev/null +++ b/python/ray/test/test_modin.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import ray # noqa F401 + + +def test_modin_import(): + import modin.pandas as pd + frame_data = [1, 2, 3, 4, 5, 6, 7, 8] + frame = pd.DataFrame(frame_data) + assert frame.sum().squeeze() == sum(frame_data) diff --git a/thirdparty/scripts/build_modin.sh b/thirdparty/scripts/build_modin.sh new file mode 100755 index 000000000000..96563fdb2106 --- /dev/null +++ b/thirdparty/scripts/build_modin.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -x + +# Cause the script to exit if a single command fails. +set -e + +if [[ -z "$1" ]]; then + PYTHON_EXECUTABLE=`which python` +else + PYTHON_EXECUTABLE=$1 +fi + +PYTHON_VERSION="$($PYTHON_EXECUTABLE -c 'import sys; print(sys.version_info[0])')" + +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../ +MODIN_VERSION=0.2.4 +MODIN_WHEELS_FNAME="modin-$MODIN_VERSION-py$PYTHON_VERSION-none-any.whl" +MODIN_WHEELS_URL="https://github.com/modin-project/modin/releases/download/v$MODIN_VERSION/" + +pushd $TP_DIR/../python/ray/ +rm -rf modin +mkdir modin +pushd modin +curl -kL "$MODIN_WHEELS_URL$MODIN_WHEELS_FNAME" -o "$MODIN_WHEELS_FNAME" +unzip "$MODIN_WHEELS_FNAME" +rm "$MODIN_WHEELS_FNAME" +popd +popd diff --git a/thirdparty/scripts/setup.sh b/thirdparty/scripts/setup.sh index 27f1ef0e3ed5..da283bd3b2bb 100755 --- a/thirdparty/scripts/setup.sh +++ b/thirdparty/scripts/setup.sh @@ -67,3 +67,8 @@ bash "$TP_SCRIPT_DIR/build_ui.sh" # rDSN (optional) ############################################## # bash "$TP_SCRIPT_DIR/build_rdsn.sh" + +############################################## +# modin +############################################## +bash "$TP_SCRIPT_DIR/build_modin.sh" $PYTHON_EXECUTABLE