Ship Modin with Ray. (#3109)

.gitignore

@@ -12,6 +12,9 @@
 /src/ray/object_manager/format/*_generated.h
 /src/ray/raylet/format/*_generated.h
 
+# Modin source files
+/python/ray/modin
+
 # Redis temporary files
 *dump.rdb
 

.travis.yml

@@ -171,6 +171,9 @@ script:
   # ray temp file tests
   - python -m pytest -v test/tempfile_test.py
 
+  # modin test files
+  - python python/ray/test/test_modin.py
+
 deploy:
   - provider: s3
     access_key_id: AKIAJ2L7XDUSZVTXI5QA

.travis/install-dependencies.sh

@@ -24,7 +24,7 @@ if [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "linux" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.22 requests \
+  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
     feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout
 elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then
   sudo apt-get update
@@ -33,7 +33,7 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.22 requests \
+  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
     feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout
 elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
   # check that brew is installed
@@ -50,7 +50,7 @@ elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.22 requests \
+  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
     feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout
 elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
   # check that brew is installed
@@ -67,7 +67,7 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.22 requests \
+  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
     feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout
 elif [[ "$LINT" == "1" ]]; then
   sudo apt-get update

.travis/test-wheels.sh

@@ -59,7 +59,7 @@ if [[ "$platform" == "linux" ]]; then
   if [[ "$NUMBER_OF_WHEELS" != "5" ]]; then
     echo "Wrong number of wheels found."
     ls -l $ROOT_DIR/../.whl/
-    exit 1
+    exit 2
   fi
 
 elif [[ "$platform" == "macosx" ]]; then
@@ -94,5 +94,5 @@ elif [[ "$platform" == "macosx" ]]; then
   done
 else
   echo "Unrecognized environment."
-  exit 1
+  exit 3
 fi

python/ray/__init__.py

@@ -46,6 +46,9 @@
             e.args += (helpful_message, )
     raise
 
+modin_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "modin")
+sys.path.insert(0, modin_path)
+
 from ray.raylet import ObjectID, _config  # noqa: E402
 from ray.profiling import profile  # noqa: E402
 from ray.worker import (error_info, init, connect, disconnect, get, put, wait,

python/ray/test/test_modin.py

@@ -0,0 +1,12 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import ray  # noqa F401
+
+
+def test_modin_import():
+    import modin.pandas as pd
+    frame_data = [1, 2, 3, 4, 5, 6, 7, 8]
+    frame = pd.DataFrame(frame_data)
+    assert frame.sum().squeeze() == sum(frame_data)

thirdparty/scripts/build_modin.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -x
+
+# Cause the script to exit if a single command fails.
+set -e
+
+if [[ -z  "$1" ]]; then
+  PYTHON_EXECUTABLE=`which python`
+else
+  PYTHON_EXECUTABLE=$1
+fi
+
+PYTHON_VERSION="$($PYTHON_EXECUTABLE -c 'import sys; print(sys.version_info[0])')"
+
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
+MODIN_VERSION=0.2.4
+MODIN_WHEELS_FNAME="modin-$MODIN_VERSION-py$PYTHON_VERSION-none-any.whl"
+MODIN_WHEELS_URL="https://github.com/modin-project/modin/releases/download/v$MODIN_VERSION/"
+
+pushd $TP_DIR/../python/ray/
+rm -rf modin
+mkdir modin
+pushd modin
+curl -kL "$MODIN_WHEELS_URL$MODIN_WHEELS_FNAME" -o "$MODIN_WHEELS_FNAME"
+unzip "$MODIN_WHEELS_FNAME"
+rm "$MODIN_WHEELS_FNAME"
+popd
+popd

thirdparty/scripts/setup.sh

@@ -67,3 +67,8 @@ bash "$TP_SCRIPT_DIR/build_ui.sh"
 # rDSN (optional)
 ##############################################
 # bash "$TP_SCRIPT_DIR/build_rdsn.sh"
+
+##############################################
+# modin
+##############################################
+bash "$TP_SCRIPT_DIR/build_modin.sh" $PYTHON_EXECUTABLE