Merge remote-tracking branch 'mars/master' into ray_channel_first_com…

…plied

# Conflicts:
#	mars/oscar/backends/ray/communication.py

.codacy.yml

@@ -7,8 +7,11 @@ exclude_paths:
   - '.github'
   - 'bin'
   - 'docs'
+  - 'mars/_version.py'
   - 'mars/lib'
   - 'mars/web/static'
   - 'misc'
+  - 'versioneer.py'
   - '*.min.js'
   - '**/tests/**'
+  - 'asv_bench/**'

.codecov.yml

@@ -17,7 +17,9 @@ comment: off
 
 # Matches 'omit:' in setup.cfg:coverage:run
 ignore:
+  - "mars/_version.py"
   - "mars/compat.py"
+  - "mars/conftest.py"
   - "mars/lib/aio/_runners.py"
   - "mars/lib/aio/lru.py"
   - "mars/lib/functools32"

.gitattributes

@@ -16,3 +16,4 @@
 *.json text
 *.yaml text
 *.yml text
+mars/_version.py export-subst

.github/workflows/benchmark-ci.yml

@@ -0,0 +1,60 @@
+name: Mars Continuous Benchmark
+
+on:
+  pull_request:
+    branches:
+      - master
+      - main
+
+jobs:
+  asv-benchmarks:
+    name: ASV Benchmarks
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    concurrency:
+      # https://git.luolix.topmunity/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-asv-benchmarks
+      cancel-in-progress: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up conda 3.8
+        env:
+          PYTHON: 3.8
+        shell: bash
+        run: |
+          source ./ci/install-conda.sh
+          python -m pip install --upgrade pip setuptools wheel coverage;
+
+      - name: Install dependencies
+        id: build
+        run: |
+          source ./ci/reload-env.sh
+          export DEFAULT_VENV=$VIRTUAL_ENV
+          pip install numpy scipy cython oss2 asv
+          pip install -e ".[dev,extra]"
+
+      - name: Run ASV benchmarks
+        run: |
+          source ./ci/reload-env.sh
+          cd asv_bench
+          asv check -E existing
+          git remote add upstream https://github.com/mars-project/mars.git
+          git fetch upstream
+          asv machine --yes
+          asv continuous -f 1.1 upstream/master HEAD
+        if: ${{ steps.build.outcome == 'success' }}
+
+      - name: Publish benchmarks artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: Benchmarks log
+          path: asv_bench/benchmarks.log
+        if: failure()

.github/workflows/cancel-prev.yml

@@ -9,4 +9,4 @@ jobs:
       - uses: styfle/cancel-workflow-action@0.9.1
         with:
           access_token: ${{ github.token }}
-          workflow_id: core-ci.yml,os-compat-ci.yml,platform-ci.yml
+          workflow_id: core-ci.yml,os-compat-ci.yml,platform-ci.yml,benchmark-ci.yml

.github/workflows/core-ci.yml

@@ -1,6 +1,11 @@
 name: Mars CI Core
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+    types: ['opened', 'reopened', 'synchronize']
 
 jobs:
   build:
@@ -56,7 +61,7 @@ jobs:
 
         if [ -z "$NO_COMMON_TESTS" ]; then
           if [[ ! "$PYTHON" =~ "3.6" ]] && [[ ! "$PYTHON" =~ "3.9" ]]; then
-            pip install h5py zarr matplotlib
+            pip install h5py zarr matplotlib prometheus-client requests
             conda install -n test --quiet --yes -c conda-forge python=$PYTHON \
               "tiledb-py>=0.4.3,<0.6.0" "tiledb<2.0.0" || true
           fi

.github/workflows/docker-cd.yml

@@ -1,11 +1,9 @@
 name: Mars CD for DockerHub
 
 on:
+  schedule:
+    - cron: '0 18 * * *'
   push:
-    branches:
-      - master
-      - main
-      - 'v*'
     tags:
       - '*'
 
@@ -14,7 +12,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
       - name: Log in to Docker Hub
         uses: docker/login-action@v1
@@ -28,13 +28,41 @@ jobs:
           DOCKER_ORG: ${{ secrets.DOCKERHUB_USERNAME }}
         run: |
           source ./ci/reload-env.sh
-          if [[ "$DOCKER_ORG" == "marsuploader" ]]; then
-            export DOCKER_ORG="marsproject"
-          fi
+
           if [[ -n "$GIT_TAG" ]]; then
-            export IMAGE_TAG="$GIT_TAG"
+            BRANCHES="$GIT_TAG"
+            echo "Will handle tag $BRANCHES"
           else
-            export IMAGE_TAG="nightly-$GIT_BRANCH"
+            MAINBRANCH=$(git rev-parse --abbrev-ref HEAD)
+            BRANCHES=$(git branch -r --list 'origin/v*' | sed 's/ *origin\///g')
+            BRANCHES="$MAINBRANCH $BRANCHES"
+          
+            echo "Will handle branches:"
+            for branch in $BRANCHES; do
+              echo "  $branch"
+            done
+          fi
+
+          if [[ "$DOCKER_ORG" == "marsuploader" ]]; then
+            export DOCKER_ORG="marsproject"
           fi
-          bash bin/kube-image-tool.sh -o "$DOCKER_ORG" -t "$IMAGE_TAG" build
-          docker push "$DOCKER_ORG/mars:$IMAGE_TAG"
+
+          for branch in $BRANCHES; do
+            if [[ -n "$GIT_TAG" ]]; then
+              export IMAGE_TAG="$GIT_TAG"
+            else
+              echo ""
+              git checkout $branch
+              git log --pretty=format:"%h - %an, %cd : %s" --since=25.hours
+
+              # consider schedule delay of Github Actions, some margin needed.
+              if [[ ! "$(git log --since=25.hours)" ]]; then
+                echo "No recent commits on branch $branch found, will skip building image."
+                continue
+              fi
+
+              export IMAGE_TAG="nightly-$branch"
+            fi
+            bash bin/kube-image-tool.sh -o "$DOCKER_ORG" -t "$IMAGE_TAG" build
+            docker push "$DOCKER_ORG/mars:$IMAGE_TAG"
+          done

.github/workflows/os-compat-ci.yml

@@ -1,6 +1,11 @@
 name: Mars CI for OS compatibility
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+    types: ['opened', 'reopened', 'synchronize']
 
 jobs:
   build:

.github/workflows/platform-ci.yml

@@ -1,6 +1,11 @@
 name: Mars CI on Platforms
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+    types: ['opened', 'reopened', 'synchronize']
 
 jobs:
   build:
@@ -82,9 +87,8 @@ jobs:
               rm -fr /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz /tmp/etcd-download-test
             fi
             if [ -n "$WITH_RAY" ]; then
-              pip install ray[default]
+              pip install ray[default]==1.9.2
               pip install xgboost_ray==0.1.5
-              pip install --upgrade numpy
             fi
             if [ -n "$RUN_DASK" ]; then
               pip install dask[complete] mimesis sklearn
@@ -132,7 +136,7 @@ jobs:
             coverage combine build/ && coverage report
           fi
           if [ -n "$WITH_RAY" ]; then
-            pytest $PYTEST_CONFIG --timeout=300 -m ray
+            pytest $PYTEST_CONFIG --durations=0 --timeout=600 -v -s -m ray
             coverage report
           fi
           if [ -n "$RUN_DASK" ]; then

.github/workflows/run-tests.sh

@@ -18,7 +18,7 @@ if [ -n "$WITH_CYTHON" ]; then
 fi
 if [ -z "$NO_COMMON_TESTS" ]; then
   mkdir -p build
-  pytest $PYTEST_CONFIG mars/remote mars/storage mars/lib
+  pytest $PYTEST_CONFIG mars/remote mars/storage mars/lib mars/metrics
   mv .coverage build/.coverage.tileable.file
 
   pytest $PYTEST_CONFIG --forked --ignore mars/tensor --ignore mars/dataframe \

.gitignore

@@ -88,3 +88,9 @@ mars/services/web/static
 
 # docs
 docs/source/savefig/
+
+# Unit / Performance Testing #
+##############################
+asv_bench/env/
+asv_bench/html/
+asv_bench/results/

MANIFEST.in

@@ -7,8 +7,9 @@ global-exclude */**/tests/*.yml
 global-exclude */**/node_modules/**/*
 include setup.cfg
 include pyproject.toml
-include mars/.git-branch
 include mars/lib/mmh3_src/*
 include mars/services/web/index.html
 include mars/services/web/static/*
 global-exclude .DS_Store
+include versioneer.py
+include mars/_version.py