[BACKPORT] Change all tests to use relative import (#2407)

.gitattributes

@@ -0,0 +1,18 @@
+* text=auto
+# enforce text on certain files
+*.py text
+*.pyx text
+*.pyi text
+*.pxd text
+*.c text
+*.cpp text
+*.h text
+*.hpp text
+*.html text
+*.csv text
+*.js text
+*.jsx text
+*.ts text
+*.json text
+*.yaml text
+*.yml text

.github/workflows/cancel-prev.yml

@@ -9,4 +9,4 @@ jobs:
       - uses: styfle/cancel-workflow-action@0.9.1
         with:
           access_token: ${{ github.token }}
-          workflow_id: core-ci.yml,os-compat-ci.yml,platform-ci.yml,checks.yml
+          workflow_id: core-ci.yml,os-compat-ci.yml,platform-ci.yml

azure-pipelines.yml

@@ -8,7 +8,7 @@ pr:
 - v*.*
 
 jobs:
-- job: Job
+- job: CI
   timeoutInMinutes: 120
   cancelTimeoutInMinutes: 2
   pool:
@@ -100,3 +100,77 @@ jobs:
     - bash: |
         bash <(curl -s https://codecov.io/bash)
       displayName: 'Upload coverage'
+
+- job: Checks
+  timeoutInMinutes: 120
+  cancelTimeoutInMinutes: 2
+
+  variables:
+    PYTHON: '3.8'
+
+  steps:
+    - bash: |
+        set -e
+        source ci/install-conda.sh
+      displayName: 'Install conda'
+
+    - bash: |
+        source ./ci/reload-env.sh
+        export DEFAULT_VENV=$VIRTUAL_ENV
+
+        pip install numpy scipy cython
+        pip install -e ".[dev,extra]"
+        pip install virtualenv flake8 codespell sphinx sphinx-intl
+        conda list -n test
+      displayName: 'Install dependencies'
+
+    - bash: |
+        source ./ci/reload-env.sh
+
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 mars --count --show-source --statistics
+
+        # special check for __init__.py
+        grep -A 10000 '\[flake8\]' setup.cfg | awk '!/(F401|F811|__init__\.py)/' > flake8_init.ini
+        flake8 --config=flake8_init.ini
+
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 mars --config="default" --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      displayName: 'Lint with flake8'
+
+    - bash: |
+        source ./ci/reload-env.sh
+        codespell
+      displayName: 'Check spellings'
+
+    - bash: |
+        source ./ci/reload-env.sh
+        git fetch origin master
+        bash ci/modecheck.sh
+      displayName: 'Check file mode changes'
+
+    - bash: |
+        source ./ci/reload-env.sh
+        python ci/importcheck.py
+      displayName: 'Check imports'
+
+    - bash: |
+        source ./ci/reload-env.sh
+        python ci/copycheck.py
+      displayName: 'Check copyright headers'
+
+    - bash: |
+        source ./ci/reload-env.sh
+        pushd mars/services/web/ui
+        npm install
+        npm run lint
+        popd
+      displayName: 'Check JS with ESLint'
+
+    - bash: |
+        source ./ci/reload-env.sh
+        pushd docs
+        pip install -r requirements-doc.txt
+        make html
+        popd
+      displayName: 'Check documentation build'

bin/fix-flake8.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+for fn in $(git diff --name-only master); do
+  autopep8 --in-place $fn
+done

ci/importcheck.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+# Copyright 1999-2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import ast
+import os
+import sys
+import textwrap
+from pathlib import PurePath
+from typing import List, NamedTuple, Optional, Tuple
+
+_IGNORES = [
+    'mars/lib/**/*.py',
+    'conftest.py',
+]
+
+
+class CheckResult(NamedTuple):
+    path: str
+    lines: List
+    absolute_imports: List[Tuple[int, int]]
+    head_disorder: Optional[Tuple[int, int]]
+    block_disorders: Optional[List[Tuple[int, int]]]
+
+    @property
+    def has_faults(self) -> bool:
+        return bool(self.absolute_imports) or bool(self.head_disorder) \
+            or bool(self.block_disorders)
+
+
+def _check_absolute_import(node: ast.AST) -> List[Tuple[int, int]]:
+    res = set()
+    if isinstance(node, ast.Import):
+        for import_name in node.names:
+            if import_name.name.startswith('mars.'):
+                res.add((node.lineno, node.end_lineno))
+    elif isinstance(node, ast.ImportFrom):
+        if node.level == 0 and node.module.startswith('mars.'):
+            res.add((node.lineno, node.end_lineno))
+    elif getattr(node, 'body', []):
+        for body_item in node.body:
+            res.update(_check_absolute_import(body_item))
+    return sorted(res)
+
+
+def check_imports(file_path) -> CheckResult:
+    with open(file_path, 'rb') as src_file:
+        body = src_file.read()
+        lines = body.splitlines()
+        parsed = ast.parse(body, filename=file_path)
+    # scan for imports
+    abs_faults = _check_absolute_import(parsed)
+
+    return CheckResult(file_path, lines, abs_faults, None, None)
+
+
+def _extract_line_block(lines: List, lineno: int, end_lineno: int, indent: str):
+    grab_lines = '\n'.join(line.decode() for line in lines[lineno - 1:end_lineno])
+    return textwrap.indent(textwrap.dedent(grab_lines), indent)
+
+
+def format_results(results: List[CheckResult], root_path):
+    rel_import_count = sum(len(res.absolute_imports) for res in results)
+    if rel_import_count > 0:
+        print(f'Do not use absolute imports for mars module in '
+              f'code ({rel_import_count}):', file=sys.stderr)
+        for res in results:
+            if not res.absolute_imports:
+                continue
+            rel_path = os.path.relpath(res.path, root_path)
+            print('  ' + rel_path, file=sys.stderr)
+            for lineno, end_lineno in res.absolute_imports:
+                print(f'    Line {lineno}-{end_lineno}', file=sys.stderr)
+                print(_extract_line_block(res.lines, lineno, end_lineno, '      '),
+                      file=sys.stderr)
+
+
+def main():
+    root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    results = []
+    for root, _dirs, files in os.walk(os.path.join(root_path, 'mars')):
+        for fn in files:
+            if '/tests' in root and not fn.startswith('test_'):
+                # allow test auxiliary files to use full imports
+                continue
+            abs_path = os.path.join(root, fn)
+            rel_path = os.path.relpath(abs_path, root_path)
+
+            if not fn.endswith('.py'):
+                continue
+            if any(PurePath(rel_path).match(patt) for patt in _IGNORES):
+                continue
+
+            check_result = check_imports(abs_path)
+            if check_result.has_faults:
+                results.append(check_result)
+    if results:
+        results = sorted(results, key=lambda x: x.path)
+        format_results(results, root_path)
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

ci/modecheck.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+if git diff origin/master -- mars | grep -q "old mode"; then
+  echo "Unexpected file mode changed. You may call"
+  echo "    git config core.fileMode false"
+  echo "before committing to the repo."
+  git diff origin/master -- mars | grep -B 2 -A 2 "old mode"
+  exit 1
+fi

docs/source/development/contributing.rst

@@ -122,6 +122,26 @@ Python code style.  Simply run command below to check your code style:
 
     flake8 mars
 
+You may fix these issues manually or use automated tools like `autopep8
+<https://github.com/hhatto/autopep8>`_ to fix them for you. To facilitate the
+process, we create a tool. You can simply run
+
+.. code-block:: bash
+
+    pip install autopep8
+    bash ./bin/fix-flake8.sh
+
+in Bash to fix these issues. For Windows users, commands above should be run in
+Git Bash or a WSL terminal.
+
+We also require relative import in code for all Mars modules. Use
+
+.. code-block:: bash
+
+    python ./ci/importcheck.py
+
+to check if your code meet the requirement.
+
 Building Documentations
 -----------------------
 Mars uses ``sphinx`` to build documents. You need to install necessary packages