From c6ec72d513f02a27b62a4fa2d400cefc78331102 Mon Sep 17 00:00:00 2001
From: Chad Dombrova <chadrik@gmail.com>
Date: Fri, 25 Oct 2019 12:39:30 -0700
Subject: [PATCH] Add a mypy lint job

---
 .gitignore                                    |  2 +
 sdks/python/.pylintrc                         |  2 +
 sdks/python/mypy.ini                          | 60 +++++++++++++++++++
 sdks/python/setup.py                          | 38 +++++++++++-
 sdks/python/test-suites/tox/py37/build.gradle |  3 +
 sdks/python/tox.ini                           |  9 ++-
 6 files changed, 112 insertions(+), 2 deletions(-)
 create mode 100644 sdks/python/mypy.ini

diff --git a/.gitignore b/.gitignore
index f9cc3887407a..63038b6288be 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,6 +35,8 @@ sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources
 **/dist/**/*
 **/distribute-*/**/*
 **/env/**/*
+**/.mypy_cache
+**/.dmypy.json
 sdks/python/**/*.c
 sdks/python/**/*.so
 sdks/python/**/*.egg
diff --git a/sdks/python/.pylintrc b/sdks/python/.pylintrc
index bb404d14d830..26b39dfaa6b3 100644
--- a/sdks/python/.pylintrc
+++ b/sdks/python/.pylintrc
@@ -143,6 +143,7 @@ disable =
   unnecessary-lambda,
   unnecessary-pass,
   unneeded-not,
+  unsubscriptable-object,
   unused-argument,
   unused-wildcard-import,
   useless-object-inheritance,
@@ -178,6 +179,7 @@ indent-after-paren=4
 ignore-long-lines=(?x)
   (^\s*(import|from)\s
    |^\s*(\#\ )?<?(https?|ftp):\/\/[^\s\/$.?#].[^\s]*>?$
+   |# type:
    )
 
 [VARIABLES]
diff --git a/sdks/python/mypy.ini b/sdks/python/mypy.ini
new file mode 100644
index 000000000000..7b98f1cd1aae
--- /dev/null
+++ b/sdks/python/mypy.ini
@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[mypy]
+python_version = 3.6
+ignore_missing_imports = true
+follow_imports = true
+warn_no_return = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+show_error_codes = true
+files = apache_beam
+color_output = true
+# uncomment this to see how close we are to being complete
+# check_untyped_defs = true
+
+[mypy-apache_beam.coders.proto2_coder_test_messages_pb2]
+ignore_errors = true
+
+[mypy-apache_beam.examples.*]
+ignore_errors = true
+
+[mypy-apache_beam.io.gcp.gcsfilesystem_test]
+# error: Cannot infer type of lambda  [misc]
+ignore_errors = true
+
+[mypy-apache_beam.io.gcp.internal.clients.storage.storage_v1_client]
+ignore_errors = true
+
+[mypy-apache_beam.io.gcp.internal.clients.bigquery.bigquery_v2_client]
+ignore_errors = true
+
+[mypy-apache_beam.portability.api.*]
+ignore_errors = true
+
+[mypy-apache_beam.runners.dataflow.internal.clients.dataflow.dataflow_v1b3_client]
+ignore_errors = true
+
+[mypy-apache_beam.typehints.typed_pipeline_test_py3]
+# error: Signature of "process" incompatible with supertype "DoFn"  [override]
+ignore_errors = true
+
+[mypy-apache_beam.typehints.typehints_test_py3]
+# error: Signature of "process" incompatible with supertype "DoFn"  [override]
+ignore_errors = true
\ No newline at end of file
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index c19a2c0c382a..182b81cc9c06 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -25,6 +25,7 @@
 import sys
 import warnings
 from distutils import log
+from distutils.errors import DistutilsError
 from distutils.version import StrictVersion
 
 # Pylint and isort disagree here.
@@ -32,6 +33,9 @@
 import setuptools
 from pkg_resources import DistributionNotFound
 from pkg_resources import get_distribution
+from pkg_resources import normalize_path
+from pkg_resources import to_filename
+from setuptools import Command
 from setuptools.command.build_py import build_py
 # TODO: (BEAM-8411): re-enable lint check.
 from setuptools.command.develop import develop  # pylint: disable-all
@@ -39,6 +43,35 @@
 from setuptools.command.test import test
 
 
+class mypy(Command):
+  user_options = []
+
+  def initialize_options(self):
+    """Abstract method that is required to be overwritten"""
+
+  def finalize_options(self):
+    """Abstract method that is required to be overwritten"""
+
+  def get_project_path(self):
+    self.run_command('egg_info')
+
+    # Build extensions in-place
+    self.reinitialize_command('build_ext', inplace=1)
+    self.run_command('build_ext')
+
+    ei_cmd = self.get_finalized_command("egg_info")
+
+    project_path = normalize_path(ei_cmd.egg_base)
+    return os.path.join(project_path, to_filename(ei_cmd.egg_name))
+
+  def run(self):
+    import subprocess
+    args = ['mypy', self.get_project_path()]
+    result = subprocess.call(args)
+    if result != 0:
+      raise DistutilsError()
+
+
 def get_version():
   global_names = {}
   exec(  # pylint: disable=exec-used
@@ -130,7 +163,9 @@ def get_version():
     'pytz>=2018.3',
     # [BEAM-5628] Beam VCF IO is not supported in Python 3.
     'pyvcf>=0.6.8,<0.7.0; python_version < "3.0"',
-    'typing>=3.6.0,<3.7.0; python_version < "3.5.0"',
+    # fixes and additions have been made since typing 3.5
+    'typing>=3.7.0,<3.8.0; python_version < "3.8.0"',
+    'typing-extensions>=3.7.0,<3.8.0; python_version < "3.8.0"',
     ]
 
 # [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms.
@@ -249,5 +284,6 @@ def run(self):
         'develop': generate_protos_first(develop),
         'egg_info': generate_protos_first(egg_info),
         'test': generate_protos_first(test),
+        'mypy': generate_protos_first(mypy),
     },
 )
diff --git a/sdks/python/test-suites/tox/py37/build.gradle b/sdks/python/test-suites/tox/py37/build.gradle
index 25274d2418d9..bcfd43e2bc43 100644
--- a/sdks/python/test-suites/tox/py37/build.gradle
+++ b/sdks/python/test-suites/tox/py37/build.gradle
@@ -32,6 +32,9 @@ check.dependsOn lint
 toxTask "lintPy37", "py37-lint"
 lint.dependsOn lintPy37
 
+toxTask "mypyPy37", "py37-mypy"
+lint.dependsOn mypyPy37
+
 toxTask "testPython37", "py37"
 test.dependsOn testPython37
 
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index c75b35e883f3..f7c99eb65b97 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -17,7 +17,7 @@
 
 [tox]
 # new environments will be excluded by default unless explicitly added to envlist.
-envlist = py27,py35,py36,py37,py27-{gcp,cython,lint,lint3},py35-{gcp,cython},py36-{gcp,cython},py37-{gcp,cython,lint},docs
+envlist = py27,py35,py36,py37,py27-{gcp,cython,lint,lint3},py35-{gcp,cython},py36-{gcp,cython},py37-{gcp,cython,lint,mypy},docs
 toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox}
 
 [pycodestyle]
@@ -188,6 +188,13 @@ commands =
   pylint --version
   time {toxinidir}/scripts/run_pylint.sh
 
+[testenv:py37-mypy]
+deps =
+  mypy==0.730
+commands =
+  mypy --version
+  python setup.py mypy
+
 [testenv:docs]
 extras = test,gcp,docs
 deps =