Merge pull request #9613 from EricCousineau-TRI/feature/py3_pydrake

pydrake: Enable testing to pass under Python3

__init__.py

@@ -14,7 +14,7 @@
 
 try:
     import drake.lcmtypes
-    __path__.append(drake.lcmtypes.__path__[0] + "/drake")
+    __path__.append(list(drake.lcmtypes.__path__)[0] + "/drake")
     from drake.lcmtypes.drake import *
 except ImportError:
     pass

bindings/pydrake/common/test/module_test.py

@@ -2,10 +2,16 @@
 
 import os
 import unittest
+
+import six
+
 import pydrake.common
 
 
 class TestCommon(unittest.TestCase):
+    if six.PY2:
+        assertRegex = unittest.TestCase.assertRegexpMatches
+
     def test_drake_demand_throws(self):
         # Drake's assertion errors should turn into SystemExit by default,
         # without the user needing to do anything special.  Here, we trigger a
@@ -16,8 +22,8 @@ def test_drake_demand_throws(self):
             self.fail("Did not get a SystemExit")
         except SystemExit as e:
             self.assertTrue(e.code is not None)
-            self.assertRegexpMatches(
-                e.message,
+            self.assertRegex(
+                str(e),
                 ".*".join([
                     "Failure at ",
                     " trigger_an_assertion_failure",

bindings/pydrake/doc/BUILD.bazel

@@ -17,20 +17,14 @@ py_library(
 # TODO(jamiesnape): Can we avoid the need to re-compile pydrake for the host
 # architecture?
 drake_py_binary(
-    name = "sphinx_build_wrapper",
-    srcs = ["sphinx_build_wrapper.py"],
-    data = ["@sphinx//:sphinx-build"],
+    name = "sphinx_build",
+    srcs = ["sphinx_build.py"],
     deps = [
         ":sphinx_pydrake",
         "//bindings/pydrake",
     ],
 )
 
-drake_runfiles_binary(
-    name = "sphinx_build",
-    target = ":sphinx_build_wrapper",
-)
-
 drake_py_binary(
     name = "refresh_doc_modules",
     srcs = ["refresh_doc_modules.py"],

bindings/pydrake/doc/serve_sphinx.py

@@ -2,7 +2,7 @@
 This program serves sphinx.zip for a web browser.
 
 Run this via:
-  $ bazel run //bindings/pydrake/doc:serve_sphinx
+  $ bazel run //doc:serve_sphinx
 """
 
 from __future__ import print_function
@@ -28,6 +28,9 @@ def str2bool(value):
 parser.add_argument(
     "--browser", type='bool', default=True, metavar='BOOL',
     help="Open browser. Disable this if you are frequently recompiling.")
+parser.add_argument(
+    "--port", type=int, default=8001, metavar='PORT',
+    help="Port for serving doc pages with a HTTP server.")
 args = parser.parse_args()
 
 # Unpack zipfile and chdir into it.
@@ -44,7 +47,7 @@ def log_request(*_):
 
 
 # Serve the current directory for local browsing.
-sockaddr = ("127.0.0.1", 8001)
+sockaddr = ("127.0.0.1", args.port)
 TCPServer.allow_reuse_address = True
 httpd = TCPServer(sockaddr, Handler)
 http_url = "http://%s:%s/index.html" % sockaddr

bindings/pydrake/doc/sphinx_build.py

@@ -0,0 +1,13 @@
+# -*- coding: UTF-8 -*-
+
+import os
+import sys
+
+os.environ['LANG'] = 'en_US.UTF-8'
+
+try:
+    from sphinx.cmd.build import main
+    sys.exit(main(sys.argv[1:]))
+except ImportError:
+    from sphinx import main
+    sys.exit(main(sys.argv))

bindings/pydrake/multibody/multibody_tree_py.cc

@@ -667,6 +667,7 @@ void init_all(py::module m) {
 }  // namespace
 
 PYBIND11_MODULE(multibody_tree, m) {
+  PYDRAKE_PREVENT_PYTHON3_MODULE_REIMPORT(m);
   m.doc() = "MultibodyTree functionality.";
 
   // TODO(eric.cousineau): Split this into separate files. See discussion in

bindings/pydrake/multibody/test/multibody_tree_test.py

@@ -43,6 +43,7 @@
 import math
 import unittest
 
+from six import text_type as unicode
 import numpy as np
 
 from pydrake.common import FindResourceOrThrow
@@ -60,7 +61,7 @@ def get_index_class(cls):
         Joint: JointIndex,
         JointActuator: JointActuatorIndex,
     }
-    for key_cls, index_cls in class_to_index_class_map.iteritems():
+    for key_cls, index_cls in class_to_index_class_map.items():
         if issubclass(cls, key_cls):
             return index_cls
     raise RuntimeError("Unknown class: {}".format(cls))
@@ -388,7 +389,7 @@ def test_multibody_add_joint(self):
         num_joints = 2
         plant = MultibodyPlant()
         instances = []
-        for i in xrange(num_joints + 1):
+        for i in range(num_joints + 1):
             instance = AddModelFromSdfFile(
                 instance_file, "instance_{}".format(i), plant)
             instances.append(instance)

bindings/pydrake/multibody/test/rigid_body_tree_test.py

@@ -120,8 +120,8 @@ def do_transform(q):
         # - Check QDotToVelocity and VelocityToQDot methods
         q = tree.getZeroConfiguration()
         v_real = np.zeros(num_v)
-        q_ad = np.array(map(AutoDiffXd, q))
-        v_real_ad = np.array(map(AutoDiffXd, v_real))
+        q_ad = np.array(list(map(AutoDiffXd, q)))
+        v_real_ad = np.array(list(map(AutoDiffXd, v_real)))
 
         kinsol = tree.doKinematics(q)
         kinsol_ad = tree.doKinematics(q_ad)
@@ -224,8 +224,8 @@ def test_constraint_api(self):
 
         q = tree.getZeroConfiguration()
         v = np.zeros(num_q)
-        q_ad = np.array(map(AutoDiffXd, q))
-        v_ad = np.array(map(AutoDiffXd, v))
+        q_ad = np.array(list(map(AutoDiffXd, q)))
+        v_ad = np.array(list(map(AutoDiffXd, v)))
         kinsol = tree.doKinematics(q, v)
         kinsol_ad = tree.doKinematics(q_ad, v_ad)
 
@@ -316,8 +316,8 @@ def assert_sane(x, nonzero=True):
         # Update kinematics.
         kinsol = tree.doKinematics(q, v)
         # AutoDiff
-        q_ad = np.array(map(AutoDiffXd, q))
-        v_ad = np.array(map(AutoDiffXd, v))
+        q_ad = np.array(list(map(AutoDiffXd, q)))
+        v_ad = np.array(list(map(AutoDiffXd, v)))
         kinsol_ad = tree.doKinematics(q_ad, v_ad)
         # Sanity checks:
         # - Actuator map.

bindings/pydrake/pydrake_pybind.h

@@ -310,5 +310,24 @@ struct overload_cast_impl {
 template <typename Return, typename... Args>
 constexpr auto overload_cast_explicit = overload_cast_impl<Return, Args...>{};
 
+#if PY_MAJOR_VERSION >= 3
+// The following works around pybind11 modules getting reconstructed /
+// reimported in Python3. See pybind/pybind11#1559 for more details.
+// Use this ONLY when necessary (e.g. when using a utility method which imports
+// the module, within the module itself).
+#define PYDRAKE_PREVENT_PYTHON3_MODULE_REIMPORT(variable) \
+  { \
+    static py::handle variable##_original; \
+    if (variable##_original) { \
+      variable = py::reinterpret_borrow<py::module>(variable##_original); \
+      return; \
+    } else { \
+      variable##_original = variable; \
+    } \
+  }
+#else  // PY_MAJOR_VERSION >= 3
+#define PYDRAKE_PREVENT_PYTHON3_MODULE_REIMPORT(variable)
+#endif  // PY_MAJOR_VERSION >= 3
+
 }  // namespace pydrake
 }  // namespace drake

bindings/pydrake/solvers/test/mathematicalprogram_test.py

@@ -3,10 +3,11 @@
 from pydrake.solvers import mathematicalprogram as mp
 from pydrake.solvers.mathematicalprogram import SolverType
 
-import numpy as np
 import unittest
 import warnings
 
+import numpy as np
+
 import pydrake
 import pydrake.symbolic as sym
 
@@ -377,7 +378,7 @@ def check_and_reset():
             self.assertTrue(np.isnan(prog.GetInitialGuess(x)).all())
 
         # Test setting individual variables
-        for i in xrange(count):
+        for i in range(count):
             prog.SetInitialGuess(x[i], x0[i])
             self.assertEqual(prog.GetInitialGuess(x[i]), x0[i])
         check_and_reset()

bindings/pydrake/symbolic.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import, division, print_function
 
+import functools
+
 from ._symbolic_py import *
 
 # Explicitly import private symbols
@@ -8,9 +10,9 @@
 
 def logical_and(*formulas):
     assert len(formulas) >= 1, "Must supply at least one operand"
-    return reduce(__logical_and, formulas)
+    return functools.reduce(__logical_and, formulas)
 
 
 def logical_or(*formulas):
     assert len(formulas) >= 1, "Must supply at least one operand"
-    return reduce(__logical_or, formulas)
+    return functools.reduce(__logical_or, formulas)

bindings/pydrake/symbolic_py.cc

@@ -339,7 +339,8 @@ PYBIND11_MODULE(_symbolic_py, m) {
     return Jacobian(f, vars);
   }, doc.Expression.Jacobian.doc);
 
-  py::class_<Formula>(m, "Formula")
+  py::class_<Formula> formula(m, "Formula", doc.Formula.doc);
+  formula
       .def("GetFreeVariables", &Formula::GetFreeVariables,
         doc.Formula.GetFreeVariables.doc)
       .def("EqualTo", &Formula::EqualTo, doc.Formula.EqualTo.doc)
@@ -376,13 +377,17 @@ PYBIND11_MODULE(_symbolic_py, m) {
            [](const Formula& self) { return std::hash<Formula>{}(self); })
       .def_static("True", &Formula::True, doc.FormulaTrue.doc)
       .def_static("False", &Formula::False, doc.FormulaFalse.doc)
+      // `True` and `False` are reserved as of Python3
+      .def_static("True_", &Formula::True, doc.FormulaTrue.doc)
+      .def_static("False_", &Formula::False, doc.FormulaFalse.doc)
       .def("__nonzero__", [](const Formula&) {
         throw std::runtime_error(
-            "You should not call `__nonzero__` on `Formula`. If you are trying "
-            "to make a map with `Variable`, `Expression`, or `Polynomial` as "
-            "keys and access the keys, please use "
-            "`pydrake.util.containers.EqualToDict`.");
+            "You should not call `__bool__` / `__nonzero__` on `Formula`. "
+            "If you are trying to make a map with `Variable`, `Expression`, "
+            "or `Polynomial` as keys (and then access the map in Python), "
+            "please use pydrake.util.containers.EqualToDict`.");
       });
+  formula.attr("__bool__") = formula.attr("__nonzero__");
 
   // Cannot overload logical operators: http://stackoverflow.com/a/471561
   // Defining custom function for clarity.

bindings/pydrake/systems/framework_py.cc

@@ -7,6 +7,7 @@ namespace drake {
 namespace pydrake {
 
 PYBIND11_MODULE(framework, m) {
+  PYDRAKE_PREVENT_PYTHON3_MODULE_REIMPORT(m);
   m.doc() = "Bindings for the core Systems framework.";
 
   // Import autodiff and symbolic modules so that their types are declared for

bindings/pydrake/systems/scalar_conversion.py

@@ -215,5 +215,6 @@ def conversion(system):
 
             converter.Add[T, U](conversion)
 
-        map(add_captured, self._T_pairs)
+        for T_pair in self._T_pairs:
+            add_captured(T_pair)
         return converter

bindings/pydrake/systems/systems_pybind.h

@@ -83,8 +83,8 @@ be destroyed when it is replaced, since it is stored using `unique_ptr<>`.
   }
   py_class.def("set_value", &Class::set_value, set_value_docstring.c_str());
   // Register instantiation.
-  py::module py_module = py::module::import("pydrake.systems.framework");
-  AddTemplateClass(py_module, "Value", py_class, GetPyParam<T>());
+  py::module py_framework = py::module::import("pydrake.systems.framework");
+  AddTemplateClass(py_framework, "Value", py_class, GetPyParam<T>());
   return py_class;
 }
 

bindings/pydrake/systems/test/custom_test.py

@@ -39,7 +39,7 @@ class CustomAdder(LeafSystem):
     # Reimplements `Adder`.
     def __init__(self, num_inputs, size):
         LeafSystem.__init__(self)
-        for i in xrange(num_inputs):
+        for i in range(num_inputs):
             self._DeclareInputPort(kUseDefaultName, PortDataType.kVectorValued,
                                    size)
         self._DeclareVectorOutputPort("sum", BasicVector(size), self._calc_sum)
@@ -49,7 +49,7 @@ def _calc_sum(self, context, sum_data):
         # since they are not stored densely.
         sum = sum_data.get_mutable_value()
         sum[:] = 0
-        for i in xrange(context.get_num_input_ports()):
+        for i in range(context.get_num_input_ports()):
             input_vector = self.EvalVectorInput(context, i)
             sum += input_vector.get_value()
 

bindings/pydrake/systems/test/lcm_test.py

@@ -4,6 +4,7 @@
 import unittest
 
 import numpy as np
+from six import text_type as unicode
 
 from robotlocomotion import quaternion_t
 

bindings/pydrake/systems/test/sensors_test.py

@@ -1,3 +1,5 @@
+from __future__ import division
+
 import pydrake.systems.sensors as mut
 
 import numpy as np
@@ -76,8 +78,8 @@ def test_image_types(self):
             self.assertEqual(image.data.shape, image.shape)
             self.assertEqual(image.data.dtype, ImageT.Traits.ChannelType)
 
-            w /= 2
-            h /= 2
+            w //= 2
+            h //= 2
             # WARNING: Resizing an image with an existing reference to
             # `image.data` will cause `image.data` + `image.mutable_data` to be
             # invalid.

bindings/pydrake/systems/test/value_test.py

@@ -35,7 +35,7 @@ def test_basic_vector_double(self):
             for wrap in [pass_through, np.array]:
                 # Ensure that we can get vectors templated on double by
                 # reference.
-                expected_init = wrap(map(float, range(n)))
+                expected_init = wrap([float(x) for x in range(n)])
                 expected_add = wrap([x + 1 for x in expected_init])
                 expected_set = wrap([x + 10 for x in expected_init])
 
@@ -140,12 +140,12 @@ def test_abstract_value_unknown(self):
         with self.assertRaises(RuntimeError) as cm:
             value.get_value()
         self.assertTrue(all(
-            s in cm.exception.message for s in [
+            s in str(cm.exception) for s in [
                 "AbstractValue",
                 "UnknownType",
                 "get_value",
                 "AddValueInstantiation",
-            ]), cm.exception.message)
+            ]), cm.exception)
 
     def test_parameters_api(self):
 

bindings/pydrake/test/math_overloads_test.py

@@ -149,7 +149,7 @@ def _check_overload(self, overload):
         def check_eval(functions, nargs):
             # Generate arguments.
             args_float = args_float_all[:nargs]
-            args_T = map(overload.to_type, args_float)
+            args_T = list(map(overload.to_type, args_float))
             # Check each supported function.
             for f_drake, f_builtin in functions:
                 if not overload.supports(f_drake):