From 1ff5761fa6ed2467b28cc60fb7419864fd1d18d5 Mon Sep 17 00:00:00 2001 From: lisamhy <763334840@qq.com> Date: Mon, 27 Feb 2023 15:43:34 +0000 Subject: [PATCH 1/4] add finfo --- paddle/fluid/pybind/pybind.cc | 92 +++++++++++++++++++ paddle/phi/common/bfloat16.h | 4 +- paddle/phi/common/float16.h | 2 +- python/paddle/__init__.py | 2 + .../tests/unittests/test_iinfo_and_finfo.py | 36 ++++++++ python/paddle/framework/dtype.py | 43 +++++++++ 6 files changed, 176 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 7fe8674ef1296d..a8566237bcb27d 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -71,6 +71,8 @@ limitations under the License. */ #include "paddle/fluid/imperative/amp_auto_cast.h" #include "paddle/fluid/imperative/layer.h" #include "paddle/fluid/memory/allocation/allocator_strategy.h" +#include "paddle/fluid/platform/bfloat16.h" +#include "paddle/fluid/platform/float16.h" #include "paddle/fluid/prim/utils/utils.h" #ifdef PADDLE_WITH_CUDA #include "paddle/fluid/memory/allocation/cuda_ipc_allocator.h" @@ -446,6 +448,73 @@ struct iinfo { } }; +struct finfo { + int64_t bits; + double eps; + double min; // lowest() + double max; + double tiny; + double smallest_normal; // min() + double resolution; + std::string dtype; + + explicit finfo(const framework::proto::VarType::Type &type) { + switch (type) { + case framework::proto::VarType::FP16: + eps = std::numeric_limits::epsilon(); + min = std::numeric_limits::lowest(); + max = std::numeric_limits::max(); + smallest_normal = std::numeric_limits::min(); + tiny = smallest_normal; + resolution = std::pow( + 10, -std::numeric_limits::digits10); + bits = 16; + dtype = "float16"; + break; + case framework::proto::VarType::FP32: + case framework::proto::VarType::COMPLEX64: + eps = std::numeric_limits::epsilon(); + min = std::numeric_limits::lowest(); + max = std::numeric_limits::max(); + smallest_normal = std::numeric_limits::min(); + tiny = smallest_normal; + resolution = std::pow(10, -std::numeric_limits::digits10); + bits = 32; + dtype = "float32"; + break; + case framework::proto::VarType::FP64: + case framework::proto::VarType::COMPLEX128: + eps = std::numeric_limits::epsilon(); + min = std::numeric_limits::lowest(); + max = std::numeric_limits::max(); + smallest_normal = std::numeric_limits::min(); + tiny = smallest_normal; + resolution = std::pow(10, -std::numeric_limits::digits10); + bits = 64; + dtype = "float64"; + break; + case framework::proto::VarType::BF16: + eps = std::numeric_limits::epsilon(); + min = std::numeric_limits::lowest(); + max = std::numeric_limits::max(); + smallest_normal = + std::numeric_limits::min(); + tiny = smallest_normal; + resolution = std::pow( + 10, -std::numeric_limits::digits10); + bits = 16; + dtype = "bfloat16"; + break; + default: + PADDLE_THROW(platform::errors::InvalidArgument( + "the argument of paddle.finfo can only be paddle.float32, " + "paddle.float64, paddle.float16, paddle.bfloat16" + "paddle.complex64, or paddle.complex128")); + break; + } + } +}; + static PyObject *GetPythonAttribute(PyObject *obj, const char *attr_name) { // NOTE(zjl): PyObject_GetAttrString would return nullptr when attr_name // is not inside obj, but it would also set the error flag of Python. @@ -667,6 +736,29 @@ PYBIND11_MODULE(libpaddle, m) { return oss.str(); }); + py::class_(m, "finfo") + .def(py::init()) + .def_readonly("min", &finfo::min) + .def_readonly("max", &finfo::max) + .def_readonly("bits", &finfo::bits) + .def_readonly("eps", &finfo::eps) + .def_readonly("resolution", &finfo::resolution) + .def_readonly("smallest_normal", &finfo::smallest_normal) + .def_readonly("tiny", &finfo::tiny) + .def_readonly("dtype", &finfo::dtype) + .def("__repr__", [](const finfo &a) { + std::ostringstream oss; + oss << "paddle.finfo(min=" << a.min; + oss << ", max=" << a.max; + oss << ", eps=" << a.eps; + oss << ", resolution=" << a.resolution; + oss << ", smallest_normal=" << a.smallest_normal; + oss << ", tiny=" << a.tiny; + oss << ", bits=" << a.bits; + oss << ", dtype=" << a.dtype << ")"; + return oss.str(); + }); + m.def("__set_bwd_prim_enabled", &paddle::prim::PrimCommonUtils::SetBwdPrimEnabled); m.def("_is_bwd_prim_enabled", diff --git a/paddle/phi/common/bfloat16.h b/paddle/phi/common/bfloat16.h index 37e4b55fbbc075..cd7360197b6aef 100644 --- a/paddle/phi/common/bfloat16.h +++ b/paddle/phi/common/bfloat16.h @@ -373,7 +373,7 @@ struct numeric_limits { static const bool tinyness_before = false; HOSTDEVICE static phi::dtype::bfloat16(min)() { - return phi::dtype::raw_uint16_to_bfloat16(0x007f); + return phi::dtype::raw_uint16_to_bfloat16(0x0080); } HOSTDEVICE static phi::dtype::bfloat16 lowest() { return phi::dtype::raw_uint16_to_bfloat16(0xff7f); @@ -382,7 +382,7 @@ struct numeric_limits { return phi::dtype::raw_uint16_to_bfloat16(0x7f7f); } HOSTDEVICE static phi::dtype::bfloat16 epsilon() { - return phi::dtype::raw_uint16_to_bfloat16(0x3400); + return phi::dtype::raw_uint16_to_bfloat16(0x3C00); } HOSTDEVICE static phi::dtype::bfloat16 round_error() { return phi::dtype::bfloat16(0.5); diff --git a/paddle/phi/common/float16.h b/paddle/phi/common/float16.h index 6f00df046c6a48..d44a13fa11fcea 100644 --- a/paddle/phi/common/float16.h +++ b/paddle/phi/common/float16.h @@ -1064,7 +1064,7 @@ struct numeric_limits { return phi::dtype::raw_uint16_to_float16(0x7bff); } HOSTDEVICE static phi::dtype::float16 epsilon() { - return phi::dtype::raw_uint16_to_float16(0x0800); + return phi::dtype::raw_uint16_to_float16(0x1400); } HOSTDEVICE static phi::dtype::float16 round_error() { return phi::dtype::float16(0.5); diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index 8f5e26bc7c44c0..7c9284e1ba22b7 100755 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -41,6 +41,7 @@ from .fluid.lazy_init import LazyGuard # noqa: F401 from .framework.dtype import iinfo # noqa: F401 +from .framework.dtype import finfo # noqa: F401 from .framework.dtype import dtype as dtype # noqa: F401 from .framework.dtype import uint8 # noqa: F401 from .framework.dtype import int8 # noqa: F401 @@ -400,6 +401,7 @@ disable_static() __all__ = [ # noqa 'iinfo', + 'finfo', 'dtype', 'uint8', 'int8', diff --git a/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py b/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py index ba6a74779b2799..92600b5f5c20b2 100644 --- a/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py +++ b/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py @@ -48,6 +48,42 @@ def test_iinfo(self): self.assertEqual(xinfo.min, xninfo.min) self.assertEqual(xinfo.dtype, xninfo.dtype) + def test_finfo(self): + + for paddle_dtype, np_dtype in [ + (paddle.float32, np.float32), + (paddle.float64, np.float64), + (paddle.complex64, np.complex64), + (paddle.complex128, np.complex128), + ]: + xinfo = paddle.finfo(paddle_dtype) + xninfo = np.finfo(np_dtype) + self.assertEqual(xinfo.dtype, xninfo.dtype) + self.assertEqual(xinfo.bits, xninfo.bits) + self.assertAlmostEqual(xinfo.max, xninfo.max) + self.assertAlmostEqual(xinfo.min, xninfo.min) + self.assertAlmostEqual(xinfo.eps, xninfo.eps) + self.assertAlmostEqual(xinfo.tiny, xninfo.tiny) + self.assertAlmostEqual(xinfo.resolution, xninfo.resolution) + + xinfo = paddle.finfo(paddle.float16) + self.assertEqual(xinfo.dtype, "float16") + self.assertEqual(xinfo.bits, 16) + self.assertAlmostEqual(xinfo.max, 65504.0) + self.assertAlmostEqual(xinfo.min, -65504.0) + self.assertAlmostEqual(xinfo.eps, 0.0009765625) + self.assertAlmostEqual(xinfo.tiny, 6.103515625e-05) + self.assertAlmostEqual(xinfo.resolution, 0.001) + + xinfo = paddle.finfo(paddle.bfloat16) + self.assertEqual(xinfo.dtype, "bfloat16") + self.assertEqual(xinfo.bits, 16) + self.assertAlmostEqual(xinfo.max, 3.3895313892515355e38) + self.assertAlmostEqual(xinfo.min, -3.3895313892515355e38) + self.assertAlmostEqual(xinfo.eps, 0.0078125) + self.assertAlmostEqual(xinfo.tiny, 1.1754943508222875e-38) + self.assertAlmostEqual(xinfo.resolution, 0.01) + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/framework/dtype.py b/python/paddle/framework/dtype.py index d3517d7395c597..8379894ea64569 100644 --- a/python/paddle/framework/dtype.py +++ b/python/paddle/framework/dtype.py @@ -13,6 +13,7 @@ # limitations under the License. from ..fluid.core import VarDesc +from ..fluid.core import finfo as core_finfo from ..fluid.core import iinfo as core_iinfo dtype = VarDesc.VarType @@ -69,3 +70,45 @@ def iinfo(dtype): """ return core_iinfo(dtype) + + +def finfo(dtype): + """ + + paddle.finfo is a function that returns an object that represents the numerical properties of a floating point + paddle.dtype. + This is similar to `numpy.finfo `_. + + Args: + dtype(paddle.dtype): One of paddle.float16, paddle.float32, paddle.float64, paddle.bfloat16, + paddle.complex64, and paddle.complex128. + + Returns: + An finfo object, which has the following 8 attributes: + + - min: double, The smallest representable number (typically -max). + - max: double, The largest representable number. + - eps: double, The smallest representable number such that 1.0 + eps != 1.0. + - resolution: The approximate decimal resolution of this type, i.e., 10**-precision. + - smallest_normal: The smallest positive normal number. + - tiny: The smallest positive normal number. Equivalent to smallest_normal. + - bits: int, The number of bits occupied by the type. + - dtype: str, The string name of the argument dtype. + + Examples: + .. code-block:: python + + import paddle + + finfo_float32 = paddle.finfo(paddle.float32) + print(finfo_float32.min) # -3.40282e+38 + print(finfo_float32.max) # 3.40282e+38 + print(finfo_float32.eps) # 1.19209e-07 + print(finfo_float32.resolution) # 1e-06 + print(finfo_float32.smallest_normal) # 1.17549e-38 + print(finfo_float32.tiny) # 1.17549e-38 + print(finfo_float32.bits) # 32 + print(finfo_float32.dtype) # float32 + + """ + return core_finfo(dtype) From d8717c4c6b17caa7a8de9cc87d3e9e53ff50bc0b Mon Sep 17 00:00:00 2001 From: lisamhy <763334840@qq.com> Date: Wed, 1 Mar 2023 03:03:37 +0000 Subject: [PATCH 2/4] fix comment --- .../tests/unittests/test_iinfo_and_finfo.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py b/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py index 92600b5f5c20b2..31f22ec5c5d153 100644 --- a/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py +++ b/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py @@ -13,6 +13,7 @@ # limitations under the License. import unittest +from distutils.version import StrictVersion import numpy as np @@ -53,8 +54,6 @@ def test_finfo(self): for paddle_dtype, np_dtype in [ (paddle.float32, np.float32), (paddle.float64, np.float64), - (paddle.complex64, np.complex64), - (paddle.complex128, np.complex128), ]: xinfo = paddle.finfo(paddle_dtype) xninfo = np.finfo(np_dtype) @@ -65,6 +64,28 @@ def test_finfo(self): self.assertAlmostEqual(xinfo.eps, xninfo.eps) self.assertAlmostEqual(xinfo.tiny, xninfo.tiny) self.assertAlmostEqual(xinfo.resolution, xninfo.resolution) + if StrictVersion(np.__version__) >= StrictVersion('1.22.0'): + self.assertAlmostEqual( + xinfo.smallest_normal, xninfo.smallest_normal + ) + + for paddle_dtype, np_dtype in [ + (paddle.complex64, np.complex64), + (paddle.complex128, np.complex128), + ]: + xinfo = paddle.finfo(paddle_dtype) + xninfo = np.finfo(np_dtype) + self.assertEqual(xinfo.dtype, xninfo.dtype) + self.assertEqual(xinfo.bits, xninfo.bits) + self.assertAlmostEqual(xinfo.max, xninfo.max, places=16) + self.assertAlmostEqual(xinfo.min, xninfo.min, places=16) + self.assertAlmostEqual(xinfo.eps, xninfo.eps, places=16) + self.assertAlmostEqual(xinfo.tiny, xninfo.tiny, places=16) + self.assertAlmostEqual(xinfo.resolution, xninfo.resolution) + if StrictVersion(np.__version__) >= StrictVersion('1.22.0'): + self.assertAlmostEqual( + xinfo.smallest_normal, xninfo.smallest_normal, places=16 + ) xinfo = paddle.finfo(paddle.float16) self.assertEqual(xinfo.dtype, "float16") @@ -74,6 +95,7 @@ def test_finfo(self): self.assertAlmostEqual(xinfo.eps, 0.0009765625) self.assertAlmostEqual(xinfo.tiny, 6.103515625e-05) self.assertAlmostEqual(xinfo.resolution, 0.001) + self.assertAlmostEqual(xinfo.smallest_normal, 6.103515625e-05) xinfo = paddle.finfo(paddle.bfloat16) self.assertEqual(xinfo.dtype, "bfloat16") @@ -83,6 +105,7 @@ def test_finfo(self): self.assertAlmostEqual(xinfo.eps, 0.0078125) self.assertAlmostEqual(xinfo.tiny, 1.1754943508222875e-38) self.assertAlmostEqual(xinfo.resolution, 0.01) + self.assertAlmostEqual(xinfo.smallest_normal, 1.1754943508222875e-38) if __name__ == '__main__': From e5864241d04a9e5bead2e06d38ce3ed6d9c83be8 Mon Sep 17 00:00:00 2001 From: lisamhy <763334840@qq.com> Date: Fri, 3 Mar 2023 09:30:04 +0000 Subject: [PATCH 3/4] fix doc --- python/paddle/framework/dtype.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/python/paddle/framework/dtype.py b/python/paddle/framework/dtype.py index 8379894ea64569..11d83a7a17aabf 100644 --- a/python/paddle/framework/dtype.py +++ b/python/paddle/framework/dtype.py @@ -75,25 +75,25 @@ def iinfo(dtype): def finfo(dtype): """ - paddle.finfo is a function that returns an object that represents the numerical properties of a floating point - paddle.dtype. + ``paddle.finfo`` is a function that returns an object that represents the numerical properties of a floating point + ``paddle.dtype``. This is similar to `numpy.finfo `_. Args: - dtype(paddle.dtype): One of paddle.float16, paddle.float32, paddle.float64, paddle.bfloat16, - paddle.complex64, and paddle.complex128. + dtype(paddle.dtype): One of ``paddle.float16``, ``paddle.float32``, ``paddle.float64``, ``paddle.bfloat16``, + ``paddle.complex64``, and ``paddle.complex128``. Returns: - An finfo object, which has the following 8 attributes: - - - min: double, The smallest representable number (typically -max). - - max: double, The largest representable number. - - eps: double, The smallest representable number such that 1.0 + eps != 1.0. - - resolution: The approximate decimal resolution of this type, i.e., 10**-precision. - - smallest_normal: The smallest positive normal number. - - tiny: The smallest positive normal number. Equivalent to smallest_normal. - - bits: int, The number of bits occupied by the type. - - dtype: str, The string name of the argument dtype. + An ``finfo`` object, which has the following 8 attributes: + + - min(double): The smallest representable number (typically `-max`). + - max(double): The largest representable number. + - eps(double): The smallest representable number such that `1.0 + eps != 1.0`. + - resolution(double): The approximate decimal resolution of this type, i.e., `10**-precision`. + - smallest_normal(double): The smallest positive normal number. + - tiny(double): The smallest positive normal number. Equivalent to smallest_normal. + - bits(int): The number of bits occupied by the type. + - dtype(str): The string name of the argument dtype. Examples: .. code-block:: python From 22dd66d080445cc164264dd486a5319041f93e24 Mon Sep 17 00:00:00 2001 From: lisamhy <763334840@qq.com> Date: Fri, 3 Mar 2023 13:31:32 +0000 Subject: [PATCH 4/4] fix typo --- python/paddle/framework/dtype.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/framework/dtype.py b/python/paddle/framework/dtype.py index 11d83a7a17aabf..42cd074d88f11c 100644 --- a/python/paddle/framework/dtype.py +++ b/python/paddle/framework/dtype.py @@ -88,7 +88,7 @@ def finfo(dtype): - min(double): The smallest representable number (typically `-max`). - max(double): The largest representable number. - - eps(double): The smallest representable number such that `1.0 + eps != 1.0`. + - eps(double): The smallest representable number such that `1.0 + eps ≠ 1.0`. - resolution(double): The approximate decimal resolution of this type, i.e., `10**-precision`. - smallest_normal(double): The smallest positive normal number. - tiny(double): The smallest positive normal number. Equivalent to smallest_normal.