Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[xdoctest][task 125-137] reformat example code with google style in static/* #56121

Merged
merged 11 commits into from
Aug 25, 2023
Merged

[xdoctest][task 125-137] reformat example code with google style in static/* #56121

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
d66c098
LGTM
DrRyanHuang Aug 9, 2023
2a819be
[Doctest]fix No.125-137, test=docs_preview
DrRyanHuang Aug 9, 2023
f8ccea3
[Doctest]fix No.125-137, test=docs_preview
DrRyanHuang Aug 9, 2023
3a1bbef
fix codestyle
DrRyanHuang Aug 9, 2023
fc25892
resolve conversation
DrRyanHuang Aug 19, 2023
17122af
change Example
DrRyanHuang Aug 20, 2023
5a671b4
Merge pull request #3 from PaddlePaddle/develop
DrRyanHuang Aug 23, 2023
3ab0c10
fix example format...
DrRyanHuang Aug 23, 2023
6524be1
Merge branch 'xdoc' of https://github.com/drryanhuang/Paddle into xdoc
DrRyanHuang Aug 23, 2023
1877907
fix my stupid fp16
DrRyanHuang Aug 24, 2023
5faf83d
fix codestyple
DrRyanHuang Aug 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions python/paddle/static/amp/bf16/amp_lists.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,11 @@ class AutoMixedPrecisionListsBF16:

Examples:
.. code-block:: python
import paddle
paddle.enable_static()
with paddle.static.amp.bf16_guard():
paddle.static.amp.bf16.AutoMixedPrecisionListsBF16(custom_fp32_list={'lstm'})

>>> import paddle
>>> paddle.enable_static()
>>> with paddle.static.amp.bf16.bf16_guard():
... paddle.static.amp.bf16.AutoMixedPrecisionListsBF16(custom_fp32_list={'lstm'})
"""

def __init__(
Expand Down
24 changes: 12 additions & 12 deletions python/paddle/static/amp/bf16/amp_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,18 +230,18 @@ def bf16_guard():
Examples:
.. code-block:: python

import numpy as np
import paddle
import paddle.nn.functional as F
paddle.enable_static()
data = paddle.static.data(name='X', shape=[None, 1, 28, 28], dtype='float32')
conv2d = paddle.static.nn.conv2d(input=data, num_filters=6, filter_size=3)

with paddle.static.amp.bf16_guard():
bn = paddle.static.nn.batch_norm(input=conv2d, act="relu")
pool = F.max_pool2d(bn, kernel_size=2, stride=2)
hidden = paddle.static.nn.fc(pool, size=10)
loss = paddle.mean(hidden)
>>> import numpy as np
>>> import paddle
>>> import paddle.nn.functional as F
>>> paddle.enable_static()
>>> data = paddle.static.data(name='X', shape=[None, 1, 28, 28], dtype='float32')
>>> conv2d = paddle.static.nn.conv2d(input=data, num_filters=6, filter_size=3)

>>> with paddle.static.amp.bf16.bf16_guard():
... bn = paddle.static.nn.batch_norm(input=conv2d, act="relu")
... pool = F.max_pool2d(bn, kernel_size=2, stride=2)
... hidden = paddle.static.nn.fc(pool, size=10)
... loss = paddle.mean(hidden)
"""
with framework.name_scope(prefix=_bf16_guard_pattern):
yield
Expand Down
173 changes: 90 additions & 83 deletions python/paddle/static/amp/bf16/decorator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,42 +135,44 @@ def amp_init(
Examples:
.. code-block:: python

import numpy as np
import paddle
import paddle.nn.functional as F
paddle.enable_static()

def run_example_code():
place = paddle.CPUPlace(0)
exe = paddle.static.Executor(place)
data = paddle.static.data(name='X', shape=[None, 1, 28, 28], dtype='float32')
conv2d = paddle.static.nn.conv2d(input=data, num_filters=6, filter_size=3)
# 1) Use bf16_guard to control the range of bf16 kernels used.
with paddle.static.amp.bf16_guard():
bn = paddle.static.nn.batch_norm(input=conv2d, act="relu")
pool = F.max_pool2d(bn, kernel_size=2, stride=2)
hidden = paddle.static.nn.fc(pool, size=10)
loss = paddle.mean(hidden)
# 2) Create the optimizer and set `multi_precision` to True.
# Setting `multi_precision` to True can avoid the poor accuracy
# or the slow convergence in a way.
optimizer = paddle.optimizer.Momentum(learning_rate=0.01, multi_precision=True)
# 3) These ops in `custom_fp32_list` will keep in the float32 computation type.
amp_list = paddle.static.amp.CustomOpLists(
custom_fp32_list=['pool2d'])
# 4) The entry of Paddle AMP.
# Enable pure bf16 training by setting `use_pure_bf16` to True.
optimizer = paddle.static.amp.bf16.decorate_bf16(
optimizer,
amp_list,
use_pure_bf16=True)
# If you don't use the default_startup_program(), you sholud pass
# your defined `startup_program` into `minimize`.
optimizer.minimize(loss)
exe.run(paddle.static.default_startup_program())
# 5) Use `amp_init` after FP32 parameters initialization(such as `exe.run(startup_program)`).
# If you want to perform the testing process, you should pass `test_program` into `amp_init`.
optimizer.amp_init(place, scope=paddle.static.global_scope())
>>> import numpy as np
>>> import paddle
>>> import paddle.nn.functional as F
>>> paddle.enable_static()

>>> def run_example_code():
... place = paddle.CPUPlace()
... exe = paddle.static.Executor(place)
... data = paddle.static.data(name='X', shape=[None, 1, 28, 28], dtype='float32')
... conv2d = paddle.static.nn.conv2d(input=data, num_filters=6, filter_size=3)
... # 1) Use bf16_guard to control the range of bf16 kernels used.
... with paddle.static.amp.bf16.bf16_guard():
... bn = paddle.static.nn.batch_norm(input=conv2d, act="relu")
... pool = F.max_pool2d(bn, kernel_size=2, stride=2)
... hidden = paddle.static.nn.fc(pool, size=10)
... loss = paddle.mean(hidden)
... # 2) Create the optimizer and set `multi_precision` to True.
... # Setting `multi_precision` to True can avoid the poor accuracy
... # or the slow convergence in a way.
... optimizer = paddle.optimizer.Momentum(learning_rate=0.01, multi_precision=True)
... # 3) These ops in `custom_black_list` will keep in the float32 computation type.
... amp_list = paddle.static.amp.CustomOpLists(
... custom_black_list=['pool2d'])
... # 4) The entry of Paddle AMP.
... # Enable pure bf16 training by setting `use_pure_bf16` to True.
... optimizer = paddle.static.amp.bf16.decorate_bf16(
... optimizer,
... amp_list,
... use_pure_bf16=True)
... # If you don't use the default_startup_program(), you sholud pass
... # your defined `startup_program` into `minimize`.
... optimizer.minimize(loss)
... exe.run(paddle.static.default_startup_program())
... # 5) Use `amp_init` after FP32 parameters initialization(such as `exe.run(startup_program)`).
... # If you want to perform the testing process, you should pass `test_program` into `amp_init`.
... optimizer.amp_init(place, scope=paddle.static.global_scope())

>>> run_example_code()

"""
assert (
Expand Down Expand Up @@ -263,63 +265,68 @@ def decorate_bf16(
An optimizer acting like a normal one but with mixed-precision training
enabled.

Examples 1:
.. code-block:: python
Examples:
.. code-block:: python
:name: example-1

# fp32&bf16 list based strategy example
import paddle
import paddle.static as static
>>> import paddle
>>> import paddle.static as static

>>> paddle.enable_static()

>>> data = static.data(name='X', shape=[None, 1], dtype='float32')
>>> hidden = static.nn.fc(x=data, size=10)
>>> loss = paddle.mean(hidden)
>>> optimizer = paddle.optimizer.Adam(learning_rate=0.001)

>>> mp_optimizer = static.amp.bf16.decorate_bf16(optimizer=optimizer)

paddle.enable_static()
>>> ops, param_grads = mp_optimizer.minimize(loss)

data = static.data(name='X', shape=[None, 1], dtype='float32')
hidden = static.nn.fc(x=data, size=10)
loss = paddle.mean(hidden)
optimizer = paddle.optimizer.Adam(learning_rate=0.001)

mp_optimizer = static.amp.decorate_bf16(optimizer=optimizer)

ops, param_grads = mp_optimizer.minimize(loss)

Examples 2:
.. code-block:: python
:name: example-2

# pure bf16 training example
import numpy as np
import paddle
import paddle.nn.functional as F

def run_example_code():
place = paddle.CPUPlace(0)
exe = paddle.static.Executor(place)
data = paddle.static.data(name='X', shape=[None, 1, 28, 28], dtype='float32')
conv2d = paddle.static.nn.conv2d(input=data, num_filters=6, filter_size=3)
# 1) Use bf16_guard to control the range of bf16 kernels used.
with paddle.static.amp.bf16_guard():
bn = paddle.static.nn.batch_norm(input=conv2d, act="relu")
pool = F.max_pool2d(bn, kernel_size=2, stride=2)
hidden = paddle.static.nn.fc(pool, size=10)
loss = paddle.mean(hidden)
# 2) Create the optimizer and set `multi_precision` to True.
# Setting `multi_precision` to True can avoid the poor accuracy
# or the slow convergence in a way.
optimizer = paddle.optimizer.Momentum(learning_rate=0.01, multi_precision=True)
# 3) These ops in `custom_fp32_list` will keep in the float32 computation type.
amp_list = paddle.static.amp.CustomOpLists(
custom_fp32_list=['pool2d'])
# 4) The entry of Paddle AMP.
# Enable pure bf16 training by setting `use_pure_bf16` to True.
optimizer = paddle.static.amp.decorate_bf16(
optimizer,
amp_list,
use_pure_bf16=True)
# If you don't use the default_startup_program(), you sholud pass
# your defined `startup_program` into `minimize`.
optimizer.minimize(loss)
exe.run(paddle.static.default_startup_program())
# 5) Use `amp_init` after FP32 parameters initialization(such as `exe.run(startup_program)`).
# If you want to perform the testing process, you should pass `test_program` into `amp_init`.
optimizer.amp_init(place, scope=paddle.static.global_scope())
>>> import numpy as np
>>> import paddle
>>> import paddle.nn.functional as F

>>> def run_example_code():
... place = paddle.CPUPlace()
... exe = paddle.static.Executor(place)
... data = paddle.static.data(name='X', shape=[None, 1, 28, 28], dtype='float32')
... conv2d = paddle.static.nn.conv2d(input=data, num_filters=6, filter_size=3)
... # 1) Use bf16_guard to control the range of bf16 kernels used.
... with paddle.static.amp.bf16.bf16_guard():
... bn = paddle.static.nn.batch_norm(input=conv2d, act="relu")
... pool = F.max_pool2d(bn, kernel_size=2, stride=2)
... hidden = paddle.static.nn.fc(pool, size=10)
... loss = paddle.mean(hidden)
... # 2) Create the optimizer and set `multi_precision` to True.
... # Setting `multi_precision` to True can avoid the poor accuracy
... # or the slow convergence in a way.
... optimizer = paddle.optimizer.Momentum(learning_rate=0.01, multi_precision=True)
... # 3) These ops in `custom_black_list` will keep in the float32 computation type.
... amp_list = paddle.static.amp.CustomOpLists(
... custom_black_list=['pool2d'])
... # 4) The entry of Paddle AMP.
... # Enable pure bf16 training by setting `use_pure_bf16` to True.
... optimizer = paddle.static.amp.bf16.decorate_bf16(
... optimizer,
... amp_list,
... use_pure_bf16=True)
... # If you don't use the default_startup_program(), you sholud pass
... # your defined `startup_program` into `minimize`.
... optimizer.minimize(loss)
... exe.run(paddle.static.default_startup_program())
... # 5) Use `amp_init` after FP32 parameters initialization(such as `exe.run(startup_program)`).
... # If you want to perform the testing process, you should pass `test_program` into `amp_init`.
... optimizer.amp_init(place, scope=paddle.static.global_scope())
>>> run_example_code()

"""
if amp_lists is None:
Expand Down
Loading