Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[xdoctest] reformat example code with google style in No. 301 #56599

Merged
merged 8 commits into from
Sep 5, 2023
Merged

[xdoctest] reformat example code with google style in No. 301 #56599

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
397fe54
Update private_helper_function.py
Liyulingyue Aug 12, 2023
5a1ebae
Merge branch 'PaddlePaddle:develop' into xdoc8
Liyulingyue Aug 23, 2023
f429590
xdoc
Liyulingyue Aug 23, 2023
a299d41
Merge remote-tracking branch 'origin/xdoc8' into xdoc8
Liyulingyue Aug 23, 2023
95d1052
Merge branch 'PaddlePaddle:develop' into xdoc8
Liyulingyue Aug 25, 2023
b9b5615
Apply suggestions from code review
Liyulingyue Aug 30, 2023
bc5e218
format and skip a to_static case
SigureMo Sep 1, 2023
dc66a21
add a missing space
SigureMo Sep 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
342 changes: 177 additions & 165 deletions python/paddle/incubate/optimizer/modelaverage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,96 +69,100 @@ class ModelAverage(Optimizer):

Examples:

.. code-block:: python

import numpy as np
import paddle
import paddle.nn as nn
import paddle.optimizer as opt

BATCH_SIZE = 16
BATCH_NUM = 4
EPOCH_NUM = 4

IMAGE_SIZE = 784
CLASS_NUM = 10

# define a random dataset
class RandomDataset(paddle.io.Dataset):
def __init__(self, num_samples):
self.num_samples = num_samples

def __getitem__(self, idx):
image = np.random.random([IMAGE_SIZE]).astype('float32')
label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64')
return image, label

def __len__(self):
return self.num_samples

class LinearNet(nn.Layer):
def __init__(self):
super().__init__()
self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM)
self.bias = self._linear.bias

@paddle.jit.to_static
def forward(self, x):
return self._linear(x)

def train(layer, loader, loss_fn, opt, model_average):
for epoch_id in range(EPOCH_NUM):
for batch_id, (image, label) in enumerate(loader()):
out = layer(image)
loss = loss_fn(out, label)
loss.backward()
opt.step()
model_average.step()
opt.clear_grad()
model_average.clear_grad()
print("Train Epoch {} batch {}: loss = {}, bias = {}".format(
epoch_id, batch_id, np.mean(loss.numpy()), layer.bias.numpy()))
def evaluate(layer, loader, loss_fn):
for batch_id, (image, label) in enumerate(loader()):
out = layer(image)
loss = loss_fn(out, label)
loss.backward()
print("Evaluate batch {}: loss = {}, bias = {}".format(
batch_id, np.mean(loss.numpy()), layer.bias.numpy()))

# create network
layer = LinearNet()
loss_fn = nn.CrossEntropyLoss()
optimizer = opt.Momentum(learning_rate=0.2, momentum=0.1, parameters=layer.parameters())
model_average = paddle.incubate.ModelAverage(0.15,
parameters=layer.parameters(),
min_average_window=2,
max_average_window=10)

# create data loader
dataset = RandomDataset(BATCH_NUM * BATCH_SIZE)
loader = paddle.io.DataLoader(dataset,
batch_size=BATCH_SIZE,
shuffle=True,
drop_last=True,
num_workers=2)
# create data loader
eval_loader = paddle.io.DataLoader(dataset,
batch_size=BATCH_SIZE,
shuffle=True,
drop_last=True,
num_workers=1)

# train
train(layer, loader, loss_fn, optimizer, model_average)

print("\nEvaluate With ModelAverage")
with model_average.apply(need_restore=False):
evaluate(layer, eval_loader, loss_fn)

print("\nEvaluate With Restored Paramters")
model_average.restore()
evaluate(layer, eval_loader, loss_fn)
.. code-block:: python

>>> # doctest: +SKIP("Cannot get source code by to_static in REPL")
>>> import numpy as np
>>> import paddle
>>> import paddle.nn as nn
>>> import paddle.optimizer as opt

>>> BATCH_SIZE = 16
>>> BATCH_NUM = 4
>>> EPOCH_NUM = 4

>>> IMAGE_SIZE = 784
>>> CLASS_NUM = 10

>>> # define a random dataset
>>> class RandomDataset(paddle.io.Dataset):
... def __init__(self, num_samples):
... self.num_samples = num_samples
... def __getitem__(self, idx):
... image = np.random.random([IMAGE_SIZE]).astype('float32')
... label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64')
... return image, label
... def __len__(self):
... return self.num_samples
...
>>> class LinearNet(nn.Layer):
... def __init__(self):
... super().__init__()
... self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM)
... self.bias = self._linear.bias
...
... @paddle.jit.to_static
... def forward(self, x):
... return self._linear(x)
...
>>> def train(layer, loader, loss_fn, opt, model_average):
... for epoch_id in range(EPOCH_NUM):
... for batch_id, (image, label) in enumerate(loader()):
... out = layer(image)
... loss = loss_fn(out, label)
... loss.backward()
... opt.step()
... model_average.step()
... opt.clear_grad()
... model_average.clear_grad()
... print("Train Epoch {} batch {}: loss = {}, bias = {}".format(
... epoch_id, batch_id, np.mean(loss.numpy()), layer.bias.numpy()))
...
>>> def evaluate(layer, loader, loss_fn):
... for batch_id, (image, label) in enumerate(loader()):
... out = layer(image)
... loss = loss_fn(out, label)
... loss.backward()
... print("Evaluate batch {}: loss = {}, bias = {}".format(
... batch_id, np.mean(loss.numpy()), layer.bias.numpy()))
...
>>> # create network
>>> layer = LinearNet()
>>> loss_fn = nn.CrossEntropyLoss()
>>> optimizer = opt.Momentum(learning_rate=0.2, momentum=0.1, parameters=layer.parameters())
>>> model_average = paddle.incubate.ModelAverage(
... 0.15,
... parameters=layer.parameters(),
... min_average_window=2,
... max_average_window=10
... )
...
>>> # create data loader
>>> dataset = RandomDataset(BATCH_NUM * BATCH_SIZE)
>>> loader = paddle.io.DataLoader(dataset,
... batch_size=BATCH_SIZE,
... shuffle=True,
... drop_last=True,
... num_workers=2)
...
>>> # create data loader
>>> eval_loader = paddle.io.DataLoader(dataset,
... batch_size=BATCH_SIZE,
... shuffle=True,
... drop_last=True,
... num_workers=1
... )
...
>>> # train
>>> train(layer, loader, loss_fn, optimizer, model_average)

>>> print("\nEvaluate With ModelAverage")
>>> with model_average.apply(need_restore=False):
... evaluate(layer, eval_loader, loss_fn)

>>> print("\nEvaluate With Restored Paramters")
>>> model_average.restore()
>>> evaluate(layer, eval_loader, loss_fn)

"""

Expand Down Expand Up @@ -317,23 +321,25 @@ def minimize(

.. code-block:: python

import paddle
inp = paddle.rand([1, 10], dtype="float32")
linear = paddle.nn.Linear(10, 1)
out = linear(inp)
loss = paddle.mean(out)
loss.backward()

sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters())
sgd.minimize(loss)

modelaverage = paddle.incubate.ModelAverage(0.15,
parameters=linear.parameters(),
min_average_window=2,
max_average_window=4)
modelaverage.minimize(loss)
sgd.clear_grad()
modelaverage.clear_grad()
>>> import paddle
>>> inp = paddle.rand([1, 10], dtype="float32")
>>> linear = paddle.nn.Linear(10, 1)
>>> out = linear(inp)
>>> loss = paddle.mean(out)
>>> loss.backward()

>>> sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters())
>>> sgd.minimize(loss)

>>> modelaverage = paddle.incubate.ModelAverage(
... 0.15,
... parameters=linear.parameters(),
... min_average_window=2,
... max_average_window=4
... )
>>> modelaverage.minimize(loss)
>>> sgd.clear_grad()
>>> modelaverage.clear_grad()

"""
if in_dynamic_mode():
Expand All @@ -352,21 +358,23 @@ def step(self):

.. code-block:: python

import paddle
inp = paddle.rand([1, 10], dtype="float32")
linear = paddle.nn.Linear(10, 1)
out = linear(inp)
loss = paddle.mean(out)
sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters())
modelaverage = paddle.incubate.ModelAverage(0.15,
parameters=linear.parameters(),
min_average_window=2,
max_average_window=4)
loss.backward()
sgd.step()
modelaverage.step()
sgd.clear_grad()
modelaverage.clear_grad()
>>> import paddle
>>> inp = paddle.rand([1, 10], dtype="float32")
>>> linear = paddle.nn.Linear(10, 1)
>>> out = linear(inp)
>>> loss = paddle.mean(out)
>>> sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters())
>>> modelaverage = paddle.incubate.ModelAverage(
... 0.15,
... parameters=linear.parameters(),
... min_average_window=2,
... max_average_window=4
... )
>>> loss.backward()
>>> sgd.step()
>>> modelaverage.step()
>>> sgd.clear_grad()
>>> modelaverage.clear_grad()
"""

params_grads = []
Expand Down Expand Up @@ -398,28 +406,30 @@ def apply(self, executor=None, need_restore=True):

.. code-block:: python

import paddle
inp = paddle.rand([1, 10], dtype="float32")
linear = paddle.nn.Linear(10, 1)
out = linear(inp)
loss = paddle.mean(out)
loss.backward()

sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters())

modelaverage = paddle.incubate.ModelAverage(0.15,
parameters=linear.parameters(),
min_average_window=2,
max_average_window=4)
sgd.step()
modelaverage.step()

with modelaverage.apply():
for param in linear.parameters():
print(param)

for param in linear.parameters():
print(param)
>>> import paddle
>>> inp = paddle.rand([1, 10], dtype="float32")
>>> linear = paddle.nn.Linear(10, 1)
>>> out = linear(inp)
>>> loss = paddle.mean(out)
>>> loss.backward()

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change

>>> sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters())

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change

>>> modelaverage = paddle.incubate.ModelAverage(
... 0.15,
... parameters=linear.parameters(),
... min_average_window=2,
... max_average_window=4
... )
>>> sgd.step()
>>> modelaverage.step()

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change

>>> with modelaverage.apply():
... for param in linear.parameters():
... print(param)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change

>>> for param in linear.parameters():
... print(param)
"""
if in_dynamic_mode():
for param in self._parameter_list:
Expand Down Expand Up @@ -472,33 +482,35 @@ def restore(self, executor=None):

.. code-block:: python

import paddle
inp = paddle.rand([1, 10], dtype="float32")
linear = paddle.nn.Linear(10, 1)
out = linear(inp)
loss = paddle.mean(out)
loss.backward()
>>> import paddle
>>> inp = paddle.rand([1, 10], dtype="float32")
>>> linear = paddle.nn.Linear(10, 1)
>>> out = linear(inp)
>>> loss = paddle.mean(out)
>>> loss.backward()

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change

sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters())
>>> sgd = paddle.optimizer.SGD(learning_rate=0.1,parameters=linear.parameters())

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change

modelaverage = paddle.incubate.ModelAverage(0.15,
parameters=linear.parameters(),
min_average_window=2,
max_average_window=4)
sgd.step()
modelaverage.step()
>>> modelaverage = paddle.incubate.ModelAverage(
... 0.15,
... parameters=linear.parameters(),
... min_average_window=2,
... max_average_window=4
... )
>>> sgd.step()
>>> modelaverage.step()

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change

with modelaverage.apply(need_restore=False):
for param in linear.parameters():
print(param)
>>> with modelaverage.apply(need_restore=False):
... for param in linear.parameters():
... print(param)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change

for param in linear.parameters():
print(param)
>>> for param in linear.parameters():
... print(param)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change

modelaverage.restore()
>>> modelaverage.restore()

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change

for param in linear.parameters():
print(param)
>>> for param in linear.parameters():
... print(param)
"""
if in_dynamic_mode():
for param in self._parameter_list:
Expand Down