Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Modify sbp.split()'s karg: axis to dim #8411

Merged
merged 21 commits into from
Jun 14, 2022
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
8a4f44a
Modify sbp.split()'s axis karg to dim
lixiang007666 Jun 13, 2022
f724b6e
Refine
lixiang007666 Jun 13, 2022
998d673
Refine
lixiang007666 Jun 13, 2022
c110556
Merge branch 'master' into sbp_split_axis_to_sbp_split_dim
lixiang007666 Jun 13, 2022
b6e4815
Refine
lixiang007666 Jun 13, 2022
c279a52
Merge branch 'master' into sbp_split_axis_to_sbp_split_dim
lixiang007666 Jun 13, 2022
13c8d60
Merge branch 'master' into sbp_split_axis_to_sbp_split_dim
lixiang007666 Jun 14, 2022
b43abc7
Refine
lixiang007666 Jun 14, 2022
3b27a05
Refine
lixiang007666 Jun 14, 2022
a10beea
Refine
lixiang007666 Jun 14, 2022
3e323f5
Refine
lixiang007666 Jun 14, 2022
67ad2da
Merge branch 'master' into sbp_split_axis_to_sbp_split_dim
mergify[bot] Jun 14, 2022
f551ab1
Merge branch 'master' into sbp_split_axis_to_sbp_split_dim
mergify[bot] Jun 14, 2022
6d9b37e
Refine
lixiang007666 Jun 14, 2022
53cf345
Merge branch 'sbp_split_axis_to_sbp_split_dim' of github.com:Oneflow-…
lixiang007666 Jun 14, 2022
eb58e0c
Merge branch 'master' into sbp_split_axis_to_sbp_split_dim
mergify[bot] Jun 14, 2022
95ffbe9
Merge branch 'master' into sbp_split_axis_to_sbp_split_dim
mergify[bot] Jun 14, 2022
5171b1a
Merge branch 'master' into sbp_split_axis_to_sbp_split_dim
mergify[bot] Jun 14, 2022
dd4b7ab
Merge branch 'master' into sbp_split_axis_to_sbp_split_dim
mergify[bot] Jun 14, 2022
4520162
Merge branch 'master' into sbp_split_axis_to_sbp_split_dim
mergify[bot] Jun 14, 2022
86ec5a1
Merge branch 'master' into sbp_split_axis_to_sbp_split_dim
mergify[bot] Jun 14, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion oneflow/api/common/sbp.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ inline Maybe<std::string> SbpToString(Symbol<SbpParallel> sbp_sym) {
} else if (sbp_sym->has_partial_sum_parallel()) {
sbp_str += "partial_sum";
} else if (sbp_sym->has_split_parallel()) {
sbp_str += "split(axis=" + std::to_string(sbp_sym->split_parallel().axis()) + ")";
sbp_str += "split(dim=" + std::to_string(sbp_sym->split_parallel().axis()) + ")";
} else {
UNIMPLEMENTED_THEN_RETURN();
}
Expand Down
26 changes: 21 additions & 5 deletions python/oneflow/framework/distribute.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,17 @@
limitations under the License.
"""
import traceback
import warnings
from contextlib import contextmanager

import oneflow._oneflow_internal


def split_sbp(axis: int) -> oneflow._oneflow_internal.sbp.sbp:
"""Generate a split scheme in which op will be splitted at `axis`.
def split_sbp(*dim, **kwarg) -> oneflow._oneflow_internal.sbp.sbp:
lixiang007666 marked this conversation as resolved.
Show resolved Hide resolved
"""Generate a split scheme in which op will be splitted at `dim`.
lixiang007666 marked this conversation as resolved.
Show resolved Hide resolved

Args:
axis (int): At `axis` the op will be splitted.
dim (int): At `dim` the op will be splitted.
lixiang007666 marked this conversation as resolved.
Show resolved Hide resolved

Returns:
SbpParallel: Split scheme object, often required by `to_global` method of `Tensor`
Expand All @@ -34,5 +35,20 @@ def split_sbp(axis: int) -> oneflow._oneflow_internal.sbp.sbp:
ct2 = t1.to_global(sbp=flow.sbp.split(0), placement=("cuda", ranks=[0, 1, 2, 3]))

"""
assert type(axis) is int
return oneflow._oneflow_internal.sbp.split(axis)
if len(kwarg) != 0:
lixiang007666 marked this conversation as resolved.
Show resolved Hide resolved
if "dim" in kwarg:
dim = kwarg["dim"]
lixiang007666 marked this conversation as resolved.
Show resolved Hide resolved
elif "axis" in kwarg:
dim = kwarg["axis"]
warnings.warn(
"This 'axis' parameter of oneflow.sbp.split() has been updated to 'dim' since OneFlow version 0.8."
)
else:
raise ValueError("Invalid keyword arguments, expected to be 'dim'.")
lixiang007666 marked this conversation as resolved.
Show resolved Hide resolved

else:
assert len(dim) == 1
dim = dim[0]
lixiang007666 marked this conversation as resolved.
Show resolved Hide resolved

assert type(dim) is int
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

最好用isinstance

return oneflow._oneflow_internal.sbp.split(dim)
16 changes: 8 additions & 8 deletions python/oneflow/framework/docstr/tensor.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,13 +319,13 @@

>>> # results on rank 0
oneflow.Size([4])
tensor([0., 1., 0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(axis=0),), dtype=oneflow.float32)
tensor([0., 1., 0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(dim=0),), dtype=oneflow.float32)

.. code-block:: python

>>> # results on rank 1
oneflow.Size([4])
tensor([0., 1., 0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(axis=0),), dtype=oneflow.float32)
tensor([0., 1., 0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(dim=0),), dtype=oneflow.float32)
""",
)

Expand Down Expand Up @@ -365,13 +365,13 @@

>>> # results on rank 0
oneflow.Size([2])
tensor([0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(axis=0),), dtype=oneflow.float32)
tensor([0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(dim=0),), dtype=oneflow.float32)

.. code-block:: python

>>> # results on rank 1
oneflow.Size([2])
tensor([0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(axis=0),), dtype=oneflow.float32)
tensor([0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(dim=0),), dtype=oneflow.float32)
""",
)

Expand Down Expand Up @@ -424,13 +424,13 @@

>>> # results on rank 0
oneflow.Size([4])
tensor([0., 1., 0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(axis=0),), dtype=oneflow.float32)
tensor([0., 1., 0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(dim=0),), dtype=oneflow.float32)

.. code-block:: python

>>> # results on rank 1
oneflow.Size([4])
tensor([0., 1., 0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(axis=0),), dtype=oneflow.float32)
tensor([0., 1., 0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(dim=0),), dtype=oneflow.float32)

For global tensor:

Expand All @@ -447,13 +447,13 @@

>>> # results on rank 0
oneflow.Size([2])
tensor([0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(axis=0),), dtype=oneflow.float32)
tensor([0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(dim=0),), dtype=oneflow.float32)

.. code-block:: python

>>> # results on rank 1
oneflow.Size([2])
tensor([0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(axis=0),), dtype=oneflow.float32)
tensor([0., 1.], placement=oneflow.placement(type="cpu", ranks=[0, 1]), sbp=(oneflow.sbp.split(dim=0),), dtype=oneflow.float32)
""",
)

Expand Down
6 changes: 3 additions & 3 deletions python/oneflow/framework/docstr/tensor_attributes.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,9 @@

``oneflow.sbp`` includes three types:

- oneflow.sbp.split(axis)
- oneflow.sbp.split(dim)

Indicates that the global tensor is evenly divided according to the dimension `axis` and distributed on each rank.
Indicates that the global tensor is evenly divided according to the dimension `dim` and distributed on each rank.

- oneflow.sbp.broadcast()

Expand All @@ -120,7 +120,7 @@

>>> s = flow.sbp.split(0)
>>> s
oneflow.sbp.split(axis=0)
oneflow.sbp.split(dim=0)
>>> b = flow.sbp.broadcast()
>>> b
oneflow.sbp.broadcast
Expand Down
2 changes: 1 addition & 1 deletion python/oneflow/test/graph/test_graph_zero.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ def one_eval_iter():

for state in linear_t_g._state():
test_case.assertEqual(
state.origin.sbp, (oneflow.sbp.split(axis=0), oneflow.sbp.split(axis=0))
state.origin.sbp, (oneflow.sbp.split(dim=0), oneflow.sbp.split(dim=0))
)

# In evaluation graph, paramters's sbp are flow.sbp.split(0).
Expand Down