Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

【PIR API adaptor No.238、239、240、241】 Migrate nn.initializer.XavierInitializer, nn.initializer.MSRAInitializer into pir #59419

Merged
merged 6 commits into from
Dec 4, 2023
Merged

【PIR API adaptor No.238、239、240、241】 Migrate nn.initializer.XavierInitializer, nn.initializer.MSRAInitializer into pir #59419

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3a4193b
add xavier and it's testcases
MarioLulab Nov 27, 2023
61651ba
add kaiming pir
MarioLulab Nov 27, 2023
c75e4f2
fix
MarioLulab Nov 27, 2023
69269cf
add testcases of kaiming
MarioLulab Nov 27, 2023
c7fae65
adapt to input dtype = fp16 and is in gaussian
MarioLulab Dec 1, 2023
a0fdfef
fix for ci-coverage
MarioLulab Dec 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 43 additions & 4 deletions python/paddle/nn/initializer/kaiming.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,15 @@
# TODO: define the initializers of Kaiming functions in neural network
import math

import paddle
from paddle import _C_ops

from ...base import core, framework, unique_name
from ...base.framework import _current_expected_place, in_dygraph_mode
from ...base.framework import (
_current_expected_place,
in_dygraph_mode,
in_pir_mode,
)
from .initializer import Initializer, calculate_gain

__all__ = []
Expand Down Expand Up @@ -87,9 +92,10 @@ def forward(self, var, block=None):
The initialization op.
"""
block = self._check_block(block)

assert isinstance(var, framework.Variable)
assert isinstance(block, framework.Block)
assert isinstance(
var, (framework.Variable, paddle.pir.core.ParameterMeta)
)
assert isinstance(block, (framework.Block, paddle.pir.Block))
f_in, f_out = self._compute_fans(var)

# If fan_in is passed, use it
Expand All @@ -112,6 +118,12 @@ def forward(self, var, block=None):
type=core.VarDesc.VarType.LOD_TENSOR,
persistable=False,
)
elif (
var.dtype in (core.DataType.FLOAT16, core.DataType.BFLOAT16)
and not self._uniform
):
out_dtype = core.DataType.FLOAT32
out_var = var
else:
out_dtype = var.dtype
out_var = var
Expand Down Expand Up @@ -144,6 +156,33 @@ def forward(self, var, block=None):
else:
out_var._share_underline_tensor_to(var)
return None
elif in_pir_mode():
MarioLulab marked this conversation as resolved.
Show resolved Hide resolved
if self._uniform:
gain = calculate_gain(self._nonlinearity, self._negative_slope)
limit = gain * math.sqrt(3.0 / float(fan_in))
out_var = _C_ops.uniform(
var.shape,
out_dtype,
-limit,
limit,
self._seed,
_current_expected_place(),
)
else:
gain = calculate_gain(self._nonlinearity, self._negative_slope)
std = gain / math.sqrt(float(fan_in))
place = _current_expected_place()
out_var = _C_ops.gaussian(
out_var.shape, 0.0, std, self._seed, out_dtype, place
)

if (
var.dtype in (core.DataType.FLOAT16, core.DataType.BFLOAT16)
and not self._uniform
):
return _C_ops.cast(out_var, var.dtype)

return out_var
0x45f marked this conversation as resolved.
Show resolved Hide resolved
else:
if self._uniform:
gain = calculate_gain(self._nonlinearity, self._negative_slope)
Expand Down
30 changes: 27 additions & 3 deletions python/paddle/nn/initializer/xavier.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,12 @@ def forward(self, var, block=None):
type=core.VarDesc.VarType.LOD_TENSOR,
persistable=False,
)
elif (
var.dtype in (core.DataType.FLOAT16, core.DataType.BFLOAT16)
and not self._uniform
):
out_dtype = core.DataType.FLOAT32
out_var = var
else:
out_dtype = var.dtype
out_var = var
Expand Down Expand Up @@ -153,14 +159,32 @@ def forward(self, var, block=None):
elif in_pir_mode():
if self._uniform:
limit = math.sqrt(6.0 / float(fan_in + fan_out))
return paddle._pir_ops.uniform(
var.shape,
var.dtype,
out_var = paddle._pir_ops.uniform(
out_var.shape,
out_dtype,
-limit,
limit,
self._seed,
_current_expected_place(),
)
else:
std = math.sqrt(2.0 / float(fan_in + fan_out))
out_var = _C_ops.gaussian(
out_var.shape,
0.0,
std,
self._seed,
out_dtype,
_current_expected_place(),
)

if (
var.dtype in (core.DataType.FLOAT16, core.DataType.BFLOAT16)
and not self._uniform
):
return _C_ops.cast(out_var, var.dtype)

return out_var
else:
if self._uniform:
limit = math.sqrt(6.0 / float(fan_in + fan_out))
Expand Down
Loading