-
Notifications
You must be signed in to change notification settings - Fork 0
/
regularizers.py
130 lines (100 loc) · 2.44 KB
/
regularizers.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
import numpy as np
class Regularizer():
""" Regularizer parent class.
Attributes
----------
reg_rate : float
Regularization rate.
Methods
-------
__init__(reg_rate)
Constuctor.
"""
def __init__(self, reg_rate):
""" Constructor.
Parameters
----------
reg_rate : float
Regularization rate.
Notes
-----
None
"""
self.reg_rate = reg_rate
class L2Regularizer(Regularizer):
""" L2 regularizer.
Attributes
----------
reg_rate : float
Regularization rate.
Methods
-------
__init__(reg_rate)
Constuctor.
loss(param)
Computes the regulariztion loss of the parameters such as
layer loss in dense due to large weights.
grad(param)
Computes the gradient of the regularization term with respect
to param.
__repr__()
Returns the string representation of class.
"""
def __init__(self, reg_rate):
""" Constructor.
Parameters
----------
reg_rate : float
Regularization rate.
Notes
-----
None
"""
super().__init__(reg_rate)
def loss(self, param):
""" Computes the regulariztion loss of the parameters such as
layer loss in dense due to large weights.
Parameters
----------
param : numpy.ndarray
The parameter tensor.
Returns
-------
float
Regularization loss.
Notes
-----
None
"""
return 0.5 * self.reg_rate * np.sum(np.power(param, 2))
def grad(self, param):
""" Computes the gradient of the regularization term with respect
to param.
Parameters
----------
param : numpy.ndarray
The parameter tensor.
Returns
-------
numpy.ndarray
Regularization gradient with respect to param.
Notes
-----
None
"""
return self.reg_rate * param
def __repr__(self, ):
""" Returns the string representation of class.
Parameters
----------
None
Returns
-------
repr_str : str
The string representation of the class.
Notes
-----
None
"""
repr_str = f"l2 with {self.reg_rate:.4e}"
return repr_str