-
Notifications
You must be signed in to change notification settings - Fork 0
/
linear.py
62 lines (49 loc) · 1.49 KB
/
linear.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
from toydl.module import Module
from toydl.tensor import NumpyTensor
class Linear(Module):
"""Fully-connected layer.
"""
def __init__(self, in_features, out_features, bias=True,
is_input_layer=False):
self.in_features = in_features
self.out_features = out_features
self.bias = bias
self.is_input_layer = is_input_layer
self.W = NumpyTensor(shape=(out_features, in_features))
self.b = NumpyTensor(shape=(out_features, ), init_value=0.01) if bias else None
def parameters(self):
res = [self.W]
if self.b:
res.append(self.b)
return res
def forward(self, x):
assert len(x.shape) == 2
assert x.shape[1] == self.in_features
self._X = x
x = self.W.matmul(x.transpose()) # (o, i) (i, b) -> (o, b)
x = x.transpose() # (b, o)
if self.bias:
return x + self.b.reshape((1, -1))
else:
return x
def backward(self, grad):
# grad : (b, o)
self.W.grad = grad.transpose().matmul(self._X).data # (o, b) (b, i)
if self.b:
self.b.grad = grad.data.sum(axis=0)
if self.is_input_layer:
return
# (i, o) (o, b) -> (i, b) -> (b, i)
input_grad = self.W.transpose().matmul(grad.transpose()).transpose()
return input_grad
class Add(Module):
def forward(self, a, b):
return a + b
def backward(self, grad):
return grad, grad
class ReLU(Module):
def forward(self, x):
self.m = (x > 0).astype(x.type)
return self.m * x
def backward(self, grad):
return grad * self.m