-
Notifications
You must be signed in to change notification settings - Fork 0
/
Tensors.py
74 lines (56 loc) · 2.34 KB
/
Tensors.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
### Defining a model using subclassing and specifying custom behavior ###
from tensorflow.keras import Model
from tensorflow.keras.layers import Dense
class IdentityModel(tf.keras.Model):
# In __init__ we define the Model's layers
def __init__(self, n_output_nodes):
super(IdentityModel, self).__init__()
self.dense_layer = tf.keras.layers.Dense(n_output_nodes, activation='sigmoid')
def call(self, inputs, isidentity=False):
x = self.dense_layer(inputs)
if isidentity:
return inputs
return x
n_output_nodes = 3
model = IdentityModel(n_output_nodes)
x_input = tf.constant([[1,2.]], shape=(1,2))
out_activate = model.call(x_input)
out_identity = model.call(x_input, isidentity=True)
print("Network output with activation: {}; network identity output: {}".format(out_activate.numpy(), out_identity.numpy()))
### Gradient computation with GradientTape ###
# y = x^2
# Example: x = 3.0
x = tf.Variable(3.0)
# Initiate the gradient tape
with tf.GradientTape() as tape:
# Define the function
y = x * x
# Access the gradient -- derivative of y with respect to x
dy_dx = tape.gradient(y, x)
### Function minimization with automatic differentiation and SGD ###
# Initialize a random value for our initial x
x = tf.Variable([tf.random.normal([1])])
print("Initializing x={}".format(x.numpy()))
learning_rate = 1e-2 # learning rate for SGD
history = []
# Define the target value
x_f = 4
# We will run SGD for a number of iterations. At each iteration, we compute the loss,
# compute the derivative of the loss with respect to x, and perform the SGD update.
for i in range(500):
with tf.GradientTape() as tape:
loss = (x - x_f)**2 # "forward pass": record the current loss on the tape
# loss minimization using gradient tape
grad = tape.gradient(loss, x) # compute the derivative of the loss with respect to x
new_x = x - learning_rate*grad # sgd update
x.assign(new_x) # update the value of x
history.append(x.numpy()[0])
# Plot the evolution of x as we optimize towards x_f!
plt.plot(history)
plt.plot([0, 500],[x_f,x_f])
plt.legend(('Predicted', 'True'))
plt.xlabel('Iteration')
plt.ylabel('x value')