How to implement a simple gradient descent with TensorFlow ?

Examples of how to implement a simple gradient descent with TensorFlow

Algorithm gradient descent with TensorFlow (1D example)

import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np

from pylab import figure, cm

x = np.arange(-10,10,0.2)
y = x**2

fig = figure(num=None, figsize=(12, 10), dpi=80, facecolor='w', edgecolor='k')

plt.plot(x,y)

plt.title('Gradient Descent with TensorFlow (1D)')
plt.xlabel('x')
plt.ylabel('y')

plt.savefig("gradient_descent_tensorflow_01.png", bbox_inches='tight')

plt.show()


def loss_function(x): 
    return x ** 2.0

def loss_function_minimzie():
    return x ** 2.0

def reset():
    x = tf.Variable(10.0) 
    return x

x = reset()
opt = tf.keras.optimizers.SGD(learning_rate=0.1)
for i in range(50):
    print ('y = {:.1f}, x = {:.1f}'.format(loss_function(x).numpy(), x.numpy()))
    opt.minimize(loss_function_minimzie, var_list=[x])

returns

y = 100.0, x = 10.0
y = 64.0, x = 8.0
y = 41.0, x = 6.4
y = 26.2, x = 5.1
y = 16.8, x = 4.1
y = 10.7, x = 3.3
y = 6.9, x = 2.6
y = 4.4, x = 2.1
y = 2.8, x = 1.7
y = 1.8, x = 1.3
y = 1.2, x = 1.1
y = 0.7, x = 0.9
y = 0.5, x = 0.7
y = 0.3, x = 0.5
y = 0.2, x = 0.4
y = 0.1, x = 0.4
y = 0.1, x = 0.3
y = 0.1, x = 0.2
y = 0.0, x = 0.2
y = 0.0, x = 0.1
y = 0.0, x = 0.1
y = 0.0, x = 0.1
y = 0.0, x = 0.1
y = 0.0, x = 0.1
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0
y = 0.0, x = 0.0

Algorithm gradient descent with TensorFlow (2D example)

def loss_function(x1,x2):
    return x1 ** 2.0 - x1 * 3  + x2 ** 2

x1_min = 0.0
x1_max = 4.0
x2_min = -1.0
x2_max = 1.0

x1, x2 = np.meshgrid(np.arange(x1_min,x1_max, 0.1), np.arange(x2_min,x2_max, 0.1))

y = f(x1,x2)

fig = figure(num=None, figsize=(12, 10), dpi=80, facecolor='w', edgecolor='k')

plt.imshow(y,extent=[x1_min,x1_max,x2_min,x2_max], cmap=cm.jet)

plt.title('Gradient Descent with TensorFlow (2D)')
plt.xlabel('x')
plt.ylabel('y')

plt.savefig("gradient_descent_tensorflow_02.png", bbox_inches='tight')

plt.show()

def loss_function_minimzie():
    return x1 ** 2.0 - x1 * 3  + x2 ** 2

def reset():
    x1 = tf.Variable(10.0) 
    x2 = tf.Variable(10.0) 
    return x1, x2

x1, x2 = reset()
opt = tf.keras.optimizers.SGD(learning_rate=0.1)
for i in range(50):
    print ('y = {:.1f}, x1 = {:.1f}, x2 = {:.1f}'.format(loss_function(x1, x2).numpy(), x1.numpy(), x2.numpy()))
    opt.minimize(loss_function_minimzie, var_list=[x1, x2])

returns

y = 170.0, x1 = 10.0, x2 = 10.0
y = 108.0, x1 = 8.3, x2 = 8.0
y = 68.3, x1 = 6.9, x2 = 6.4
y = 42.9, x1 = 5.9, x2 = 5.1
y = 26.6, x1 = 5.0, x2 = 4.1
y = 16.2, x1 = 4.3, x2 = 3.3
y = 9.6, x1 = 3.7, x2 = 2.6
y = 5.3, x1 = 3.3, x2 = 2.1
y = 2.6, x1 = 2.9, x2 = 1.7
y = 0.9, x1 = 2.6, x2 = 1.3
y = -0.3, x1 = 2.4, x2 = 1.1
y = -1.0, x1 = 2.2, x2 = 0.9
y = -1.4, x1 = 2.1, x2 = 0.7
y = -1.7, x1 = 2.0, x2 = 0.5
y = -1.9, x1 = 1.9, x2 = 0.4
y = -2.0, x1 = 1.8, x2 = 0.4
y = -2.1, x1 = 1.7, x2 = 0.3
y = -2.2, x1 = 1.7, x2 = 0.2
y = -2.2, x1 = 1.7, x2 = 0.2
y = -2.2, x1 = 1.6, x2 = 0.1
y = -2.2, x1 = 1.6, x2 = 0.1
y = -2.2, x1 = 1.6, x2 = 0.1
y = -2.2, x1 = 1.6, x2 = 0.1
y = -2.2, x1 = 1.6, x2 = 0.1
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0
y = -2.2, x1 = 1.5, x2 = 0.0

References

• Regression using tensorflow and gradient descent optimizer
• 3 different ways to Perform Gradient Descent in Tensorflow 2.0 and MS Excel
• tf.keras.optimizers.Optimizer
• Implement Gradient Descent in Python
• Gradient Descent in Python