Simple SGD Example

[JackYumCha]

import tensorflow as tf
import numpy as np

def build_graph(rate):
    x = tf.Variable(2,dtype=tf.float32)
    # y = x ** 2 + tf.exp(x / (x ** 2 + 1))
    y = tf.exp(x / (x ** 2 + 1) + x) - x ** 3
    dydx = tf.gradients(y,x)[0]
    update = x.assign(x - rate * dydx)
    return update, x, y, dydx

with tf.device('/cpu:0'):
    sess = tf.InteractiveSession()
    update, x, y, dydx = build_graph(0.1)
    tf.global_variables_initializer().run()
    # sess.run(tf.global_variables_initializer())
    for i in range(0,100):
        sess.run([update])
        #print('round:', i, 'x:', sess.run([x]), 'y:', sess.run([y]))
        print(i)
    print('x:', sess.run([x]), 'y:', sess.run([y]))

[JackYumCha]

import tensorflow as tf
import numpy as np

#np.random.rand(1)[0] * 10 - 5
def build_graph(rate, epsilon, v):
    x = tf.Variable(20,dtype=tf.float32)
    # y = x ** 2 + tf.exp(x / (x ** 2 + 1))
    y = tf.exp(x / (x ** 2 + 1) + x) - x ** 3
    dydx = tf.clip_by_norm(tf.gradients(y,x),1)[0]
    reg = tf.Variable(0,dtype=tf.float32)
    updateReg = reg.assign(v * reg + dydx ** 2)
    with tf.control_dependencies([updateReg]):
        updateX = x.assign(x - rate / tf.sqrt(reg + epsilon) * dydx)
    return updateX, x, y, dydx

with tf.device('/cpu:0'):
    sess = tf.InteractiveSession()
    updateX, x, y, dydx = build_graph(0.3, 1, 0.9) # 0.9 can make it converge
    tf.global_variables_initializer().run()
    # sess.run(tf.global_variables_initializer())
    for i in range(0,500):
        sess.run([updateX])
        print('round:', i, 'x:', sess.run([x]), 'y:', sess.run([y]))
        #print(i)
    print('x:', sess.run([x]), 'y:', sess.run([y]))

[JackYumCha]

import tensorflow as tf
import numpy as np 

# x (-inf,-2]:0, (-2,2]:1, (2,+inf):2

#x = np.array([np.arange(start = -6, stop=6, step=0.2)]).T

x = np.arange(start = -6, stop=6, step=0.2)

def classify(input):
    if input <= -2:
        return 0
    elif input >2:
        return 2
    else:
        return 1

y = np.vectorize(classify)(x)

x = np.array([x]).T
y = np.array([y]).T

print(x)

print(y)

class layer:
    def __init__(self, inputs, variables, outputs, previousLayer=None):
        self.outputs = outputs
        self.inputs = inputs
        self.variables = variables
        self.grad_inputs = tf.gradients(outputs, inputs)
        self.previousLayer = previousLayer
        if variables != None:
            self.grad_varaibles = tf.gradients(outputs, variables)
        else:
            self.grad_varaibles = None

    def back_propagation_update(self, optimizer, next_layer_grad = None):
        updates=[]
        if self.grad_varaibles != None:
            varaiables_grads = self.grad_varaibles
            inputs_grads = self.grad_inputs
            if next_layer_grad != None:
                varaiables_grads = tf.matmul(self.grad_varaibles, next_layer_grad)
                inputs_grads = tf.matmul(self.grad_inputs, next_layer_grad)
            update = optimizer.update(self.variables, varaiables_grads)
            updates.append(update)
            if self.previousLayer != None:
                with tf.control_dependencies([update]):
                    updates.extend(self.previousLayer.back_propagation_update(optimizer, inputs_grads))
        else:
            inputs_grads = self.grad_inputs
            if self.previousLayer != None:
                if next_layer_grad != None:
                    inputs_grads = tf.matmul(self.grad_inputs, next_layer_grad)
                    updates.extend(self.previousLayer.back_propagation_update(optimizer,inputs_grads))
        return updates

class sgd_optimizer:
    def __init__(self, learning_rate):
        self.learning_rate = learning_rate

    def update(self, variables, gradients):
        return variables.assign(variables - self.learning_rate * gradients) 

def offset_layer(x):
    b = tf.Variable([0,0],dtype=tf.float32)
    y = x + b
    return b,y

def linear_layer(x):
    k = tf.Variable(4,dtype=tf.float32)
    y = k * x
    return k, y

def logistic_regression_layer(x):
    y = tf.reduce_sum(1 / (1 + tf.exp(-x)), axis=1)
    return None, y

def loss(y, y_data):
    y = tf.reduce_sum((y - y_data) ** 2)
    #tf.nn.l2_loss(y-y_data)
    return None, y

def build_graph():
    x_ = tf.placeholder(tf.float32, shape=[None,1])
    y_ = tf.placeholder(tf.float32, shape=[None,1])

    var1,y1 = linear_layer(x_)
    layer1 = layer(x_,var1,y1)

    var2,y2 = linear_layer(layer1.outputs)
    layer2 = layer(layer1.outputs,var2,y2,layer1)

    var3,y3 = logistic_regression_layer(layer2.outputs)
    layer3 = layer(layer2.outputs,var3,y3,layer2)

    var4,y4 = loss(layer3.outputs,y_)
    layer4 = layer([layer3.outputs, y_],var4,y4,layer3)

    sgd = sgd_optimizer(0.01)

    update = layer4.back_propagation_update(sgd)

    return x_, y_, y3, y4, update

with tf.device('/cpu:0'):
    sess = tf.InteractiveSession()
    x_, y_, y2, y3, update = build_graph()
    sess.run(tf.global_variables_initializer())

    for i in range(0,100):
        sess.run(update, {x_: x, y_: y})
        print(sess.run([y2,y3]))

[JackYumCha]

import tensorflow as tf
import numpy as np

# we try to set a logic function, if x > 2, return 1, x <= 2 and x > 0, return 0, if x <= 0, return -1

# we have data

x_data = np.array([np.arange(-10, 12, 0.1)]).T

def target_function(value):
    if value > 2 :
        return np.array([0,0,1])
    elif value <= 0 :
        return np.array([1,0,0])
    else:
        return np.array([0,1,0])

y_data = np.apply_along_axis(target_function, 1, x_data)

print(y_data)

x = tf.placeholder(tf.float32, [None, 1])
y_ = tf.placeholder(tf.float32, [None, 3])
W = tf.Variable(tf.zeros([1,3]))
b = tf.Variable(tf.zeros([3]))
y = tf.nn.softmax(tf.matmul(x, W) + b)

cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1]))

train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)

sess = tf.InteractiveSession()

tf.global_variables_initializer().run()

for _ in range(100):

    # feeding placeholder x and target y_ to the training
    sess.run(train_step, feed_dict={x: x_data, y_: y_data})
    print(sess.run(W))
    print(sess.run(b))
    print(_)

print('y:', sess.run(tf.transpose([tf.argmax(y, 1),tf.argmax(y_, 1)]), feed_dict={x: x_data, y_: y_data}) )

correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))

accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

print(sess.run(accuracy, feed_dict={x: x_data, y_: y_data}))