Machine Learning Baby

Author: Dan-Andrei-Iliescu

3layer_nn.py

@@ -0,0 +1,50 @@
+import numpy as np
+
+# sigmoid function
+def nonlin(x, deriv = False):
+	if(deriv == True):
+		return x * (1 - x)
+	return 1/(1 + np.exp(-x))
+	
+# input dataset
+X = np.array([[0, 0, 1],
+				[0, 1, 1],
+				[1, 0, 1],
+				[1, 1, 1]])
+				
+# output dataset
+y = np.array([[0], [1], [1], [0]])
+
+np.random.seed(1)
+
+# initialise weights randomly
+syn0 = 2 * np.random.random((3, 4)) - 1
+syn1 = 2 * np.random.random((4, 1)) - 1
+
+for j in xrange(60000):
+
+	# forward propagation
+	l0 = X
+	l1 = nonlin(np.dot(l0, syn0))
+	l2 = nonlin(np.dot(l1, syn1))
+	
+	# error
+	l2_error = y - l2
+	
+	if(j % 10000) == 0:
+		print "Error:" + str(np.mean(np.abs(l2_error)))
+	
+	# multiply how much we missed by the
+	# slope of the sigmoid at the values in l1
+	l2_delta = l2_error * nonlin(l2, deriv=True)
+	
+	l1_error = l2_delta.dot(syn1.T)
+	
+	l1_delta = l1_error * nonlin(l1, deriv = True)
+	
+	# update weights
+	syn1 += l1.T.dot(l2_delta)
+	syn0 += l0.T.dot(l1_delta)
+
+print "Output After Training:"
+print l2

basic_nn.py

@@ -0,0 +1,42 @@
+import numpy as math
+
+# sigmoid function
+def nonlin(x, deriv = False):
+	if(deriv == True):
+		return x * (1 - x)
+	return 1/(1 + math.exp(-x))
+	
+# input dataset
+X = math.array([[0, 0, 1],
+				[0, 1, 1],
+				[1, 0, 1],
+				[1, 1, 1]])
+				
+# output dataset
+y = math.array([[0, 0, 1, 1]]).T
+
+def derivative(x, y):
+
+	m, n = np.shape(x)
+	#numIterations= 1000
+	#alpha = 0.005
+	syn0 = np.ones([n, 1])
+
+	for iter in xrange(10000):
+
+		# forward propagation
+		l0 = X
+		l1 = nonlin(math.dot(l0, syn0))
+	
+		# error
+		l1_error = y - l1
+	
+		# multiply how much we missed by the
+		# slope of the sigmoid at the values in l1
+		l1_delta = l1_error * nonlin(l1, True)
+	
+		# update weights
+		syn0 += math.dot(l0.T, l1_delta)
+
+	print "Output After Training:"
+	print l1

deriv_nn.py

@@ -0,0 +1,40 @@
+import numpy as np
+
+# sigmoid function
+def nonlin(x, deriv = False):
+	if(deriv == True):
+		return x * (1 - x)
+	return 1/(1 + np.exp(-x))
+
+def derivative(x, y):
+
+	m, n = np.shape(x)
+	#numIterations= 1000
+	#alpha = 0.005
+	np.random.seed(1)
+
+	# initialise weights randomly
+	syn0 = 2*np.random.random((n, 1)) - 1
+
+	for iter in xrange(100):
+
+		# forward propagation
+		l0 = x
+		l1 = nonlin(np.dot(l0, syn0))
+		print l1
+		print x
+		print y
+		# error
+		l1_error = y - l1
+	
+		# multiply how much we missed by the
+		# slope of the sigmoid at the values in l1
+		l1_delta = l1_error * nonlin(l1, True)
+	
+		# update weights
+		syn0 += np.dot(l0.T, l1_delta)
+	
+	return syn0[0];
+	
+	print "Output After Training:"
+	print l1

final_rnn.py

@@ -0,0 +1,127 @@
+import copy, numpy as np
+np.random.seed(0)
+
+# compute sigmoid nonlinearity
+def sigmoid(x):
+	output = 1 / (1 + np.exp(-x))
+	return output
+
+# convert output of sigmoid function to its derivate#
+def sigmoid_output_to_derivative(output):
+	return output * (1 - output)
+	
+
+# training dataset generation
+int2binary = {}
+binary_dim = 8
+
+largest_number = pow(2, binary_dim)
+binary = np.unpackbits(
+	np.array([range(largest_number)], dtype = np.uint8).T, axis = 1)
+for i in range(largest_number):
+	int2binary[i] = binary[i]
+	
+# input variables
+alpha = 0.1
+input_dim = 1
+hidden_dim = 32
+output_dim = 1
+
+# initialise neural network weights
+synapse_0 = 2 * np.random.random((input_dim, hidden_dim)) - 1
+synapse_1 = 2 * np.random.random((hidden_dim, output_dim)) - 1
+synapse_h = 2 * np.random.random((hidden_dim, hidden_dim)) - 1
+
+synapse_0_update = np.zeros_like(synapse_0)
+synapse_1_update = np.zeros_like(synapse_1)
+synapse_h_update = np.zeros_like(synapse_h)
+
+# training logic
+for j in range(60000):
+
+	# generate a simple addition problem (a + b = c)
+	# a_int = np.random.randint(largest_number / 2) # int version
+	# a = int2binary[a_int] # binary encoding
+	
+	# b_int = np.random.randint(largest_number / 2) # int version
+	# b = int2binary[b_int] # binary encoding
+	
+	# true answer
+	# c_int = 2 * a_int
+	# c = int2binary[c_int]
+	
+	a = np.array([0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7])
+	c = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1, 0.35])
+	
+	# where we'll store our best guess in binary
+	d = np.zeros_like(c)
+	
+	overallError = 0
+	
+	layer_2_deltas = list()
+	layer_1_values = list()
+	layer_1_values.append(np.zeros(hidden_dim))
+	
+	# moving along the positions in the binary encoding
+	for position in range(binary_dim):
+	
+		# generate IO
+		X = np.array([[a[binary_dim - position - 1]]])
+		y = np.array([[c[binary_dim - position - 1]]]).T
+		
+		# hidden layer (input ~+ prev_hidden)
+		layer_1 = sigmoid(np.dot(X, synapse_0) + np.dot(layer_1_values[-1], synapse_h))
+		
+		#  output layer (new binary representation)
+		layer_2 = sigmoid(np.dot(layer_1, synapse_1))
+		
+		# error
+		layer_2_error = y - layer_2
+		layer_2_deltas.append((layer_2_error) * sigmoid_output_to_derivative(layer_2))
+		overallError += np.abs(layer_2_error[0])
+		
+		# decode estimate so we can print it out
+		d[binary_dim - position - 1] = layer_2[0][0]
+		
+		# store hidden layer so we use it the next time
+		layer_1_values.append(copy.deepcopy(layer_1))
+		
+	future_layer_1_delta = np.zeros(hidden_dim)
+	
+	for position in range(binary_dim):
+	
+		X = np.array([[a[position]]])
+		layer_1 = layer_1_values[-position-1]
+		prev_layer_1 = layer_1_values[-position-2]
+		
+		# error at output layer
+		layer_2_delta = layer_2_deltas[-position-1]
+		# error at hidden layer
+		layer_1_delta = (future_layer_1_delta.dot(synapse_h.T) + layer_2_delta.dot(synapse_1.T)) * sigmoid_output_to_derivative(layer_1)
+		
+		# let's update all our weights so we can try again
+		synapse_1_update += np.atleast_2d(layer_1).T.dot(layer_2_delta)
+		synapse_h_update += np.atleast_2d(prev_layer_1).T.dot(layer_1_delta)
+		synapse_0_update += X.T.dot(layer_1_delta)
+		
+		future_layer_layer_1_delta = layer_1_delta
+		
+		
+	synapse_0 += synapse_0_update * alpha
+	synapse_1 += synapse_1_update * alpha
+	synapse_h += synapse_h_update * alpha
+	
+	synapse_0_update *= 0
+	synapse_1_update *= 0
+	synapse_h_update *= 0
+	
+	# print out progress
+	if(j % 1000 == 0):
+		print "Error:" + str(overallError)
+		print "Pred:" + str(d)
+		print "True:" + str(c)
+		out = 0
+		for index, x in enumerate(reversed(d)):
+			out += x * pow(2, index)
+		print str(a) + " -> " + str(d[0])
+		print "------------"

lin_reg.py

@@ -0,0 +1,56 @@
+import numpy as np
+import random
+
+# sigmoid function
+def nonlin(x):
+	return 1/(1 + np.exp(-(x/3)))
+
+# m denotes the number of examples here, not the number of features
+def gradientDescent(x, y, theta, alpha, m, numIterations):
+    xTrans = x.transpose()
+    for i in range(0, numIterations):
+        hypothesis = np.dot(x, theta)
+        #print "Theta:" + str(theta)
+        #print "X:" + str(np.shape(x))
+        #print "Hypo:" + str(np.shape(hypothesis))
+        #print "Y:" + str(np.shape(y))
+        loss = hypothesis - y
+        #print "Loss:" + str(loss)
+        # avg cost per example (the 2 in 2*m doesn't really matter here.
+        # But to be consistent with the gradient, I include it)
+        cost = np.sum(loss ** 2) / (2 * m)
+        #print("Iteration %d | Cost: %f" % (i, cost))
+        # avg gradient per example
+        gradient = np.dot(xTrans, loss) / m
+        #print gradient
+        # update
+        theta = theta - alpha * gradient
+        #print theta
+    return theta
+    
+
+def derivative(x, y):
+
+	m, n = np.shape(x)
+	numIterations= 1000
+	alpha = 0.005
+	theta = np.ones([n, 1])
+	
+	#print theta
+	theta = gradientDescent(x, y, theta, alpha, m, numIterations)
+	#print "Theta:" + str(theta)
+	#print "X:" + str(x)
+	#print "Y:" + str(y)
+	theta = nonlin(theta)
+	#print "Theta:" + str(theta)
+	return theta * np.ones_like(y)
+	#print m
+	#print n
+
+	
+# gen data
+x = np.array([[0],
+				[1],
+				[2],
+				[3]])
+y = np.array([[0, 2, 4, 8]]).T