-
Notifications
You must be signed in to change notification settings - Fork 0
/
two_layer_perceptron.py
131 lines (111 loc) · 4.61 KB
/
two_layer_perceptron.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
from forward_mode import *
from random import uniform
def vplus(u, v): return [u[i]+v[i] for i in range(len(u))]
def ktimesv(k, u): return [k*u[i] for i in range(len(u))]
def dot(u, v):
sum = 0
for i in range(len(u)): sum += u[i]*v[i]
return sum
def mtimesv(m, v): return [dot(m[i], v) for i in range(len(m))]
def vminus(u, v): return vplus(u, ktimesv(-1, v))
def distance(u, v): return dot(vminus(u, v), vminus(u, v))
def naive_gradient_descent(f, x0, learning_rate, n):
x = x0
for i in range(n): x = vminus(x, ktimesv(learning_rate, gradient(f)(x)))
return x
def fc_layer(point, weights, biases):
return vplus(mtimesv(weights, point), biases)
def sigmoid(x): return 1/(exp(-x)+1)
def sigmoid_layer(point): return [sigmoid(point[i]) for i in range(len(point))]
def two_layer_perceptron(point, weights1, biases1, weights2, biases2):
hidden = sigmoid_layer(fc_layer(point, weights1, biases1))
return fc_layer(hidden, weights2, biases2)
def cost(points, labels, weights1, biases1, weights2, biases2):
cost = 0
for i in range(len(points)):
cost += distance(two_layer_perceptron(points[i],
weights1,
biases1,
weights2,
biases2),
[labels[i]])
return cost
def pack(weights1, biases1, weights2, biases2):
parameters = []
for bias in biases1: parameters.append(bias)
for row in weights1:
for weight in row: parameters.append(weight)
for bias in biases2: parameters.append(bias)
for row in weights2:
for weight in row: parameters.append(weight)
return parameters
def unpack(parameters, number_of_inputs, number_of_hidden):
k = 0
biases1 = []
for j in range(number_of_hidden):
biases1.append(parameters[k])
k = k+1
weights1 = []
for j in range(number_of_hidden):
row = []
for i in range(number_of_inputs):
row.append(parameters[k])
k = k+1
weights1.append(row)
biases2 = []
biases2.append(parameters[k])
k = k+1
row = []
for i in range(number_of_hidden):
row.append(parameters[k])
k = k+1
weights2 = [row]
return weights1, biases1, weights2, biases2
def initialize(points, labels, number_of_hidden):
number_of_inputs = len(points[0])
weights1 = [[uniform(-1, 1) for i in range(number_of_inputs)]
for j in range(number_of_hidden)]
biases1 = [uniform(-1, 1) for j in range(number_of_hidden)]
weights2 = [[uniform(-1, 1) for j in range(number_of_hidden)]]
biases2 = [uniform(-1, 1)]
return weights1, biases1, weights2, biases2
def step(points, labels, weights1, biases1, weights2, biases2):
number_of_inputs = len(points[0])
number_of_hidden = len(biases1)
def loss(parameters):
weights1, biases1, weights2, biases2 = unpack(
parameters, number_of_inputs, number_of_hidden)
return cost(points, labels, weights1, biases1, weights2, biases2)
parameters = pack(weights1, biases1, weights2, biases2)
parameters = vminus(parameters, ktimesv(0.01, gradient(loss)(parameters)))
weights1, biases1, weights2, biases2 = unpack(
parameters, number_of_inputs, number_of_hidden)
return weights1, biases1, weights2, biases2
def train(points, labels, number_of_hidden):
number_of_inputs = len(points[0])
def loss(parameters):
weights1, biases1, weights2, biases2 = unpack(
parameters, number_of_inputs, number_of_hidden)
return cost(points, labels, weights1, biases1, weights2, biases2)
weights1 = [[uniform(-1, 1) for i in range(number_of_inputs)]
for j in range(number_of_hidden)]
biases1 = [uniform(-1, 1) for j in range(number_of_hidden)]
weights2 = [[uniform(-1, 1) for j in range(number_of_hidden)]]
biases2 = [uniform(-1, 1)]
parameters = pack(weights1, biases1, weights2, biases2)
parameters = naive_gradient_descent(loss, parameters, 0.02, 5000)
#parameters = naive_gradient_descent(loss, parameters, 0.1, 10000)
weights1, biases1, weights2, biases2 = unpack(
parameters, number_of_inputs, number_of_hidden)
return weights1, biases1, weights2, biases2
def classify(point, weights1, biases1, weights2, biases2):
if two_layer_perceptron(point, weights1, biases1, weights2, biases2)[0]<0:
return -1
else: return +1
def all_labels(labels):
red = False
blue = False
for label in labels:
if label<0: red = True
else: blue = True
return red and blue