diff --git a/MNIST30epoch b/MNIST30epoch new file mode 100644 index 0000000..8992447 Binary files /dev/null and b/MNIST30epoch differ diff --git a/MNISTDrawingPrediction.py b/MNISTDrawingPrediction.py new file mode 100644 index 0000000..e5659b8 --- /dev/null +++ b/MNISTDrawingPrediction.py @@ -0,0 +1,41 @@ +import tkinter +from PIL import Image, ImageDraw +from sobek.network import network +import numpy as np + +class Sketchpad(tkinter.Canvas): + def __init__(self, parent, predictionLabel, **kwargs, ): + super().__init__(parent, **kwargs) + self.bind("", self.test) + self.bind("", self.add_line) + self.PILImage = Image.new("F", (560, 560), 100) + self.draw = ImageDraw.Draw(self.PILImage) + self.MNISTNN = network.networkFromFile("MNIST30epoch") + self.predictionLabel = predictionLabel + + def add_line(self, event): + self.create_oval((event.x+32, event.y+32, event.x-32, event.y-32), fill="black") + self.draw.ellipse([event.x-32, event.y-32, event.x+32, event.y+32], fill="black") + smallerImage = self.PILImage.reduce(20) + imageAsArray = np.array(smallerImage.getdata()) + imageAsArray = (100 - imageAsArray)/100 + self.predictionLabel['text'] = ( "Predicted number : " + str(np.argmax(self.MNISTNN.process(imageAsArray)))) + + def test(self, event): + self.PILImage = Image.new("F", (560, 560), 100) + self.draw = ImageDraw.Draw(self.PILImage) + self.delete("all") + +window = tkinter.Tk() +window.title("Number guesser") +window.resizable(False, False) +window.columnconfigure(0, weight=1) +window.rowconfigure(0, weight=1) + +predictionLabel = tkinter.Label(window, text="Predicted number :") + +sketch = Sketchpad(window, predictionLabel, width=560, height=560) +sketch.grid(column=0, row=0, sticky=(tkinter.N, tkinter.W, tkinter.E, tkinter.S)) +predictionLabel.grid(column=0, row=1) + +window.mainloop() \ No newline at end of file diff --git a/MNISTLearning.py b/MNISTLearning.py new file mode 100644 index 0000000..6ed386c --- /dev/null +++ b/MNISTLearning.py @@ -0,0 +1,57 @@ +import numpy as np +from sobek.network import network +import gzip +import time + +print("--- Data loading ---") + +def getData(fileName): + with open(fileName, 'rb') as f: + data = f.read() + return np.frombuffer(gzip.decompress(data), dtype=np.uint8).copy() + + +tempTrainImages = getData("./MNIST/train-images-idx3-ubyte.gz")[0x10:].reshape((-1, 784)).tolist() +trainImages = [] +for image in tempTrainImages: + for pixel in range(784): + if image[pixel] !=0: + image[pixel] = image[pixel]/256 + trainImages.append(np.array(image, dtype=np.float64)) +tempTrainLabels = getData("./MNIST/train-labels-idx1-ubyte.gz")[8:] +trainLabels = [] +for label in tempTrainLabels: + trainLabels.append(np.zeros(10)) + trainLabels[-1][label] = 1.0 + +myNetwork = network(784, 30, 10) + +learningRate = 3.0 + +print("--- Learning ---") + +startTime = time.perf_counter() + +""" +for i in range(1): + print("Epoch: " + str(i)) + batchEnd = 10 + while batchEnd < 1000: + batchImages = trainImages[:batchEnd] + batchLabels = trainLabels[:batchEnd] + myNetwork.train(batchImages, batchLabels, learningRate) + batchEnd += 10 + if (batchEnd%100) == 0: + print(batchEnd) +""" + +myNetwork.train(trainImages, trainLabels, learningRate, 10, 30) + +endTime = time.perf_counter() + +print("Learning time : " + str(endTime - startTime)) + +print(trainLabels[121]) +print(myNetwork.process(trainImages[121])) + +myNetwork.saveToFile("MNIST30epoch") \ No newline at end of file diff --git a/MNISTLoadTest.py b/MNISTLoadTest.py new file mode 100644 index 0000000..d929474 --- /dev/null +++ b/MNISTLoadTest.py @@ -0,0 +1,30 @@ +import numpy as np +from sobek.network import network +import gzip + +print("--- Data loading ---") + +def getData(fileName): + with open(fileName, 'rb') as f: + data = f.read() + return np.frombuffer(gzip.decompress(data), dtype=np.uint8).copy() + + +tempTrainImages = getData("./MNIST/t10k-images-idx3-ubyte.gz")[0x10:].reshape((-1, 784)).tolist() +trainImages = [] +for image in tempTrainImages: + for pixel in range(784): + if image[pixel] !=0: + image[pixel] = image[pixel]/256 + trainImages.append(np.array(image, dtype=np.float64)) +tempTrainLabels = getData("./MNIST/t10k-labels-idx1-ubyte.gz")[8:] +trainLabels = [] +for label in tempTrainLabels: + trainLabels.append(np.zeros(10)) + trainLabels[-1][label] = 1.0 + +print("--- Testing ---") + +myNetwork = network.networkFromFile("MNIST30epoch") + +print(myNetwork.accuracy(trainImages, trainLabels)) \ No newline at end of file diff --git a/sobek/network.py b/sobek/network.py index a312fd9..84edaf2 100755 --- a/sobek/network.py +++ b/sobek/network.py @@ -1,4 +1,8 @@ +import random import numpy as np +import matplotlib.pyplot as plt +import matplotlib.animation as animation +import pickle class network: @@ -26,9 +30,9 @@ class network: def process(self, _input, __storeValues=False): if type(_input) != np.ndarray: - raise TypeError("The input must be a vector!") + raise TypeError("The input must be a numpy array!") if _input.size != self.__inputLayerSize: - raise ValueError("The input vector has the wrong size!") + raise ValueError("The input vector has the wrong size! " + str(_input.size) + " != " + str(self.__inputLayerSize)) if _input.dtype != np.float64: print(_input.dtype) raise TypeError("The input vector must contain floats!") @@ -59,58 +63,100 @@ class network: - def train(self, inputs, desiredOutputs, learningRate): + def train(self, inputs, desiredOutputs, learningRate, batchSize, epochs=1, visualize=False): + if (type(inputs) != list or type(desiredOutputs) != list): + raise TypeError("The inputs and desired outputs must be lists of numpy arrays !") if (len(inputs) != len(desiredOutputs)): - raise ValueError("The inputs and desired outputs vectors must have the same amount of data !") + raise ValueError("The inputs and desired outputs lists must have the same amount of data ! " + str(len(inputs)) + " != " + str(len(desiredOutputs))) + if (len(inputs) == 0): + raise ValueError("The list is empty !") + if (visualize == False): + if (self.__inputLayerSize != 2): + raise ValueError("Visualization is only possible for 2 inputs networks") + if (len(self.weights[-1]) != 1): + raise ValueError("Visualization is only possible for 1 output networks") - for _input, desiredOutput in zip(inputs, desiredOutputs): + errorSumsWeights = [] + errorSumsBiases = [] - errorSumsWeights = [np.zeros(layer.shape) for layer in self.weights] - errorSumsBiases = [np.zeros(layer.shape) for layer in self.biases] - self.__errors = [np.zeros(len(layer)) for layer in self.weights] + if (visualize): + vizualisationData = [] + fig, graph = plt.subplots() - #rempli self.activations et self.outputs - self.process(_input, True) - self.__desiredOutput = desiredOutput + for epoch in range(epochs): + randomState = random.getstate() - #Somme de matrice ? - for layerNumber in range(len(errorSumsWeights)-1, -1, -1): - for neuronNumber in range(len(errorSumsWeights[layerNumber])): - errorSumsBiases[layerNumber][neuronNumber] += self.__Error(layerNumber, neuronNumber) - for weightNumber in range(len(errorSumsWeights[layerNumber][neuronNumber])): - #print("layer : " + str(layerNumber) + " neuron : " + str(neuronNumber) + " weight : " + str(weightNumber)) - errorSumsWeights[layerNumber][neuronNumber][weightNumber] += self.__PartialDerivative(layerNumber, neuronNumber, weightNumber) + random.shuffle(inputs) - total = 0 - - - errorSumsWeights = np.multiply(errorSumsWeights, -(learningRate/len(inputs))) - self.weights = np.add(self.weights, errorSumsWeights) + random.setstate(randomState) - errorSumsBiases = np.multiply(errorSumsBiases, -(learningRate/len(inputs))) - self.biases = np.add(self.biases, errorSumsBiases) + random.shuffle(desiredOutputs) - #print(self.__biases) - - """ - for layerNumber in range(len(errorSumsWeights)): - for neuronNumber in range(len(errorSumsWeights[layerNumber])): + if (visualize and epoch%10 == 0): + vizualisationFrame = np.empty((30, 30)) + for x in range(30): + for y in range(30): + vizualisationFrame[x][y] = self.process(np.array([float(x), float(y)])) + vizualisationData.append([graph.imshow(vizualisationFrame, animated=True)]) - errorSumsBiases[layerNumber][neuronNumber] = errorSumsBiases[layerNumber][neuronNumber] / len(inputs) - total += errorSumsBiases[layerNumber][neuronNumber] - self.biases[layerNumber][neuronNumber] -= learningRate * errorSumsBiases[layerNumber][neuronNumber] + inputBatches = [inputs[j:j+batchSize] for j in range(0, len(inputs), batchSize)] + desiredOutputsBatches = [desiredOutputs[j:j+batchSize] for j in range(0, len(inputs), batchSize)] + + for inputBatch, desiredOutputsBatch in zip(inputBatches, desiredOutputsBatches): - for weightNumber in range(len(errorSumsWeights[layerNumber][neuronNumber])): + for _input, desiredOutput in zip(inputBatch, desiredOutputsBatch): - #Probablement faisable avec une multiplication de matrices - errorSumsWeights[layerNumber][neuronNumber][weightNumber] = errorSumsWeights[layerNumber][neuronNumber][weightNumber] / len(inputs) - - total += errorSumsWeights[layerNumber][neuronNumber][weightNumber] + errorSumsWeights = [np.zeros(layer.shape) for layer in self.weights] + errorSumsBiases = [np.zeros(layer.shape) for layer in self.biases] + self.__errors = [np.zeros(len(layer)) for layer in self.weights] - #Probablement faisable avec une somme de matrices - self.weights[layerNumber][neuronNumber][weightNumber] -= learningRate * errorSumsWeights[layerNumber][neuronNumber][weightNumber] + #rempli self.activations et self.outputs + self.process(_input, True) + self.__desiredOutput = desiredOutput - #print("Error : " + str(total))""" + #A optimiser + for layerNumber in range(len(errorSumsWeights)-1, -1, -1): + for neuronNumber in range(len(errorSumsWeights[layerNumber])): + errorSumsBiases[layerNumber][neuronNumber] += self.__Error(layerNumber, neuronNumber) + #for weightNumber in range(len(errorSumsWeights[layerNumber][neuronNumber])): + #print("layer : " + str(layerNumber) + " neuron : " + str(neuronNumber) + " weight : " + str(weightNumber)) + #errorSumsWeights[layerNumber][neuronNumber][weightNumber] += self.__PartialDerivative(layerNumber, neuronNumber, weightNumber) + #errorSumsWeights[layerNumber][neuronNumber][weightNumber] = errorSumsBiases[layerNumber][neuronNumber] * self.outputs[layerNumber][weightNumber] + errorSumsWeights[layerNumber][neuronNumber] = np.dot(errorSumsBiases[layerNumber][neuronNumber],self.outputs[layerNumber]) + + total = 0 + + for layerNumber in range(len(errorSumsWeights)): + errorSumsWeights[layerNumber] = np.multiply(errorSumsWeights[layerNumber], -(learningRate/len(inputBatch))) + self.weights[layerNumber] = np.add(self.weights[layerNumber], errorSumsWeights[layerNumber]) + + errorSumsBiases[layerNumber] = np.multiply(errorSumsBiases[layerNumber], -(learningRate/len(inputBatch))) + self.biases[layerNumber] = np.add(self.biases[layerNumber], errorSumsBiases[layerNumber]) + + #print(self.__biases) + """ + + for layerNumber in range(len(errorSumsWeights)): + for neuronNumber in range(len(errorSumsWeights[layerNumber])): + + errorSumsBiases[layerNumber][neuronNumber] = errorSumsBiases[layerNumber][neuronNumber] / len(inputBatch) + total += errorSumsBiases[layerNumber][neuronNumber] + self.biases[layerNumber][neuronNumber] -= learningRate * errorSumsBiases[layerNumber][neuronNumber] + + for weightNumber in range(len(errorSumsWeights[layerNumber][neuronNumber])): + + #Probablement faisable avec une multiplication de matrices + errorSumsWeights[layerNumber][neuronNumber][weightNumber] = errorSumsWeights[layerNumber][neuronNumber][weightNumber] / len(inputBatch) + + #total += errorSumsWeights[layerNumber][neuronNumber][weightNumber] + + #Probablement faisable avec une somme de matrices + self.weights[layerNumber][neuronNumber][weightNumber] -= learningRate * errorSumsWeights[layerNumber][neuronNumber][weightNumber] + + #print("Error : " + str(total))""" + if (visualize): + ani = animation.ArtistAnimation(fig, vizualisationData, interval=100) + plt.show() def __Error(self, layer, neuron): if (self.__errors[layer][neuron] == 0 ): @@ -122,20 +168,38 @@ class network: def __ErrorHiddenLayer(self, layer, neuron): upperLayerLinksSum = 0 - #Probablement faisable avec une multiplication de matrices for upperLayerNeuron in range(len(self.weights[layer+1])): upperLayerLinksSum += self.weights[layer+1][upperLayerNeuron][neuron] * self.__errors[layer+1][upperLayerNeuron] return network.__sigmoid(self.activations[layer][neuron], derivative=True) * upperLayerLinksSum - def __PartialDerivative(self, layer, neuron, weight): - return self.__Error(layer, neuron) * self.outputs[layer][weight] + #def __PartialDerivative(self, layer, neuron, weight): + # return self.__Error(layer, neuron) * self.outputs[layer][weight] + def accuracy(self, inputs, desiredOutputs): + if (type(inputs) != list or type(desiredOutputs) != list): + raise TypeError("The inputs and desired outputs must be lists of numpy arrays !") + if (len(inputs) != len(desiredOutputs)): + raise ValueError("The inputs and desired outputs lists must have the same amount of data !") + if (len(inputs) == 0): + raise ValueError("The list is empty !") + sum = 0 + for i in range(len(desiredOutputs)): + if (np.argmax(desiredOutputs[i]) == np.argmax(self.process(inputs[i]))): + sum += 1 + return sum/len(desiredOutputs) def saveToFile(self, fileName): - np.savez(fileName, biases=self.biases, weights=self.weights) + with open(fileName, "wb") as file: + pickle.dump(self, file) def loadFromFile(self, fileName): - data = np.load(fileName) - self.biases = data['biases'] - self.weights = data['weights'] \ No newline at end of file + with open(fileName, "rb") as file: + fromNetwork = pickle.load(file) + self.weights = fromNetwork.weights + self.biases = fromNetwork.biases + self.__inputLayerSize = fromNetwork.__inputLayerSize + + def networkFromFile(fileName): + with open(fileName, "rb") as file: + return pickle.load(file) \ No newline at end of file diff --git a/testLearning.py b/testLearning.py index db19caa..8f25fd4 100644 --- a/testLearning.py +++ b/testLearning.py @@ -8,7 +8,7 @@ myNetwork = network(10, 10) learningRate = 3 -for j in range(10000): +for j in range(1000): rand = [] inputs = [] desiredOutputs = [] @@ -36,4 +36,13 @@ test[0][1] = 1.0 test[1][5] = 1.0 print(test[0]) print(myNetwork.process(test[0])) -print(myNetwork.process(test[1])) \ No newline at end of file +print(test[1]) +print(myNetwork.process(test[1])) + +print("Save and load test :") + +myNetwork.saveToFile("test") + +myNetwork2 = network.networkFromFile("test") + +print(myNetwork.process(test[0]).all() == myNetwork2.process(test[0]).all()) \ No newline at end of file diff --git a/testLearningNAND.py b/testLearningNAND.py index 14ba696..5d65b65 100644 --- a/testLearningNAND.py +++ b/testLearningNAND.py @@ -1,10 +1,11 @@ import numpy as np import random from sobek.network import network +import time random.seed() -myNetwork = network(2, 1) +myNetwork = network(2, 2, 1) learningRate = 3 @@ -23,23 +24,29 @@ result.append(np.ones(1)) result.append(np.ones(1)) result.append(np.zeros(1)) -for j in range(10000): - inputs = [] - desiredOutputs = [] - - if (j%1000 == 0): - print(j) +learningTime = 0 - random.shuffle(test) +nbRep = 1 - for i in range(4): - if (test[i][0] == 1.0) and (test[i][1] == 1.0): - result[i][0] = 0.0 - else: - result[i][0] = 1.0 - - myNetwork.train(test, result, learningRate) +for i in range(nbRep): + if (i%(nbRep/10) == 0): print(i) + startTime = time.perf_counter() + + #for j in range(10000): + # inputs = [] + # desiredOutputs = [] + + #if (j%1000 == 0): + # print(j) + + # myNetwork.train(test, result, learningRate) + + myNetwork.train(test, result, learningRate, len(test), 10000, visualize=False) + + endTime = time.perf_counter() + learningTime += endTime - startTime +learningTime = learningTime / nbRep test = [] result = [] test.append(np.zeros(2)) @@ -55,9 +62,13 @@ result.append(np.ones(1)) result.append(np.ones(1)) result.append(np.zeros(1)) -print(myNetwork.weights) -print(myNetwork.biases) +#print(myNetwork.weights) +#print(myNetwork.biases) print("0 0 : " + str(myNetwork.process(test[0])) + " == 1 ?") print("0 1 : " + str(myNetwork.process(test[1])) + " == 1 ?") print("1 0 : " + str(myNetwork.process(test[2])) + " == 1 ?") -print("1 1 : " + str(myNetwork.process(test[3])) + " == 0 ?") \ No newline at end of file +print("1 1 : " + str(myNetwork.process(test[3])) + " == 0 ?") + +myNetwork.saveToFile("NAND") + +print("Learning time : " + str(endTime - startTime)) \ No newline at end of file