Jalon 2 complete

MNISTDrawingPrediction.py

@@ -0,0 +1,41 @@
+import tkinter
+from PIL import Image, ImageDraw
+from sobek.network import network
+import numpy as np
+
+class Sketchpad(tkinter.Canvas):
+    def __init__(self, parent, predictionLabel, **kwargs, ):
+        super().__init__(parent, **kwargs)
+        self.bind("<Button-3>", self.test)
+        self.bind("<B1-Motion>", self.add_line)
+        self.PILImage = Image.new("F", (560, 560), 100)
+        self.draw = ImageDraw.Draw(self.PILImage)
+        self.MNISTNN = network.networkFromFile("MNIST30epoch")
+        self.predictionLabel = predictionLabel
+
+    def add_line(self, event):
+        self.create_oval((event.x+32, event.y+32, event.x-32, event.y-32), fill="black")
+        self.draw.ellipse([event.x-32, event.y-32, event.x+32, event.y+32], fill="black")
+        smallerImage = self.PILImage.reduce(20)
+        imageAsArray = np.array(smallerImage.getdata())
+        imageAsArray = (100 - imageAsArray)/100
+        self.predictionLabel['text'] =  ( "Predicted number : "  + str(np.argmax(self.MNISTNN.process(imageAsArray))))
+
+    def test(self, event):
+        self.PILImage = Image.new("F", (560, 560), 100)
+        self.draw = ImageDraw.Draw(self.PILImage)
+        self.delete("all")
+
+window = tkinter.Tk()
+window.title("Number guesser")
+window.resizable(False, False)
+window.columnconfigure(0, weight=1)
+window.rowconfigure(0, weight=1)
+
+predictionLabel = tkinter.Label(window, text="Predicted number :")
+
+sketch = Sketchpad(window, predictionLabel, width=560, height=560)
+sketch.grid(column=0, row=0, sticky=(tkinter.N, tkinter.W, tkinter.E, tkinter.S))
+predictionLabel.grid(column=0, row=1)
+
+window.mainloop()

MNISTLearning.py

@@ -0,0 +1,57 @@
+import numpy as np
+from sobek.network import network
+import gzip
+import time
+
+print("--- Data loading ---")
+
+def getData(fileName):
+    with open(fileName, 'rb') as f:
+        data = f.read()
+        return np.frombuffer(gzip.decompress(data), dtype=np.uint8).copy()
+
+
+tempTrainImages = getData("./MNIST/train-images-idx3-ubyte.gz")[0x10:].reshape((-1, 784)).tolist()
+trainImages = []
+for image in tempTrainImages:
+    for pixel in range(784):
+        if image[pixel] !=0:
+            image[pixel] = image[pixel]/256
+    trainImages.append(np.array(image, dtype=np.float64))
+tempTrainLabels = getData("./MNIST/train-labels-idx1-ubyte.gz")[8:]
+trainLabels = []
+for label in tempTrainLabels:
+    trainLabels.append(np.zeros(10))
+    trainLabels[-1][label] = 1.0
+
+myNetwork = network(784, 30, 10)
+
+learningRate = 3.0
+
+print("--- Learning ---")
+
+startTime = time.perf_counter()
+
+"""
+for i in range(1):
+    print("Epoch: " + str(i))
+    batchEnd = 10
+    while batchEnd < 1000:
+        batchImages = trainImages[:batchEnd]
+        batchLabels = trainLabels[:batchEnd]
+        myNetwork.train(batchImages, batchLabels, learningRate)
+        batchEnd += 10
+        if (batchEnd%100) == 0:
+            print(batchEnd)
+"""
+
+myNetwork.train(trainImages, trainLabels, learningRate, 10, 30)
+
+endTime = time.perf_counter()
+
+print("Learning time : " + str(endTime - startTime))
+
+print(trainLabels[121])
+print(myNetwork.process(trainImages[121]))
+
+myNetwork.saveToFile("MNIST30epoch")

MNISTLoadTest.py

@@ -0,0 +1,30 @@
+import numpy as np
+from sobek.network import network
+import gzip
+
+print("--- Data loading ---")
+
+def getData(fileName):
+    with open(fileName, 'rb') as f:
+        data = f.read()
+        return np.frombuffer(gzip.decompress(data), dtype=np.uint8).copy()
+
+
+tempTrainImages = getData("./MNIST/t10k-images-idx3-ubyte.gz")[0x10:].reshape((-1, 784)).tolist()
+trainImages = []
+for image in tempTrainImages:
+    for pixel in range(784):
+        if image[pixel] !=0:
+            image[pixel] = image[pixel]/256
+    trainImages.append(np.array(image, dtype=np.float64))
+tempTrainLabels = getData("./MNIST/t10k-labels-idx1-ubyte.gz")[8:]
+trainLabels = []
+for label in tempTrainLabels:
+    trainLabels.append(np.zeros(10))
+    trainLabels[-1][label] = 1.0
+
+print("--- Testing ---")
+
+myNetwork = network.networkFromFile("MNIST30epoch")
+
+print(myNetwork.accuracy(trainImages, trainLabels))

sobek/network.py

@@ -1,4 +1,8 @@
+import random
 import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.animation as animation
+import pickle
 
 class network:
 
@@ -26,9 +30,9 @@ class network:
 
     def process(self, _input, __storeValues=False):
         if type(_input) != np.ndarray:
-            raise TypeError("The input must be a vector!")
+            raise TypeError("The input must be a numpy array!")
         if _input.size != self.__inputLayerSize:
-            raise ValueError("The input vector has the wrong size!")
+            raise ValueError("The input vector has the wrong size! " + str(_input.size) + " != " + str(self.__inputLayerSize))
         if _input.dtype != np.float64:
             print(_input.dtype)
             raise TypeError("The input vector must contain floats!")
@@ -59,58 +63,100 @@ class network:
 
 
 
-    def train(self, inputs, desiredOutputs, learningRate):
+    def train(self, inputs, desiredOutputs, learningRate, batchSize, epochs=1, visualize=False):
+        if (type(inputs) != list or type(desiredOutputs) != list):
+            raise TypeError("The inputs and desired outputs must be lists of numpy arrays !")
         if (len(inputs) != len(desiredOutputs)):
-            raise ValueError("The inputs and desired outputs vectors must have the same amount of data !")
+            raise ValueError("The inputs and desired outputs lists must have the same amount of data ! " + str(len(inputs)) + " != " + str(len(desiredOutputs)))
+        if (len(inputs) == 0):
+            raise ValueError("The list is empty !")
+        if (visualize == False):
+            if (self.__inputLayerSize != 2):
+                raise ValueError("Visualization is only possible for 2 inputs networks")
+            if (len(self.weights[-1]) != 1):
+                raise ValueError("Visualization is only possible for 1 output networks")
 
-        for _input, desiredOutput in zip(inputs, desiredOutputs):
+        errorSumsWeights = []
+        errorSumsBiases = []
 
-            errorSumsWeights = [np.zeros(layer.shape) for layer in self.weights]
+        if (visualize):
-            errorSumsBiases = [np.zeros(layer.shape) for layer in self.biases]
+            vizualisationData = []
-            self.__errors = [np.zeros(len(layer)) for layer in self.weights]
+            fig, graph = plt.subplots()
 
-            #rempli self.activations et self.outputs
+        for epoch in range(epochs):
-            self.process(_input, True)
+            randomState = random.getstate()
-            self.__desiredOutput = desiredOutput
 
-            #Somme de matrice ?
+            random.shuffle(inputs)
-            for layerNumber in range(len(errorSumsWeights)-1, -1, -1):
-                for neuronNumber in range(len(errorSumsWeights[layerNumber])):
-                    errorSumsBiases[layerNumber][neuronNumber] += self.__Error(layerNumber, neuronNumber)
-                    for weightNumber in range(len(errorSumsWeights[layerNumber][neuronNumber])):
-                        #print("layer : " + str(layerNumber) + " neuron : " + str(neuronNumber) + " weight : " + str(weightNumber))
-                        errorSumsWeights[layerNumber][neuronNumber][weightNumber] += self.__PartialDerivative(layerNumber, neuronNumber, weightNumber)
 
-        total = 0
+            random.setstate(randomState)
-        
-        
-        errorSumsWeights = np.multiply(errorSumsWeights, -(learningRate/len(inputs)))
-        self.weights = np.add(self.weights, errorSumsWeights)
 
-        errorSumsBiases = np.multiply(errorSumsBiases, -(learningRate/len(inputs)))
+            random.shuffle(desiredOutputs)
-        self.biases = np.add(self.biases, errorSumsBiases)
 
-        #print(self.__biases)
+            if (visualize and epoch%10 == 0):
-        
+                vizualisationFrame = np.empty((30, 30))
-        """
+                for x in range(30):
-        for layerNumber in range(len(errorSumsWeights)):
+                    for y in range(30):
-                for neuronNumber in range(len(errorSumsWeights[layerNumber])):
+                        vizualisationFrame[x][y] = self.process(np.array([float(x), float(y)]))
+                vizualisationData.append([graph.imshow(vizualisationFrame, animated=True)])
 
-                    errorSumsBiases[layerNumber][neuronNumber] = errorSumsBiases[layerNumber][neuronNumber] / len(inputs)
+            inputBatches = [inputs[j:j+batchSize] for j in range(0, len(inputs), batchSize)]
-                    total += errorSumsBiases[layerNumber][neuronNumber]
+            desiredOutputsBatches = [desiredOutputs[j:j+batchSize] for j in range(0, len(inputs), batchSize)]
-                    self.biases[layerNumber][neuronNumber] -= learningRate * errorSumsBiases[layerNumber][neuronNumber]
+
+            for inputBatch, desiredOutputsBatch in zip(inputBatches, desiredOutputsBatches):
                     
-                    for weightNumber in range(len(errorSumsWeights[layerNumber][neuronNumber])):
+                for _input, desiredOutput in zip(inputBatch, desiredOutputsBatch):
 
-                        #Probablement faisable avec une multiplication de matrices
+                    errorSumsWeights = [np.zeros(layer.shape) for layer in self.weights]
-                        errorSumsWeights[layerNumber][neuronNumber][weightNumber] = errorSumsWeights[layerNumber][neuronNumber][weightNumber] / len(inputs)
+                    errorSumsBiases = [np.zeros(layer.shape) for layer in self.biases]
-                        
+                    self.__errors = [np.zeros(len(layer)) for layer in self.weights]
-                        total += errorSumsWeights[layerNumber][neuronNumber][weightNumber]
 
-                        #Probablement faisable avec une somme de matrices
+                    #rempli self.activations et self.outputs
-                        self.weights[layerNumber][neuronNumber][weightNumber] -= learningRate * errorSumsWeights[layerNumber][neuronNumber][weightNumber]
+                    self.process(_input, True)
+                    self.__desiredOutput = desiredOutput
 
-        #print("Error : " + str(total))"""
+                    #A optimiser
+                    for layerNumber in range(len(errorSumsWeights)-1, -1, -1):
+                        for neuronNumber in range(len(errorSumsWeights[layerNumber])):
+                            errorSumsBiases[layerNumber][neuronNumber] += self.__Error(layerNumber, neuronNumber)
+                            #for weightNumber in range(len(errorSumsWeights[layerNumber][neuronNumber])):
+                                #print("layer : " + str(layerNumber) + " neuron : " + str(neuronNumber) + " weight : " + str(weightNumber))
+                                #errorSumsWeights[layerNumber][neuronNumber][weightNumber] += self.__PartialDerivative(layerNumber, neuronNumber, weightNumber)
+                                #errorSumsWeights[layerNumber][neuronNumber][weightNumber] = errorSumsBiases[layerNumber][neuronNumber] * self.outputs[layerNumber][weightNumber]
+                            errorSumsWeights[layerNumber][neuronNumber] = np.dot(errorSumsBiases[layerNumber][neuronNumber],self.outputs[layerNumber])
+
+                total = 0
+                
+                for layerNumber in range(len(errorSumsWeights)):
+                    errorSumsWeights[layerNumber] = np.multiply(errorSumsWeights[layerNumber], -(learningRate/len(inputBatch)))
+                    self.weights[layerNumber] = np.add(self.weights[layerNumber], errorSumsWeights[layerNumber])
+
+                    errorSumsBiases[layerNumber] = np.multiply(errorSumsBiases[layerNumber], -(learningRate/len(inputBatch)))
+                    self.biases[layerNumber] = np.add(self.biases[layerNumber], errorSumsBiases[layerNumber])
+
+                #print(self.__biases)
+                """
+                
+                for layerNumber in range(len(errorSumsWeights)):
+                        for neuronNumber in range(len(errorSumsWeights[layerNumber])):
+
+                            errorSumsBiases[layerNumber][neuronNumber] = errorSumsBiases[layerNumber][neuronNumber] / len(inputBatch)
+                            total += errorSumsBiases[layerNumber][neuronNumber]
+                            self.biases[layerNumber][neuronNumber] -= learningRate * errorSumsBiases[layerNumber][neuronNumber]
+                            
+                            for weightNumber in range(len(errorSumsWeights[layerNumber][neuronNumber])):
+
+                                #Probablement faisable avec une multiplication de matrices
+                                errorSumsWeights[layerNumber][neuronNumber][weightNumber] = errorSumsWeights[layerNumber][neuronNumber][weightNumber] / len(inputBatch)
+                                
+                                #total += errorSumsWeights[layerNumber][neuronNumber][weightNumber]
+
+                                #Probablement faisable avec une somme de matrices
+                                self.weights[layerNumber][neuronNumber][weightNumber] -= learningRate * errorSumsWeights[layerNumber][neuronNumber][weightNumber]
+
+                #print("Error : " + str(total))"""
+        if (visualize):
+            ani = animation.ArtistAnimation(fig, vizualisationData, interval=100)
+            plt.show()
 
     def __Error(self, layer, neuron):
         if (self.__errors[layer][neuron] == 0 ):
@@ -122,20 +168,38 @@ class network:
 
     def __ErrorHiddenLayer(self, layer, neuron):
         upperLayerLinksSum = 0
-        #Probablement faisable avec une multiplication de matrices
         for upperLayerNeuron in range(len(self.weights[layer+1])):
             upperLayerLinksSum += self.weights[layer+1][upperLayerNeuron][neuron] * self.__errors[layer+1][upperLayerNeuron]
         return network.__sigmoid(self.activations[layer][neuron], derivative=True) * upperLayerLinksSum
 
-    def __PartialDerivative(self, layer, neuron, weight):
+    #def __PartialDerivative(self, layer, neuron, weight):
-        return self.__Error(layer, neuron) * self.outputs[layer][weight]
+    #    return self.__Error(layer, neuron) * self.outputs[layer][weight]
 
+    def accuracy(self, inputs, desiredOutputs):
+        if (type(inputs) != list or type(desiredOutputs) != list):
+            raise TypeError("The inputs and desired outputs must be lists of numpy arrays !")
+        if (len(inputs) != len(desiredOutputs)):
+            raise ValueError("The inputs and desired outputs lists must have the same amount of data !")
+        if (len(inputs) == 0):
+            raise ValueError("The list is empty !")
+        sum = 0
+        for i in range(len(desiredOutputs)):
+            if (np.argmax(desiredOutputs[i]) == np.argmax(self.process(inputs[i]))):
+                sum += 1
+        return sum/len(desiredOutputs)
 
 
     def saveToFile(self, fileName):
-        np.savez(fileName, biases=self.biases, weights=self.weights)
+        with open(fileName, "wb") as file:
+            pickle.dump(self, file)
 
     def loadFromFile(self, fileName):
-        data = np.load(fileName)
+        with open(fileName, "rb") as file:
-        self.biases = data['biases']
+            fromNetwork = pickle.load(file)
-        self.weights = data['weights']
+            self.weights = fromNetwork.weights
+            self.biases = fromNetwork.biases
+            self.__inputLayerSize = fromNetwork.__inputLayerSize
+
+    def networkFromFile(fileName):
+        with open(fileName, "rb") as file:
+            return pickle.load(file)

testLearning.py

@@ -8,7 +8,7 @@ myNetwork = network(10, 10)
 
 learningRate = 3
 
-for j in range(10000):
+for j in range(1000):
     rand = []
     inputs = []
     desiredOutputs = []
@@ -36,4 +36,13 @@ test[0][1] = 1.0
 test[1][5] = 1.0
 print(test[0])
 print(myNetwork.process(test[0]))
-print(myNetwork.process(test[1]))
+print(test[1])
+print(myNetwork.process(test[1]))
+
+print("Save and load test :")
+
+myNetwork.saveToFile("test")
+
+myNetwork2 = network.networkFromFile("test")
+
+print(myNetwork.process(test[0]).all() == myNetwork2.process(test[0]).all())

testLearningNAND.py

@@ -1,10 +1,11 @@
 import numpy as np
 import random
 from sobek.network import network
+import time
 
 random.seed()
 
-myNetwork = network(2, 1)
+myNetwork = network(2, 2, 1)
 
 learningRate = 3
 
@@ -23,23 +24,29 @@ result.append(np.ones(1))
 result.append(np.ones(1))
 result.append(np.zeros(1))
 
-for j in range(10000):
+learningTime = 0
-    inputs = []
-    desiredOutputs = []
-    
-    if (j%1000 == 0):
-        print(j)
 
-    random.shuffle(test)
+nbRep = 1
 
-    for i in range(4):
+for i in range(nbRep):
-        if (test[i][0] == 1.0) and (test[i][1] == 1.0):
+    if (i%(nbRep/10) == 0): print(i)
-            result[i][0] = 0.0
-        else:
-            result[i][0] = 1.0
-    
-    myNetwork.train(test, result, learningRate)
 
+    startTime = time.perf_counter()
+
+    #for j in range(10000):
+    #    inputs = []
+    #    desiredOutputs = []
+        
+        #if (j%1000 == 0):
+        #    print(j)
+        
+    #    myNetwork.train(test, result, learningRate)
+
+    myNetwork.train(test, result, learningRate, len(test), 10000, visualize=False)
+
+    endTime = time.perf_counter()
+    learningTime += endTime - startTime
+learningTime = learningTime / nbRep
 test = []
 result = []
 test.append(np.zeros(2))
@@ -55,9 +62,13 @@ result.append(np.ones(1))
 result.append(np.ones(1))
 result.append(np.zeros(1))
 
-print(myNetwork.weights)
+#print(myNetwork.weights)
-print(myNetwork.biases)
+#print(myNetwork.biases)
 print("0 0 : " + str(myNetwork.process(test[0])) + " == 1 ?")
 print("0 1 : " + str(myNetwork.process(test[1])) + " == 1 ?")
 print("1 0 : " + str(myNetwork.process(test[2])) + " == 1 ?")
-print("1 1 : " + str(myNetwork.process(test[3])) + " == 0 ?")
+print("1 1 : " + str(myNetwork.process(test[3])) + " == 0 ?")
+
+myNetwork.saveToFile("NAND")
+
+print("Learning time : " + str(endTime - startTime))