{ "cells": [ { "cell_type": "code", "execution_count": 20, "id": "80ccac6d-7dad-4928-b68b-efc6765f2c35", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Output of hidden layer:\n", "Input Number: 1\n", "0.4987756666498182\n", "0.49816081852766714\n", "0.5035945902547627\n", "Input Number: 2\n", "0.5042451142413316\n", "0.5011971512746239\n", "0.49008839881299443\n", "Input Number: 3\n", "0.48926121232079367\n", "0.5051796832546991\n", "0.49827281093681053\n", "Input Number: 4\n", "0.505354870554823\n", "0.48986247672074507\n", "0.4953537613194172\n", "Input Number: 5\n", "0.48880104035355204\n", "0.49675908780253053\n", "0.4908471862728163\n", "Input Number: 6\n", "0.49203607654596787\n", "0.5003089438172074\n", "0.5029701472597984\n", "Input Number: 7\n", "0.49807772957261226\n", "0.5085135987531676\n", "0.49134034032414436\n", "Input Number: 8\n", "0.5040177001538083\n", "0.5062444974079586\n", "0.49072545801232065\n" ] } ], "source": [ "#########################################################Imports#########################################################\n", "import numpy as np\n", "import random\n", "\n", "##########################################################Variables#########################################################\n", "n_1 = 8 #Input dimension, size of first layer\n", "n_2 = 3 #Size of second layer\n", "n_3 = 8 #Size of third layer\n", "learning_rate = 0.01 #Learning rate for actualisation of the weights\n", "training_set_size = 8 #Number of training inputs\n", "training_epochs = 5000 #Number of iterations for training\n", "activation_hidden_layer = 1 # 0 = linear activation, 1 = sigmoid activation\n", "activation_output_layer = 1\n", "\n", "##########################################################Define activation function##########################################################\n", "def activation(activation_function, x, w):\n", " #Linear activation function\n", " if(activation_function == 0):\n", " return np.dot(x,w)\n", " \n", " #Sigmoid activation function\n", " elif(activation_function == 1):\n", " return 1.0/(1.0 + np.exp(-np.dot(x,w)))\n", " \n", " else:\n", " print(\"Error: Activation function not specified.\")\n", " exit()\n", "\n", "\n", "##########################################################Define weight_sum function########################################################## \n", " \n", "def weight_sum(w, delta, j):\n", " result = 0.0\n", " for i in range(n_3):\n", " result += w[i][j] * delta_k[i]\n", " \n", " return result\n", "\n", "##########################################################Set input data and target output#########################################################\n", "input_data = np.identity(8)\n", "\n", "target_output = np.identity(8)\n", "\n", "\n", "##########################################################Set arrays for output#########################################################\n", "output_hidden_layer = np.zeros((8,3)) # 8 Training iterations times 3 nodes\n", "\n", "output_output_layer = np.zeros((8,8))\n", "\n", "##########################################################Set arrays for weights#########################################################\n", "weight_input_hidden = np.zeros((3,8)) # 3 Hidden nodes * 8 connections each\n", "\n", "weight_hidden_output = np.zeros((8,3)) #8 Output nodes * 3 connections each\n", "\n", "\n", "##########################################################Set arrays for weight updates#########################################################\n", "\n", "delta_k = np.arange(0, n_3, 1) #Initialise array with correct size\n", "delta_j = np.arange(0, n_2, 1)\n", "delta_w_kj = np.zeros((8,3)) # 3 connections for each output node\n", "delta_w_ji = np.zeros((3,8)) # 3 Hidden nodes * 8 connections each\n", "\n", "################################################Initialise weights with random values from -0.05 to 0.05#######################################\n", "# To ensure that random numbers stay the same for multiple executions\n", "random.seed(3)\n", "\n", "# Initalise weights input-hidden\n", "for i in range(3):\n", " for j in range(8):\n", " sign = random.random()\n", " weight_input_hidden[i][j] = random.random()*0.05\n", " if(sign < 0.5):\n", " weight_input_hidden[i][j] *= -1\n", "\n", "# Initalise weights hidden-output\n", "for i in range(8):\n", " for j in range(3):\n", " sign = random.random()\n", " weight_hidden_output[i][j] = random.random()*0.05\n", " if(sign < 0.5):\n", " weight_hidden_output[i][j] *= -1\n", " \n", " \n", "#################################################Backpropagation algorithm for 1 hidden layer################################################\n", "for i in range(training_epochs): # Loop over epochs\n", " for j in range(training_set_size): # Loop over training set\n", " #Compute output for hidden layer\n", " for k in range(n_2):\n", " output_hidden_layer[j][k] = activation(activation_hidden_layer, input_data[j], weight_input_hidden[k])\n", " \n", " #Compute output for output layer\n", " for k in range(n_3):\n", " output_output_layer[j][k] = activation(activation_output_layer, output_hidden_layer[k], weight_hidden_output[k])\n", "\n", " #Update weights \n", " #Calculate delta_ks and weight steps for output layer - hidden layer\n", " for k in range(n_3):\n", " delta_k[k] = output_output_layer[j][k] * (1 - output_output_layer[j][k]) * (target_output[j][k] - output_output_layer[j][k])\n", " for l in range(3):\n", " delta_w_kj[k][l] = delta_k[k] * output_hidden_layer[j][l]\n", " \n", " #Calculate delta_js and weight steps for hidden layer - input layer\n", " for k in range(n_2):\n", " delta_j[k] = output_hidden_layer[j][k] * (1 - output_hidden_layer[j][k]) * weight_sum(weight_hidden_output, delta_k, k)\n", " for l in range(8):\n", " delta_w_ji[k][l] = delta_j[k] * input_data[j][l]\n", " \n", " #Update weights\n", " #Update weights input-hidden\n", " for k in range(3):\n", " for l in range(8):\n", " weight_input_hidden[k][l] += learning_rate * delta_w_ji[k][l]\n", " \n", " #Update weights hidden_output\n", " for k in range(8):\n", " for l in range(3):\n", " weight_hidden_output[k][l] += learning_rate * delta_w_kj[k][l]\n", " \n", " \n", "#############################################Print out hidden output for comparison of results######################################################\n", "print(\"Output of hidden layer:\")\n", "for i in range(training_set_size):\n", " print(\"Input Number: \", i + 1)\n", " for j in range(n_2):\n", " print(activation(activation_hidden_layer, input_data[i], weight_input_hidden[j]))" ] }, { "cell_type": "code", "execution_count": null, "id": "3755481e-c9c7-4825-96c6-d9e61637a333", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.6" } }, "nbformat": 4, "nbformat_minor": 5 }