{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "892317fe-34c5-4f50-9330-1c6db398be23",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Algorithm completed. Final weights: \n",
      "[-6.36596803 -8.70783535  1.61165206 -3.44519417 -0.1840602 ]\n"
     ]
    }
   ],
   "source": [
    "#Imports\n",
    "import numpy as np\n",
    "import random \n",
    "\n",
    "#Define activation function\n",
    "def activation(x, w):\n",
    "    #Linear activation function\n",
    "    if(activation_function == 0):\n",
    "        return np.dot(x,w)\n",
    "    \n",
    "    #Sigmoid activation function\n",
    "    elif(activation_function == 1):\n",
    "        return 1.0/(1.0 + np.exp(-np.dot(x,w)))\n",
    "    \n",
    "    else:\n",
    "        print(\"Error: Activation function not specified.\")\n",
    "        exit()\n",
    "\n",
    "#Variables\n",
    "n = 5 #Input dimension\n",
    "training_set_size = 20 #Number of training examples\n",
    "eta = 0.1 * 1.0/training_set_size # Learning Rate\n",
    "activation_function = 1 # 0 = linear activation function, 1 = sigmoid activation function\n",
    "\n",
    "x = np.mgrid[0.0:training_set_size, 0:n] # Training set, only use x[0] subarray\n",
    "t = np.arange(0, training_set_size, 1.0) #Target values\n",
    "w = np.arange(0, n, 1.0) # Weight vector\n",
    "w_step = np.arange(0, n, 1.0) # Weight step vector\n",
    "w_grad = np.arange(0, n, 1.0) # Weight gradient\n",
    "output = np.arange(0, training_set_size, 1.0) # Output vector\n",
    "\n",
    "number_iterations = 0\n",
    "max_iterations = 100 # Secondary exit condition for gradient descent algorithm\n",
    "\n",
    "# To ensure that random numbers stay the same for multiple executions\n",
    "random.seed(3)\n",
    "\n",
    "#Initialise x and t array\n",
    "for i in range(training_set_size):\n",
    "    #Initialise target values randomly between -1 and 1\n",
    "    random_number = random.random()\n",
    "    t[i] = random.random()*1.0 \n",
    "    if(random_number < 0.5):\n",
    "        t[i] *= -1\n",
    "        \n",
    "    #Initialise inputs\n",
    "    for j in range(n):\n",
    "        random_number = random.random()\n",
    "        x[0][i][j] = random.random()\n",
    "        if(random_number < 0.5):\n",
    "            x[0][i][j] *= -1\n",
    "        \n",
    "#Initialise weight vector\n",
    "for i in range(n):\n",
    "    w[i] = random.random()*0.25\n",
    "\n",
    "print(\"Weights at the beginning: \")\n",
    "print(w)\n",
    "    \n",
    "#Gradient descent\n",
    "while(number_iterations < max_iterations): #Loop until gradient of weights is equal to zero or max iterations are reached\n",
    "    #Initalise step vector, output vector and gradient\n",
    "    for i in range(n):\n",
    "        w_step[i] = 0.0\n",
    "        w_grad[i] = 0.0\n",
    "        output[i] = 0.0\n",
    "        \n",
    "    #Loop over training sample\n",
    "    for i in range(training_set_size):\n",
    "        #Compute the output\n",
    "        output[i] = activation(x[0][i], w)\n",
    "        \n",
    "        #Update weight step\n",
    "        for j in range(n):\n",
    "            w_step[j] += eta*(t[i] - output[i])*x[0][i][j]\n",
    "    \n",
    "    #Update weights\n",
    "    for i in range(n):\n",
    "        w[i] += w_step[i]\n",
    "    \n",
    "    #Calculate gradient of the weights and check if training is finished\n",
    "    for i in range(n): #Loop over vector entries\n",
    "        result = 0.0\n",
    "        for j in range(training_set_size): # Loop over sample\n",
    "            result += (t[j] - output[j])*x[0][j][i]\n",
    "            \n",
    "        w_grad[i] = -1.0 * (1.0/training_set_size) *result\n",
    "    \n",
    "    # Check, if weights have reached their optimum\n",
    "    cancellation = True\n",
    "    for i in range(n):\n",
    "        if(abs(w_grad[i]) >= 0.01):\n",
    "            cancellation = False\n",
    "        \n",
    "    if(cancellation):\n",
    "        break\n",
    "    \n",
    "    \n",
    "print(\"Algorithm completed. Final weights: \")\n",
    "print(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b3dce80-2634-4604-9c1c-0b6afe5fb8a8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}