diff --git a/lecture13_15/handout_13.ipynb b/lecture13_15/handout_13.ipynb index 43d9bc4..1ac3ee2 100644 --- a/lecture13_15/handout_13.ipynb +++ b/lecture13_15/handout_13.ipynb @@ -2,29 +2,29 @@ "cells": [ { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Iteration 0, Loss: 466.56000000000006\n", - "Iteration 20, Loss: 5.32959636083938\n", - "Iteration 40, Loss: 0.41191523404899866\n", - "Iteration 60, Loss: 0.031836212079467595\n", - "Iteration 80, Loss: 0.002460565465389601\n", - "Iteration 100, Loss: 0.000190172825660145\n", - "Iteration 120, Loss: 1.4698126966451542e-05\n", - "Iteration 140, Loss: 1.1359926717815175e-06\n", - "Iteration 160, Loss: 8.779889800154524e-08\n", - "Iteration 180, Loss: 6.7858241357822796e-09\n", + "Iteration 0, Loss: 0.0\n", + "Iteration 20, Loss: 0.0\n", + "Iteration 40, Loss: 0.0\n", + "Iteration 60, Loss: 0.0\n", + "Iteration 80, Loss: 0.0\n", + "Iteration 100, Loss: 0.0\n", + "Iteration 120, Loss: 0.0\n", + "Iteration 140, Loss: 0.0\n", + "Iteration 160, Loss: 0.0\n", + "Iteration 180, Loss: 0.0\n", "Final weights:\n", - " [[-0.00698895 -0.01397789 -0.02096684 -0.02795579]\n", - " [ 0.25975286 0.11950572 -0.02074143 -0.16098857]\n", - " [ 0.53548461 0.27096922 0.00645383 -0.25806156]]\n", + " [[0. 0. 0. 0.]\n", + " [0. 0. 0. 0.]\n", + " [0. 0. 0. 0.]]\n", "Final biases:\n", - " [-0.00698895 -0.04024714 -0.06451539]\n" + " [0. 0. 0.]\n" ] } ], @@ -114,7 +114,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.2" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/lecture13_15/notes_13.ipynb b/lecture13_15/notes_13.ipynb index 83114b4..7dc7486 100644 --- a/lecture13_15/notes_13.ipynb +++ b/lecture13_15/notes_13.ipynb @@ -368,6 +368,99 @@ "source": [ "# Backpropagation of a Layer" ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iteration 0, Loss: 466.56000000000006\n", + "Iteration 20, Loss: 5.329595763793193\n", + "Iteration 40, Loss: 0.41191524253483786\n", + "Iteration 60, Loss: 0.03183621475376345\n", + "Iteration 80, Loss: 0.002460565405431671\n", + "Iteration 100, Loss: 0.0001901729121621426\n", + "Iteration 120, Loss: 1.4698120139337557e-05\n", + "Iteration 140, Loss: 1.1359948840900371e-06\n", + "Iteration 160, Loss: 8.779778427447647e-08\n", + "Iteration 180, Loss: 6.785903626216421e-09\n", + "Final weights:\n", + " [[-0.00698895 -0.0139779 -0.02096685 -0.0279558 ]\n", + " [ 0.25975286 0.11950571 -0.02074143 -0.16098857]\n", + " [ 0.53548461 0.27096922 0.00645383 -0.25806156]]\n", + "Final biases:\n", + " [-0.00698895 -0.04024714 -0.06451539]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "\n", + "# Initial inputs\n", + "inputs = np.array([1, 2, 3, 4])\n", + "\n", + "# Initial weights and biases\n", + "weights = np.array([\n", + " [0.1, 0.2, 0.3, 0.4],\n", + " [0.5, 0.6, 0.7, 0.8],\n", + " [0.9, 1.0, 1.1, 1.2]\n", + "])\n", + "\n", + "biases = np.array([0.1, 0.2, 0.3])\n", + "\n", + "learning_rate = 0.001\n", + "\n", + "# Add the derivative function to the ReLU class\n", + "class Activation_ReLU:\n", + " def forward(self, inputs):\n", + " return np.maximum(0, inputs)\n", + " \n", + " def derivative(self, inputs):\n", + " return np.where(inputs > 0, 1, 0)\n", + " \n", + "relu = Activation_ReLU()\n", + "\n", + "num_iterations = 200\n", + "\n", + "# Training loop\n", + "# A single layer of 3 neurons, each with 4 inputs\n", + "# The neuron layer is then fed into a ReLU activation layer\n", + "for iteration in range(num_iterations):\n", + " # Forward pass\n", + " neuron_outputs = np.dot(weights, inputs) + biases\n", + " relu_outputs = relu.forward(neuron_outputs)\n", + " \n", + " # Calculate the squared loss assuming the desired output is a sum of 0. Trivial but just an example\n", + " final_output = np.sum(relu_outputs)\n", + " loss = final_output**2\n", + "\n", + " # Backward pass\n", + " dL_dfinal_output = 2 * final_output\n", + " dfinal_output_drelu_output = np.ones_like(relu_outputs)\n", + " drelu_output_dneuron_output = relu.derivative(neuron_outputs)\n", + "\n", + " dL_dneuron_output = dL_dfinal_output * dfinal_output_drelu_output * drelu_output_dneuron_output\n", + "\n", + " # Get the gradient of the Loss with respect to the weights and biases\n", + " dL_dW = np.outer(dL_dneuron_output, inputs)\n", + " dL_db = dL_dneuron_output\n", + "\n", + " # Update the weights and biases\n", + " weights -= learning_rate * dL_dW\n", + " biases -= learning_rate * dL_db\n", + "\n", + " # Print the loss every 20 iterations\n", + " if iteration % 20 == 0:\n", + " print(f\"Iteration {iteration}, Loss: {loss}\")\n", + "\n", + "# Final weights and biases\n", + "print(\"Final weights:\\n\", weights)\n", + "print(\"Final biases:\\n\", biases)\n" + ] } ], "metadata": {