diff --git a/lecture07/notes_07.ipynb b/lecture07/notes_07.ipynb
index 676f3fb..dc7152b 100644
--- a/lecture07/notes_07.ipynb
+++ b/lecture07/notes_07.ipynb
@@ -5,7 +5,7 @@
    "metadata": {},
    "source": [
     "# Previous Class Definitions\n",
-    "The previously defined Layer_Dense, Activation_ReLU, and Activation_Softmax"
+    "The previously defined Layer_Dense, Activation_ReLU, and Activation_Softmax classes."
    ]
   },
   {
@@ -389,12 +389,12 @@
    ],
    "source": [
     "# Create dataset\n",
-    "X, y = vertical_data(samples=100, classes=3)\n",
+    "X, Y = vertical_data(samples=100, classes=3)\n",
     "\n",
     "# Create model\n",
-    "dense1 = Layer_Dense(2, 3) # first dense layer, 2 inputs\n",
+    "dense1 = Layer_Dense(2, 3) # first dense layer, 2 inputs, 3 neurons\n",
     "activation1 = Activation_ReLU()\n",
-    "dense2 = Layer_Dense(3, 3) # second dense layer, 3 inputs, 3 outputs\n",
+    "dense2 = Layer_Dense(3, 3) # second dense layer, 3 inputs, 3 neurons\n",
     "activation2 = Activation_Softmax()\n",
     "\n",
     "# Create loss function\n",
@@ -422,12 +422,12 @@
     "\n",
     " # Perform a forward pass through activation function\n",
     " # it takes the output of second dense layer here and returns loss\n",
-    " loss = loss_function.calculate(activation2.output, y)\n",
+    " loss = loss_function.calculate(activation2.output, Y)\n",
     "\n",
     " # Calculate accuracy from output of activation2 and targets\n",
     " # calculate values along first axis\n",
     " predictions = np.argmax(activation2.output, axis=1)\n",
-    " accuracy = np.mean(predictions == y)\n",
+    " accuracy = np.mean(predictions == Y)\n",
     "\n",
     " # If loss is smaller - print and save weights and biases aside\n",
     " if loss < lowest_loss:\n",
@@ -449,7 +449,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -804,7 +804,7 @@
    ],
    "source": [
     "# Create dataset\n",
-    "X, y = vertical_data(samples=100, classes=3)\n",
+    "X, Y = vertical_data(samples=100, classes=3)\n",
     "\n",
     "# Create model\n",
     "dense1 = Layer_Dense(2, 3) # first dense layer, 2 inputs\n",
@@ -821,6 +821,7 @@
     "best_dense1_biases = dense1.biases.copy()\n",
     "best_dense2_weights = dense2.weights.copy()\n",
     "best_dense2_biases = dense2.biases.copy()\n",
+    "\n",
     "for iteration in range(10000):\n",
     " # Update weights with some small random values\n",
     " dense1.weights += 0.05 * np.random.randn(2, 3)\n",
@@ -836,12 +837,12 @@
     "\n",
     " # Perform a forward pass through activation function\n",
     " # it takes the output of second dense layer here and returns loss\n",
-    " loss = loss_function.calculate(activation2.output, y)\n",
+    " loss = loss_function.calculate(activation2.output, Y)\n",
     "\n",
     " # Calculate accuracy from output of activation2 and targets\n",
     " # calculate values along first axis\n",
     " predictions = np.argmax(activation2.output, axis=1)\n",
-    " accuracy = np.mean(predictions == y)\n",
+    " accuracy = np.mean(predictions == Y)\n",
     "\n",
     " # If loss is smaller - print and save weights and biases aside\n",
     " if loss < lowest_loss:\n",
@@ -868,133 +869,72 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "New set of weights found, iteration: 0 loss: 1.0986983 acc: 0.3333333333333333\n",
-      "New set of weights found, iteration: 42 loss: 1.0984432 acc: 0.3333333333333333\n",
-      "New set of weights found, iteration: 48 loss: 1.0983725 acc: 0.3333333333333333\n",
-      "New set of weights found, iteration: 54 loss: 1.097728 acc: 0.38666666666666666\n",
-      "New set of weights found, iteration: 55 loss: 1.0976882 acc: 0.3333333333333333\n",
-      "New set of weights found, iteration: 56 loss: 1.0973428 acc: 0.38333333333333336\n",
-      "New set of weights found, iteration: 57 loss: 1.0970833 acc: 0.3333333333333333\n",
-      "New set of weights found, iteration: 64 loss: 1.0964628 acc: 0.3566666666666667\n",
-      "New set of weights found, iteration: 65 loss: 1.0957834 acc: 0.34\n",
-      "New set of weights found, iteration: 84 loss: 1.0957702 acc: 0.34\n",
-      "New set of weights found, iteration: 90 loss: 1.0955024 acc: 0.3566666666666667\n",
-      "New set of weights found, iteration: 95 loss: 1.0942755 acc: 0.39\n",
-      "New set of weights found, iteration: 100 loss: 1.0938662 acc: 0.3466666666666667\n",
-      "New set of weights found, iteration: 101 loss: 1.091843 acc: 0.33666666666666667\n",
-      "New set of weights found, iteration: 104 loss: 1.0912626 acc: 0.34\n",
-      "New set of weights found, iteration: 105 loss: 1.0882009 acc: 0.36666666666666664\n",
-      "New set of weights found, iteration: 106 loss: 1.0867509 acc: 0.41\n",
-      "New set of weights found, iteration: 110 loss: 1.0861986 acc: 0.38333333333333336\n",
-      "New set of weights found, iteration: 111 loss: 1.0858816 acc: 0.3433333333333333\n",
-      "New set of weights found, iteration: 114 loss: 1.0845512 acc: 0.32666666666666666\n",
-      "New set of weights found, iteration: 115 loss: 1.0842649 acc: 0.3433333333333333\n",
-      "New set of weights found, iteration: 124 loss: 1.0840762 acc: 0.32666666666666666\n",
-      "New set of weights found, iteration: 125 loss: 1.0813359 acc: 0.39\n",
-      "New set of weights found, iteration: 133 loss: 1.0780971 acc: 0.37\n",
-      "New set of weights found, iteration: 137 loss: 1.077851 acc: 0.38666666666666666\n",
-      "New set of weights found, iteration: 138 loss: 1.0777876 acc: 0.4066666666666667\n",
-      "New set of weights found, iteration: 143 loss: 1.0771211 acc: 0.39666666666666667\n",
-      "New set of weights found, iteration: 144 loss: 1.0768937 acc: 0.38333333333333336\n",
-      "New set of weights found, iteration: 146 loss: 1.0742698 acc: 0.38333333333333336\n",
-      "New set of weights found, iteration: 148 loss: 1.0733455 acc: 0.41\n",
-      "New set of weights found, iteration: 162 loss: 1.0730222 acc: 0.4033333333333333\n",
-      "New set of weights found, iteration: 179 loss: 1.0726937 acc: 0.4166666666666667\n",
-      "New set of weights found, iteration: 191 loss: 1.0725039 acc: 0.42\n",
-      "New set of weights found, iteration: 222 loss: 1.0716708 acc: 0.4033333333333333\n",
-      "New set of weights found, iteration: 253 loss: 1.0708596 acc: 0.39\n",
-      "New set of weights found, iteration: 272 loss: 1.0706216 acc: 0.4066666666666667\n",
-      "New set of weights found, iteration: 290 loss: 1.0698603 acc: 0.44\n",
-      "New set of weights found, iteration: 300 loss: 1.0697052 acc: 0.4166666666666667\n",
-      "New set of weights found, iteration: 325 loss: 1.069674 acc: 0.43666666666666665\n",
-      "New set of weights found, iteration: 381 loss: 1.0691994 acc: 0.3933333333333333\n",
-      "New set of weights found, iteration: 398 loss: 1.0687411 acc: 0.4266666666666667\n",
-      "New set of weights found, iteration: 406 loss: 1.0684437 acc: 0.43\n",
-      "New set of weights found, iteration: 550 loss: 1.0684316 acc: 0.43333333333333335\n",
-      "New set of weights found, iteration: 570 loss: 1.0684133 acc: 0.41\n",
-      "New set of weights found, iteration: 594 loss: 1.068293 acc: 0.4\n",
-      "New set of weights found, iteration: 596 loss: 1.0681537 acc: 0.4066666666666667\n",
-      "New set of weights found, iteration: 597 loss: 1.0677991 acc: 0.42\n",
-      "New set of weights found, iteration: 642 loss: 1.0676459 acc: 0.39666666666666667\n",
-      "New set of weights found, iteration: 661 loss: 1.0675713 acc: 0.3933333333333333\n",
-      "New set of weights found, iteration: 681 loss: 1.0674102 acc: 0.38333333333333336\n",
-      "New set of weights found, iteration: 695 loss: 1.0673658 acc: 0.4033333333333333\n",
-      "New set of weights found, iteration: 701 loss: 1.0666102 acc: 0.4066666666666667\n",
-      "New set of weights found, iteration: 719 loss: 1.0663345 acc: 0.42\n",
-      "New set of weights found, iteration: 737 loss: 1.066033 acc: 0.4033333333333333\n",
-      "New set of weights found, iteration: 752 loss: 1.0657896 acc: 0.4266666666666667\n",
-      "New set of weights found, iteration: 903 loss: 1.0655118 acc: 0.4166666666666667\n",
-      "New set of weights found, iteration: 981 loss: 1.065493 acc: 0.41333333333333333\n",
-      "New set of weights found, iteration: 1006 loss: 1.0654801 acc: 0.41\n",
-      "New set of weights found, iteration: 1048 loss: 1.0651859 acc: 0.39666666666666667\n",
-      "New set of weights found, iteration: 1175 loss: 1.064625 acc: 0.4266666666666667\n",
-      "New set of weights found, iteration: 1209 loss: 1.0643268 acc: 0.41333333333333333\n",
-      "New set of weights found, iteration: 1245 loss: 1.0643263 acc: 0.43666666666666665\n",
-      "New set of weights found, iteration: 1302 loss: 1.0640283 acc: 0.4\n",
-      "New set of weights found, iteration: 1303 loss: 1.0634205 acc: 0.44333333333333336\n",
-      "New set of weights found, iteration: 1352 loss: 1.0630084 acc: 0.43\n",
-      "New set of weights found, iteration: 1577 loss: 1.0626279 acc: 0.42333333333333334\n",
-      "New set of weights found, iteration: 1594 loss: 1.0625374 acc: 0.43333333333333335\n",
-      "New set of weights found, iteration: 1600 loss: 1.0623267 acc: 0.44333333333333336\n",
-      "New set of weights found, iteration: 1794 loss: 1.0622777 acc: 0.41\n",
-      "New set of weights found, iteration: 1851 loss: 1.0618818 acc: 0.43333333333333335\n",
-      "New set of weights found, iteration: 1877 loss: 1.0616083 acc: 0.43333333333333335\n",
-      "New set of weights found, iteration: 1958 loss: 1.0614555 acc: 0.43666666666666665\n",
-      "New set of weights found, iteration: 1998 loss: 1.0613961 acc: 0.4066666666666667\n",
-      "New set of weights found, iteration: 2031 loss: 1.0606906 acc: 0.46\n",
-      "New set of weights found, iteration: 2130 loss: 1.0606595 acc: 0.43\n",
-      "New set of weights found, iteration: 2431 loss: 1.06059 acc: 0.4066666666666667\n",
-      "New set of weights found, iteration: 3294 loss: 1.0603732 acc: 0.4\n",
-      "New set of weights found, iteration: 3492 loss: 1.0603614 acc: 0.4266666666666667\n",
-      "New set of weights found, iteration: 3662 loss: 1.0598251 acc: 0.4\n",
-      "New set of weights found, iteration: 3756 loss: 1.0595479 acc: 0.39\n",
-      "New set of weights found, iteration: 3769 loss: 1.0593852 acc: 0.42333333333333334\n",
-      "New set of weights found, iteration: 3875 loss: 1.0583456 acc: 0.4033333333333333\n",
-      "New set of weights found, iteration: 3981 loss: 1.0582583 acc: 0.42333333333333334\n",
-      "New set of weights found, iteration: 4146 loss: 1.0579673 acc: 0.4166666666666667\n",
-      "New set of weights found, iteration: 4153 loss: 1.0578284 acc: 0.4166666666666667\n",
-      "New set of weights found, iteration: 4301 loss: 1.0575745 acc: 0.41\n",
-      "New set of weights found, iteration: 4405 loss: 1.057048 acc: 0.43333333333333335\n",
-      "New set of weights found, iteration: 4498 loss: 1.056719 acc: 0.4066666666666667\n",
-      "New set of weights found, iteration: 4594 loss: 1.0565504 acc: 0.43666666666666665\n",
-      "New set of weights found, iteration: 5092 loss: 1.0562842 acc: 0.4266666666666667\n",
-      "New set of weights found, iteration: 5117 loss: 1.0557985 acc: 0.4\n",
-      "New set of weights found, iteration: 5497 loss: 1.0555316 acc: 0.44\n",
-      "New set of weights found, iteration: 6021 loss: 1.0554525 acc: 0.3933333333333333\n",
-      "New set of weights found, iteration: 6154 loss: 1.0551611 acc: 0.4033333333333333\n",
-      "New set of weights found, iteration: 6168 loss: 1.0548483 acc: 0.42\n",
-      "New set of weights found, iteration: 6210 loss: 1.0546328 acc: 0.44666666666666666\n",
-      "New set of weights found, iteration: 6233 loss: 1.0541582 acc: 0.44\n",
-      "New set of weights found, iteration: 6323 loss: 1.0541245 acc: 0.4533333333333333\n",
-      "New set of weights found, iteration: 6386 loss: 1.0537696 acc: 0.4633333333333333\n",
-      "New set of weights found, iteration: 6702 loss: 1.0534701 acc: 0.4533333333333333\n",
-      "New set of weights found, iteration: 6997 loss: 1.0533447 acc: 0.4166666666666667\n",
-      "New set of weights found, iteration: 7101 loss: 1.0529538 acc: 0.41\n",
-      "New set of weights found, iteration: 7182 loss: 1.0524737 acc: 0.42\n",
-      "New set of weights found, iteration: 7476 loss: 1.0522219 acc: 0.44333333333333336\n",
-      "New set of weights found, iteration: 7719 loss: 1.0521553 acc: 0.44666666666666666\n",
-      "New set of weights found, iteration: 7858 loss: 1.0520765 acc: 0.4266666666666667\n",
-      "New set of weights found, iteration: 7877 loss: 1.0507878 acc: 0.41\n",
-      "New set of weights found, iteration: 7953 loss: 1.0506427 acc: 0.41333333333333333\n",
-      "New set of weights found, iteration: 8026 loss: 1.0503834 acc: 0.42\n",
-      "New set of weights found, iteration: 8763 loss: 1.0503162 acc: 0.41333333333333333\n",
-      "New set of weights found, iteration: 9308 loss: 1.0501956 acc: 0.41\n",
-      "New set of weights found, iteration: 9399 loss: 1.0493395 acc: 0.4066666666666667\n",
-      "New set of weights found, iteration: 9529 loss: 1.0491025 acc: 0.4166666666666667\n",
-      "New set of weights found, iteration: 9822 loss: 1.0488548 acc: 0.4533333333333333\n"
+      "New set of weights found, iteration: 0 loss: 1.0990145 acc: 0.3333333333333333\n",
+      "New set of weights found, iteration: 2 loss: 1.0988153 acc: 0.33666666666666667\n",
+      "New set of weights found, iteration: 6 loss: 1.0986578 acc: 0.3333333333333333\n",
+      "New set of weights found, iteration: 10 loss: 1.0986433 acc: 0.3333333333333333\n",
+      "New set of weights found, iteration: 18 loss: 1.098215 acc: 0.3333333333333333\n",
+      "New set of weights found, iteration: 37 loss: 1.0981511 acc: 0.3333333333333333\n",
+      "New set of weights found, iteration: 39 loss: 1.0980632 acc: 0.37666666666666665\n",
+      "New set of weights found, iteration: 41 loss: 1.0969162 acc: 0.3566666666666667\n",
+      "New set of weights found, iteration: 42 loss: 1.096638 acc: 0.3333333333333333\n",
+      "New set of weights found, iteration: 44 loss: 1.0957416 acc: 0.3433333333333333\n",
+      "New set of weights found, iteration: 59 loss: 1.0945884 acc: 0.3333333333333333\n",
+      "New set of weights found, iteration: 60 loss: 1.0926462 acc: 0.3566666666666667\n",
+      "New set of weights found, iteration: 61 loss: 1.0917169 acc: 0.35333333333333333\n",
+      "New set of weights found, iteration: 63 loss: 1.0914608 acc: 0.3566666666666667\n",
+      "New set of weights found, iteration: 66 loss: 1.0909171 acc: 0.38333333333333336\n",
+      "New set of weights found, iteration: 67 loss: 1.0909047 acc: 0.38\n",
+      "New set of weights found, iteration: 69 loss: 1.0905784 acc: 0.3933333333333333\n",
+      "New set of weights found, iteration: 70 loss: 1.0899522 acc: 0.42\n",
+      "New set of weights found, iteration: 77 loss: 1.0879942 acc: 0.39666666666666667\n",
+      "New set of weights found, iteration: 79 loss: 1.0872517 acc: 0.4066666666666667\n",
+      "New set of weights found, iteration: 81 loss: 1.0858692 acc: 0.3933333333333333\n",
+      "New set of weights found, iteration: 85 loss: 1.0848513 acc: 0.37333333333333335\n",
+      "New set of weights found, iteration: 95 loss: 1.0845288 acc: 0.36333333333333334\n",
+      "New set of weights found, iteration: 99 loss: 1.0844362 acc: 0.36333333333333334\n",
+      "New set of weights found, iteration: 105 loss: 1.0842372 acc: 0.3933333333333333\n",
+      "New set of weights found, iteration: 107 loss: 1.0820792 acc: 0.39666666666666667\n",
+      "New set of weights found, iteration: 108 loss: 1.0811542 acc: 0.37666666666666665\n",
+      "New set of weights found, iteration: 109 loss: 1.079949 acc: 0.39666666666666667\n",
+      "New set of weights found, iteration: 121 loss: 1.0790045 acc: 0.3933333333333333\n",
+      "New set of weights found, iteration: 122 loss: 1.0788108 acc: 0.4066666666666667\n",
+      "New set of weights found, iteration: 124 loss: 1.0781832 acc: 0.4166666666666667\n",
+      "New set of weights found, iteration: 128 loss: 1.0773427 acc: 0.4266666666666667\n",
+      "New set of weights found, iteration: 138 loss: 1.0766457 acc: 0.4166666666666667\n",
+      "New set of weights found, iteration: 140 loss: 1.0765723 acc: 0.4266666666666667\n",
+      "New set of weights found, iteration: 177 loss: 1.0755141 acc: 0.4033333333333333\n",
+      "New set of weights found, iteration: 181 loss: 1.0748584 acc: 0.43666666666666665\n",
+      "New set of weights found, iteration: 185 loss: 1.0729859 acc: 0.4033333333333333\n",
+      "New set of weights found, iteration: 195 loss: 1.072569 acc: 0.4166666666666667\n",
+      "New set of weights found, iteration: 205 loss: 1.0725149 acc: 0.41\n",
+      "New set of weights found, iteration: 207 loss: 1.0712609 acc: 0.4166666666666667\n",
+      "New set of weights found, iteration: 230 loss: 1.0707617 acc: 0.44\n",
+      "New set of weights found, iteration: 260 loss: 1.0705951 acc: 0.4166666666666667\n",
+      "New set of weights found, iteration: 263 loss: 1.0704489 acc: 0.38666666666666666\n",
+      "New set of weights found, iteration: 278 loss: 1.0694778 acc: 0.4166666666666667\n",
+      "New set of weights found, iteration: 339 loss: 1.0694373 acc: 0.42\n",
+      "New set of weights found, iteration: 344 loss: 1.0693406 acc: 0.41333333333333333\n",
+      "New set of weights found, iteration: 378 loss: 1.0693176 acc: 0.4066666666666667\n",
+      "New set of weights found, iteration: 385 loss: 1.0691156 acc: 0.42\n",
+      "New set of weights found, iteration: 387 loss: 1.0687498 acc: 0.4266666666666667\n",
+      "New set of weights found, iteration: 418 loss: 1.0683544 acc: 0.41\n",
+      "New set of weights found, iteration: 476 loss: 1.0682718 acc: 0.43\n",
+      "New set of weights found, iteration: 523 loss: 1.0680497 acc: 0.41\n",
+      "New set of weights found, iteration: 655 loss: 1.0680199 acc: 0.43\n"
      ]
     }
    ],
    "source": [
     "# Create dataset\n",
-    "X, y = spiral_data(samples=100, classes=3)\n",
+    "X, Y = spiral_data(samples=100, classes=3)\n",
     "\n",
     "# Create model\n",
     "dense1 = Layer_Dense(2, 3) # first dense layer, 2 inputs\n",
@@ -1011,6 +951,7 @@
     "best_dense1_biases = dense1.biases.copy()\n",
     "best_dense2_weights = dense2.weights.copy()\n",
     "best_dense2_biases = dense2.biases.copy()\n",
+    "\n",
     "for iteration in range(10000):\n",
     " # Update weights with some small random values\n",
     " dense1.weights += 0.05 * np.random.randn(2, 3)\n",
@@ -1026,12 +967,12 @@
     "\n",
     " # Perform a forward pass through activation function\n",
     " # it takes the output of second dense layer here and returns loss\n",
-    " loss = loss_function.calculate(activation2.output, y)\n",
+    " loss = loss_function.calculate(activation2.output, Y)\n",
     "\n",
     " # Calculate accuracy from output of activation2 and targets\n",
     " # calculate values along first axis\n",
     " predictions = np.argmax(activation2.output, axis=1)\n",
-    " accuracy = np.mean(predictions == y)\n",
+    " accuracy = np.mean(predictions == Y)\n",
     "\n",
     " # If loss is smaller - print and save weights and biases aside\n",
     " if loss < lowest_loss:\n",
diff --git a/lecture12/handout_12.ipynb b/lecture12/handout_12.ipynb
index 43d9bc4..4099331 100644
--- a/lecture12/handout_12.ipynb
+++ b/lecture12/handout_12.ipynb
@@ -1,5 +1,53 @@
 {
  "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "# Initial parameters\n",
+    "weights = np.array([-3.0, -1.0, 2.0])\n",
+    "bias = 1.0\n",
+    "inputs = np.array([1.0, -2.0, 3.0])\n",
+    "target_output = 0.0\n",
+    "learning_rate = 0.001\n",
+    "\n",
+    "def relu(x):\n",
+    "    return np.maximum(0, x)\n",
+    "\n",
+    "def relu_derivative(x):\n",
+    "    return np.where(x > 0, 1.0, 0.0)\n",
+    "\n",
+    "for iteration in range(200):\n",
+    "    # Forward pass\n",
+    "    linear_output = np.dot(weights, inputs) + bias\n",
+    "    output = relu(linear_output)\n",
+    "    loss = (output - target_output) ** 2\n",
+    "\n",
+    "    # Backward pass\n",
+    "    dloss_doutput = 2 * (output - target_output)\n",
+    "    doutput_dlinear = relu_derivative(linear_output)\n",
+    "    dlinear_dweights = inputs\n",
+    "    dlinear_dbias = 1.0\n",
+    "\n",
+    "    dloss_dlinear = dloss_doutput * doutput_dlinear\n",
+    "    dloss_dweights = dloss_dlinear * dlinear_dweights\n",
+    "    dloss_dbias = dloss_dlinear * dlinear_dbias\n",
+    "\n",
+    "    # Update weights and bias\n",
+    "    weights -= learning_rate * dloss_dweights\n",
+    "    bias -= learning_rate * dloss_dbias\n",
+    "\n",
+    "    # Print the loss for this iteration\n",
+    "    print(f\"Iteration {iteration + 1}, Loss: {loss}\")\n",
+    "\n",
+    "print(\"Final weights:\", weights)\n",
+    "print(\"Final bias:\", bias)\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 17,
diff --git a/lecture12/notes_12.ipynb b/lecture12/notes_12.ipynb
new file mode 100644
index 0000000..80b8958
--- /dev/null
+++ b/lecture12/notes_12.ipynb
@@ -0,0 +1,387 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Previous Class Definitions\n",
+    "The previously defined Layer_Dense, Activation_ReLU, Activation_Softmax, Loss, and Loss_CategoricalCrossEntropy classes."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# imports\n",
+    "import matplotlib.pyplot as plt\n",
+    "import numpy as np\n",
+    "import nnfs\n",
+    "from nnfs.datasets import spiral_data, vertical_data\n",
+    "nnfs.init()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class Layer_Dense:\n",
+    "    def __init__(self, n_inputs, n_neurons):\n",
+    "        # Initialize the weights and biases\n",
+    "        self.weights = 0.01 * np.random.randn(n_inputs, n_neurons)  # Normal distribution of weights\n",
+    "        self.biases = np.zeros((1, n_neurons))\n",
+    "\n",
+    "    def forward(self, inputs):\n",
+    "        # Calculate the output values from inputs, weights, and biases\n",
+    "        self.output = np.dot(inputs, self.weights) + self.biases        # Weights are already transposed\n",
+    "\n",
+    "class Activation_ReLU:\n",
+    "    def forward(self, inputs):\n",
+    "        self.output = np.maximum(0, inputs)\n",
+    "        \n",
+    "class Activation_Softmax:\n",
+    "    def forward(self, inputs):\n",
+    "        # Get the unnormalized probabilities\n",
+    "        # Subtract max from the row to prevent larger numbers\n",
+    "        exp_values = np.exp(inputs - np.max(inputs, axis=1, keepdims=True))\n",
+    "\n",
+    "        # Normalize the probabilities with element wise division\n",
+    "        probabilities = exp_values / np.sum(exp_values, axis=1,keepdims=True)\n",
+    "        self.output = probabilities\n",
+    "\n",
+    "# Base class for Loss functions\n",
+    "class Loss:\n",
+    "    '''Calculates the data and regularization losses given\n",
+    "    model output and ground truth values'''\n",
+    "    def calculate(self, output, y):\n",
+    "        sample_losses = self.forward(output, y)\n",
+    "        data_loss = np.average(sample_losses)\n",
+    "        return data_loss\n",
+    "\n",
+    "class Loss_CategoricalCrossEntropy(Loss):\n",
+    "    def forward(self, y_pred, y_true):\n",
+    "        '''y_pred is the neural network output\n",
+    "        y_true is the ideal output of the neural network'''\n",
+    "        samples = len(y_pred)\n",
+    "        # Bound the predicted values \n",
+    "        y_pred_clipped = np.clip(y_pred, 1e-7, 1-1e-7)\n",
+    "        \n",
+    "        if len(y_true.shape) == 1:     # Categorically labeled\n",
+    "            correct_confidences = y_pred_clipped[range(samples), y_true]\n",
+    "        elif len(y_true.shape) == 2:   # One hot encoded\n",
+    "            correct_confidences = np.sum(y_pred_clipped*y_true, axis=1)\n",
+    "\n",
+    "        # Calculate the losses\n",
+    "        negative_log_likelihoods = -np.log(correct_confidences)\n",
+    "        return negative_log_likelihoods"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Backpropagation of a Single Neuron\n",
+    "Backpropagation helps us find the gradient of the neural network with respect to each of the parameters (weights and biases) of each neuron.\n",
+    "\n",
+    "Imagine a layer that has 3 inputs and 1 neuron. There are 3 inputs (x0, x1, x2), three weights (w0, w1, w2), 1 bias (b0), and 1 output (z). There is a ReLU activation layer after the neuron output going into a square loss function (loss = z^2).\n",
+    "\n",
+    "Loss = (ReLU(sum(mul(x0, w0), mul(x1, w1), mul(x2, w2(, b0)))))^2\n",
+    "\n",
+    "$\\frac{\\delta Loss()}{\\delta w0} = \\frac{\\delta Loss()}{\\delta ReLU()} * \\frac{\\delta ReLU()}{\\delta sum()} * \\frac{\\delta sum()}{\\delta mul(x0, w0)} * \\frac{\\delta mul(x0, w0)}{\\delta w0}$\n",
+    "\n",
+    "$\\frac{\\delta Loss()}{\\delta ReLU()} = 2 * ReLU(sum(...))$\n",
+    "\n",
+    "$\\frac{\\delta ReLU()}{\\delta sum()}$ = 0 if sum(...) is less than 0 and 1 if sum(...) is greater than 0\n",
+    "\n",
+    "$\\frac{\\delta sum()}{\\delta mul(x0, w0)} = 1$\n",
+    "\n",
+    "$\\frac{\\delta mul(x0, w0)}{\\delta w0} = x0$\n",
+    "\n",
+    "This is repeated for w0, w1, w2, b0.\n",
+    "\n",
+    "We then use numerical differentiation to approximate the gradient. Then, we update the parameters using small step sizes, such that $w0[i+1] = w0[i] - step*\\frac{\\delta Loss()}{\\delta w0}$\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Iteration 1, Loss: 36.0\n",
+      "Iteration 2, Loss: 33.872399999999985\n",
+      "Iteration 3, Loss: 31.870541159999995\n",
+      "Iteration 4, Loss: 29.98699217744401\n",
+      "Iteration 5, Loss: 28.21476093975706\n",
+      "Iteration 6, Loss: 26.54726856821742\n",
+      "Iteration 7, Loss: 24.978324995835766\n",
+      "Iteration 8, Loss: 23.502105988581878\n",
+      "Iteration 9, Loss: 22.113131524656684\n",
+      "Iteration 10, Loss: 20.80624545154949\n",
+      "Iteration 11, Loss: 19.576596345362915\n",
+      "Iteration 12, Loss: 18.419619501351963\n",
+      "Iteration 13, Loss: 17.331019988822064\n",
+      "Iteration 14, Loss: 16.306756707482677\n",
+      "Iteration 15, Loss: 15.343027386070442\n",
+      "Iteration 16, Loss: 14.43625446755368\n",
+      "Iteration 17, Loss: 13.583071828521266\n",
+      "Iteration 18, Loss: 12.780312283455652\n",
+      "Iteration 19, Loss: 12.024995827503426\n",
+      "Iteration 20, Loss: 11.314318574097976\n",
+      "Iteration 21, Loss: 10.645642346368787\n",
+      "Iteration 22, Loss: 10.016484883698395\n",
+      "Iteration 23, Loss: 9.424510627071816\n",
+      "Iteration 24, Loss: 8.867522049011871\n",
+      "Iteration 25, Loss: 8.34345149591527\n",
+      "Iteration 26, Loss: 7.850353512506679\n",
+      "Iteration 27, Loss: 7.386397619917536\n",
+      "Iteration 28, Loss: 6.949861520580408\n",
+      "Iteration 29, Loss: 6.539124704714106\n",
+      "Iteration 30, Loss: 6.152662434665503\n",
+      "Iteration 31, Loss: 5.789040084776769\n",
+      "Iteration 32, Loss: 5.446907815766464\n",
+      "Iteration 33, Loss: 5.124995563854669\n",
+      "Iteration 34, Loss: 4.822108326030859\n",
+      "Iteration 35, Loss: 4.537121723962434\n",
+      "Iteration 36, Loss: 4.268977830076255\n",
+      "Iteration 37, Loss: 4.016681240318748\n",
+      "Iteration 38, Loss: 3.7792953790159096\n",
+      "Iteration 39, Loss: 3.55593902211607\n",
+      "Iteration 40, Loss: 3.345783025909011\n",
+      "Iteration 41, Loss: 3.148047249077789\n",
+      "Iteration 42, Loss: 2.9619976566572896\n",
+      "Iteration 43, Loss: 2.786943595148845\n",
+      "Iteration 44, Loss: 2.622235228675548\n",
+      "Iteration 45, Loss: 2.4672611266608238\n",
+      "Iteration 46, Loss: 2.3214459940751673\n",
+      "Iteration 47, Loss: 2.1842485358253243\n",
+      "Iteration 48, Loss: 2.055159447358047\n",
+      "Iteration 49, Loss: 1.9336995240191863\n",
+      "Iteration 50, Loss: 1.8194178821496518\n",
+      "Iteration 51, Loss: 1.7118902853146072\n",
+      "Iteration 52, Loss: 1.6107175694525138\n",
+      "Iteration 53, Loss: 1.5155241610978685\n",
+      "Iteration 54, Loss: 1.4259566831769857\n",
+      "Iteration 55, Loss: 1.3416826432012259\n",
+      "Iteration 56, Loss: 1.2623891989880334\n",
+      "Iteration 57, Loss: 1.18778199732784\n",
+      "Iteration 58, Loss: 1.1175840812857638\n",
+      "Iteration 59, Loss: 1.0515348620817762\n",
+      "Iteration 60, Loss: 0.9893891517327436\n",
+      "Iteration 61, Loss: 0.930916252865338\n",
+      "Iteration 62, Loss: 0.8758991023209965\n",
+      "Iteration 63, Loss: 0.8241334653738256\n",
+      "Iteration 64, Loss: 0.775427177570232\n",
+      "Iteration 65, Loss: 0.7295994313758314\n",
+      "Iteration 66, Loss: 0.6864801049815188\n",
+      "Iteration 67, Loss: 0.6459091307771113\n",
+      "Iteration 68, Loss: 0.6077359011481849\n",
+      "Iteration 69, Loss: 0.5718187093903269\n",
+      "Iteration 70, Loss: 0.538024223665358\n",
+      "Iteration 71, Loss: 0.5062269920467352\n",
+      "Iteration 72, Loss: 0.4763089768167732\n",
+      "Iteration 73, Loss: 0.44815911628690125\n",
+      "Iteration 74, Loss: 0.4216729125143454\n",
+      "Iteration 75, Loss: 0.3967520433847474\n",
+      "Iteration 76, Loss: 0.3733039976207088\n",
+      "Iteration 77, Loss: 0.35124173136132447\n",
+      "Iteration 78, Loss: 0.3304833450378703\n",
+      "Iteration 79, Loss: 0.3109517793461324\n",
+      "Iteration 80, Loss: 0.29257452918677557\n",
+      "Iteration 81, Loss: 0.275283374511837\n",
+      "Iteration 82, Loss: 0.2590141270781873\n",
+      "Iteration 83, Loss: 0.24370639216786646\n",
+      "Iteration 84, Loss: 0.22930334439074573\n",
+      "Iteration 85, Loss: 0.21575151673725296\n",
+      "Iteration 86, Loss: 0.20300060209808138\n",
+      "Iteration 87, Loss: 0.1910032665140845\n",
+      "Iteration 88, Loss: 0.17971497346310233\n",
+      "Iteration 89, Loss: 0.16909381853143318\n",
+      "Iteration 90, Loss: 0.159100373856225\n",
+      "Iteration 91, Loss: 0.14969754176132244\n",
+      "Iteration 92, Loss: 0.1408504170432283\n",
+      "Iteration 93, Loss: 0.13252615739597354\n",
+      "Iteration 94, Loss: 0.1246938614938715\n",
+      "Iteration 95, Loss: 0.11732445427958361\n",
+      "Iteration 96, Loss: 0.11039057903166032\n",
+      "Iteration 97, Loss: 0.10386649581088914\n",
+      "Iteration 98, Loss: 0.09772798590846545\n",
+      "Iteration 99, Loss: 0.09195226194127527\n",
+      "Iteration 100, Loss: 0.08651788326054573\n",
+      "Iteration 101, Loss: 0.08140467635984756\n",
+      "Iteration 102, Loss: 0.07659365998698067\n",
+      "Iteration 103, Loss: 0.07206697468175016\n",
+      "Iteration 104, Loss: 0.06780781647805846\n",
+      "Iteration 105, Loss: 0.06380037452420505\n",
+      "Iteration 106, Loss: 0.060029772389824425\n",
+      "Iteration 107, Loss: 0.05648201284158581\n",
+      "Iteration 108, Loss: 0.05314392588264792\n",
+      "Iteration 109, Loss: 0.05000311986298341\n",
+      "Iteration 110, Loss: 0.04704793547908098\n",
+      "Iteration 111, Loss: 0.044267402492267266\n",
+      "Iteration 112, Loss: 0.04165119900497416\n",
+      "Iteration 113, Loss: 0.03918961314378044\n",
+      "Iteration 114, Loss: 0.03687350700698295\n",
+      "Iteration 115, Loss: 0.03469428274287037\n",
+      "Iteration 116, Loss: 0.032643850632766785\n",
+      "Iteration 117, Loss: 0.030714599060370343\n",
+      "Iteration 118, Loss: 0.028899366255902458\n",
+      "Iteration 119, Loss: 0.027191413710178605\n",
+      "Iteration 120, Loss: 0.025584401159906987\n",
+      "Iteration 121, Loss: 0.02407236305135653\n",
+      "Iteration 122, Loss: 0.02264968639502141\n",
+      "Iteration 123, Loss: 0.02131108992907558\n",
+      "Iteration 124, Loss: 0.020051604514267202\n",
+      "Iteration 125, Loss: 0.018866554687474092\n",
+      "Iteration 126, Loss: 0.01775154130544445\n",
+      "Iteration 127, Loss: 0.01670242521429262\n",
+      "Iteration 128, Loss: 0.015715311884128023\n",
+      "Iteration 129, Loss: 0.014786536951776045\n",
+      "Iteration 130, Loss: 0.01391265261792606\n",
+      "Iteration 131, Loss: 0.013090414848206555\n",
+      "Iteration 132, Loss: 0.01231677133067759\n",
+      "Iteration 133, Loss: 0.011588850145034609\n",
+      "Iteration 134, Loss: 0.01090394910146302\n",
+      "Iteration 135, Loss: 0.010259525709566512\n",
+      "Iteration 136, Loss: 0.00965318774013127\n",
+      "Iteration 137, Loss: 0.009082684344689475\n",
+      "Iteration 138, Loss: 0.008545897699918257\n",
+      "Iteration 139, Loss: 0.008040835145853137\n",
+      "Iteration 140, Loss: 0.00756562178873318\n",
+      "Iteration 141, Loss: 0.0071184935410191314\n",
+      "Iteration 142, Loss: 0.006697790572744897\n",
+      "Iteration 143, Loss: 0.0063019511498957235\n",
+      "Iteration 144, Loss: 0.0059295058369368625\n",
+      "Iteration 145, Loss: 0.005579072041973895\n",
+      "Iteration 146, Loss: 0.005249348884293221\n",
+      "Iteration 147, Loss: 0.004939112365231496\n",
+      "Iteration 148, Loss: 0.0046472108244463226\n",
+      "Iteration 149, Loss: 0.004372560664721515\n",
+      "Iteration 150, Loss: 0.004114142329436494\n",
+      "Iteration 151, Loss: 0.0038709965177668067\n",
+      "Iteration 152, Loss: 0.003642220623566796\n",
+      "Iteration 153, Loss: 0.003426965384714043\n",
+      "Iteration 154, Loss: 0.0032244317304774253\n",
+      "Iteration 155, Loss: 0.003033867815206219\n",
+      "Iteration 156, Loss: 0.0028545662273275238\n",
+      "Iteration 157, Loss: 0.002685861363292454\n",
+      "Iteration 158, Loss: 0.002527126956721865\n",
+      "Iteration 159, Loss: 0.0023777737535795648\n",
+      "Iteration 160, Loss: 0.002237247324743051\n",
+      "Iteration 161, Loss: 0.0021050260078507234\n",
+      "Iteration 162, Loss: 0.001980618970786757\n",
+      "Iteration 163, Loss: 0.001863564389613244\n",
+      "Iteration 164, Loss: 0.0017534277341871227\n",
+      "Iteration 165, Loss: 0.001649800155096659\n",
+      "Iteration 166, Loss: 0.0015522969659304577\n",
+      "Iteration 167, Loss: 0.0014605562152439574\n",
+      "Iteration 168, Loss: 0.001374237342923055\n",
+      "Iteration 169, Loss: 0.0012930199159562866\n",
+      "Iteration 170, Loss: 0.0012166024389232565\n",
+      "Iteration 171, Loss: 0.0011447012347829103\n",
+      "Iteration 172, Loss: 0.0010770493918072343\n",
+      "Iteration 173, Loss: 0.0010133957727514104\n",
+      "Iteration 174, Loss: 0.0009535040825818146\n",
+      "Iteration 175, Loss: 0.0008971519913012098\n",
+      "Iteration 176, Loss: 0.0008441303086153165\n",
+      "Iteration 177, Loss: 0.0007942422073761319\n",
+      "Iteration 178, Loss: 0.0007473024929202092\n",
+      "Iteration 179, Loss: 0.0007031369155886454\n",
+      "Iteration 180, Loss: 0.0006615815238773228\n",
+      "Iteration 181, Loss: 0.0006224820558161947\n",
+      "Iteration 182, Loss: 0.0005856933663174615\n",
+      "Iteration 183, Loss: 0.0005510788883681067\n",
+      "Iteration 184, Loss: 0.0005185101260655349\n",
+      "Iteration 185, Loss: 0.0004878661776150635\n",
+      "Iteration 186, Loss: 0.00045903328651800607\n",
+      "Iteration 187, Loss: 0.0004319044192847727\n",
+      "Iteration 188, Loss: 0.00040637886810505474\n",
+      "Iteration 189, Loss: 0.0003823618770000461\n",
+      "Iteration 190, Loss: 0.00035976429006934636\n",
+      "Iteration 191, Loss: 0.00033850222052625716\n",
+      "Iteration 192, Loss: 0.0003184967392931672\n",
+      "Iteration 193, Loss: 0.0002996735820009388\n",
+      "Iteration 194, Loss: 0.000281962873304691\n",
+      "Iteration 195, Loss: 0.0002652988674923804\n",
+      "Iteration 196, Loss: 0.0002496197044235683\n",
+      "Iteration 197, Loss: 0.00023486717989213552\n",
+      "Iteration 198, Loss: 0.00022098652956051033\n",
+      "Iteration 199, Loss: 0.0002079262256634926\n",
+      "Iteration 200, Loss: 0.00019563778572677975\n",
+      "Final weights: [-3.3990955  -0.20180899  0.80271349]\n",
+      "Final bias: 0.6009044964039992\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "# Initial parameters\n",
+    "weights = np.array([-3.0, -1.0, 2.0])\n",
+    "bias = 1.0\n",
+    "inputs = np.array([1.0, -2.0, 3.0])\n",
+    "target_output = 0.0\n",
+    "learning_rate = 0.001\n",
+    "\n",
+    "def relu(x):\n",
+    "    return np.maximum(0, x)\n",
+    "\n",
+    "def relu_derivative(x):\n",
+    "    return np.where(x > 0, 1.0, 0.0)\n",
+    "\n",
+    "for iteration in range(200):\n",
+    "    # Forward pass\n",
+    "    linear_output = np.dot(weights, inputs) + bias\n",
+    "    output = relu(linear_output)\n",
+    "    loss = (output - target_output) ** 2\n",
+    "\n",
+    "    # Backward pass to calculate gradient\n",
+    "    dloss_doutput = 2 * (output - target_output)\n",
+    "    doutput_dlinear = relu_derivative(linear_output)\n",
+    "    dlinear_dweights = inputs\n",
+    "    dlinear_dbias = 1.0\n",
+    "\n",
+    "    dloss_dlinear = dloss_doutput * doutput_dlinear\n",
+    "    dloss_dweights = dloss_dlinear * dlinear_dweights\n",
+    "    dloss_dbias = dloss_dlinear * dlinear_dbias\n",
+    "\n",
+    "    # Update weights and bias\n",
+    "    weights -= learning_rate * dloss_dweights\n",
+    "    bias -= learning_rate * dloss_dbias\n",
+    "\n",
+    "    # Print the loss for this iteration\n",
+    "    print(f\"Iteration {iteration + 1}, Loss: {loss}\")\n",
+    "\n",
+    "print(\"Final weights:\", weights)\n",
+    "print(\"Final bias:\", bias)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/lecture12/notes_12.pdf b/lecture12/notes_12.pdf
new file mode 100644
index 0000000..48d6427
Binary files /dev/null and b/lecture12/notes_12.pdf differ
diff --git a/lecture12/notes_12.py b/lecture12/notes_12.py
new file mode 100644
index 0000000..c7d732e
--- /dev/null
+++ b/lecture12/notes_12.py
@@ -0,0 +1,130 @@
+# %% [markdown]
+# # Previous Class Definitions
+# The previously defined Layer_Dense, Activation_ReLU, Activation_Softmax, Loss, and Loss_CategoricalCrossEntropy classes.
+
+# %%
+# imports
+import matplotlib.pyplot as plt
+import numpy as np
+import nnfs
+from nnfs.datasets import spiral_data, vertical_data
+nnfs.init()
+
+# %%
+class Layer_Dense:
+    def __init__(self, n_inputs, n_neurons):
+        # Initialize the weights and biases
+        self.weights = 0.01 * np.random.randn(n_inputs, n_neurons)  # Normal distribution of weights
+        self.biases = np.zeros((1, n_neurons))
+
+    def forward(self, inputs):
+        # Calculate the output values from inputs, weights, and biases
+        self.output = np.dot(inputs, self.weights) + self.biases        # Weights are already transposed
+
+class Activation_ReLU:
+    def forward(self, inputs):
+        self.output = np.maximum(0, inputs)
+        
+class Activation_Softmax:
+    def forward(self, inputs):
+        # Get the unnormalized probabilities
+        # Subtract max from the row to prevent larger numbers
+        exp_values = np.exp(inputs - np.max(inputs, axis=1, keepdims=True))
+
+        # Normalize the probabilities with element wise division
+        probabilities = exp_values / np.sum(exp_values, axis=1,keepdims=True)
+        self.output = probabilities
+
+# Base class for Loss functions
+class Loss:
+    '''Calculates the data and regularization losses given
+    model output and ground truth values'''
+    def calculate(self, output, y):
+        sample_losses = self.forward(output, y)
+        data_loss = np.average(sample_losses)
+        return data_loss
+
+class Loss_CategoricalCrossEntropy(Loss):
+    def forward(self, y_pred, y_true):
+        '''y_pred is the neural network output
+        y_true is the ideal output of the neural network'''
+        samples = len(y_pred)
+        # Bound the predicted values 
+        y_pred_clipped = np.clip(y_pred, 1e-7, 1-1e-7)
+        
+        if len(y_true.shape) == 1:     # Categorically labeled
+            correct_confidences = y_pred_clipped[range(samples), y_true]
+        elif len(y_true.shape) == 2:   # One hot encoded
+            correct_confidences = np.sum(y_pred_clipped*y_true, axis=1)
+
+        # Calculate the losses
+        negative_log_likelihoods = -np.log(correct_confidences)
+        return negative_log_likelihoods
+
+# %% [markdown]
+# # Backpropagation of a Single Neuron
+# Backpropagation helps us find the gradient of the neural network with respect to each of the parameters (weights and biases) of each neuron.
+# 
+# Imagine a layer that has 3 inputs and 1 neuron. There are 3 inputs (x0, x1, x2), three weights (w0, w1, w2), 1 bias (b0), and 1 output (z). There is a ReLU activation layer after the neuron output going into a square loss function (loss = z^2).
+# 
+# Loss = (ReLU(sum(mul(x0, w0), mul(x1, w1), mul(x2, w2(, b0)))))^2
+# 
+# $\frac{\delta Loss()}{\delta w0} = \frac{\delta Loss()}{\delta ReLU()} * \frac{\delta ReLU()}{\delta sum()} * \frac{\delta sum()}{\delta mul(x0, w0)} * \frac{\delta mul(x0, w0)}{\delta w0}$
+# 
+# $\frac{\delta Loss()}{\delta ReLU()} = 2 * ReLU(sum(...))$
+# 
+# $\frac{\delta ReLU()}{\delta sum()}$ = 0 if sum(...) is less than 0 and 1 if sum(...) is greater than 0
+# 
+# $\frac{\delta sum()}{\delta mul(x0, w0)} = 1$
+# 
+# $\frac{\delta mul(x0, w0)}{\delta w0} = x0$
+# 
+# This is repeated for w0, w1, w2, b0.
+# 
+# We then use numerical differentiation to approximate the gradient. Then, we update the parameters using small step sizes, such that $w0[i+1] = w0[i] - step*\frac{\delta Loss()}{\delta w0}$
+# 
+
+# %%
+import numpy as np
+
+# Initial parameters
+weights = np.array([-3.0, -1.0, 2.0])
+bias = 1.0
+inputs = np.array([1.0, -2.0, 3.0])
+target_output = 0.0
+learning_rate = 0.001
+
+def relu(x):
+    return np.maximum(0, x)
+
+def relu_derivative(x):
+    return np.where(x > 0, 1.0, 0.0)
+
+for iteration in range(200):
+    # Forward pass
+    linear_output = np.dot(weights, inputs) + bias
+    output = relu(linear_output)
+    loss = (output - target_output) ** 2
+
+    # Backward pass to calculate gradient
+    dloss_doutput = 2 * (output - target_output)
+    doutput_dlinear = relu_derivative(linear_output)
+    dlinear_dweights = inputs
+    dlinear_dbias = 1.0
+
+    dloss_dlinear = dloss_doutput * doutput_dlinear
+    dloss_dweights = dloss_dlinear * dlinear_dweights
+    dloss_dbias = dloss_dlinear * dlinear_dbias
+
+    # Update weights and bias
+    weights -= learning_rate * dloss_dweights
+    bias -= learning_rate * dloss_dbias
+
+    # Print the loss for this iteration
+    print(f"Iteration {iteration + 1}, Loss: {loss}")
+
+print("Final weights:", weights)
+print("Final bias:", bias)
+
+
+