judsonupchurch/Neural-Networks-From-Scratch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lecture 13

Browse Source
This commit is contained in:
judsonupchurch 2024-12-18 20:38:38 +00:00
parent 741460a482
commit 5de460e8d0
2 changed files with 109 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
lecture13_15/handout_13.ipynb
View File

@ -2,29 +2,29 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 17, "execution_count": null,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"Iteration 0, Loss: 466.56000000000006\n", "Iteration 0, Loss: 0.0\n",
"Iteration 20, Loss: 5.32959636083938\n", "Iteration 20, Loss: 0.0\n",
"Iteration 40, Loss: 0.41191523404899866\n", "Iteration 40, Loss: 0.0\n",
"Iteration 60, Loss: 0.031836212079467595\n", "Iteration 60, Loss: 0.0\n",
"Iteration 80, Loss: 0.002460565465389601\n", "Iteration 80, Loss: 0.0\n",
"Iteration 100, Loss: 0.000190172825660145\n", "Iteration 100, Loss: 0.0\n",
"Iteration 120, Loss: 1.4698126966451542e-05\n", "Iteration 120, Loss: 0.0\n",
"Iteration 140, Loss: 1.1359926717815175e-06\n", "Iteration 140, Loss: 0.0\n",
"Iteration 160, Loss: 8.779889800154524e-08\n", "Iteration 160, Loss: 0.0\n",
"Iteration 180, Loss: 6.7858241357822796e-09\n", "Iteration 180, Loss: 0.0\n",
"Final weights:\n", "Final weights:\n",
" [[-0.00698895 -0.01397789 -0.02096684 -0.02795579]\n", " [[0. 0. 0. 0.]\n",
" [ 0.25975286 0.11950572 -0.02074143 -0.16098857]\n", " [0. 0. 0. 0.]\n",
" [ 0.53548461 0.27096922 0.00645383 -0.25806156]]\n", " [0. 0. 0. 0.]]\n",
"Final biases:\n", "Final biases:\n",
" [-0.00698895 -0.04024714 -0.06451539]\n" " [0. 0. 0.]\n"
] ]
} }
], ],
@ -114,7 +114,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.12.2" "version": "3.10.12"
} }
}, },
"nbformat": 4, "nbformat": 4,

93
lecture13_15/notes_13.ipynb
View File

@ -368,6 +368,99 @@
"source": [ "source": [
"# Backpropagation of a Layer" "# Backpropagation of a Layer"
] ]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Iteration 0, Loss: 466.56000000000006\n",
"Iteration 20, Loss: 5.329595763793193\n",
"Iteration 40, Loss: 0.41191524253483786\n",
"Iteration 60, Loss: 0.03183621475376345\n",
"Iteration 80, Loss: 0.002460565405431671\n",
"Iteration 100, Loss: 0.0001901729121621426\n",
"Iteration 120, Loss: 1.4698120139337557e-05\n",
"Iteration 140, Loss: 1.1359948840900371e-06\n",
"Iteration 160, Loss: 8.779778427447647e-08\n",
"Iteration 180, Loss: 6.785903626216421e-09\n",
"Final weights:\n",
" [[-0.00698895 -0.0139779 -0.02096685 -0.0279558 ]\n",
" [ 0.25975286 0.11950571 -0.02074143 -0.16098857]\n",
" [ 0.53548461 0.27096922 0.00645383 -0.25806156]]\n",
"Final biases:\n",
" [-0.00698895 -0.04024714 -0.06451539]\n"
]
}
],
"source": [
"import numpy as np\n",
"\n",
"# Initial inputs\n",
"inputs = np.array([1, 2, 3, 4])\n",
"\n",
"# Initial weights and biases\n",
"weights = np.array([\n",
" [0.1, 0.2, 0.3, 0.4],\n",
" [0.5, 0.6, 0.7, 0.8],\n",
" [0.9, 1.0, 1.1, 1.2]\n",
"])\n",
"\n",
"biases = np.array([0.1, 0.2, 0.3])\n",
"\n",
"learning_rate = 0.001\n",
"\n",
"# Add the derivative function to the ReLU class\n",
"class Activation_ReLU:\n",
" def forward(self, inputs):\n",
" return np.maximum(0, inputs)\n",
" \n",
" def derivative(self, inputs):\n",
" return np.where(inputs > 0, 1, 0)\n",
" \n",
"relu = Activation_ReLU()\n",
"\n",
"num_iterations = 200\n",
"\n",
"# Training loop\n",
"# A single layer of 3 neurons, each with 4 inputs\n",
"# The neuron layer is then fed into a ReLU activation layer\n",
"for iteration in range(num_iterations):\n",
" # Forward pass\n",
" neuron_outputs = np.dot(weights, inputs) + biases\n",
" relu_outputs = relu.forward(neuron_outputs)\n",
" \n",
" # Calculate the squared loss assuming the desired output is a sum of 0. Trivial but just an example\n",
" final_output = np.sum(relu_outputs)\n",
" loss = final_output**2\n",
"\n",
" # Backward pass\n",
" dL_dfinal_output = 2 * final_output\n",
" dfinal_output_drelu_output = np.ones_like(relu_outputs)\n",
" drelu_output_dneuron_output = relu.derivative(neuron_outputs)\n",
"\n",
" dL_dneuron_output = dL_dfinal_output * dfinal_output_drelu_output * drelu_output_dneuron_output\n",
"\n",
" # Get the gradient of the Loss with respect to the weights and biases\n",
" dL_dW = np.outer(dL_dneuron_output, inputs)\n",
" dL_db = dL_dneuron_output\n",
"\n",
" # Update the weights and biases\n",
" weights -= learning_rate * dL_dW\n",
" biases -= learning_rate * dL_db\n",
"\n",
" # Print the loss every 20 iterations\n",
" if iteration % 20 == 0:\n",
" print(f\"Iteration {iteration}, Loss: {loss}\")\n",
"\n",
"# Final weights and biases\n",
"print(\"Final weights:\\n\", weights)\n",
"print(\"Final biases:\\n\", biases)\n"
]
} }
], ],
"metadata": { "metadata": {