judsonupchurch/Neural-Networks-From-Scratch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lecture 13

Browse Source
This commit is contained in:
judsonupchurch 2024-12-18 20:38:38 +00:00
parent 741460a482
commit 5de460e8d0
2 changed files with 109 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
lecture13_15/handout_13.ipynb
View File

@ -2,29 +2,29 @@
"cells": [
{
"cell_type": "code",
"execution_count": 17,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Iteration 0, Loss: 466.56000000000006\n",
"Iteration 20, Loss: 5.32959636083938\n",
"Iteration 40, Loss: 0.41191523404899866\n",
"Iteration 60, Loss: 0.031836212079467595\n",
"Iteration 80, Loss: 0.002460565465389601\n",
"Iteration 100, Loss: 0.000190172825660145\n",
"Iteration 120, Loss: 1.4698126966451542e-05\n",
"Iteration 140, Loss: 1.1359926717815175e-06\n",
"Iteration 160, Loss: 8.779889800154524e-08\n",
"Iteration 180, Loss: 6.7858241357822796e-09\n",
"Iteration 0, Loss: 0.0\n",
"Iteration 20, Loss: 0.0\n",
"Iteration 40, Loss: 0.0\n",
"Iteration 60, Loss: 0.0\n",
"Iteration 80, Loss: 0.0\n",
"Iteration 100, Loss: 0.0\n",
"Iteration 120, Loss: 0.0\n",
"Iteration 140, Loss: 0.0\n",
"Iteration 160, Loss: 0.0\n",
"Iteration 180, Loss: 0.0\n",
"Final weights:\n",
" [[-0.00698895 -0.01397789 -0.02096684 -0.02795579]\n",
" [ 0.25975286 0.11950572 -0.02074143 -0.16098857]\n",
" [ 0.53548461 0.27096922 0.00645383 -0.25806156]]\n",
" [[0. 0. 0. 0.]\n",
" [0. 0. 0. 0.]\n",
" [0. 0. 0. 0.]]\n",
"Final biases:\n",
" [-0.00698895 -0.04024714 -0.06451539]\n"
" [0. 0. 0.]\n"
]
}
],
@ -114,7 +114,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
"version": "3.10.12"
}
},
"nbformat": 4,

93
lecture13_15/notes_13.ipynb
View File

@ -368,6 +368,99 @@
"source": [
"# Backpropagation of a Layer"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Iteration 0, Loss: 466.56000000000006\n",
"Iteration 20, Loss: 5.329595763793193\n",
"Iteration 40, Loss: 0.41191524253483786\n",
"Iteration 60, Loss: 0.03183621475376345\n",
"Iteration 80, Loss: 0.002460565405431671\n",
"Iteration 100, Loss: 0.0001901729121621426\n",
"Iteration 120, Loss: 1.4698120139337557e-05\n",
"Iteration 140, Loss: 1.1359948840900371e-06\n",
"Iteration 160, Loss: 8.779778427447647e-08\n",
"Iteration 180, Loss: 6.785903626216421e-09\n",
"Final weights:\n",
" [[-0.00698895 -0.0139779 -0.02096685 -0.0279558 ]\n",
" [ 0.25975286 0.11950571 -0.02074143 -0.16098857]\n",
" [ 0.53548461 0.27096922 0.00645383 -0.25806156]]\n",
"Final biases:\n",
" [-0.00698895 -0.04024714 -0.06451539]\n"
]
}
],
"source": [
"import numpy as np\n",
"\n",
"# Initial inputs\n",
"inputs = np.array([1, 2, 3, 4])\n",
"\n",
"# Initial weights and biases\n",
"weights = np.array([\n",
" [0.1, 0.2, 0.3, 0.4],\n",
" [0.5, 0.6, 0.7, 0.8],\n",
" [0.9, 1.0, 1.1, 1.2]\n",
"])\n",
"\n",
"biases = np.array([0.1, 0.2, 0.3])\n",
"\n",
"learning_rate = 0.001\n",
"\n",
"# Add the derivative function to the ReLU class\n",
"class Activation_ReLU:\n",
" def forward(self, inputs):\n",
" return np.maximum(0, inputs)\n",
" \n",
" def derivative(self, inputs):\n",
" return np.where(inputs > 0, 1, 0)\n",
" \n",
"relu = Activation_ReLU()\n",
"\n",
"num_iterations = 200\n",
"\n",
"# Training loop\n",
"# A single layer of 3 neurons, each with 4 inputs\n",
"# The neuron layer is then fed into a ReLU activation layer\n",
"for iteration in range(num_iterations):\n",
" # Forward pass\n",
" neuron_outputs = np.dot(weights, inputs) + biases\n",
" relu_outputs = relu.forward(neuron_outputs)\n",
" \n",
" # Calculate the squared loss assuming the desired output is a sum of 0. Trivial but just an example\n",
" final_output = np.sum(relu_outputs)\n",
" loss = final_output**2\n",
"\n",
" # Backward pass\n",
" dL_dfinal_output = 2 * final_output\n",
" dfinal_output_drelu_output = np.ones_like(relu_outputs)\n",
" drelu_output_dneuron_output = relu.derivative(neuron_outputs)\n",
"\n",
" dL_dneuron_output = dL_dfinal_output * dfinal_output_drelu_output * drelu_output_dneuron_output\n",
"\n",
" # Get the gradient of the Loss with respect to the weights and biases\n",
" dL_dW = np.outer(dL_dneuron_output, inputs)\n",
" dL_db = dL_dneuron_output\n",
"\n",
" # Update the weights and biases\n",
" weights -= learning_rate * dL_dW\n",
" biases -= learning_rate * dL_db\n",
"\n",
" # Print the loss every 20 iterations\n",
" if iteration % 20 == 0:\n",
" print(f\"Iteration {iteration}, Loss: {loss}\")\n",
"\n",
"# Final weights and biases\n",
"print(\"Final weights:\\n\", weights)\n",
"print(\"Final biases:\\n\", biases)\n"
]
}
],
"metadata": {