"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"plt.scatter(X[:, 0], X[:, 1], c=y, cmap='brg')\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"\n",
"The neural network will not be aware of the color differences as the data have no class encodings
"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"DENSE LAYER CLASS \n",
"
"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[[ 0.0000000e+00 0.0000000e+00 0.0000000e+00]\n",
" [-1.0475188e-04 1.1395361e-04 -4.7983500e-05]\n",
" [-2.7414842e-04 3.1729150e-04 -8.6921798e-05]\n",
" [-4.2188365e-04 5.2666257e-04 -5.5912682e-05]\n",
" [-5.7707680e-04 7.1401405e-04 -8.9430439e-05]]\n"
]
}
],
"source": [
"import numpy as np\n",
"import nnfs\n",
"from nnfs.datasets import spiral_data\n",
"nnfs.init()\n",
"# Dense layer\n",
"class Layer_Dense:\n",
" # Layer initialization\n",
" def __init__(self, n_inputs, n_neurons):\n",
" # Initialize weights and biases\n",
" self.weights = 0.01 * np.random.randn(n_inputs, n_neurons)\n",
" self.biases = np.zeros((1, n_neurons))\n",
"\n",
" # Forward pass\n",
" def forward(self, inputs):\n",
" # Calculate output values from inputs, weights and biases\n",
" self.output = np.dot(inputs, self.weights) + self.biases\n",
"\n",
"# Create dataset\n",
"X, y = spiral_data(samples=100, classes=3)\n",
"# Create Dense layer with 2 input features and 3 output values\n",
"dense1 = Layer_Dense(2, 3)\n",
"# Perform a forward pass of our training data through this layer\n",
"dense1.forward(X)\n",
"\n",
"\n",
"# Let's see output of the first few samples:\n",
"print(dense1.output[:5])\n",
" "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"ACTIVATION FUNCTION: RELU \n",
"
"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[0. 2. 0. 3.3 0. 1.1 2.2 0. ]\n"
]
}
],
"source": [
"import numpy as np\n",
"inputs = [0, 2, -1, 3.3, -2.7, 1.1, 2.2, -100]\n",
"output = np.maximum(0, inputs)\n",
"print(output)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"# ReLU activation\n",
"class Activation_ReLU:\n",
" # Forward pass\n",
" def forward(self, inputs):\n",
" # Calculate output values from input\n",
" self.output = np.maximum(0, inputs)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[[0. 0. 0. ]\n",
" [0.00013767 0. 0. ]\n",
" [0.00022187 0. 0. ]\n",
" [0.0004077 0. 0. ]\n",
" [0.00054541 0. 0. ]]\n"
]
}
],
"source": [
"# Create dataset\n",
"X, y = spiral_data(samples=100, classes=3)\n",
"# Create Dense layer with 2 input features and 3 output values\n",
"dense1 = Layer_Dense(2, 3)\n",
"# Create ReLU activation (to be used with Dense layer):\n",
"activation1 = Activation_ReLU()\n",
"# Make a forward pass of our training data through this layer\n",
"dense1.forward(X)\n",
"# Forward pass through activation func.\n",
"# Takes in output from previous layer\n",
"activation1.forward(dense1.output)\n",
"# Let's see output of the first few samples:\n",
"print(activation1.output[:5])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"ACTIVATION FUNCTION: SOFTMAX \n",
"
"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"45\n",
"[12 15 18]\n",
"(3,)\n",
"[ 6 15 24]\n",
"(3,)\n",
"[[12 15 18]]\n",
"(1, 3)\n",
"[[ 6]\n",
" [15]\n",
" [24]]\n",
"(3, 1)\n",
"[7 8 9]\n",
"[3 6 9]\n"
]
}
],
"source": [
"### TRY THESE EXERCISES FOR YOURSELF!\n",
"\n",
"A = [[1, 2, 3], [4, 5, 6], [7, 8,9]]\n",
"print(np.sum(A))\n",
"\n",
"print(np.sum(A, axis = 0))\n",
"print(np.sum(A, axis = 0).shape)\n",
"\n",
"print(np.sum(A, axis = 1))\n",
"print(np.sum(A, axis = 1).shape)\n",
"\n",
"print(np.sum(A, axis = 0,keepdims = True))\n",
"print(np.sum(A, axis = 0,keepdims = True).shape)\n",
"\n",
"print(np.sum(A, axis = 1,keepdims = True))\n",
"print(np.sum(A, axis = 1,keepdims = True).shape)\n",
"\n",
"print(np.max(A, axis = 0))\n",
"print(np.max(A, axis = 1))"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[[0.06414769 0.17437149 0.47399085 0.28748998]\n",
" [0.04517666 0.90739747 0.00224921 0.04517666]\n",
" [0.00522984 0.34875873 0.63547983 0.0105316 ]]\n"
]
},
{
"data": {
"text/plain": [
"array([1., 1., 1.])"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"inputs = [[1, 2, 3, 2.5],\n",
" [2., 5., -1., 2],\n",
" [-1.5, 2.7, 3.3, -0.8]]\n",
"\n",
"# Get unnormalized probabilities\n",
"exp_values = np.exp(inputs - np.max(inputs, axis=1,keepdims=True))\n",
" # Normalize them for each sample\n",
"probabilities = exp_values / np.sum(exp_values, axis=1,keepdims=True)\n",
"print(probabilities)\n",
"np.sum(probabilities, axis = 1)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"# Softmax activation\n",
"class Activation_Softmax:\n",
" # Forward pass\n",
" def forward(self, inputs):\n",
" # Get unnormalized probabilities\n",
" exp_values = np.exp(inputs - np.max(inputs, axis=1, keepdims=True))\n",
" # Normalize them for each sample\n",
" probabilities = exp_values / np.sum(exp_values, axis=1,keepdims=True)\n",
" self.output = probabilities"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[[0.33333334 0.33333334 0.33333334]\n",
" [0.33333337 0.333333 0.3333336 ]\n",
" [0.33333334 0.3333327 0.33333394]\n",
" [0.33333334 0.333333 0.33333364]\n",
" [0.3333334 0.33333305 0.33333358]]\n"
]
}
],
"source": [
"# Create dataset\n",
"X, y = spiral_data(samples=100, classes=3)\n",
"# Create Dense layer with 2 input features and 3 output values\n",
"dense1 = Layer_Dense(2, 3)\n",
"# Create ReLU activation (to be used with Dense layer):\n",
"activation1 = Activation_ReLU()\n",
"# Create second Dense layer with 3 input features (as we take output\n",
"# of previous layer here) and 3 output values\n",
"dense2 = Layer_Dense(3, 3)\n",
"# Create Softmax activation (to be used with Dense layer):\n",
"activation2 = Activation_Softmax()\n",
"\n",
"# Make a forward pass of our training data through this layer\n",
"dense1.forward(X)\n",
"\n",
"# Make a forward pass through activation function\n",
"# it takes the output of first dense layer here\n",
"activation1.forward(dense1.output)\n",
"# Make a forward pass through second Dense layer\n",
"# it takes outputs of activation function of first layer as inputs\n",
"dense2.forward(activation1.output)\n",
"# Make a forward pass through activation function\n",
"# it takes the output of second dense layer here\n",
"activation2.forward(dense2.output)\n",
"# Let's see output of the first few samples:\n",
"print(activation2.output[:5])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"CALCULATING NETWORK ERROR WITH LOSS \n",
"
"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"CROSS ENTROPY LOSS BUILDING BLOCKS IN PYTHON\n",
"
"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[0.7 0.5 0.9]\n"
]
}
],
"source": [
"softmax_outputs = np.array([[0.7, 0.1, 0.2],\n",
" [0.1, 0.5, 0.4],\n",
" [0.02, 0.9, 0.08]])\n",
"class_targets = [0, 1, 1]\n",
"print(softmax_outputs[[0, 1, 2], class_targets])"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[0.35667494 0.69314718 0.10536052]\n",
"0.38506088005216804\n"
]
}
],
"source": [
"print(-np.log(softmax_outputs[\n",
" range(len(softmax_outputs)), class_targets\n",
"]))\n",
"neg_log = -np.log(softmax_outputs[\n",
" range(len(softmax_outputs)), class_targets\n",
" ])\n",
"average_loss = np.mean(neg_log)\n",
"print(average_loss)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"IF DATA IS ONE HOT ENCODED, HOW TO EXTRACT THE RELEVANT PREDICTIONS\n",
"
"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0. , 0.7, 0. ],\n",
" [0.8, 0. , 0. ],\n",
" [0. , 0. , 0.7]])"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"y_true_check = np.array([\n",
" [0, 1, 0],\n",
" [1, 0, 0],\n",
" [0, 0, 1]\n",
"])\n",
"\n",
"y_pred_clipped_check = np.array([\n",
" [0.2, 0.7, 0.1],\n",
" [0.8, 0.1, 0.1],\n",
" [0.1, 0.2, 0.7]\n",
"])\n",
"\n",
"y_true_check*y_pred_clipped_check\n",
"\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"IMPLEMENTING THE LOSS CLASS\n",
"
"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"# Common loss class\n",
"class Loss:\n",
" # Calculates the data and regularization losses\n",
" # given model output and ground truth values\n",
" def calculate(self, output, y):\n",
" # Calculate sample losses\n",
" sample_losses = self.forward(output, y)\n",
" # Calculate mean loss\n",
" data_loss = np.mean(sample_losses)\n",
" # Return loss\n",
" return data_loss"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"IMPLEMENTING THE CATEGORICAL CROSS ENTROPY CLASS\n",
"
"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"# Cross-entropy loss\n",
"class Loss_CategoricalCrossentropy(Loss):\n",
" # Forward pass\n",
" def forward(self, y_pred, y_true):\n",
" # Number of samples in a batch\n",
" samples = len(y_pred)\n",
" # Clip data to prevent division by 0\n",
" # Clip both sides to not drag mean towards any value\n",
" y_pred_clipped = np.clip(y_pred, 1e-7, 1 - 1e-7)\n",
" # Probabilities for target values -\n",
" # only if categorical labels\n",
" if len(y_true.shape) == 1:\n",
" correct_confidences = y_pred_clipped[\n",
" range(samples),\n",
" y_true\n",
" ]\n",
" # Mask values - only for one-hot encoded labels\n",
" elif len(y_true.shape) == 2:\n",
" correct_confidences = np.sum(\n",
" y_pred_clipped*y_true,\n",
" axis=1\n",
" )\n",
" # Losses\n",
" negative_log_likelihoods = -np.log(correct_confidences)\n",
" return negative_log_likelihoods"
]
},
{
"cell_type": "code",
"execution_count": 138,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.38506088005216804\n"
]
}
],
"source": [
"softmax_outputs = np.array([[0.7, 0.1, 0.2],\n",
" [0.1, 0.5, 0.4],\n",
" [0.02, 0.9, 0.08]])\n",
"class_targets = np.array([[1, 0, 0],\n",
" [0, 1, 0],\n",
" [0, 1, 0]])\n",
"loss_function = Loss_CategoricalCrossentropy()\n",
"loss = loss_function.calculate(softmax_outputs, class_targets)\n",
"print(loss)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"FULL CODE UPTO THIS POINT\n",
"
"
]
},
{
"cell_type": "code",
"execution_count": 139,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[[0.33333334 0.33333334 0.33333334]\n",
" [0.3333341 0.33333302 0.3333329 ]\n",
" [0.3333341 0.33333302 0.33333296]\n",
" [0.3333341 0.333333 0.33333293]\n",
" [0.3333364 0.33333203 0.33333158]]\n",
"loss: 1.0986193\n",
"acc: 0.28\n"
]
}
],
"source": [
"# Create dataset\n",
"X, y = spiral_data(samples=100, classes=3)\n",
"# Create Dense layer with 2 input features and 3 output values\n",
"dense1 = Layer_Dense(2, 3)\n",
"# Create ReLU activation (to be used with Dense layer):\n",
"activation1 = Activation_ReLU()\n",
"# Create second Dense layer with 3 input features (as we take output\n",
"# of previous layer here) and 3 output values\n",
"dense2 = Layer_Dense(3, 3)\n",
"# Create Softmax activation (to be used with Dense layer):\n",
"activation2 = Activation_Softmax()\n",
"# Create loss function\n",
"loss_function = Loss_CategoricalCrossentropy()\n",
"# Perform a forward pass of our training data through this layer\n",
"dense1.forward(X)\n",
"# Perform a forward pass through activation function\n",
"# it takes the output of first dense layer here\n",
"activation1.forward(dense1.output)\n",
"\n",
"# Perform a forward pass through second Dense layer\n",
"# it takes outputs of activation function of first layer as inputs\n",
"dense2.forward(activation1.output)\n",
"# Perform a forward pass through activation function\n",
"# it takes the output of second dense layer here\n",
"activation2.forward(dense2.output)\n",
"# Let's see output of the first few samples:\n",
"print(activation2.output[:5])\n",
"# Perform a forward pass through activation function\n",
"# it takes the output of second dense layer here and returns loss\n",
"loss = loss_function.calculate(activation2.output, y)\n",
"# Print loss value\n",
"print('loss:', loss)\n",
"\n",
"# Calculate accuracy from output of activation2 and targets\n",
"# calculate values along first axis\n",
"predictions = np.argmax(activation2.output, axis=1)\n",
"if len(y.shape) == 2:\n",
" y = np.argmax(y, axis=1)\n",
"accuracy = np.mean(predictions == y)\n",
"# Print accuracy\n",
"print('acc:', accuracy)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"INTRODUCING ACCRUACY \n",
"
"
]
},
{
"cell_type": "code",
"execution_count": 109,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"acc: 0.6666666666666666\n"
]
}
],
"source": [
"import numpy as np\n",
"# Probabilities of 3 samples\n",
"softmax_outputs = np.array([[0.7, 0.2, 0.1],\n",
" [0.5, 0.1, 0.4],\n",
" [0.02, 0.9, 0.08]])\n",
"# Target (ground-truth) labels for 3 samples\n",
"class_targets = np.array([0, 1, 1])\n",
"# Calculate values along second axis (axis of index 1)\n",
"predictions = np.argmax(softmax_outputs, axis=1)\n",
"# If targets are one-hot encoded - convert them\n",
"if len(class_targets.shape) == 2:\n",
" class_targets = np.argmax(class_targets, axis=1)\n",
"# True evaluates to 1; False to 0\n",
"accuracy = np.mean(predictions == class_targets)\n",
"print('acc:', accuracy)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 4
}