Lecture 5

lecture03/handout_3.ipynb

@@ -556,21 +556,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
-     "ename": "ValueError",
+     "name": "stdout",
-     "evalue": "shapes (300,2) and (3,2) not aligned: 2 (dim 1) != 3 (dim 0)",
+     "output_type": "stream",
-     "output_type": "error",
+     "text": [
-     "traceback": [
+      "[[ 0.0000000e+00  0.0000000e+00  0.0000000e+00]\n",
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      " [-1.0475188e-04  1.1395361e-04 -4.7983500e-05]\n",
-      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
+      " [-2.7414842e-04  3.1729150e-04 -8.6921798e-05]\n",
-      "\u001b[0;32m/tmp/ipykernel_21709/3680822285.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0mdense1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLayer_Dense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;31m# Perform a forward pass of our training data through this layer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m \u001b[0mdense1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      " [-4.2188365e-04  5.2666257e-04 -5.5912682e-05]\n",
-      "\u001b[0;32m/tmp/ipykernel_21709/3680822285.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m     14\u001b[0m  \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m  \u001b[0;31m# Calculate output values from inputs, weights and biases\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m    \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweights\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbiases\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;31m# Create dataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      " [-5.7707680e-04  7.1401405e-04 -8.9430439e-05]]\n"
-      "\u001b[0;32m~/.local/lib/python3.10/site-packages/nnfs/core.py\u001b[0m in \u001b[0;36mdot\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0morig_dot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0morig_dot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'float64'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'float32'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m         \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdot\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/.local/lib/python3.10/site-packages/nnfs/core.py\u001b[0m in \u001b[0;36mdot\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0morig_dot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0morig_dot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'float64'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'float32'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m         \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdot\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;31mValueError\u001b[0m: shapes (300,2) and (3,2) not aligned: 2 (dim 1) != 3 (dim 0)"
      ]
     }
    ],

lecture03/notes_03.ipynb

@@ -269,6 +269,146 @@
     "print(dense1.output[:5])\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Python Array Summation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "45\n",
+      "[12 15 18]\n",
+      "[ 6 15 24]\n",
+      "[[12 15 18]]\n",
+      "[[ 6]\n",
+      " [15]\n",
+      " [24]]\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "A = [\n",
+    "    [1, 2, 3],\n",
+    "    [4, 5, 6],\n",
+    "    [7, 8, 9]\n",
+    "]\n",
+    "\n",
+    "# Sum the elements in the array resulting in a scalar\n",
+    "print(np.sum(A))\n",
+    "\n",
+    "# Sum the rows resulting in a row vector\n",
+    "'''\n",
+    "  [1, 2, 3]\n",
+    "+ [4, 5, 6]\n",
+    "+ [7, 8, 9]\n",
+    "= [12, 15, 18]\n",
+    "'''\n",
+    "print(np.sum(A, axis=0))\n",
+    "\n",
+    "# Sum the columns resulting in a column vector\n",
+    "'''\n",
+    "1   +   2   +   3   =   6\n",
+    "4   +   5   +   6   =   15\n",
+    "7   +   8   +   9   =   24\n",
+    "'''\n",
+    "print(np.sum(A, axis=1))\n",
+    "\n",
+    "# keepdims will keep the dimension of A\n",
+    "print(np.sum(A, axis=0, keepdims=True))\n",
+    "print(np.sum(A, axis=1, keepdims=True))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Python Array Broadcasting Rules\n",
+    "Check if arrays are broadcastable by scanning right to left for their dimensions. If they share the sime size in that dimension, either of them is size 1, or one size does not exist, continue left.\n",
+    "\n",
+    "Given A is 3x3 and B is 3x3, check if it is broadcastable.\n",
+    "\n",
+    "3 x 3\n",
+    "\n",
+    "3 x 1\n",
+    "\n",
+    "Start by comparing 3 and 1. This is fine because B has size 1.\n",
+    "\n",
+    "Move to 3 and 3. This is fine because they are equal. The arrays are broadcastable.\n",
+    "\n",
+    "Once the arrays are deemed broadcastable, Python will maniupulate one or the other to perform the operation."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[[ 2  3  4]\n",
+      " [ 6  7  8]\n",
+      " [10 11 12]]\n",
+      "[[ 2  3  4]\n",
+      " [ 6  7  8]\n",
+      " [10 11 12]]\n",
+      "[[-2 -1  0]\n",
+      " [-2 -1  0]\n",
+      " [-2 -1  0]]\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "A = np.array([\n",
+    "    [1, 2, 3],\n",
+    "    [4, 5, 6],\n",
+    "    [7, 8, 9]\n",
+    "])  # 3 x 3\n",
+    "\n",
+    "B = np.array([\n",
+    "    [1],\n",
+    "    [2],\n",
+    "    [3]\n",
+    "])  # 3 x 1\n",
+    "\n",
+    "print(A+B)\n",
+    "\n",
+    "# Adding the arrays will broadcast B into a 3x3 in order to add them.\n",
+    "B_broadcast = np.array([\n",
+    "    [1, 1, 1],\n",
+    "    [2, 2, 2],\n",
+    "    [3, 3, 3]\n",
+    "])  # 3 x 1\n",
+    "\n",
+    "print(A+B_broadcast)\n",
+    "\n",
+    "# Example\n",
+    "# Take each row and subtract the maximum value from the row\n",
+    "A = np.array([\n",
+    "    [1, 2, 3],\n",
+    "    [4, 5, 6],\n",
+    "    [7, 8, 9]\n",
+    "])  # 3 x 3\n",
+    "\n",
+    "# Gets the max value in each row by comparing columns (axis1). Use keepdims so it is a column vector\n",
+    "result = A - np.max(A, axis=1, keepdims=True)\n",
+    "print(result)\n"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

lecture03/notes_03.py

@@ -163,6 +163,97 @@ dense1.forward(X)
 print(dense1.output[:5])
 
 
+# %% [markdown]
+# # Python Array Summation
+
+# %%
+import numpy as np
+
+A = [
+    [1, 2, 3],
+    [4, 5, 6],
+    [7, 8, 9]
+]
+
+# Sum the elements in the array resulting in a scalar
+print(np.sum(A))
+
+# Sum the rows resulting in a row vector
+'''
+  [1, 2, 3]
++ [4, 5, 6]
++ [7, 8, 9]
+= [12, 15, 18]
+'''
+print(np.sum(A, axis=0))
+
+# Sum the columns resulting in a column vector
+'''
+1   +   2   +   3   =   6
+4   +   5   +   6   =   15
+7   +   8   +   9   =   24
+'''
+print(np.sum(A, axis=1))
+
+# keepdims will keep the dimension of A
+print(np.sum(A, axis=0, keepdims=True))
+print(np.sum(A, axis=1, keepdims=True))
+
+# %% [markdown]
+# # Python Array Broadcasting Rules
+# Check if arrays are broadcastable by scanning right to left for their dimensions. If they share the sime size in that dimension, either of them is size 1, or one size does not exist, continue left.
+# 
+# Given A is 3x3 and B is 3x3, check if it is broadcastable.
+# 
+# 3 x 3
+# 
+# 3 x 1
+# 
+# Start by comparing 3 and 1. This is fine because B has size 1.
+# 
+# Move to 3 and 3. This is fine because they are equal. The arrays are broadcastable.
+# 
+# Once the arrays are deemed broadcastable, Python will maniupulate one or the other to perform the operation.
+
+# %%
+import numpy as np
+
+A = np.array([
+    [1, 2, 3],
+    [4, 5, 6],
+    [7, 8, 9]
+])  # 3 x 3
+
+B = np.array([
+    [1],
+    [2],
+    [3]
+])  # 3 x 1
+
+print(A+B)
+
+# Adding the arrays will broadcast B into a 3x3 in order to add them.
+B_broadcast = np.array([
+    [1, 1, 1],
+    [2, 2, 2],
+    [3, 3, 3]
+])  # 3 x 1
+
+print(A+B_broadcast)
+
+# Example
+# Take each row and subtract the maximum value from the row
+A = np.array([
+    [1, 2, 3],
+    [4, 5, 6],
+    [7, 8, 9]
+])  # 3 x 3
+
+# Gets the max value in each row by comparing the columns (axis1). Use keepdims so it is a column vector
+result = A - np.max(A, axis=1, keepdims=True)
+print(result)
+
+
 # %% [markdown]
 # # Activation Function: ReLU
 # Rectified Linear Unit