From 036d06a652f075a094f0bdeaa6142bda1a7a2360 Mon Sep 17 00:00:00 2001 From: judsonupchurch Date: Tue, 21 Jan 2025 02:13:32 +0000 Subject: [PATCH] Lecture 26, RMSProp optimizer --- lecture23_24/notes_23.ipynb | 1322 ++++++----------------------------- lecture23_24/notes_23.pdf | Bin 88600 -> 80689 bytes lecture23_24/notes_23.py | 10 +- lecture25_27/notes_25.ipynb | 446 +++++++++--- 4 files changed, 558 insertions(+), 1220 deletions(-) diff --git a/lecture23_24/notes_23.ipynb b/lecture23_24/notes_23.ipynb index e0f9209..b8b6bdd 100644 --- a/lecture23_24/notes_23.ipynb +++ b/lecture23_24/notes_23.ipynb @@ -9,7 +9,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -227,7 +227,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -261,1020 +261,120 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "epoch: 0, acc: 0.353, loss: 1.099\n", - "epoch: 100, acc: 0.467, loss: 1.079\n", - "epoch: 200, acc: 0.450, loss: 1.067\n", - "epoch: 300, acc: 0.447, loss: 1.065\n" + "epoch: 0, acc: 0.360, loss: 1.099, lr: 1.0\n", + "epoch: 100, acc: 0.400, loss: 1.088, lr: 0.9099181073703367\n", + "epoch: 200, acc: 0.423, loss: 1.078, lr: 0.8340283569641367\n", + "epoch: 300, acc: 0.423, loss: 1.076, lr: 0.7698229407236336\n", + "epoch: 400, acc: 0.420, loss: 1.076, lr: 0.7147962830593281\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch: 400, acc: 0.433, loss: 1.064\n", - "epoch: 500, acc: 0.430, loss: 1.062\n", - "epoch: 600, acc: 0.423, loss: 1.061\n", - "epoch: 700, acc: 0.433, loss: 1.059\n", - "epoch: 800, acc: 0.453, loss: 1.056\n", - "epoch: 900, acc: 0.467, loss: 1.054\n", - "epoch: 1000, acc: 0.467, loss: 1.051\n", - "epoch: 1100, acc: 0.463, loss: 1.048\n", - "epoch: 1200, acc: 0.463, loss: 1.045\n", - "epoch: 1300, acc: 0.467, loss: 1.041\n", - "epoch: 1400, acc: 0.470, loss: 1.037\n", - "epoch: 1500, acc: 0.480, loss: 1.032\n", - "epoch: 1600, acc: 0.480, loss: 1.026\n", - "epoch: 1700, acc: 0.490, loss: 1.020\n", - "epoch: 1800, acc: 0.493, loss: 1.014\n", - "epoch: 1900, acc: 0.493, loss: 1.006\n", - "epoch: 2000, acc: 0.507, loss: 0.998\n", - "epoch: 2100, acc: 0.510, loss: 0.989\n", - "epoch: 2200, acc: 0.527, loss: 0.980\n", - "epoch: 2300, acc: 0.540, loss: 0.971\n", - "epoch: 2400, acc: 0.433, loss: 0.976\n", - "epoch: 2500, acc: 0.430, loss: 0.973\n", - "epoch: 2600, acc: 0.430, loss: 0.968\n", - "epoch: 2700, acc: 0.433, loss: 0.964\n", - "epoch: 2800, acc: 0.433, loss: 0.958\n", - "epoch: 2900, acc: 0.440, loss: 0.953\n", - "epoch: 3000, acc: 0.443, loss: 0.948\n", - "epoch: 3100, acc: 0.447, loss: 0.943\n", - "epoch: 3200, acc: 0.457, loss: 0.939\n", - "epoch: 3300, acc: 0.467, loss: 0.935\n", - "epoch: 3400, acc: 0.470, loss: 0.930\n", - "epoch: 3500, acc: 0.477, loss: 0.927\n", - "epoch: 3600, acc: 0.487, loss: 0.923\n", - "epoch: 3700, acc: 0.483, loss: 0.919\n", - "epoch: 3800, acc: 0.480, loss: 0.915\n", - "epoch: 3900, acc: 0.480, loss: 0.911\n", - "epoch: 4000, acc: 0.487, loss: 0.907\n", - "epoch: 4100, acc: 0.503, loss: 0.904\n", - "epoch: 4200, acc: 0.513, loss: 0.900\n", - "epoch: 4300, acc: 0.517, loss: 0.896\n", - "epoch: 4400, acc: 0.523, loss: 0.893\n", - "epoch: 4500, acc: 0.533, loss: 0.889\n", - "epoch: 4600, acc: 0.533, loss: 0.886\n", - "epoch: 4700, acc: 0.537, loss: 0.882\n", - "epoch: 4800, acc: 0.537, loss: 0.878\n", - "epoch: 4900, acc: 0.537, loss: 0.875\n", - "epoch: 5000, acc: 0.547, loss: 0.871\n", - "epoch: 5100, acc: 0.543, loss: 0.868\n", - "epoch: 5200, acc: 0.547, loss: 0.865\n", - "epoch: 5300, acc: 0.550, loss: 0.861\n", - "epoch: 5400, acc: 0.553, loss: 0.857\n", - "epoch: 5500, acc: 0.557, loss: 0.854\n", - "epoch: 5600, acc: 0.567, loss: 0.850\n", - "epoch: 5700, acc: 0.563, loss: 0.846\n", - "epoch: 5800, acc: 0.567, loss: 0.843\n", - "epoch: 5900, acc: 0.567, loss: 0.840\n", - "epoch: 6000, acc: 0.567, loss: 0.837\n", - "epoch: 6100, acc: 0.583, loss: 0.833\n", - "epoch: 6200, acc: 0.587, loss: 0.830\n", - "epoch: 6300, acc: 0.593, loss: 0.827\n", - "epoch: 6400, acc: 0.600, loss: 0.824\n", - "epoch: 6500, acc: 0.600, loss: 0.821\n", - "epoch: 6600, acc: 0.597, loss: 0.818\n", - "epoch: 6700, acc: 0.597, loss: 0.815\n", - "epoch: 6800, acc: 0.597, loss: 0.812\n", - "epoch: 6900, acc: 0.597, loss: 0.809\n", - "epoch: 7000, acc: 0.603, loss: 0.805\n", - "epoch: 7100, acc: 0.603, loss: 0.803\n", - "epoch: 7200, acc: 0.610, loss: 0.800\n", - "epoch: 7300, acc: 0.607, loss: 0.797\n", - "epoch: 7400, acc: 0.613, loss: 0.794\n", - "epoch: 7500, acc: 0.617, loss: 0.792\n", - "epoch: 7600, acc: 0.620, loss: 0.788\n", - "epoch: 7700, acc: 0.620, loss: 0.785\n", - "epoch: 7800, acc: 0.620, loss: 0.783\n", - "epoch: 7900, acc: 0.627, loss: 0.780\n", - "epoch: 8000, acc: 0.627, loss: 0.779\n", - "epoch: 8100, acc: 0.627, loss: 0.775\n", - "epoch: 8200, acc: 0.627, loss: 0.772\n", - "epoch: 8300, acc: 0.630, loss: 0.769\n", - "epoch: 8400, acc: 0.630, loss: 0.765\n", - "epoch: 8500, acc: 0.633, loss: 0.762\n", - "epoch: 8600, acc: 0.630, loss: 0.759\n", - "epoch: 8700, acc: 0.643, loss: 0.755\n", - "epoch: 8800, acc: 0.647, loss: 0.751\n", - "epoch: 8900, acc: 0.647, loss: 0.748\n", - "epoch: 9000, acc: 0.647, loss: 0.743\n", - "epoch: 9100, acc: 0.647, loss: 0.740\n", - "epoch: 9200, acc: 0.647, loss: 0.736\n", - "epoch: 9300, acc: 0.650, loss: 0.732\n", - "epoch: 9400, acc: 0.650, loss: 0.729\n", - "epoch: 9500, acc: 0.647, loss: 0.725\n", - "epoch: 9600, acc: 0.650, loss: 0.722\n", - "epoch: 9700, acc: 0.653, loss: 0.718\n", - "epoch: 9800, acc: 0.657, loss: 0.714\n", - "epoch: 9900, acc: 0.660, loss: 0.711\n", - "epoch: 10000, acc: 0.660, loss: 0.708\n", - "epoch: 10100, acc: 0.663, loss: 0.705\n", - "epoch: 10200, acc: 0.670, loss: 0.701\n", - "epoch: 10300, acc: 0.677, loss: 0.698\n", - "epoch: 10400, acc: 0.680, loss: 0.695\n", - "epoch: 10500, acc: 0.683, loss: 0.692\n", - "epoch: 10600, acc: 0.683, loss: 0.689\n", - "epoch: 10700, acc: 0.680, loss: 0.686\n", - "epoch: 10800, acc: 0.690, loss: 0.683\n", - "epoch: 10900, acc: 0.690, loss: 0.680\n", - "epoch: 11000, acc: 0.690, loss: 0.677\n", - "epoch: 11100, acc: 0.697, loss: 0.674\n", - "epoch: 11200, acc: 0.707, loss: 0.671\n", - "epoch: 11300, acc: 0.707, loss: 0.668\n", - "epoch: 11400, acc: 0.713, loss: 0.665\n", - "epoch: 11500, acc: 0.713, loss: 0.662\n", - "epoch: 11600, acc: 0.717, loss: 0.658\n", - "epoch: 11700, acc: 0.720, loss: 0.655\n", - "epoch: 11800, acc: 0.720, loss: 0.654\n", - "epoch: 11900, acc: 0.727, loss: 0.650\n", - "epoch: 12000, acc: 0.733, loss: 0.647\n", - "epoch: 12100, acc: 0.733, loss: 0.644\n", - "epoch: 12200, acc: 0.733, loss: 0.641\n", - "epoch: 12300, acc: 0.733, loss: 0.639\n", - "epoch: 12400, acc: 0.733, loss: 0.637\n", - "epoch: 12500, acc: 0.727, loss: 0.634\n", - "epoch: 12600, acc: 0.723, loss: 0.630\n", - "epoch: 12700, acc: 0.733, loss: 0.627\n", - "epoch: 12800, acc: 0.733, loss: 0.626\n", - "epoch: 12900, acc: 0.737, loss: 0.622\n", - "epoch: 13000, acc: 0.737, loss: 0.619\n", - "epoch: 13100, acc: 0.740, loss: 0.617\n", - "epoch: 13200, acc: 0.743, loss: 0.614\n", - "epoch: 13300, acc: 0.743, loss: 0.611\n", - "epoch: 13400, acc: 0.740, loss: 0.608\n", - "epoch: 13500, acc: 0.740, loss: 0.606\n", - "epoch: 13600, acc: 0.737, loss: 0.604\n", - "epoch: 13700, acc: 0.743, loss: 0.602\n", - "epoch: 13800, acc: 0.747, loss: 0.600\n", - "epoch: 13900, acc: 0.747, loss: 0.598\n", - "epoch: 14000, acc: 0.757, loss: 0.596\n", - "epoch: 14100, acc: 0.753, loss: 0.594\n", - "epoch: 14200, acc: 0.760, loss: 0.592\n", - "epoch: 14300, acc: 0.760, loss: 0.590\n", - "epoch: 14400, acc: 0.757, loss: 0.589\n", - "epoch: 14500, acc: 0.763, loss: 0.587\n", - "epoch: 14600, acc: 0.763, loss: 0.585\n", - "epoch: 14700, acc: 0.773, loss: 0.584\n", - "epoch: 14800, acc: 0.777, loss: 0.582\n", - "epoch: 14900, acc: 0.777, loss: 0.580\n", - "epoch: 15000, acc: 0.783, loss: 0.579\n", - "epoch: 15100, acc: 0.783, loss: 0.577\n", - "epoch: 15200, acc: 0.783, loss: 0.576\n", - "epoch: 15300, acc: 0.790, loss: 0.575\n", - "epoch: 15400, acc: 0.787, loss: 0.573\n", - "epoch: 15500, acc: 0.790, loss: 0.572\n", - "epoch: 15600, acc: 0.790, loss: 0.570\n", - "epoch: 15700, acc: 0.790, loss: 0.569\n", - "epoch: 15800, acc: 0.793, loss: 0.568\n", - "epoch: 15900, acc: 0.793, loss: 0.566\n", - "epoch: 16000, acc: 0.793, loss: 0.565\n", - "epoch: 16100, acc: 0.793, loss: 0.564\n", - "epoch: 16200, acc: 0.790, loss: 0.562\n", - "epoch: 16300, acc: 0.790, loss: 0.561\n", - "epoch: 16400, acc: 0.790, loss: 0.560\n", - "epoch: 16500, acc: 0.790, loss: 0.559\n", - "epoch: 16600, acc: 0.793, loss: 0.558\n", - "epoch: 16700, acc: 0.793, loss: 0.557\n", - "epoch: 16800, acc: 0.793, loss: 0.555\n", - "epoch: 16900, acc: 0.797, loss: 0.554\n", - "epoch: 17000, acc: 0.797, loss: 0.553\n", - "epoch: 17100, acc: 0.797, loss: 0.552\n", - "epoch: 17200, acc: 0.797, loss: 0.551\n", - "epoch: 17300, acc: 0.797, loss: 0.550\n", - "epoch: 17400, acc: 0.797, loss: 0.549\n", - "epoch: 17500, acc: 0.797, loss: 0.548\n", - "epoch: 17600, acc: 0.797, loss: 0.547\n", - "epoch: 17700, acc: 0.797, loss: 0.546\n", - "epoch: 17800, acc: 0.797, loss: 0.545\n", - "epoch: 17900, acc: 0.793, loss: 0.544\n", - "epoch: 18000, acc: 0.790, loss: 0.543\n", - "epoch: 18100, acc: 0.793, loss: 0.542\n", - "epoch: 18200, acc: 0.793, loss: 0.542\n", - "epoch: 18300, acc: 0.793, loss: 0.541\n", - "epoch: 18400, acc: 0.793, loss: 0.540\n", - "epoch: 18500, acc: 0.793, loss: 0.539\n", - "epoch: 18600, acc: 0.793, loss: 0.538\n", - "epoch: 18700, acc: 0.790, loss: 0.537\n", - "epoch: 18800, acc: 0.793, loss: 0.536\n", - "epoch: 18900, acc: 0.793, loss: 0.535\n", - "epoch: 19000, acc: 0.793, loss: 0.535\n", - "epoch: 19100, acc: 0.793, loss: 0.534\n", - "epoch: 19200, acc: 0.793, loss: 0.533\n", - "epoch: 19300, acc: 0.793, loss: 0.532\n", - "epoch: 19400, acc: 0.793, loss: 0.531\n", - "epoch: 19500, acc: 0.793, loss: 0.531\n", - "epoch: 19600, acc: 0.793, loss: 0.530\n", - "epoch: 19700, acc: 0.793, loss: 0.529\n", - "epoch: 19800, acc: 0.793, loss: 0.528\n", - "epoch: 19900, acc: 0.793, loss: 0.528\n", - "epoch: 20000, acc: 0.793, loss: 0.527\n", - "epoch: 20100, acc: 0.797, loss: 0.526\n", - "epoch: 20200, acc: 0.797, loss: 0.525\n", - "epoch: 20300, acc: 0.797, loss: 0.525\n", - "epoch: 20400, acc: 0.797, loss: 0.524\n", - "epoch: 20500, acc: 0.797, loss: 0.523\n", - "epoch: 20600, acc: 0.797, loss: 0.523\n", - "epoch: 20700, acc: 0.797, loss: 0.522\n", - "epoch: 20800, acc: 0.800, loss: 0.521\n", - "epoch: 20900, acc: 0.800, loss: 0.521\n", - "epoch: 21000, acc: 0.800, loss: 0.520\n", - "epoch: 21100, acc: 0.800, loss: 0.519\n", - "epoch: 21200, acc: 0.800, loss: 0.519\n", - "epoch: 21300, acc: 0.800, loss: 0.518\n", - "epoch: 21400, acc: 0.800, loss: 0.518\n", - "epoch: 21500, acc: 0.800, loss: 0.517\n", - "epoch: 21600, acc: 0.800, loss: 0.516\n", - "epoch: 21700, acc: 0.800, loss: 0.516\n", - "epoch: 21800, acc: 0.800, loss: 0.515\n", - "epoch: 21900, acc: 0.800, loss: 0.515\n", - "epoch: 22000, acc: 0.800, loss: 0.514\n", - "epoch: 22100, acc: 0.797, loss: 0.513\n", - "epoch: 22200, acc: 0.800, loss: 0.513\n", - "epoch: 22300, acc: 0.797, loss: 0.512\n", - "epoch: 22400, acc: 0.797, loss: 0.512\n", - "epoch: 22500, acc: 0.797, loss: 0.511\n", - "epoch: 22600, acc: 0.797, loss: 0.510\n", - "epoch: 22700, acc: 0.797, loss: 0.510\n", - "epoch: 22800, acc: 0.797, loss: 0.509\n", - "epoch: 22900, acc: 0.797, loss: 0.509\n", - "epoch: 23000, acc: 0.797, loss: 0.508\n", - "epoch: 23100, acc: 0.800, loss: 0.508\n", - "epoch: 23200, acc: 0.797, loss: 0.507\n", - "epoch: 23300, acc: 0.797, loss: 0.507\n", - "epoch: 23400, acc: 0.797, loss: 0.506\n", - "epoch: 23500, acc: 0.797, loss: 0.506\n", - "epoch: 23600, acc: 0.800, loss: 0.505\n", - "epoch: 23700, acc: 0.803, loss: 0.505\n", - "epoch: 23800, acc: 0.803, loss: 0.504\n", - "epoch: 23900, acc: 0.800, loss: 0.504\n", - "epoch: 24000, acc: 0.803, loss: 0.503\n", - "epoch: 24100, acc: 0.807, loss: 0.503\n", - "epoch: 24200, acc: 0.807, loss: 0.502\n", - "epoch: 24300, acc: 0.807, loss: 0.502\n", - "epoch: 24400, acc: 0.807, loss: 0.501\n", - "epoch: 24500, acc: 0.807, loss: 0.501\n", - "epoch: 24600, acc: 0.807, loss: 0.500\n", - "epoch: 24700, acc: 0.807, loss: 0.500\n", - "epoch: 24800, acc: 0.807, loss: 0.499\n", - "epoch: 24900, acc: 0.807, loss: 0.499\n", - "epoch: 25000, acc: 0.807, loss: 0.498\n", - "epoch: 25100, acc: 0.807, loss: 0.498\n", - "epoch: 25200, acc: 0.807, loss: 0.497\n", - "epoch: 25300, acc: 0.807, loss: 0.497\n", - "epoch: 25400, acc: 0.807, loss: 0.497\n", - "epoch: 25500, acc: 0.803, loss: 0.496\n", - "epoch: 25600, acc: 0.803, loss: 0.496\n", - "epoch: 25700, acc: 0.803, loss: 0.495\n", - "epoch: 25800, acc: 0.803, loss: 0.495\n", - "epoch: 25900, acc: 0.803, loss: 0.494\n", - "epoch: 26000, acc: 0.800, loss: 0.494\n", - "epoch: 26100, acc: 0.803, loss: 0.493\n", - "epoch: 26200, acc: 0.800, loss: 0.493\n", - "epoch: 26300, acc: 0.800, loss: 0.492\n", - "epoch: 26400, acc: 0.800, loss: 0.492\n", - "epoch: 26500, acc: 0.800, loss: 0.492\n", - "epoch: 26600, acc: 0.800, loss: 0.491\n", - "epoch: 26700, acc: 0.803, loss: 0.491\n", - "epoch: 26800, acc: 0.800, loss: 0.490\n", - "epoch: 26900, acc: 0.800, loss: 0.490\n", - "epoch: 27000, acc: 0.800, loss: 0.490\n", - "epoch: 27100, acc: 0.803, loss: 0.489\n", - "epoch: 27200, acc: 0.803, loss: 0.489\n", - "epoch: 27300, acc: 0.803, loss: 0.488\n", - "epoch: 27400, acc: 0.803, loss: 0.488\n", - "epoch: 27500, acc: 0.803, loss: 0.488\n", - "epoch: 27600, acc: 0.803, loss: 0.487\n", - "epoch: 27700, acc: 0.803, loss: 0.487\n", - "epoch: 27800, acc: 0.803, loss: 0.487\n", - "epoch: 27900, acc: 0.803, loss: 0.486\n", - "epoch: 28000, acc: 0.803, loss: 0.486\n", - "epoch: 28100, acc: 0.803, loss: 0.485\n", - "epoch: 28200, acc: 0.803, loss: 0.485\n", - "epoch: 28300, acc: 0.803, loss: 0.485\n", - "epoch: 28400, acc: 0.803, loss: 0.484\n", - "epoch: 28500, acc: 0.803, loss: 0.484\n", - "epoch: 28600, acc: 0.803, loss: 0.484\n", - "epoch: 28700, acc: 0.800, loss: 0.483\n", - "epoch: 28800, acc: 0.803, loss: 0.483\n", - "epoch: 28900, acc: 0.800, loss: 0.483\n", - "epoch: 29000, acc: 0.800, loss: 0.482\n", - "epoch: 29100, acc: 0.800, loss: 0.482\n", - "epoch: 29200, acc: 0.800, loss: 0.482\n", - "epoch: 29300, acc: 0.800, loss: 0.481\n", - "epoch: 29400, acc: 0.800, loss: 0.481\n", - "epoch: 29500, acc: 0.800, loss: 0.481\n", - "epoch: 29600, acc: 0.800, loss: 0.480\n", - "epoch: 29700, acc: 0.800, loss: 0.480\n", - "epoch: 29800, acc: 0.800, loss: 0.480\n", - "epoch: 29900, acc: 0.800, loss: 0.479\n", - "epoch: 30000, acc: 0.800, loss: 0.479\n", - "epoch: 30100, acc: 0.800, loss: 0.479\n", - "epoch: 30200, acc: 0.800, loss: 0.478\n", - "epoch: 30300, acc: 0.800, loss: 0.478\n", - "epoch: 30400, acc: 0.800, loss: 0.478\n", - "epoch: 30500, acc: 0.800, loss: 0.477\n", - "epoch: 30600, acc: 0.800, loss: 0.477\n", - "epoch: 30700, acc: 0.800, loss: 0.477\n", - "epoch: 30800, acc: 0.800, loss: 0.476\n", - "epoch: 30900, acc: 0.800, loss: 0.476\n", - "epoch: 31000, acc: 0.800, loss: 0.476\n", - "epoch: 31100, acc: 0.800, loss: 0.476\n", - "epoch: 31200, acc: 0.800, loss: 0.475\n", - "epoch: 31300, acc: 0.800, loss: 0.475\n", - "epoch: 31400, acc: 0.800, loss: 0.475\n", - "epoch: 31500, acc: 0.800, loss: 0.474\n", - "epoch: 31600, acc: 0.800, loss: 0.474\n", - "epoch: 31700, acc: 0.800, loss: 0.474\n", - "epoch: 31800, acc: 0.800, loss: 0.473\n", - "epoch: 31900, acc: 0.800, loss: 0.473\n", - "epoch: 32000, acc: 0.800, loss: 0.473\n", - "epoch: 32100, acc: 0.800, loss: 0.473\n", - "epoch: 32200, acc: 0.800, loss: 0.472\n", - "epoch: 32300, acc: 0.800, loss: 0.472\n", - "epoch: 32400, acc: 0.800, loss: 0.472\n", - "epoch: 32500, acc: 0.800, loss: 0.471\n", - "epoch: 32600, acc: 0.800, loss: 0.471\n", - "epoch: 32700, acc: 0.800, loss: 0.471\n", - "epoch: 32800, acc: 0.800, loss: 0.471\n", - "epoch: 32900, acc: 0.800, loss: 0.470\n", - "epoch: 33000, acc: 0.800, loss: 0.470\n", - "epoch: 33100, acc: 0.800, loss: 0.470\n", - "epoch: 33200, acc: 0.800, loss: 0.469\n", - "epoch: 33300, acc: 0.800, loss: 0.469\n", - "epoch: 33400, acc: 0.800, loss: 0.469\n", - "epoch: 33500, acc: 0.800, loss: 0.469\n", - "epoch: 33600, acc: 0.800, loss: 0.468\n", - "epoch: 33700, acc: 0.800, loss: 0.468\n", - "epoch: 33800, acc: 0.800, loss: 0.468\n", - "epoch: 33900, acc: 0.800, loss: 0.468\n", - "epoch: 34000, acc: 0.800, loss: 0.467\n", - "epoch: 34100, acc: 0.800, loss: 0.467\n", - "epoch: 34200, acc: 0.800, loss: 0.467\n", - "epoch: 34300, acc: 0.800, loss: 0.467\n", - "epoch: 34400, acc: 0.800, loss: 0.466\n", - "epoch: 34500, acc: 0.800, loss: 0.466\n", - "epoch: 34600, acc: 0.800, loss: 0.466\n", - "epoch: 34700, acc: 0.800, loss: 0.465\n", - "epoch: 34800, acc: 0.800, loss: 0.465\n", - "epoch: 34900, acc: 0.800, loss: 0.465\n", - "epoch: 35000, acc: 0.800, loss: 0.464\n", - "epoch: 35100, acc: 0.800, loss: 0.464\n", - "epoch: 35200, acc: 0.800, loss: 0.464\n", - "epoch: 35300, acc: 0.800, loss: 0.464\n", - "epoch: 35400, acc: 0.800, loss: 0.463\n", - "epoch: 35500, acc: 0.800, loss: 0.463\n", - "epoch: 35600, acc: 0.800, loss: 0.463\n", - "epoch: 35700, acc: 0.800, loss: 0.462\n", - "epoch: 35800, acc: 0.800, loss: 0.462\n", - "epoch: 35900, acc: 0.800, loss: 0.462\n", - "epoch: 36000, acc: 0.800, loss: 0.462\n", - "epoch: 36100, acc: 0.800, loss: 0.461\n", - "epoch: 36200, acc: 0.800, loss: 0.461\n", - "epoch: 36300, acc: 0.800, loss: 0.461\n", - "epoch: 36400, acc: 0.800, loss: 0.460\n", - "epoch: 36500, acc: 0.800, loss: 0.460\n", - "epoch: 36600, acc: 0.800, loss: 0.460\n", - "epoch: 36700, acc: 0.800, loss: 0.460\n", - "epoch: 36800, acc: 0.800, loss: 0.459\n", - "epoch: 36900, acc: 0.800, loss: 0.459\n", - "epoch: 37000, acc: 0.800, loss: 0.459\n", - "epoch: 37100, acc: 0.800, loss: 0.458\n", - "epoch: 37200, acc: 0.797, loss: 0.458\n", - "epoch: 37300, acc: 0.800, loss: 0.458\n", - "epoch: 37400, acc: 0.800, loss: 0.457\n", - "epoch: 37500, acc: 0.800, loss: 0.457\n", - "epoch: 37600, acc: 0.800, loss: 0.457\n", - "epoch: 37700, acc: 0.800, loss: 0.456\n", - "epoch: 37800, acc: 0.800, loss: 0.456\n", - "epoch: 37900, acc: 0.800, loss: 0.456\n", - "epoch: 38000, acc: 0.800, loss: 0.455\n", - "epoch: 38100, acc: 0.800, loss: 0.455\n", - "epoch: 38200, acc: 0.800, loss: 0.455\n", - "epoch: 38300, acc: 0.800, loss: 0.454\n", - "epoch: 38400, acc: 0.800, loss: 0.454\n", - "epoch: 38500, acc: 0.800, loss: 0.454\n", - "epoch: 38600, acc: 0.800, loss: 0.454\n", - "epoch: 38700, acc: 0.800, loss: 0.453\n", - "epoch: 38800, acc: 0.800, loss: 0.453\n", - "epoch: 38900, acc: 0.800, loss: 0.453\n", - "epoch: 39000, acc: 0.800, loss: 0.453\n", - "epoch: 39100, acc: 0.800, loss: 0.452\n", - "epoch: 39200, acc: 0.800, loss: 0.452\n", - "epoch: 39300, acc: 0.800, loss: 0.452\n", - "epoch: 39400, acc: 0.800, loss: 0.451\n", - "epoch: 39500, acc: 0.800, loss: 0.451\n", - "epoch: 39600, acc: 0.800, loss: 0.451\n", - "epoch: 39700, acc: 0.803, loss: 0.451\n", - "epoch: 39800, acc: 0.807, loss: 0.450\n", - "epoch: 39900, acc: 0.807, loss: 0.450\n", - "epoch: 40000, acc: 0.807, loss: 0.450\n", - "epoch: 40100, acc: 0.807, loss: 0.450\n", - "epoch: 40200, acc: 0.807, loss: 0.449\n", - "epoch: 40300, acc: 0.807, loss: 0.449\n", - "epoch: 40400, acc: 0.807, loss: 0.449\n", - "epoch: 40500, acc: 0.803, loss: 0.449\n", - "epoch: 40600, acc: 0.803, loss: 0.448\n", - "epoch: 40700, acc: 0.803, loss: 0.448\n", - "epoch: 40800, acc: 0.803, loss: 0.448\n", - "epoch: 40900, acc: 0.807, loss: 0.448\n", - "epoch: 41000, acc: 0.807, loss: 0.447\n", - "epoch: 41100, acc: 0.807, loss: 0.447\n", - "epoch: 41200, acc: 0.807, loss: 0.447\n", - "epoch: 41300, acc: 0.807, loss: 0.447\n", - "epoch: 41400, acc: 0.807, loss: 0.446\n", - "epoch: 41500, acc: 0.807, loss: 0.446\n", - "epoch: 41600, acc: 0.810, loss: 0.446\n", - "epoch: 41700, acc: 0.807, loss: 0.446\n", - "epoch: 41800, acc: 0.807, loss: 0.446\n", - "epoch: 41900, acc: 0.807, loss: 0.445\n", - "epoch: 42000, acc: 0.807, loss: 0.445\n", - "epoch: 42100, acc: 0.807, loss: 0.445\n", - "epoch: 42200, acc: 0.807, loss: 0.445\n", - "epoch: 42300, acc: 0.807, loss: 0.444\n", - "epoch: 42400, acc: 0.807, loss: 0.444\n", - "epoch: 42500, acc: 0.807, loss: 0.444\n", - "epoch: 42600, acc: 0.807, loss: 0.444\n", - "epoch: 42700, acc: 0.807, loss: 0.444\n", - "epoch: 42800, acc: 0.807, loss: 0.443\n", - "epoch: 42900, acc: 0.807, loss: 0.443\n", - "epoch: 43000, acc: 0.807, loss: 0.443\n", - "epoch: 43100, acc: 0.807, loss: 0.443\n", - "epoch: 43200, acc: 0.807, loss: 0.443\n", - "epoch: 43300, acc: 0.807, loss: 0.442\n", - "epoch: 43400, acc: 0.807, loss: 0.442\n", - "epoch: 43500, acc: 0.807, loss: 0.442\n", - "epoch: 43600, acc: 0.807, loss: 0.442\n", - "epoch: 43700, acc: 0.807, loss: 0.442\n", - "epoch: 43800, acc: 0.807, loss: 0.441\n", - "epoch: 43900, acc: 0.807, loss: 0.441\n", - "epoch: 44000, acc: 0.807, loss: 0.441\n", - "epoch: 44100, acc: 0.807, loss: 0.441\n", - "epoch: 44200, acc: 0.807, loss: 0.441\n", - "epoch: 44300, acc: 0.807, loss: 0.440\n", - "epoch: 44400, acc: 0.807, loss: 0.440\n", - "epoch: 44500, acc: 0.807, loss: 0.440\n", - "epoch: 44600, acc: 0.807, loss: 0.440\n", - "epoch: 44700, acc: 0.807, loss: 0.440\n", - "epoch: 44800, acc: 0.807, loss: 0.439\n", - "epoch: 44900, acc: 0.810, loss: 0.439\n", - "epoch: 45000, acc: 0.807, loss: 0.439\n", - "epoch: 45100, acc: 0.807, loss: 0.439\n", - "epoch: 45200, acc: 0.810, loss: 0.439\n", - "epoch: 45300, acc: 0.810, loss: 0.438\n", - "epoch: 45400, acc: 0.810, loss: 0.438\n", - "epoch: 45500, acc: 0.810, loss: 0.438\n", - "epoch: 45600, acc: 0.810, loss: 0.438\n", - "epoch: 45700, acc: 0.810, loss: 0.438\n", - "epoch: 45800, acc: 0.810, loss: 0.437\n", - "epoch: 45900, acc: 0.810, loss: 0.437\n", - "epoch: 46000, acc: 0.810, loss: 0.437\n", - "epoch: 46100, acc: 0.810, loss: 0.437\n", - "epoch: 46200, acc: 0.810, loss: 0.437\n", - "epoch: 46300, acc: 0.810, loss: 0.437\n", - "epoch: 46400, acc: 0.810, loss: 0.436\n", - "epoch: 46500, acc: 0.810, loss: 0.436\n", - "epoch: 46600, acc: 0.810, loss: 0.436\n", - "epoch: 46700, acc: 0.810, loss: 0.436\n", - "epoch: 46800, acc: 0.810, loss: 0.436\n", - "epoch: 46900, acc: 0.810, loss: 0.435\n", - "epoch: 47000, acc: 0.810, loss: 0.435\n", - "epoch: 47100, acc: 0.810, loss: 0.435\n", - "epoch: 47200, acc: 0.813, loss: 0.435\n", - "epoch: 47300, acc: 0.813, loss: 0.435\n", - "epoch: 47400, acc: 0.813, loss: 0.435\n", - "epoch: 47500, acc: 0.813, loss: 0.434\n", - "epoch: 47600, acc: 0.813, loss: 0.434\n", - "epoch: 47700, acc: 0.813, loss: 0.434\n", - "epoch: 47800, acc: 0.813, loss: 0.434\n", - "epoch: 47900, acc: 0.817, loss: 0.434\n", - "epoch: 48000, acc: 0.817, loss: 0.433\n", - "epoch: 48100, acc: 0.817, loss: 0.433\n", - "epoch: 48200, acc: 0.813, loss: 0.433\n", - "epoch: 48300, acc: 0.817, loss: 0.433\n", - "epoch: 48400, acc: 0.817, loss: 0.433\n", - "epoch: 48500, acc: 0.813, loss: 0.433\n", - "epoch: 48600, acc: 0.817, loss: 0.432\n", - "epoch: 48700, acc: 0.817, loss: 0.432\n", - "epoch: 48800, acc: 0.817, loss: 0.432\n", - "epoch: 48900, acc: 0.817, loss: 0.432\n", - "epoch: 49000, acc: 0.817, loss: 0.432\n", - "epoch: 49100, acc: 0.817, loss: 0.432\n", - "epoch: 49200, acc: 0.817, loss: 0.431\n", - "epoch: 49300, acc: 0.817, loss: 0.431\n", - "epoch: 49400, acc: 0.817, loss: 0.431\n", - "epoch: 49500, acc: 0.817, loss: 0.431\n", - "epoch: 49600, acc: 0.817, loss: 0.431\n", - "epoch: 49700, acc: 0.817, loss: 0.431\n", - "epoch: 49800, acc: 0.817, loss: 0.430\n", - "epoch: 49900, acc: 0.817, loss: 0.430\n", - "epoch: 50000, acc: 0.817, loss: 0.430\n", - "epoch: 50100, acc: 0.820, loss: 0.430\n", - "epoch: 50200, acc: 0.820, loss: 0.430\n", - "epoch: 50300, acc: 0.820, loss: 0.429\n", - "epoch: 50400, acc: 0.820, loss: 0.429\n", - "epoch: 50500, acc: 0.820, loss: 0.429\n", - "epoch: 50600, acc: 0.820, loss: 0.429\n", - "epoch: 50700, acc: 0.820, loss: 0.429\n", - "epoch: 50800, acc: 0.820, loss: 0.429\n", - "epoch: 50900, acc: 0.820, loss: 0.428\n", - "epoch: 51000, acc: 0.820, loss: 0.428\n", - "epoch: 51100, acc: 0.820, loss: 0.428\n", - "epoch: 51200, acc: 0.820, loss: 0.428\n", - "epoch: 51300, acc: 0.820, loss: 0.428\n", - "epoch: 51400, acc: 0.820, loss: 0.428\n", - "epoch: 51500, acc: 0.820, loss: 0.427\n", - "epoch: 51600, acc: 0.820, loss: 0.427\n", - "epoch: 51700, acc: 0.820, loss: 0.427\n", - "epoch: 51800, acc: 0.820, loss: 0.427\n", - "epoch: 51900, acc: 0.820, loss: 0.427\n", - "epoch: 52000, acc: 0.820, loss: 0.427\n", - "epoch: 52100, acc: 0.820, loss: 0.426\n", - "epoch: 52200, acc: 0.820, loss: 0.426\n", - "epoch: 52300, acc: 0.820, loss: 0.426\n", - "epoch: 52400, acc: 0.820, loss: 0.426\n", - "epoch: 52500, acc: 0.820, loss: 0.426\n", - "epoch: 52600, acc: 0.820, loss: 0.426\n", - "epoch: 52700, acc: 0.820, loss: 0.426\n", - "epoch: 52800, acc: 0.820, loss: 0.425\n", - "epoch: 52900, acc: 0.820, loss: 0.425\n", - "epoch: 53000, acc: 0.820, loss: 0.425\n", - "epoch: 53100, acc: 0.820, loss: 0.425\n", - "epoch: 53200, acc: 0.820, loss: 0.425\n", - "epoch: 53300, acc: 0.820, loss: 0.425\n", - "epoch: 53400, acc: 0.820, loss: 0.424\n", - "epoch: 53500, acc: 0.820, loss: 0.424\n", - "epoch: 53600, acc: 0.820, loss: 0.424\n", - "epoch: 53700, acc: 0.820, loss: 0.424\n", - "epoch: 53800, acc: 0.820, loss: 0.424\n", - "epoch: 53900, acc: 0.820, loss: 0.424\n", - "epoch: 54000, acc: 0.820, loss: 0.424\n", - "epoch: 54100, acc: 0.820, loss: 0.423\n", - "epoch: 54200, acc: 0.820, loss: 0.423\n", - "epoch: 54300, acc: 0.820, loss: 0.423\n", - "epoch: 54400, acc: 0.820, loss: 0.423\n", - "epoch: 54500, acc: 0.820, loss: 0.423\n", - "epoch: 54600, acc: 0.823, loss: 0.423\n", - "epoch: 54700, acc: 0.823, loss: 0.422\n", - "epoch: 54800, acc: 0.823, loss: 0.422\n", - "epoch: 54900, acc: 0.823, loss: 0.422\n", - "epoch: 55000, acc: 0.823, loss: 0.422\n", - "epoch: 55100, acc: 0.823, loss: 0.422\n", - "epoch: 55200, acc: 0.823, loss: 0.422\n", - "epoch: 55300, acc: 0.823, loss: 0.422\n", - "epoch: 55400, acc: 0.823, loss: 0.421\n", - "epoch: 55500, acc: 0.823, loss: 0.421\n", - "epoch: 55600, acc: 0.823, loss: 0.421\n", - "epoch: 55700, acc: 0.823, loss: 0.421\n", - "epoch: 55800, acc: 0.823, loss: 0.421\n", - "epoch: 55900, acc: 0.823, loss: 0.421\n", - "epoch: 56000, acc: 0.823, loss: 0.421\n", - "epoch: 56100, acc: 0.823, loss: 0.420\n", - "epoch: 56200, acc: 0.823, loss: 0.420\n", - "epoch: 56300, acc: 0.823, loss: 0.420\n", - "epoch: 56400, acc: 0.823, loss: 0.420\n", - "epoch: 56500, acc: 0.823, loss: 0.420\n", - "epoch: 56600, acc: 0.823, loss: 0.420\n", - "epoch: 56700, acc: 0.823, loss: 0.420\n", - "epoch: 56800, acc: 0.823, loss: 0.419\n", - "epoch: 56900, acc: 0.820, loss: 0.419\n", - "epoch: 57000, acc: 0.820, loss: 0.419\n", - "epoch: 57100, acc: 0.820, loss: 0.419\n", - "epoch: 57200, acc: 0.820, loss: 0.419\n", - "epoch: 57300, acc: 0.820, loss: 0.419\n", - "epoch: 57400, acc: 0.820, loss: 0.419\n", - "epoch: 57500, acc: 0.820, loss: 0.418\n", - "epoch: 57600, acc: 0.820, loss: 0.418\n", - "epoch: 57700, acc: 0.820, loss: 0.418\n", - "epoch: 57800, acc: 0.820, loss: 0.418\n", - "epoch: 57900, acc: 0.820, loss: 0.418\n", - "epoch: 58000, acc: 0.820, loss: 0.418\n", - "epoch: 58100, acc: 0.820, loss: 0.418\n", - "epoch: 58200, acc: 0.820, loss: 0.417\n", - "epoch: 58300, acc: 0.820, loss: 0.417\n", - "epoch: 58400, acc: 0.820, loss: 0.417\n", - "epoch: 58500, acc: 0.820, loss: 0.417\n", - "epoch: 58600, acc: 0.820, loss: 0.417\n", - "epoch: 58700, acc: 0.823, loss: 0.417\n", - "epoch: 58800, acc: 0.823, loss: 0.417\n", - "epoch: 58900, acc: 0.823, loss: 0.417\n", - "epoch: 59000, acc: 0.823, loss: 0.416\n", - "epoch: 59100, acc: 0.823, loss: 0.416\n", - "epoch: 59200, acc: 0.823, loss: 0.416\n", - "epoch: 59300, acc: 0.823, loss: 0.416\n", - "epoch: 59400, acc: 0.823, loss: 0.416\n", - "epoch: 59500, acc: 0.823, loss: 0.416\n", - "epoch: 59600, acc: 0.823, loss: 0.416\n", - "epoch: 59700, acc: 0.823, loss: 0.415\n", - "epoch: 59800, acc: 0.823, loss: 0.415\n", - "epoch: 59900, acc: 0.823, loss: 0.415\n", - "epoch: 60000, acc: 0.823, loss: 0.415\n", - "epoch: 60100, acc: 0.823, loss: 0.415\n", - "epoch: 60200, acc: 0.823, loss: 0.415\n", - "epoch: 60300, acc: 0.820, loss: 0.415\n", - "epoch: 60400, acc: 0.820, loss: 0.415\n", - "epoch: 60500, acc: 0.820, loss: 0.414\n", - "epoch: 60600, acc: 0.820, loss: 0.414\n", - "epoch: 60700, acc: 0.823, loss: 0.414\n", - "epoch: 60800, acc: 0.820, loss: 0.414\n", - "epoch: 60900, acc: 0.823, loss: 0.414\n", - "epoch: 61000, acc: 0.823, loss: 0.414\n", - "epoch: 61100, acc: 0.823, loss: 0.414\n", - "epoch: 61200, acc: 0.823, loss: 0.414\n", - "epoch: 61300, acc: 0.827, loss: 0.413\n", - "epoch: 61400, acc: 0.827, loss: 0.413\n", - "epoch: 61500, acc: 0.827, loss: 0.413\n", - "epoch: 61600, acc: 0.827, loss: 0.413\n", - "epoch: 61700, acc: 0.827, loss: 0.413\n", - "epoch: 61800, acc: 0.827, loss: 0.413\n", - "epoch: 61900, acc: 0.827, loss: 0.413\n", - "epoch: 62000, acc: 0.827, loss: 0.413\n", - "epoch: 62100, acc: 0.827, loss: 0.412\n", - "epoch: 62200, acc: 0.827, loss: 0.412\n", - "epoch: 62300, acc: 0.827, loss: 0.412\n", - "epoch: 62400, acc: 0.827, loss: 0.412\n", - "epoch: 62500, acc: 0.827, loss: 0.412\n", - "epoch: 62600, acc: 0.827, loss: 0.412\n", - "epoch: 62700, acc: 0.827, loss: 0.412\n", - "epoch: 62800, acc: 0.827, loss: 0.412\n", - "epoch: 62900, acc: 0.827, loss: 0.412\n", - "epoch: 63000, acc: 0.827, loss: 0.411\n", - "epoch: 63100, acc: 0.827, loss: 0.411\n", - "epoch: 63200, acc: 0.827, loss: 0.411\n", - "epoch: 63300, acc: 0.827, loss: 0.411\n", - "epoch: 63400, acc: 0.827, loss: 0.411\n", - "epoch: 63500, acc: 0.830, loss: 0.411\n", - "epoch: 63600, acc: 0.830, loss: 0.411\n", - "epoch: 63700, acc: 0.830, loss: 0.411\n", - "epoch: 63800, acc: 0.830, loss: 0.410\n", - "epoch: 63900, acc: 0.830, loss: 0.410\n", - "epoch: 64000, acc: 0.830, loss: 0.410\n", - "epoch: 64100, acc: 0.830, loss: 0.410\n", - "epoch: 64200, acc: 0.830, loss: 0.410\n", - "epoch: 64300, acc: 0.830, loss: 0.410\n", - "epoch: 64400, acc: 0.830, loss: 0.410\n", - "epoch: 64500, acc: 0.830, loss: 0.410\n", - "epoch: 64600, acc: 0.830, loss: 0.410\n", - "epoch: 64700, acc: 0.833, loss: 0.409\n", - "epoch: 64800, acc: 0.833, loss: 0.409\n", - "epoch: 64900, acc: 0.833, loss: 0.409\n", - "epoch: 65000, acc: 0.833, loss: 0.409\n", - "epoch: 65100, acc: 0.833, loss: 0.409\n", - "epoch: 65200, acc: 0.833, loss: 0.409\n", - "epoch: 65300, acc: 0.833, loss: 0.409\n", - "epoch: 65400, acc: 0.833, loss: 0.409\n", - "epoch: 65500, acc: 0.837, loss: 0.409\n", - "epoch: 65600, acc: 0.837, loss: 0.408\n", - "epoch: 65700, acc: 0.837, loss: 0.408\n", - "epoch: 65800, acc: 0.837, loss: 0.408\n", - "epoch: 65900, acc: 0.837, loss: 0.408\n", - "epoch: 66000, acc: 0.837, loss: 0.408\n", - "epoch: 66100, acc: 0.837, loss: 0.408\n", - "epoch: 66200, acc: 0.833, loss: 0.408\n", - "epoch: 66300, acc: 0.833, loss: 0.408\n", - "epoch: 66400, acc: 0.833, loss: 0.408\n", - "epoch: 66500, acc: 0.833, loss: 0.408\n", - "epoch: 66600, acc: 0.833, loss: 0.407\n", - "epoch: 66700, acc: 0.833, loss: 0.407\n", - "epoch: 66800, acc: 0.833, loss: 0.407\n", - "epoch: 66900, acc: 0.833, loss: 0.407\n", - "epoch: 67000, acc: 0.830, loss: 0.407\n", - "epoch: 67100, acc: 0.833, loss: 0.407\n", - "epoch: 67200, acc: 0.830, loss: 0.407\n", - "epoch: 67300, acc: 0.830, loss: 0.407\n", - "epoch: 67400, acc: 0.830, loss: 0.407\n", - "epoch: 67500, acc: 0.830, loss: 0.406\n", - "epoch: 67600, acc: 0.830, loss: 0.406\n", - "epoch: 67700, acc: 0.830, loss: 0.406\n", - "epoch: 67800, acc: 0.830, loss: 0.406\n", - "epoch: 67900, acc: 0.830, loss: 0.406\n", - "epoch: 68000, acc: 0.830, loss: 0.406\n", - "epoch: 68100, acc: 0.830, loss: 0.406\n", - "epoch: 68200, acc: 0.830, loss: 0.406\n", - "epoch: 68300, acc: 0.830, loss: 0.406\n", - "epoch: 68400, acc: 0.833, loss: 0.406\n", - "epoch: 68500, acc: 0.833, loss: 0.405\n", - "epoch: 68600, acc: 0.830, loss: 0.405\n", - "epoch: 68700, acc: 0.833, loss: 0.405\n", - "epoch: 68800, acc: 0.837, loss: 0.405\n", - "epoch: 68900, acc: 0.837, loss: 0.405\n", - "epoch: 69000, acc: 0.837, loss: 0.405\n", - "epoch: 69100, acc: 0.837, loss: 0.405\n", - "epoch: 69200, acc: 0.837, loss: 0.405\n", - "epoch: 69300, acc: 0.837, loss: 0.405\n", - "epoch: 69400, acc: 0.833, loss: 0.405\n", - "epoch: 69500, acc: 0.837, loss: 0.404\n", - "epoch: 69600, acc: 0.837, loss: 0.404\n", - "epoch: 69700, acc: 0.837, loss: 0.404\n", - "epoch: 69800, acc: 0.837, loss: 0.404\n", - "epoch: 69900, acc: 0.837, loss: 0.404\n", - "epoch: 70000, acc: 0.837, loss: 0.404\n", - "epoch: 70100, acc: 0.837, loss: 0.404\n", - "epoch: 70200, acc: 0.837, loss: 0.404\n", - "epoch: 70300, acc: 0.840, loss: 0.404\n", - "epoch: 70400, acc: 0.840, loss: 0.404\n", - "epoch: 70500, acc: 0.840, loss: 0.404\n", - "epoch: 70600, acc: 0.840, loss: 0.403\n", - "epoch: 70700, acc: 0.840, loss: 0.403\n", - "epoch: 70800, acc: 0.840, loss: 0.403\n", - "epoch: 70900, acc: 0.840, loss: 0.403\n", - "epoch: 71000, acc: 0.840, loss: 0.403\n", - "epoch: 71100, acc: 0.840, loss: 0.403\n", - "epoch: 71200, acc: 0.840, loss: 0.403\n", - "epoch: 71300, acc: 0.840, loss: 0.403\n", - "epoch: 71400, acc: 0.840, loss: 0.403\n", - "epoch: 71500, acc: 0.840, loss: 0.403\n", - "epoch: 71600, acc: 0.840, loss: 0.402\n", - "epoch: 71700, acc: 0.840, loss: 0.402\n", - "epoch: 71800, acc: 0.840, loss: 0.402\n", - "epoch: 71900, acc: 0.840, loss: 0.402\n", - "epoch: 72000, acc: 0.840, loss: 0.402\n", - "epoch: 72100, acc: 0.840, loss: 0.402\n", - "epoch: 72200, acc: 0.840, loss: 0.402\n", - "epoch: 72300, acc: 0.840, loss: 0.402\n", - "epoch: 72400, acc: 0.840, loss: 0.402\n", - "epoch: 72500, acc: 0.840, loss: 0.402\n", - "epoch: 72600, acc: 0.840, loss: 0.401\n", - "epoch: 72700, acc: 0.840, loss: 0.401\n", - "epoch: 72800, acc: 0.840, loss: 0.401\n", - "epoch: 72900, acc: 0.840, loss: 0.401\n", - "epoch: 73000, acc: 0.840, loss: 0.401\n", - "epoch: 73100, acc: 0.840, loss: 0.401\n", - "epoch: 73200, acc: 0.840, loss: 0.401\n", - "epoch: 73300, acc: 0.840, loss: 0.401\n", - "epoch: 73400, acc: 0.840, loss: 0.401\n", - "epoch: 73500, acc: 0.840, loss: 0.401\n", - "epoch: 73600, acc: 0.840, loss: 0.401\n", - "epoch: 73700, acc: 0.840, loss: 0.400\n", - "epoch: 73800, acc: 0.840, loss: 0.400\n", - "epoch: 73900, acc: 0.840, loss: 0.400\n", - "epoch: 74000, acc: 0.837, loss: 0.400\n", - "epoch: 74100, acc: 0.837, loss: 0.400\n", - "epoch: 74200, acc: 0.837, loss: 0.400\n", - "epoch: 74300, acc: 0.837, loss: 0.400\n", - "epoch: 74400, acc: 0.837, loss: 0.400\n", - "epoch: 74500, acc: 0.837, loss: 0.400\n", - "epoch: 74600, acc: 0.833, loss: 0.399\n", - "epoch: 74700, acc: 0.833, loss: 0.399\n", - "epoch: 74800, acc: 0.833, loss: 0.399\n", - "epoch: 74900, acc: 0.837, loss: 0.399\n", - "epoch: 75000, acc: 0.837, loss: 0.399\n", - "epoch: 75100, acc: 0.837, loss: 0.399\n", - "epoch: 75200, acc: 0.837, loss: 0.399\n", - "epoch: 75300, acc: 0.837, loss: 0.399\n", - "epoch: 75400, acc: 0.837, loss: 0.399\n", - "epoch: 75500, acc: 0.837, loss: 0.399\n", - "epoch: 75600, acc: 0.837, loss: 0.398\n", - "epoch: 75700, acc: 0.837, loss: 0.398\n", - "epoch: 75800, acc: 0.837, loss: 0.398\n", - "epoch: 75900, acc: 0.837, loss: 0.398\n", - "epoch: 76000, acc: 0.837, loss: 0.398\n", - "epoch: 76100, acc: 0.837, loss: 0.398\n", - "epoch: 76200, acc: 0.837, loss: 0.398\n", - "epoch: 76300, acc: 0.837, loss: 0.398\n", - "epoch: 76400, acc: 0.837, loss: 0.398\n", - "epoch: 76500, acc: 0.837, loss: 0.398\n", - "epoch: 76600, acc: 0.837, loss: 0.397\n", - "epoch: 76700, acc: 0.837, loss: 0.397\n", - "epoch: 76800, acc: 0.837, loss: 0.397\n", - "epoch: 76900, acc: 0.837, loss: 0.397\n", - "epoch: 77000, acc: 0.837, loss: 0.397\n", - "epoch: 77100, acc: 0.837, loss: 0.397\n", - "epoch: 77200, acc: 0.837, loss: 0.397\n", - "epoch: 77300, acc: 0.837, loss: 0.397\n", - "epoch: 77400, acc: 0.837, loss: 0.397\n", - "epoch: 77500, acc: 0.837, loss: 0.397\n", - "epoch: 77600, acc: 0.837, loss: 0.397\n", - "epoch: 77700, acc: 0.837, loss: 0.396\n", - "epoch: 77800, acc: 0.837, loss: 0.396\n", - "epoch: 77900, acc: 0.837, loss: 0.396\n", - "epoch: 78000, acc: 0.837, loss: 0.396\n", - "epoch: 78100, acc: 0.837, loss: 0.396\n", - "epoch: 78200, acc: 0.837, loss: 0.396\n", - "epoch: 78300, acc: 0.837, loss: 0.396\n", - "epoch: 78400, acc: 0.837, loss: 0.396\n", - "epoch: 78500, acc: 0.837, loss: 0.396\n", - "epoch: 78600, acc: 0.837, loss: 0.396\n", - "epoch: 78700, acc: 0.837, loss: 0.396\n", - "epoch: 78800, acc: 0.837, loss: 0.396\n", - "epoch: 78900, acc: 0.837, loss: 0.395\n", - "epoch: 79000, acc: 0.837, loss: 0.395\n", - "epoch: 79100, acc: 0.837, loss: 0.395\n", - "epoch: 79200, acc: 0.837, loss: 0.395\n", - "epoch: 79300, acc: 0.837, loss: 0.395\n", - "epoch: 79400, acc: 0.837, loss: 0.395\n", - "epoch: 79500, acc: 0.837, loss: 0.395\n", - "epoch: 79600, acc: 0.837, loss: 0.395\n", - "epoch: 79700, acc: 0.837, loss: 0.395\n", - "epoch: 79800, acc: 0.837, loss: 0.395\n", - "epoch: 79900, acc: 0.837, loss: 0.395\n", - "epoch: 80000, acc: 0.837, loss: 0.394\n", - "epoch: 80100, acc: 0.837, loss: 0.394\n", - "epoch: 80200, acc: 0.837, loss: 0.394\n", - "epoch: 80300, acc: 0.837, loss: 0.394\n", - "epoch: 80400, acc: 0.837, loss: 0.394\n", - "epoch: 80500, acc: 0.837, loss: 0.394\n", - "epoch: 80600, acc: 0.837, loss: 0.394\n", - "epoch: 80700, acc: 0.837, loss: 0.394\n", - "epoch: 80800, acc: 0.837, loss: 0.394\n", - "epoch: 80900, acc: 0.837, loss: 0.394\n", - "epoch: 81000, acc: 0.837, loss: 0.394\n", - "epoch: 81100, acc: 0.837, loss: 0.394\n", - "epoch: 81200, acc: 0.837, loss: 0.393\n", - "epoch: 81300, acc: 0.837, loss: 0.393\n", - "epoch: 81400, acc: 0.837, loss: 0.393\n", - "epoch: 81500, acc: 0.837, loss: 0.393\n", - "epoch: 81600, acc: 0.837, loss: 0.393\n", - "epoch: 81700, acc: 0.837, loss: 0.393\n", - "epoch: 81800, acc: 0.837, loss: 0.393\n", - "epoch: 81900, acc: 0.837, loss: 0.393\n", - "epoch: 82000, acc: 0.837, loss: 0.393\n", - "epoch: 82100, acc: 0.837, loss: 0.393\n", - "epoch: 82200, acc: 0.837, loss: 0.393\n", - "epoch: 82300, acc: 0.837, loss: 0.393\n", - "epoch: 82400, acc: 0.837, loss: 0.392\n", - "epoch: 82500, acc: 0.837, loss: 0.392\n", - "epoch: 82600, acc: 0.837, loss: 0.392\n", - "epoch: 82700, acc: 0.837, loss: 0.392\n", - "epoch: 82800, acc: 0.837, loss: 0.392\n", - "epoch: 82900, acc: 0.837, loss: 0.392\n", - "epoch: 83000, acc: 0.837, loss: 0.392\n", - "epoch: 83100, acc: 0.837, loss: 0.392\n", - "epoch: 83200, acc: 0.837, loss: 0.392\n", - "epoch: 83300, acc: 0.837, loss: 0.392\n", - "epoch: 83400, acc: 0.837, loss: 0.392\n", - "epoch: 83500, acc: 0.837, loss: 0.392\n", - "epoch: 83600, acc: 0.837, loss: 0.392\n", - "epoch: 83700, acc: 0.837, loss: 0.391\n", - "epoch: 83800, acc: 0.837, loss: 0.391\n", - "epoch: 83900, acc: 0.837, loss: 0.391\n", - "epoch: 84000, acc: 0.837, loss: 0.391\n", - "epoch: 84100, acc: 0.837, loss: 0.391\n", - "epoch: 84200, acc: 0.837, loss: 0.391\n", - "epoch: 84300, acc: 0.837, loss: 0.391\n", - "epoch: 84400, acc: 0.837, loss: 0.391\n", - "epoch: 84500, acc: 0.837, loss: 0.391\n", - "epoch: 84600, acc: 0.837, loss: 0.391\n", - "epoch: 84700, acc: 0.837, loss: 0.391\n", - "epoch: 84800, acc: 0.837, loss: 0.391\n", - "epoch: 84900, acc: 0.837, loss: 0.391\n", - "epoch: 85000, acc: 0.837, loss: 0.390\n", - "epoch: 85100, acc: 0.837, loss: 0.390\n", - "epoch: 85200, acc: 0.837, loss: 0.390\n", - "epoch: 85300, acc: 0.837, loss: 0.390\n", - "epoch: 85400, acc: 0.837, loss: 0.390\n", - "epoch: 85500, acc: 0.837, loss: 0.390\n", - "epoch: 85600, acc: 0.837, loss: 0.390\n", - "epoch: 85700, acc: 0.837, loss: 0.390\n", - "epoch: 85800, acc: 0.837, loss: 0.390\n", - "epoch: 85900, acc: 0.837, loss: 0.390\n", - "epoch: 86000, acc: 0.837, loss: 0.390\n", - "epoch: 86100, acc: 0.837, loss: 0.390\n", - "epoch: 86200, acc: 0.837, loss: 0.390\n", - "epoch: 86300, acc: 0.837, loss: 0.390\n", - "epoch: 86400, acc: 0.837, loss: 0.389\n", - "epoch: 86500, acc: 0.837, loss: 0.389\n", - "epoch: 86600, acc: 0.837, loss: 0.389\n", - "epoch: 86700, acc: 0.837, loss: 0.389\n", - "epoch: 86800, acc: 0.837, loss: 0.389\n", - "epoch: 86900, acc: 0.837, loss: 0.389\n", - "epoch: 87000, acc: 0.837, loss: 0.389\n", - "epoch: 87100, acc: 0.837, loss: 0.389\n", - "epoch: 87200, acc: 0.837, loss: 0.389\n", - "epoch: 87300, acc: 0.837, loss: 0.389\n", - "epoch: 87400, acc: 0.837, loss: 0.389\n", - "epoch: 87500, acc: 0.837, loss: 0.389\n", - "epoch: 87600, acc: 0.837, loss: 0.389\n", - "epoch: 87700, acc: 0.837, loss: 0.389\n", - "epoch: 87800, acc: 0.837, loss: 0.388\n", - "epoch: 87900, acc: 0.837, loss: 0.388\n", - "epoch: 88000, acc: 0.837, loss: 0.388\n", - "epoch: 88100, acc: 0.837, loss: 0.388\n", - "epoch: 88200, acc: 0.837, loss: 0.388\n", - "epoch: 88300, acc: 0.837, loss: 0.388\n", - "epoch: 88400, acc: 0.837, loss: 0.388\n", - "epoch: 88500, acc: 0.837, loss: 0.388\n", - "epoch: 88600, acc: 0.837, loss: 0.388\n", - "epoch: 88700, acc: 0.837, loss: 0.388\n", - "epoch: 88800, acc: 0.837, loss: 0.388\n", - "epoch: 88900, acc: 0.837, loss: 0.388\n", - "epoch: 89000, acc: 0.837, loss: 0.388\n", - "epoch: 89100, acc: 0.837, loss: 0.388\n", - "epoch: 89200, acc: 0.837, loss: 0.388\n", - "epoch: 89300, acc: 0.837, loss: 0.387\n", - "epoch: 89400, acc: 0.837, loss: 0.387\n", - "epoch: 89500, acc: 0.837, loss: 0.387\n", - "epoch: 89600, acc: 0.837, loss: 0.387\n", - "epoch: 89700, acc: 0.837, loss: 0.387\n", - "epoch: 89800, acc: 0.837, loss: 0.387\n", - "epoch: 89900, acc: 0.837, loss: 0.387\n", - "epoch: 90000, acc: 0.837, loss: 0.387\n", - "epoch: 90100, acc: 0.837, loss: 0.387\n", - "epoch: 90200, acc: 0.837, loss: 0.387\n", - "epoch: 90300, acc: 0.837, loss: 0.387\n", - "epoch: 90400, acc: 0.840, loss: 0.387\n", - "epoch: 90500, acc: 0.840, loss: 0.387\n", - "epoch: 90600, acc: 0.840, loss: 0.387\n", - "epoch: 90700, acc: 0.843, loss: 0.387\n", - "epoch: 90800, acc: 0.843, loss: 0.386\n", - "epoch: 90900, acc: 0.843, loss: 0.386\n", - "epoch: 91000, acc: 0.843, loss: 0.386\n", - "epoch: 91100, acc: 0.843, loss: 0.386\n", - "epoch: 91200, acc: 0.843, loss: 0.386\n", - "epoch: 91300, acc: 0.843, loss: 0.386\n", - "epoch: 91400, acc: 0.843, loss: 0.386\n", - "epoch: 91500, acc: 0.843, loss: 0.386\n", - "epoch: 91600, acc: 0.843, loss: 0.386\n", - "epoch: 91700, acc: 0.843, loss: 0.386\n", - "epoch: 91800, acc: 0.843, loss: 0.386\n", - "epoch: 91900, acc: 0.843, loss: 0.386\n", - "epoch: 92000, acc: 0.843, loss: 0.386\n", - "epoch: 92100, acc: 0.843, loss: 0.386\n", - "epoch: 92200, acc: 0.840, loss: 0.386\n", - "epoch: 92300, acc: 0.840, loss: 0.386\n", - "epoch: 92400, acc: 0.840, loss: 0.385\n", - "epoch: 92500, acc: 0.840, loss: 0.385\n", - "epoch: 92600, acc: 0.840, loss: 0.385\n", - "epoch: 92700, acc: 0.840, loss: 0.385\n", - "epoch: 92800, acc: 0.840, loss: 0.385\n", - "epoch: 92900, acc: 0.840, loss: 0.385\n", - "epoch: 93000, acc: 0.840, loss: 0.385\n", - "epoch: 93100, acc: 0.840, loss: 0.385\n", - "epoch: 93200, acc: 0.840, loss: 0.385\n", - "epoch: 93300, acc: 0.840, loss: 0.385\n", - "epoch: 93400, acc: 0.840, loss: 0.385\n", - "epoch: 93500, acc: 0.840, loss: 0.385\n", - "epoch: 93600, acc: 0.840, loss: 0.385\n", - "epoch: 93700, acc: 0.840, loss: 0.385\n", - "epoch: 93800, acc: 0.840, loss: 0.385\n", - "epoch: 93900, acc: 0.840, loss: 0.385\n", - "epoch: 94000, acc: 0.840, loss: 0.384\n", - "epoch: 94100, acc: 0.840, loss: 0.384\n", - "epoch: 94200, acc: 0.840, loss: 0.384\n", - "epoch: 94300, acc: 0.840, loss: 0.384\n", - "epoch: 94400, acc: 0.840, loss: 0.384\n", - "epoch: 94500, acc: 0.840, loss: 0.384\n", - "epoch: 94600, acc: 0.840, loss: 0.384\n", - "epoch: 94700, acc: 0.840, loss: 0.384\n", - "epoch: 94800, acc: 0.840, loss: 0.384\n", - "epoch: 94900, acc: 0.840, loss: 0.384\n", - "epoch: 95000, acc: 0.840, loss: 0.384\n", - "epoch: 95100, acc: 0.840, loss: 0.384\n", - "epoch: 95200, acc: 0.840, loss: 0.384\n", - "epoch: 95300, acc: 0.840, loss: 0.384\n", - "epoch: 95400, acc: 0.840, loss: 0.384\n", - "epoch: 95500, acc: 0.840, loss: 0.384\n", - "epoch: 95600, acc: 0.840, loss: 0.384\n", - "epoch: 95700, acc: 0.840, loss: 0.383\n", - "epoch: 95800, acc: 0.840, loss: 0.383\n", - "epoch: 95900, acc: 0.840, loss: 0.383\n", - "epoch: 96000, acc: 0.840, loss: 0.383\n", - "epoch: 96100, acc: 0.840, loss: 0.383\n", - "epoch: 96200, acc: 0.840, loss: 0.383\n", - "epoch: 96300, acc: 0.840, loss: 0.383\n", - "epoch: 96400, acc: 0.840, loss: 0.383\n", - "epoch: 96500, acc: 0.840, loss: 0.383\n", - "epoch: 96600, acc: 0.840, loss: 0.383\n", - "epoch: 96700, acc: 0.840, loss: 0.383\n", - "epoch: 96800, acc: 0.840, loss: 0.383\n", - "epoch: 96900, acc: 0.840, loss: 0.383\n", - "epoch: 97000, acc: 0.840, loss: 0.383\n", - "epoch: 97100, acc: 0.840, loss: 0.383\n", - "epoch: 97200, acc: 0.840, loss: 0.383\n", - "epoch: 97300, acc: 0.840, loss: 0.383\n", - "epoch: 97400, acc: 0.840, loss: 0.382\n", - "epoch: 97500, acc: 0.840, loss: 0.382\n", - "epoch: 97600, acc: 0.840, loss: 0.382\n", - "epoch: 97700, acc: 0.840, loss: 0.382\n", - "epoch: 97800, acc: 0.840, loss: 0.382\n", - "epoch: 97900, acc: 0.840, loss: 0.382\n", - "epoch: 98000, acc: 0.840, loss: 0.382\n", - "epoch: 98100, acc: 0.843, loss: 0.382\n", - "epoch: 98200, acc: 0.843, loss: 0.382\n", - "epoch: 98300, acc: 0.843, loss: 0.382\n", - "epoch: 98400, acc: 0.843, loss: 0.382\n", - "epoch: 98500, acc: 0.843, loss: 0.382\n", - "epoch: 98600, acc: 0.843, loss: 0.382\n", - "epoch: 98700, acc: 0.843, loss: 0.382\n", - "epoch: 98800, acc: 0.843, loss: 0.382\n", - "epoch: 98900, acc: 0.843, loss: 0.382\n", - "epoch: 99000, acc: 0.843, loss: 0.382\n", - "epoch: 99100, acc: 0.843, loss: 0.382\n", - "epoch: 99200, acc: 0.843, loss: 0.381\n", - "epoch: 99300, acc: 0.843, loss: 0.381\n", - "epoch: 99400, acc: 0.843, loss: 0.381\n", - "epoch: 99500, acc: 0.843, loss: 0.381\n", - "epoch: 99600, acc: 0.843, loss: 0.381\n", - "epoch: 99700, acc: 0.843, loss: 0.381\n", - "epoch: 99800, acc: 0.843, loss: 0.381\n", - "epoch: 99900, acc: 0.843, loss: 0.381\n", - "epoch: 100000, acc: 0.843, loss: 0.381\n" + "epoch: 500, acc: 0.403, loss: 1.074, lr: 0.66711140760507\n", + "epoch: 600, acc: 0.403, loss: 1.072, lr: 0.6253908692933083\n", + "epoch: 700, acc: 0.410, loss: 1.070, lr: 0.5885815185403178\n", + "epoch: 800, acc: 0.413, loss: 1.068, lr: 0.5558643690939411\n", + "epoch: 900, acc: 0.427, loss: 1.066, lr: 0.526592943654555\n", + "epoch: 1000, acc: 0.440, loss: 1.063, lr: 0.5002501250625312\n", + "epoch: 1100, acc: 0.437, loss: 1.059, lr: 0.4764173415912339\n", + "epoch: 1200, acc: 0.447, loss: 1.056, lr: 0.45475216007276037\n", + "epoch: 1300, acc: 0.440, loss: 1.052, lr: 0.43497172683775553\n", + "epoch: 1400, acc: 0.427, loss: 1.048, lr: 0.4168403501458941\n", + "epoch: 1500, acc: 0.417, loss: 1.041, lr: 0.4001600640256102\n", + "epoch: 1600, acc: 0.427, loss: 1.032, lr: 0.3847633705271258\n", + "epoch: 1700, acc: 0.440, loss: 1.025, lr: 0.3705075954057058\n", + "epoch: 1800, acc: 0.473, loss: 1.017, lr: 0.35727045373347627\n", + "epoch: 1900, acc: 0.460, loss: 1.008, lr: 0.3449465332873405\n", + "epoch: 2000, acc: 0.463, loss: 1.000, lr: 0.33344448149383127\n", + "epoch: 2100, acc: 0.493, loss: 1.006, lr: 0.32268473701193934\n", + "epoch: 2200, acc: 0.463, loss: 1.014, lr: 0.31259768677711786\n", + "epoch: 2300, acc: 0.480, loss: 1.015, lr: 0.3031221582297666\n", + "epoch: 2400, acc: 0.497, loss: 1.012, lr: 0.29420417769932333\n", + "epoch: 2500, acc: 0.493, loss: 1.010, lr: 0.2857959416976279\n", + "epoch: 2600, acc: 0.503, loss: 1.006, lr: 0.2778549597110308\n", + "epoch: 2700, acc: 0.487, loss: 1.003, lr: 0.2703433360367667\n", + "epoch: 2800, acc: 0.483, loss: 0.998, lr: 0.26322716504343247\n", + "epoch: 2900, acc: 0.483, loss: 0.996, lr: 0.25647601949217746\n", + "epoch: 3000, acc: 0.490, loss: 0.991, lr: 0.25006251562890724\n", + "epoch: 3100, acc: 0.490, loss: 0.988, lr: 0.2439619419370578\n", + "epoch: 3200, acc: 0.490, loss: 0.983, lr: 0.23815194093831865\n", + "epoch: 3300, acc: 0.490, loss: 0.980, lr: 0.23261223540358225\n", + "epoch: 3400, acc: 0.493, loss: 0.974, lr: 0.22732439190725165\n", + "epoch: 3500, acc: 0.510, loss: 0.969, lr: 0.22227161591464767\n", + "epoch: 3600, acc: 0.517, loss: 0.966, lr: 0.21743857360295715\n", + "epoch: 3700, acc: 0.520, loss: 0.961, lr: 0.21281123643328367\n", + "epoch: 3800, acc: 0.520, loss: 0.957, lr: 0.20837674515524068\n", + "epoch: 3900, acc: 0.520, loss: 0.951, lr: 0.20412329046744235\n", + "epoch: 4000, acc: 0.523, loss: 0.947, lr: 0.2000400080016003\n", + "epoch: 4100, acc: 0.537, loss: 0.942, lr: 0.19611688566385566\n", + "epoch: 4200, acc: 0.543, loss: 0.938, lr: 0.19234468166955185\n", + "epoch: 4300, acc: 0.543, loss: 0.934, lr: 0.18871485185884126\n", + "epoch: 4400, acc: 0.553, loss: 0.929, lr: 0.18521948508983144\n", + "epoch: 4500, acc: 0.553, loss: 0.924, lr: 0.18185124568103292\n", + "epoch: 4600, acc: 0.557, loss: 0.920, lr: 0.1786033220217896\n", + "epoch: 4700, acc: 0.567, loss: 0.916, lr: 0.1754693805930865\n", + "epoch: 4800, acc: 0.573, loss: 0.911, lr: 0.17244352474564578\n", + "epoch: 4900, acc: 0.577, loss: 0.907, lr: 0.16952025767079165\n", + "epoch: 5000, acc: 0.583, loss: 0.903, lr: 0.16669444907484582\n", + "epoch: 5100, acc: 0.580, loss: 0.898, lr: 0.16396130513198884\n", + "epoch: 5200, acc: 0.587, loss: 0.894, lr: 0.16131634134537828\n", + "epoch: 5300, acc: 0.590, loss: 0.890, lr: 0.15875535799333226\n", + "epoch: 5400, acc: 0.593, loss: 0.886, lr: 0.1562744178777934\n", + "epoch: 5500, acc: 0.600, loss: 0.882, lr: 0.15386982612709646\n", + "epoch: 5600, acc: 0.603, loss: 0.877, lr: 0.15153811183512653\n", + "epoch: 5700, acc: 0.617, loss: 0.873, lr: 0.14927601134497687\n", + "epoch: 5800, acc: 0.620, loss: 0.869, lr: 0.14708045300779526\n", + "epoch: 5900, acc: 0.627, loss: 0.865, lr: 0.14494854326714016\n", + "epoch: 6000, acc: 0.637, loss: 0.860, lr: 0.1428775539362766\n", + "epoch: 6100, acc: 0.640, loss: 0.857, lr: 0.1408649105507818\n", + "epoch: 6200, acc: 0.637, loss: 0.852, lr: 0.13890818169190167\n", + "epoch: 6300, acc: 0.640, loss: 0.848, lr: 0.13700506918755992\n", + "epoch: 6400, acc: 0.647, loss: 0.844, lr: 0.13515339910798757\n", + "epoch: 6500, acc: 0.653, loss: 0.840, lr: 0.13335111348179757\n", + "epoch: 6600, acc: 0.643, loss: 0.836, lr: 0.13159626266614027\n", + "epoch: 6700, acc: 0.647, loss: 0.833, lr: 0.12988699831146902\n", + "epoch: 6800, acc: 0.653, loss: 0.830, lr: 0.12822156686754713\n", + "epoch: 6900, acc: 0.657, loss: 0.827, lr: 0.126598303582732\n", + "epoch: 7000, acc: 0.657, loss: 0.823, lr: 0.12501562695336915\n", + "epoch: 7100, acc: 0.647, loss: 0.821, lr: 0.12347203358439313\n", + "epoch: 7200, acc: 0.647, loss: 0.818, lr: 0.12196609342602757\n", + "epoch: 7300, acc: 0.660, loss: 0.816, lr: 0.12049644535486204\n", + "epoch: 7400, acc: 0.663, loss: 0.813, lr: 0.11906179307060363\n", + "epoch: 7500, acc: 0.677, loss: 0.810, lr: 0.11766090128250381\n", + "epoch: 7600, acc: 0.670, loss: 0.806, lr: 0.11629259216187929\n", + "epoch: 7700, acc: 0.670, loss: 0.804, lr: 0.11495574203931487\n", + "epoch: 7800, acc: 0.670, loss: 0.803, lr: 0.11364927832708263\n", + "epoch: 7900, acc: 0.693, loss: 0.804, lr: 0.11237217664906168\n", + "epoch: 8000, acc: 0.687, loss: 0.803, lr: 0.11112345816201799\n", + "epoch: 8100, acc: 0.687, loss: 0.801, lr: 0.10990218705352237\n", + "epoch: 8200, acc: 0.680, loss: 0.799, lr: 0.10870746820306555\n", + "epoch: 8300, acc: 0.677, loss: 0.796, lr: 0.1075384449940854\n", + "epoch: 8400, acc: 0.670, loss: 0.793, lr: 0.10639429726566654\n", + "epoch: 8500, acc: 0.667, loss: 0.791, lr: 0.10527423939362038\n", + "epoch: 8600, acc: 0.667, loss: 0.789, lr: 0.10417751849150952\n", + "epoch: 8700, acc: 0.670, loss: 0.787, lr: 0.10310341272296113\n", + "epoch: 8800, acc: 0.667, loss: 0.785, lr: 0.1020512297173181\n", + "epoch: 8900, acc: 0.667, loss: 0.783, lr: 0.10102030508132134\n", + "epoch: 9000, acc: 0.670, loss: 0.781, lr: 0.1000100010001\n", + "epoch: 9100, acc: 0.670, loss: 0.778, lr: 0.09901970492127933\n", + "epoch: 9200, acc: 0.663, loss: 0.776, lr: 0.09804882831650162\n", + "epoch: 9300, acc: 0.663, loss: 0.774, lr: 0.09709680551509856\n", + "epoch: 9400, acc: 0.673, loss: 0.772, lr: 0.09616309260505818\n", + "epoch: 9500, acc: 0.680, loss: 0.769, lr: 0.09524716639679968\n", + "epoch: 9600, acc: 0.680, loss: 0.768, lr: 0.09434852344560807\n", + "epoch: 9700, acc: 0.680, loss: 0.766, lr: 0.09346667912889055\n", + "epoch: 9800, acc: 0.683, loss: 0.765, lr: 0.09260116677470137\n", + "epoch: 9900, acc: 0.680, loss: 0.763, lr: 0.09175153683824203\n", + "epoch: 10000, acc: 0.680, loss: 0.761, lr: 0.09091735612328393\n" ] } ], @@ -1325,7 +425,8 @@ " if not epoch % 100:\n", " print(f'epoch: {epoch}, ' +\n", " f'acc: {accuracy:.3f}, ' +\n", - " f'loss: {loss:.3f}')\n", + " f'loss: {loss:.3f}, ' +\n", + " f'lr: {optimizer.current_learning_rate}')\n", " \n", " # Backward pass\n", " loss_activation.backward(loss_activation.output, y)\n", @@ -1352,7 +453,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -1408,114 +509,114 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "epoch: 0, acc: 0.317, loss: 1.099\n", - "epoch: 100, acc: 0.437, loss: 1.033\n", - "epoch: 200, acc: 0.483, loss: 0.913\n", - "epoch: 300, acc: 0.750, loss: 0.622\n", - "epoch: 400, acc: 0.810, loss: 0.471\n", - "epoch: 500, acc: 0.850, loss: 0.403\n", - "epoch: 600, acc: 0.873, loss: 0.380\n", - "epoch: 700, acc: 0.843, loss: 0.378\n", - "epoch: 800, acc: 0.897, loss: 0.289\n", - "epoch: 900, acc: 0.910, loss: 0.264\n", - "epoch: 1000, acc: 0.883, loss: 0.293\n", - "epoch: 1100, acc: 0.923, loss: 0.236\n", - "epoch: 1200, acc: 0.927, loss: 0.227\n", - "epoch: 1300, acc: 0.903, loss: 0.238\n", - "epoch: 1400, acc: 0.927, loss: 0.215\n", - "epoch: 1500, acc: 0.927, loss: 0.204\n", - "epoch: 1600, acc: 0.930, loss: 0.198\n", - "epoch: 1700, acc: 0.927, loss: 0.195\n", - "epoch: 1800, acc: 0.927, loss: 0.191\n", - "epoch: 1900, acc: 0.930, loss: 0.190\n", - "epoch: 2000, acc: 0.930, loss: 0.185\n", - "epoch: 2100, acc: 0.930, loss: 0.183\n", - "epoch: 2200, acc: 0.930, loss: 0.181\n", - "epoch: 2300, acc: 0.930, loss: 0.179\n", - "epoch: 2400, acc: 0.933, loss: 0.177\n", - "epoch: 2500, acc: 0.930, loss: 0.175\n", - "epoch: 2600, acc: 0.930, loss: 0.174\n", - "epoch: 2700, acc: 0.930, loss: 0.172\n", - "epoch: 2800, acc: 0.930, loss: 0.171\n", - "epoch: 2900, acc: 0.930, loss: 0.170\n", - "epoch: 3000, acc: 0.933, loss: 0.168\n", - "epoch: 3100, acc: 0.930, loss: 0.167\n", - "epoch: 3200, acc: 0.933, loss: 0.167\n", - "epoch: 3300, acc: 0.937, loss: 0.166\n", - "epoch: 3400, acc: 0.937, loss: 0.165\n", - "epoch: 3500, acc: 0.940, loss: 0.164\n", - "epoch: 3600, acc: 0.940, loss: 0.163\n", - "epoch: 3700, acc: 0.940, loss: 0.162\n", - "epoch: 3800, acc: 0.943, loss: 0.161\n", - "epoch: 3900, acc: 0.940, loss: 0.161\n", - "epoch: 4000, acc: 0.940, loss: 0.160\n", - "epoch: 4100, acc: 0.940, loss: 0.159\n", - "epoch: 4200, acc: 0.940, loss: 0.159\n", - "epoch: 4300, acc: 0.940, loss: 0.158\n", - "epoch: 4400, acc: 0.940, loss: 0.158\n", - "epoch: 4500, acc: 0.940, loss: 0.157\n", - "epoch: 4600, acc: 0.940, loss: 0.157\n", - "epoch: 4700, acc: 0.940, loss: 0.156\n", - "epoch: 4800, acc: 0.943, loss: 0.156\n", - "epoch: 4900, acc: 0.940, loss: 0.155\n", - "epoch: 5000, acc: 0.940, loss: 0.155\n", - "epoch: 5100, acc: 0.940, loss: 0.154\n", - "epoch: 5200, acc: 0.940, loss: 0.154\n", - "epoch: 5300, acc: 0.940, loss: 0.154\n", - "epoch: 5400, acc: 0.940, loss: 0.153\n", - "epoch: 5500, acc: 0.940, loss: 0.153\n", - "epoch: 5600, acc: 0.940, loss: 0.153\n", - "epoch: 5700, acc: 0.940, loss: 0.152\n", - "epoch: 5800, acc: 0.940, loss: 0.152\n", - "epoch: 5900, acc: 0.940, loss: 0.152\n", - "epoch: 6000, acc: 0.940, loss: 0.151\n", - "epoch: 6100, acc: 0.940, loss: 0.151\n", - "epoch: 6200, acc: 0.940, loss: 0.151\n", - "epoch: 6300, acc: 0.940, loss: 0.151\n", - "epoch: 6400, acc: 0.940, loss: 0.150\n", - "epoch: 6500, acc: 0.940, loss: 0.150\n", - "epoch: 6600, acc: 0.940, loss: 0.150\n", - "epoch: 6700, acc: 0.940, loss: 0.150\n", - "epoch: 6800, acc: 0.940, loss: 0.150\n", - "epoch: 6900, acc: 0.940, loss: 0.149\n", - "epoch: 7000, acc: 0.940, loss: 0.149\n", - "epoch: 7100, acc: 0.940, loss: 0.149\n", - "epoch: 7200, acc: 0.940, loss: 0.149\n", - "epoch: 7300, acc: 0.940, loss: 0.149\n", - "epoch: 7400, acc: 0.940, loss: 0.148\n", - "epoch: 7500, acc: 0.940, loss: 0.148\n", - "epoch: 7600, acc: 0.943, loss: 0.148\n", - "epoch: 7700, acc: 0.943, loss: 0.148\n", - "epoch: 7800, acc: 0.940, loss: 0.147\n", - "epoch: 7900, acc: 0.943, loss: 0.147\n", - "epoch: 8000, acc: 0.943, loss: 0.147\n", - "epoch: 8100, acc: 0.943, loss: 0.147\n", - "epoch: 8200, acc: 0.940, loss: 0.147\n", - "epoch: 8300, acc: 0.943, loss: 0.147\n", - "epoch: 8400, acc: 0.943, loss: 0.146\n", - "epoch: 8500, acc: 0.943, loss: 0.146\n", - "epoch: 8600, acc: 0.943, loss: 0.146\n", - "epoch: 8700, acc: 0.943, loss: 0.146\n", - "epoch: 8800, acc: 0.943, loss: 0.146\n", - "epoch: 8900, acc: 0.943, loss: 0.146\n", - "epoch: 9000, acc: 0.943, loss: 0.146\n", - "epoch: 9100, acc: 0.943, loss: 0.145\n", - "epoch: 9200, acc: 0.943, loss: 0.145\n", - "epoch: 9300, acc: 0.943, loss: 0.145\n", - "epoch: 9400, acc: 0.943, loss: 0.145\n", - "epoch: 9500, acc: 0.943, loss: 0.145\n", - "epoch: 9600, acc: 0.943, loss: 0.145\n", - "epoch: 9700, acc: 0.943, loss: 0.145\n", - "epoch: 9800, acc: 0.943, loss: 0.145\n", - "epoch: 9900, acc: 0.943, loss: 0.145\n", - "epoch: 10000, acc: 0.943, loss: 0.144\n" + "epoch: 0, acc: 0.343, loss: 1.099, lr: 1.0\n", + "epoch: 100, acc: 0.443, loss: 1.047, lr: 0.9099181073703367\n", + "epoch: 200, acc: 0.397, loss: 1.013, lr: 0.8340283569641367\n", + "epoch: 300, acc: 0.540, loss: 0.905, lr: 0.7698229407236336\n", + "epoch: 400, acc: 0.553, loss: 0.844, lr: 0.7147962830593281\n", + "epoch: 500, acc: 0.560, loss: 0.813, lr: 0.66711140760507\n", + "epoch: 600, acc: 0.567, loss: 0.792, lr: 0.6253908692933083\n", + "epoch: 700, acc: 0.570, loss: 0.776, lr: 0.5885815185403178\n", + "epoch: 800, acc: 0.570, loss: 0.768, lr: 0.5558643690939411\n", + "epoch: 900, acc: 0.573, loss: 0.762, lr: 0.526592943654555\n", + "epoch: 1000, acc: 0.573, loss: 0.757, lr: 0.5002501250625312\n", + "epoch: 1100, acc: 0.573, loss: 0.753, lr: 0.4764173415912339\n", + "epoch: 1200, acc: 0.573, loss: 0.751, lr: 0.45475216007276037\n", + "epoch: 1300, acc: 0.577, loss: 0.749, lr: 0.43497172683775553\n", + "epoch: 1400, acc: 0.573, loss: 0.747, lr: 0.4168403501458941\n", + "epoch: 1500, acc: 0.577, loss: 0.745, lr: 0.4001600640256102\n", + "epoch: 1600, acc: 0.577, loss: 0.744, lr: 0.3847633705271258\n", + "epoch: 1700, acc: 0.580, loss: 0.743, lr: 0.3705075954057058\n", + "epoch: 1800, acc: 0.583, loss: 0.742, lr: 0.35727045373347627\n", + "epoch: 1900, acc: 0.583, loss: 0.741, lr: 0.3449465332873405\n", + "epoch: 2000, acc: 0.580, loss: 0.740, lr: 0.33344448149383127\n", + "epoch: 2100, acc: 0.583, loss: 0.740, lr: 0.32268473701193934\n", + "epoch: 2200, acc: 0.583, loss: 0.739, lr: 0.31259768677711786\n", + "epoch: 2300, acc: 0.583, loss: 0.738, lr: 0.3031221582297666\n", + "epoch: 2400, acc: 0.583, loss: 0.738, lr: 0.29420417769932333\n", + "epoch: 2500, acc: 0.583, loss: 0.737, lr: 0.2857959416976279\n", + "epoch: 2600, acc: 0.583, loss: 0.737, lr: 0.2778549597110308\n", + "epoch: 2700, acc: 0.583, loss: 0.737, lr: 0.2703433360367667\n", + "epoch: 2800, acc: 0.583, loss: 0.736, lr: 0.26322716504343247\n", + "epoch: 2900, acc: 0.583, loss: 0.736, lr: 0.25647601949217746\n", + "epoch: 3000, acc: 0.583, loss: 0.736, lr: 0.25006251562890724\n", + "epoch: 3100, acc: 0.583, loss: 0.735, lr: 0.2439619419370578\n", + "epoch: 3200, acc: 0.583, loss: 0.735, lr: 0.23815194093831865\n", + "epoch: 3300, acc: 0.583, loss: 0.735, lr: 0.23261223540358225\n", + "epoch: 3400, acc: 0.583, loss: 0.735, lr: 0.22732439190725165\n", + "epoch: 3500, acc: 0.583, loss: 0.734, lr: 0.22227161591464767\n", + "epoch: 3600, acc: 0.583, loss: 0.734, lr: 0.21743857360295715\n", + "epoch: 3700, acc: 0.580, loss: 0.734, lr: 0.21281123643328367\n", + "epoch: 3800, acc: 0.583, loss: 0.734, lr: 0.20837674515524068\n", + "epoch: 3900, acc: 0.583, loss: 0.734, lr: 0.20412329046744235\n", + "epoch: 4000, acc: 0.583, loss: 0.734, lr: 0.2000400080016003\n", + "epoch: 4100, acc: 0.583, loss: 0.733, lr: 0.19611688566385566\n", + "epoch: 4200, acc: 0.583, loss: 0.733, lr: 0.19234468166955185\n", + "epoch: 4300, acc: 0.580, loss: 0.733, lr: 0.18871485185884126\n", + "epoch: 4400, acc: 0.580, loss: 0.733, lr: 0.18521948508983144\n", + "epoch: 4500, acc: 0.583, loss: 0.733, lr: 0.18185124568103292\n", + "epoch: 4600, acc: 0.583, loss: 0.733, lr: 0.1786033220217896\n", + "epoch: 4700, acc: 0.583, loss: 0.733, lr: 0.1754693805930865\n", + "epoch: 4800, acc: 0.583, loss: 0.732, lr: 0.17244352474564578\n", + "epoch: 4900, acc: 0.580, loss: 0.732, lr: 0.16952025767079165\n", + "epoch: 5000, acc: 0.580, loss: 0.732, lr: 0.16669444907484582\n", + "epoch: 5100, acc: 0.580, loss: 0.732, lr: 0.16396130513198884\n", + "epoch: 5200, acc: 0.583, loss: 0.732, lr: 0.16131634134537828\n", + "epoch: 5300, acc: 0.583, loss: 0.731, lr: 0.15875535799333226\n", + "epoch: 5400, acc: 0.583, loss: 0.731, lr: 0.1562744178777934\n", + "epoch: 5500, acc: 0.583, loss: 0.731, lr: 0.15386982612709646\n", + "epoch: 5600, acc: 0.583, loss: 0.731, lr: 0.15153811183512653\n", + "epoch: 5700, acc: 0.583, loss: 0.731, lr: 0.14927601134497687\n", + "epoch: 5800, acc: 0.583, loss: 0.731, lr: 0.14708045300779526\n", + "epoch: 5900, acc: 0.580, loss: 0.730, lr: 0.14494854326714016\n", + "epoch: 6000, acc: 0.580, loss: 0.730, lr: 0.1428775539362766\n", + "epoch: 6100, acc: 0.583, loss: 0.730, lr: 0.1408649105507818\n", + "epoch: 6200, acc: 0.580, loss: 0.730, lr: 0.13890818169190167\n", + "epoch: 6300, acc: 0.583, loss: 0.730, lr: 0.13700506918755992\n", + "epoch: 6400, acc: 0.580, loss: 0.730, lr: 0.13515339910798757\n", + "epoch: 6500, acc: 0.583, loss: 0.729, lr: 0.13335111348179757\n", + "epoch: 6600, acc: 0.580, loss: 0.729, lr: 0.13159626266614027\n", + "epoch: 6700, acc: 0.583, loss: 0.729, lr: 0.12988699831146902\n", + "epoch: 6800, acc: 0.583, loss: 0.729, lr: 0.12822156686754713\n", + "epoch: 6900, acc: 0.580, loss: 0.729, lr: 0.126598303582732\n", + "epoch: 7000, acc: 0.580, loss: 0.729, lr: 0.12501562695336915\n", + "epoch: 7100, acc: 0.583, loss: 0.728, lr: 0.12347203358439313\n", + "epoch: 7200, acc: 0.580, loss: 0.728, lr: 0.12196609342602757\n", + "epoch: 7300, acc: 0.580, loss: 0.728, lr: 0.12049644535486204\n", + "epoch: 7400, acc: 0.583, loss: 0.728, lr: 0.11906179307060363\n", + "epoch: 7500, acc: 0.580, loss: 0.728, lr: 0.11766090128250381\n", + "epoch: 7600, acc: 0.580, loss: 0.728, lr: 0.11629259216187929\n", + "epoch: 7700, acc: 0.580, loss: 0.727, lr: 0.11495574203931487\n", + "epoch: 7800, acc: 0.580, loss: 0.727, lr: 0.11364927832708263\n", + "epoch: 7900, acc: 0.583, loss: 0.727, lr: 0.11237217664906168\n", + "epoch: 8000, acc: 0.580, loss: 0.727, lr: 0.11112345816201799\n", + "epoch: 8100, acc: 0.580, loss: 0.727, lr: 0.10990218705352237\n", + "epoch: 8200, acc: 0.580, loss: 0.727, lr: 0.10870746820306555\n", + "epoch: 8300, acc: 0.580, loss: 0.727, lr: 0.1075384449940854\n", + "epoch: 8400, acc: 0.580, loss: 0.726, lr: 0.10639429726566654\n", + "epoch: 8500, acc: 0.583, loss: 0.726, lr: 0.10527423939362038\n", + "epoch: 8600, acc: 0.580, loss: 0.726, lr: 0.10417751849150952\n", + "epoch: 8700, acc: 0.580, loss: 0.726, lr: 0.10310341272296113\n", + "epoch: 8800, acc: 0.580, loss: 0.726, lr: 0.1020512297173181\n", + "epoch: 8900, acc: 0.580, loss: 0.726, lr: 0.10102030508132134\n", + "epoch: 9000, acc: 0.580, loss: 0.726, lr: 0.1000100010001\n", + "epoch: 9100, acc: 0.580, loss: 0.726, lr: 0.09901970492127933\n", + "epoch: 9200, acc: 0.580, loss: 0.725, lr: 0.09804882831650162\n", + "epoch: 9300, acc: 0.580, loss: 0.725, lr: 0.09709680551509856\n", + "epoch: 9400, acc: 0.580, loss: 0.725, lr: 0.09616309260505818\n", + "epoch: 9500, acc: 0.580, loss: 0.725, lr: 0.09524716639679968\n", + "epoch: 9600, acc: 0.583, loss: 0.725, lr: 0.09434852344560807\n", + "epoch: 9700, acc: 0.583, loss: 0.725, lr: 0.09346667912889055\n", + "epoch: 9800, acc: 0.587, loss: 0.725, lr: 0.09260116677470137\n", + "epoch: 9900, acc: 0.587, loss: 0.725, lr: 0.09175153683824203\n", + "epoch: 10000, acc: 0.587, loss: 0.724, lr: 0.09091735612328393\n" ] } ], @@ -1566,7 +667,8 @@ " if not epoch % 100:\n", " print(f'epoch: {epoch}, ' +\n", " f'acc: {accuracy:.3f}, ' +\n", - " f'loss: {loss:.3f}')\n", + " f'loss: {loss:.3f}, ' +\n", + " f'lr: {optimizer.current_learning_rate}')\n", " \n", " # Backward pass\n", " loss_activation.backward(loss_activation.output, y)\n", diff --git a/lecture23_24/notes_23.pdf b/lecture23_24/notes_23.pdf index 49ef51cbe5a0b172139abfb15b7bca44dac82b53..d0846b43309787315cecadead2c0725d2fd8bb81 100644 GIT binary patch delta 38721 zcmY&E-HoBc9nZkk$^F_{ei|U)utb458lf8HV>hk*vt6GMyWCCA!YxSU@8ybs zGx*#)em;B2^7$ClzzULb2bk#VM%($aiJmVOM>fWCWvKMeVnl0ds+1tEsw9ySq5%k* zFG-Z=S7ThybUuWy>u`jun6(+heNGPTkp+icl|2_AWaj0i-kh6dQo40z0wTK|0xk0| z+PfW0`HywX{_*VhK5{m+sSM_-7s37Ly?@`&Uqp3`F+y$s-s^&l4)Hau-{=KwC4 z>4+}0hv-ykyRpYAw5v%(oW^9w7VkRgRBS{t6KY%iUo}Lr(P}T~TF40{v@zJ^H^PXS z+bsM5B0JO8%E~(lP1a8|sR_R?Q*5p46QZLkfwMZ1I)HKNqtREooX4(3s|QKhO<^cY ze0-E5#@@;$#`?80MbDEjWU(`@jm^l{W$X z6yeI+qTYz&4BGG6g`L*KfGA$Q5<5eN*t(yzgLw*ma-nGhkzy5&5 zB|1%)EOwEJSuf|t`R>!s>qq?v-EGS-Q}dj?d^i=Ek3;>6g9?bajZa_8A~n|-S8ghZ z=DJOHqFB(>CHi(ilc^XSf1}n!HZdY_JhP2rQcb*oYi$2as;6Y)nud40AeN}m1%Jvx z5(8n#{@Q6Y5|LDi)xD%hCA1V_fDP0cFu2GJIJmFH===_M51lAgZ$z|~)A`qdbry^8 z9VDq7*twNK@&x{9#xFscEae_2Gmx0i@Qes4fKHkbVC9rvmO1_ad(bR^Si}LGS@}ll z`Eo1;=Xu>1qT(&<6ox z7Dpu5@QEda`Hm8B9?qhW|1yE%0-6E4{~{d|=$jHSI3%+?wcz?Hu$#Y8e>_tWWLdD` zotCqS4&ge5pW$o^5hrv@v-x5iCySX<$eDJOsqdI7BBQf7E70j>QLR6opyKRWOOvN5 ze4br>bGdxXenLjYBBvx6neU6T8~7xLY;gIT<)~59kY*PA{@aR(+W`0tC9n>OsibZ) z9W%SGcMeN^Ue`(T@5AO$6b1*v-5Xjo?!vFJMktxNC~KR;o@AdjGt44r%9(SYhF31Y z-%gfJErY=7)T&yy)?S(q9tz^&rCGkaQ>FK|(Nh$i?X zJbvLzCdI1q56*)lmACTMW< zJZ8HcyAoXNyAFl1*)l!qgYnqJVt|vi$+0)p8U{D(e^$vkW=%wcDr9Hyi7h9*8ZUT>+VAm*$G{k5L8`nBoE4 z)^%&B6eOcC43Q|(HD25VsxszDOqDzrPMwbh>9YIx=jo4XYfHKCEdZFN_s;F!ZM6Q$L#P_~0b zeC^wsMmZj8b^+WubIIeMKf12B9q&z(0^t6j`8rX7j&NmR%dG~ z$%L9HFB*{U`y(^0neYoXHUItQuP%V&-~pCk`SScCH3p>gN%D`a@7~-dCjI&LA8nn` zPFzN&x#6Lf8kx@7k2^wey-Fb_$hL%d=Ai;n98x=Eyv$@x&}-@2r=UoOe|gFGEpI={ zIuJozJt@7GTpWloa1OMaWw9zbVqO~5=nqxhlwryDcImik=#M9{5DRZzSA2bZtyqu>v*7P>&4uI|lXM`gBqIRcT6Z#T89TC0FI~{PW z7068^vgFV*4PEH6>)-FMgdP-bCaUz;qJX=f%F8QjQ3Ug6kl1J7oAe zmDJkDh#$lvP$sJfmTk#RJ>vM}pZauez+P;utDK$f@)D`zos*2ku+~Cr?2JTFmz!x> z=QN+$3T3rLvmTOWx+V(4HLmvH^bJ8lTLCYlQ%jS_7mzpXy>IE`ziw#5HCN=RfuXxnmJ2SZIz#mvdD4t+tVK4v9Kt)^2#4F zsWyA8-nWM$z?u_P=MuM^gDHc3Sf;@HxV7ytSisbo%E7M*p(nKT5E}=mUChMJ%?Mq= zLQ8H00|);*P{=KK+XamMB|KzikIn8)bdc)|c8}3$!W}$hYE!We)&C9hMv{ej^NY>l;*Y4O^pRbPp zYj#4kxdd*DueUaOvH>HOxk{Poj_`{ zE!TL2rU|n~7M$7|xMUcDABdxg2=clt7e>r6*~`h%XY;DirJ+TL3L}c2P0;zk-N|9* zK118Xwsq#>w-K#Oq4HkC+q(+oTDclCvdKb_SJrj!g_6M#eeAe!VQaJTY~-hZX|-Hv zl66706CdJH!K_>n@nlG*69q;h@xNk@04(pAW8yjG`uV=^2y6|O0zBsKent0@aWx0x zFn_{#SgBEE<*dx{JV3*qs*VCc)$`i_vL+n*0Tl-0yhc~2ly$-H1h*ow`+H3l%!Xdl zS&wfYxxL(pMU8q={ALMzSHVIm7sJC5JyhpX-$H#TNsG6NX%QzH4Yhs7X zy2NSp5HWb#VUq@ptQ`@P7j?cNEYi~hE8=xdObzKR6l6K33Hde2nPjM`O1n|Fh5&cp z&oe~?)w4O*G%rgB1U()8K2)Fuwd}BNwT2iwpko`Wcz%a*oDK|p6y}iuMJrq=otz?l z7pr}7hyH%Lh~H)X_dEztXavnJM&ca7>8`_z{(w&@$2x2Tj}4vN2hWt~1}--Z*8ne>Kh^XH*6$vVv%#1>kED?Gbm#JI6O;`2Vwf4N~XlD9j@2=^E^d^#hqw*SdXs z;EydRKC|BVRvu0W-5wKc6*7tQ)HG(OipgnPn@P?ZDxT4)nwVKGFI_ViG3)F)_QGT} zMj=)UqYF@~H|2|XY#U-9NH7_2^#`PW$o8V5DB-W2{vh?K_!VlB>~_M3-?Q0d)1?#M z^E7KIYQS;hF?;mqz$Qhk7kvMnV9IFma1G^VcQLbcLiX)xgGk$i>GzAGsW}UIEC%)8 z*6>j0cHQ@(Qf6w<$2y0_47D&PdiN!*lFxtgrtbi=Z&2=Y&>aV(9d_j|6Y)oQ0OtDy zC2L$<6!$QU{M-<};ZshL6}L)vk0(i4x3t0$&&}o=iASsi@$RZc3*U)<=c7t!w9rw^ zLa-~x4^A*N;{79LeqMNI#r4@E8yiCW8+UF!B8GESu>)?ID5T$*=z&vX>-p#VnovMKP4sYhMwWNkPWNgCO3WI9uJGLLqN0Gsa9?@=}1${;9V=(w>ZQHZ@>T$_On;h_XIbxZjwE zt(*oQ=$FqsUWGCiE9hpeLXTKL%xF={XM(0DA8cY=$Sm4X0%75Yl){xc~L!ZMIYM~Hbi2Gdz;&q}$&LQPk z*>k^~L26Jp(GqWokW19+mYIAgy*xPKweDG4KxK}}^-_8W75{eu#&E^F&!WgBPJpO% z*?y8yv8Af_Nrp8(laeb2TML=(fM>Q_vyQtjTK z(H!U?gbXqexLA_oOw1RB2SApc94j2Mr`;oy$%(~)_>T-*D2Aq@Cwn8W>ziU@caaVpm>Z2XU`-WL&;Tswra7MJa$170 z-roh?R;{jgJekG`jYMxsOgC#hcy7MHA+;Lh*N2qH5x0zS=vLiyAB3WLm_Gbfrt`b1 z%y7KfA(jif+|hZrrk~m3tkqxpd6G)0n4CYhP(NrnDTx>iRyH_bJ0UD>0ppH?cQcaH61MozPv?2SCzZ#Q@x z$una7mHuCRP1Sokt9$(&XsbtQQ&H{{&O^n4`cA9|A9Ok_)qoz(Uoaui=)Pk~D;!*X z+->8EBhH_WU)Gy6oC8x#wmb0#nGXt69$gcw4?kSzdh0%5gLmUj`=GIE(Mr_*sx>Hd-CiYo4%ZG{a^5_VujGaFdGzgFEX+?^kwAB% zk}AcgjV831mjYuVFj3UBdjhu;>YLG)K}|LRgrNGB`&W7cnsyj{W4x}+6ZG=8AwsMs z(HW~Ea2(SuBaZy=(W_S;Y|ij<5N*>q&GMa6?uT)lr}T~oxExG&>UJAL6cN2#5%Lvj zl@*2fl^m%$C9YJFoqFX2L}Bw^KhbDY@kHgLvLm`{{sP&Dr_D6wA_wT?`*KkKhKsAW z(!A_gRTo!gbiZ5-wwg24kqj4>)>R`k;5)zRj0|IBIxvsWA-R=ye0$odMf-+8R2|_5 z4O&p);3Ebv{1WjtedO9)vZ6}y47mC=?QeRpHBx!5ds6iDZOkX9AZS)V>D4w1FmHx7 zlMg?)^8jq&<}1uXN7vjjqJ-kbIKNj+N3Uok@ONnZ7vp{^8^1PdP-Mdg@Ku{%w+*}0 zM5S@A*)SrCaKTk9I+ZieSmBv2p46l1MgLr>A^F!1DbNa_`>WC)m|q!bxEG_g6Pgevc@s9n(sBgYM)3Sm|Vg8=m!Y@{uuPR-8TvRBbVQv8W0$* zeV-)&{+bQGf`KUUqFX%tOb4Y2WIfY`Uuh$*Jw9GRc_#&>8xWjJ8zMe!&dBj+G?}N~*rX(!5L4XvX7(nT=M849XsU6oL)~~3_|Ym@Qr?{y1YYP$>~0>r zpTfyoBs*}Ltt$#oNP8!_Q({zWIEcU|4}b~j`nktTh(RFo+CR$KE#OX(VC*{IYd2wtW59ydS2Cuq9I0EPLJzhF~IzA zuyAb_@mW-ZKkUM}k>fCA`n>5KTH7UX7G1M?&?i`lI>=XBE8$Q($b`XhtTr7>7SBjp z9%-~u`TZ@sdH;lrjRu-=Qh46goMaz|2k{kR<1&Q~f+a&_{pW|A)Aw+Tc&G|3}O46|AUvG%RLU`Q*V>xNO?$ln|yd3xT=FDT= z`h7S6c)dL?Sw4N9OabMzEatnxrKl#JaN9pM_OtA*${>eNvPgGJ;o5}^TMnTqQ-(*9 zt!$>28mUaNu~3h+?Y?DZ#+DSP09}I6Q-d;* z+XUItk~ZO9DRGCRqTV_E#LN7thsNKTMeaRL=lEA=1@1kZ`;(7to6U>PcFoOUj<%tQ zm?=M%MkVGdj7v_J-`QKOgP{vslVlu=tnFBZ_ExF5InTKL{@`Ii8gIZmi%m#plu=*N3HtDZ#soC^n+Id-Hy{1xf}6wA%bO zLF<8#qKe?3GuZGdf&ez#cb5O@p!){|JlyQTs<>nMquOq;xi##=@Y-L)e*VA9bQ3o&mk*x^~! z<1)uFyTG;EXzB9ww?g^5@X8Ol+jj5?48(ge#2nSoh zdI|}CRx93Izo9eT1pe9O4!t)&yL?!K-N9hXEiWIEsE7xJwv`9oBy~sQR(s1MTp%C2 zv3p`y9Nc9QFfx{9pgtsk=cnR;inW98#up%I;ir}`P1I1WMfVzf`c9=1r^u#VLhg0; z2Q1&AW4?Rs^+wIoC#qumIcw{Z1uBOaCHuSHe+iD_5dERvV`xC1UqN!Dsx(V+^t8962g@P}@->Wh zHO*<1lGKi3pZ_q`&0%R9^QUpPh3XOfN-thy-7WoE%Fwmx=Pw0Rr*|Hw-3ER?oMAUt zAat_me^!6l0}{{#@kIy=Qi3dpwJY;wfnMsaW9MuvZ6@VWgfYwoDbA)-yvGaE+HtVQ<`m$ZzoPRS)|kV&uAjxdR?~UR zDu=gpn_3cW{2lPXauoqrduOjE}dYTx4FGhA$}q z@wilaS}R&NIEw|_OWt1)J4p#a2%Okruzy)kFB?hHa#Kz)&y&m_k7<+f@GE3#O)d*O zH-nn+2L?w{G@qzchRlE4I#Zf&uNAO2B`vrQ6{510e|hqD*DQ~U^|xik%r#(tJ%&G- z49>%^0*L2xtfjouLy4Jf96Ph9u&Ku9w2(YIbB*~#XVvjRBoyc2i#)v!*AVS&9qTbp z<4^(GKWrvVLGpjE&zM`2TG^27qO-j4?fv+9W@jY0jDVautDyPF4;RlmX;ATyWo#&b zOk%s}@56bJ0=F}SpxoS*+fdT!_I5T?q-=SI3G8$f$xD?q*7sg5_*wl+qqms%&rR#| zv7r`+mrK3Sz1BR@!HC++il}TWe-U0cZ*!mBKHy|lKO+b5L!<@!n)2`k13bheAKd?w z4kSZj{1nE10PH5+_&S=ACu0+N2sV^anc{<2bCX z?aVmCKm9AkFeZ}9F-unE@UMS$siQ`ZnDj*N_+6P%?&n{32MI3pd?R|~AH3Uj3Shn5 zv&N(BBy;w-A%^>1jdG4^@+@32t=|alK;J-713iYm{^3}_>zYGbUvKX?Vb~X*;g4=V z=*i8S7tbqohaQyH51yT?T`yDiLKvA>hNS&Pu1??&&ygNuF$W1~-_Q9MmCFQ4T+SyE zE@PGlFtNw@S7@1JKl(s1-?d%z>F55ueZNutfQXQ_+5nbQ zi+w4jyZ|YNH$0MC(@lV6b9wWUxmLL%6s8AfA*kX0QcdJ2qu5+l z-V~bjK6%b*XQH$hG~gVnm>J?~sbgA<_+cOfddPQ})4O_)qkq77kN#r#5pE?B3k27F za2k4RC(5{i!q!p6-s4h_e1NQNiLpzEN6bw>#q|z*APWc}z1}EKC=aZiqixYH{Zuo% z;qzM}ru^@hc8*;tWXWYhy-mJGH=T~2Z#6?o7HU4~Enp}p0Q9XFOW?iqI0OS7sr`m(yt&0bIS zoE0h^TgW>6Pr}OtxsY)=KeE>k$(Ng*}KwhQ<~6@kOeA)SbDo<>wu=?okSC79nB-bFz!N?xiy*mAf)u zZ^emm)uJ&Nh}Q|v5B#bEK$Ur&TSA_*K$W_Kt0M=wq=t>Bc+gNu1J0mxGQ35FA>`Q; z{hZ2RM9RP0r+9(rCx_7!c>FLUDteK;U$NhmkD0`*0%#fLfjBv=9c)lg7`7CSi9M-8 zraAkW`yAP-S(LDOVqz|YxmoWRm(ChDXF?)iRm%a?1QO}-+fLRjK&p$;ijhDCNH(Zb zRY>lGi1KS#9->?bEZwXW8et^oux}E+O?f*08>49~3z%_(_4?mTt_r7~M(3<$(fO)c z>Kw;#)ld>1+&I6B0UDC9hz&HN$ml`GDRh7QBZL^UGx&OrUPa|UT=t$^cBSneHqzm_ zERwc?Iz)GyiCoU;fEVe4i*oD9GLAY#NxwLkivTZrxt=_TLF>lv=8A@a&M_8HHB_W3 zcH=sWUW^T%=q1|B+a2g#{l#$@BG_l_f8i{GR2RRq2Ab!Vbc z8R*XLN-p)&x$pUTS$~@oTEIA3AViCLOv*Ll*v4Vnn3~HG0dG?9m~VgO(oSncTKw9P z$0);lb~QY!TV}Yt{kWnt7f?f0(Wy6ckEC-%jmcna z&9`$~N@2b01E%gWnw!tE6e>Hr1Ba|tpXDo(Y2tf^vy)fqMfx6c$(OzR)sC8hN)z?l zAUPTzu9go$N-|PB__T2~O`3a+&arbxcd*QU%h zjHNy#5?#x@9nUbpqh9~o<+6DHIv9fQOdZ(qF*2QUCP6ZqqmXnP;;zGf{&lsB;vvTu}F!cmuZ?lPyaRN z)F)7nCyx<*jEg5xf?@Y9C)*|D z6r=zpMeABZ&x`&ntX}Hc)HP8n8M?iIcrxSn8yF7Qa1))3a87pDl{z;A8T;+bNK^u@ z5;RN~K4M)_1`al2m&?-#;~Nx-0>#_EY<|LpHbqIlW*y#lSWXBH#@vfDC+-+hk$}s% zC4!eKW>K!drJ3NaKX~)2Yuev6vF>#+)4;j9i4PSSjfbU^n?a_G&ogY0m2Og9EECb0 z9ypv?)HyQte<|@Ux=Ey~$B!MPM>%G(aZxmj1-k zp~W;D5}_0+__N;o(8%Vo^%us8r0;Q>1(0ilrqX98`1aP1OQe&U@mL+0iah;xN7({N ziK(vG@`Fh>-jZg!uNea@vHOYxH{dR!S(FPxYX*MRwfv0KvKsJi}8+kWWyI>RZ5TN8c@PL%QQAQCScpeqEW*o!?^zu!aJ+CK&8NrP z)oOA5!;E>I8T9!LGp)>~nbd1kV#&pWj2dwxe zxeYEYznJ06j6J2s0UlNyufLs9)h-fp=sR2x*{NY~9&@hqPLRK8$jvw>EtVy8#FbhJ zDJ|`^gAv8`^TMn%`t=}#EzA*?dn9<`keiR@9|OP~i-p9(4Ey~9Z#`anqH7Zkf>inR z+;XCm*YBKOpEmmNzq);ZYVJb(fepS7Ha8=V_dkezduMHV09GF#;;sLY@4mm!^!>p- z8#Jgxj{-;!9$|d`E^8@->7xJZ=d07}bAxzkjW~^>&Odt+=ZBv?T5ft#{8+#LP*v+e z&u1~)JRB>ZfIn3zXu3VzScU3G<lo1R-pDY2>Xx%yoXeQ~R2pXL%iOqPA9JZnm?fM)3g@MN`gnlh%17BT3%Y~a_4qM4 z<4pgO>7z6F6F0)lu9!0yr^9)ycMaT;3vb2F@RpCc=8 z8nUe1_RBXOIjEWp9S#(fI86-n2&;A4j zrS8wn4KQ1Jexp{axy8eU>q#?h;yy``UL!*bLy($5VJKsx_H#ieQp|8IZ=`})U#N{m zh4>Q&Br!2MG6-6~NH_fs-MM_!@GlBsW1F_M|ERQe<29cU9!yJf;{Em2@pLAd<-_Xo72%sM4XLP0ZS`HMzZ%>JK{fB zTkDaT9lvxU_NcL;XLU3C8rIQR(s1Dhe8RzNB@Y{_Wq!D7hD@NAkiiFIJTp~_j# z1~=#jm7}pyeF+8w?d@zy8ycew^mX$3fKxixV zSwbcsceSiAMGvF?ZRL&!WwFfbYLAi7Pu@-`VXNL}>0m{+q9xrth|DJKf2toRAJH?q zbZ@ygk7|Rz?8Hec1^gMTR^MOKJ!JAoZT)nDwRbX=o9N?hoJT2Gb!|HtNo8xdx9cw;)e}5z1DY0A&Z{w-x&<~GD_R|>v#lHIu1Y((HQ}rzYiG1; zIW8_*xDunF!Ce{w&#Y8YmQ!@3jMb3bsa^+tu;82+xU0=_4KojB=UeYk{ z*Kns23G0j4$h2S{-G!_iLH&GYB~8LM!?&uHa>6+sTVPgSwl;>ot z|J*6L#t33f>M`L5yJ*lAc{(id^tlW)q_3`0IR{hecgDgWidqx^J~l;>MGUkI5)(Ai zl>eq+j}W3>hZWi-3N9Q~3LK9~VHyJuapf!=7k(!!^l}Ic+zSb_H4)jFMjC1lN<@yl zU_8`LDHKP!ifokg$7*7Tr5Da^~8TMR8Rw z&R-&f5XKAvx1?tc(Y8S-sw}}M?0x_1FQz6R$?)hgzwj*$)@fsvO!I<4F4;wms80_h z=c-(iSOl_wns`dbumu?%SU8k?E1pnC3E>};sI3GuJ!5fTyjIItu1R9q6=1u5Vx9=K^U>i>V1o!57SH2F!?1^P^MeEh3PBIxzw0F1$ zzil!& zbn?ZG=cNtJUly^|PN2C=9H|O|&tX%3@&s~!v`vNQe}mf{y0#?0D{ukKY;3Im3!g@^ zr{cCGki}o#V88q2sNCDeA;B2oxgZ^uI-)=g6fTy&6mVeKzZdmJ|$4*znyQveu9YEXmpI1KH zAEHvXPa9lAPdWV6N4+k$iX)Y_INec?F_)R6+CM#ybZc0vHr=DoIge-NJp@--tu~QY zXI|5tx8Z_&{<;AnYv^;}nMj9_jXODDW;p4y;DEM0n+B67HVEs#R{XxIHMTlYr9iP6 zkyE&wF&UvKBY%`w5#64yn9MOWv7amgGE>5kiZtMZmCxpXMyd|G}5o?RPuC z@GoF-*JeXkpArBYv7v$=LzgWD17z3J2|>SkDtD6!1OB^Z4>+uKLzfC^>fUctuRV5- zDj_w3Y&PPEXa-_YKTrHJWu^g+WRHu$R~no3@(u)+N~nkqm=y@?h39>dy&9U__`1fh z*sR1S#Ewxryww?vGVT8Qz-p08g}iai;1z2t;0YYsD_6@l|3RcQf0PlzsNHX7M{Qq z*hPrXh76*Lu0+CL5MjbgKz%i3kJ63hJ&`E)VBlnfyTV4ae{*sR*yQgEEguo(ki+rO zZB&8Q7!PKSTg%_oXDU0Czm!roNxK4|qTB>T>Dve%!N<^O!K0Wp78>y%(J3Ci_~mRd z6?W&Q4+KcoG_9#ssJAKQr#7dNQKD#ZEH71tw~B2ex&H9WK^C5C0TAxg_wcjW*+_L= z9la|;Q=5E^<0F_T5rxJ?4an_pRtUO(S>mf?KcmKLOqV*9Lsjx@$v56wU4E%$i6h4p zjk4?W!hY2jTWQ&=%gQw_DIvoCAPM0ge?YiTL=azTkfFy^@}@+m3velt6H%AtC^$l) zs*FRZjH91jG~H0F2I`Louf))poNM?llnz~XpcKE@Qk)F2g{in(89Xtg)Uzd4{09Gv zJMDmno|*mGO;!ETLTh3AuCQ=eU1^%C5VoWH#60k}&UG_u+T>*r6)2|PCjc>Y!5;W+ zVXP^2aL7Kx)&{{Jp+vY)PlK*W8LV@5ZtXZntHffO2wbuAA^z8ndV!FM*YCR!I5KJb zkw>C5E$Ui}Lq&&=zKagB@1i^gJ%cU*HpaDHfFKO?c+lzkR`3fhs+EmBxmWo={>#F_ z{{KDTy0Q-cm;Y*>BWV?I`GO4;Qq4o2(}y~UBhfnK5atw0BYk}k2!tU;Hi*~ugworL zj7{HVQ&v1bx#>`Z2~HJre>YwnzzO*EU%7M(pwAtAFMU1TVDZ!2woWrFeUY~Ac2HWC zeN5fn;(*U)0O+3dG4~kxMEBp0oymG8b%TU%7IIsT_i)d2NXFfLi=LbkxH+Ul>T@U* zcglQvA6_7yXU6vYiVdrtg6;@&0qL*kry{@Nv$T^_#hocDQEiaPYF+)V z!%GFwzhhSGG_@x?kkDz?r^J#Gt4WATi|dmww?UPF*5^I? zKyIC(wA(hBfJKy|ZG0oF6In-or0Kbr1OCPwOy*!EzFtYpDHcW@6urN$;ft)43*6OW z)&EVhatGRIB3?(^2<`-8+MAt^Py+i73Yfd$D_jtw4!xpzFUVB9)>h3VJQZt@y;=E2 zUPKEc3nJVhUz@vMjpH*{PI1wI+Rs`UehBmBpX2S=mZ<#|BREs2W282yVHTGn0qk+j z1$)7%M&vG8uSlKX%&{w-cgJu?nY&tQf+X!dEYh z6;k>r)d|VnQ7h6ex>mAvR=b@G<(qm?Jhij@^%}qj7^^JXl{!U{v2cq444nu@ z%;hZ7-LUUC@h4YyASD(P?__kX+62Hp4siYwA=f1>11R_SvdtH~J^qD(?|pr}-H(MovK%!7+e&17x>F5{ zpQuFS98#f2`Q@M%6zu772`6$Nn6z|4S=qXTrP5UFO+V&sTpodBo7Kn00QxgHa|@^t zs`eB+vZ%7GEYhX2#zeCzy_{=p^p_B|U$s6@S8z?+rtJ@`2qW5n!qO3&U(C6VIr{aD zylaj6cJW90cC&CM?i=001os1GP_RQiY6e|EX&S=(@+W#$$sn>W8?P32lMa^LWGn11 zYl@D%ONl^fELea+G9V5P78{Kwa2~S+VHZ!XuXC?!ANBq;_rUbMaQg-SbVJ(q-R=Dg zsaxN17phx;^SofCFqyg;Mc2>N!5{`LA1*v@@YlyyxWrN0s>kb1du}Xfb$|6Jl3L)s z=lEa^Bcv(Se>SRe8}#U-0tvUn_LtJ$2=ha6Vhk+CCR4jAM^A|Hz)aZmavk1TTd1e<=J402{xY8WOb zT(H574iLeZa|6HsfMU}DgpgSHm$Bw875E_#Xi8J7b>R@jwl!r?sPo+?Lg7PNyodXk zYvd)Q0y|w6)irEux`Yn_XurRpr@yi@x45LT6#?M#-g>|og#y=M#Mc2VwjxUWs^WwY z$+l+`f>jpTp=u^dGM%F;^N*jxtYe!TR5iKL&;?Q@X;~oWGSt}5%7Q0S*aNUotgatO z@0xY7hu?w5Bk!dgI``*|3)ZJsC!1>FtMG{8scKvVsWqCUGA0G%dm5w|@eHwQtP>no zg7ttPPYnloj}F+5iMQYToOH?0ZpDwC&ZXKg-CYfj{}Ol9S@%lKblW#sc#hSjS4dg@ z+GAqeaJ74_^q7b*I%J!oJ{P>BZeEg3&8P3m&rc<_w8T4XFopaPnd$K1&GaoOTjvjiMe}D`{REA_J{wyp= zC^iN^adP$b@*2lQ#PbDX%*#G0-)a6U3Rz?%d+T%y8dRNj!~#%?2cycq6*~s)~Zu(scc*ysIsKTyw$(a zsd|gy#t++ewiNGxRVx-2B`aAJmYNu`kgzFiW`sew7A?(hAyujDx zal; zy9Fe(jAMF2`?~rjUJ0CF_neNtZ&&>|(HfogbD|ObL>OM#sbCoZGSBh1g2w$x*JJ*l z@U=e@EA{OTt9?ASQxYHJHjlXIL!9z;XU{*>g7MDBr7C7psKmF&Ac^*9u@1H~)aVVm z)YC(}stE(nZluh_;>lJBJo*^}c`SDn3}bd_NT_u$96OIqJdqfl($8KGUs?chy0WW`F|#=`V`O;b&EOHFzf#(~BMDF3%t@R9 z6JA{J*5L=E7|}O7ezox5G_=@50zBmvrwo+8Qfh{vh?K!KUgtHTF}m4AE`BB@fx6v$ z3_x|>qsMkTxGG2None5-9!apVs`uKc5w+V_^i;s}$7cBewtRrEik=^|XV3~J3C-)s zo(3G9(;Ai3m1haDzHw{`SD~;+HT^Wr9i%*S%IJW5#g;=3KS~dYTS0!@D7G(1ZKT+-4SF26M9jbwc5UG!+&t zJMj1lkhT6p7PoDbem!{17i}VoB5#-G!t4yJg2}0|pSk#iEd)dp8+@a8;+#YG_UBXY zp9@Ce4|oo#N`p=OTHBG4&9v-6DyEr3Cd*vSF4zk)ujA9IL=J4NkYDVg!*n)gb`U~; zILt&;)C)s~(1C)UXgvr67zw;no#*2U6wBLk zK>s`2A+U!chjhjlsu$P;k7b9moj{}Z1`;Ml@ke~|Jtv(7E%$HH!T3}Q5TQJtX5P$c zYkg-akqe+8KN2|%Ua!@qi&43P4X?ek%4XdSMv}0M<=EHh%I0lzJ7a0_kDPq^(9!nR zlu=!T%`l8(Sf4pnB67W+fyJLc~*!@h26c@CdANK%g=( zTX!sv(~3pggyi-F3;|-_!=^F0?A;d;UwA9Ja^JKqYZDSeV(tRxt zX&=w6)G9mS(S%ipypw#^*qmAIhtOuD zit$tQrkhxWEGXV2HUae%(RjJ^W*!SjTg@j=8a%Gx=NRmpHyRE6Txd z&=b5#Z|L@uPIKtwLMig4)XZi1==&saR6|d*D|vzWgU3k{j4}Mon3IgQ(In z1I$46LTCq0r>(GR=o62%D(Yp&cRr@AylStr*mdcHGN=m|0AR8ssEP?0GfsI%F1TVC z@(-4-2CNGHBh=)jd*_Lke3Zo{um9{IsvV5r&l8(#)iVH639MMB44pYUqqZ*^mL0W>^k6WB7SF^9XB}> z^O$Q3E7DL8*<^MI`;hg{K;!KzlJ~egqnKhyMLlCmJ58&47{@{R5{*GYeNU*nu0siz zp-El_U%$i)I6bDF-(44(1vK!u`Q0>1a-T0$d|@Pp0ba8ha4q!I*4)aujN@E8olv(+ zTjWRc4UdNtiK|Mw(?-?{JPZ?OIDX<#81uhXxG|RVnp~g2cJi1#Vv!sQHOsh}idZLX z=(iVqt`)1@TGek4-&sA1I`>XJ-M1EIksggZll=SQ>bOTWWVuQ>EM2L~Q=N)^t6#dk z@oj!B0O$x<;Fq@FT;NJ7z!a$j>x*c}KA;)=YvFbp$Vag}`NP>;+W62@sXcjb!b6J5 z%|!03qK$X2^ zl-B4OA@S3I9T!rgO8oZQ2usPNcpm?{Z?yp43|X<#AT_9J!unVAjOvV^(O-=YR{C8> zR$~+^6ud>Ka!lkXhvIK=rSA?0h-j}k*s}O;wm}_EJ@p=TFbHRhtzn<GD-ow4M}O127XPSViJ;FT~?*E=foEppmb6G4kMAoopkh zH%nGL5aPTleqaaeF!{{^Q~#Rrq<79(FLO=NE!$f|4?Jl*@=5{41Ts?mxkFWUu2&e# zprnhgO0n%s%wkf5aK(|lXIGt)95XRNwYdZ#t$oVUj;~Q!I03>GoG2q)?96);?=a}! zk%5l>L>JjVuwLCHD;QuHaOnAHtt|c6HE=Pal!Rov@rDjz*b^cH%$NN5ea*Ib5WKxy z+^6)ZyE80td6JGz8DXR5 z2#>da?!T}Ag~@<3r39WW{SQTRufmNW+`Q7fs4@W<_^l&=thg6sP&dcz<|*qqqP^&X zUfja#pn03sQIU;v>^E^I`kQw{N>d8ERf7%Gg-l7l{=P7;3uU0 z!MLe5*AVA&3)#NfzFO0^cC+xH)%gxr!zYv9yRU)p##P`O0PU3^Lu)07hQMz`r^_&C zkiZ$;=W|g2<~ADS#@qV8BsxZM=p9C%MLc=0UO18vqm)bA3DY!Na)oJ0^-G3-$epW$ z%rJv;%>8NOyR0z|C%qY^4TyQU*l3OtG!I5jP7X=iIo=A1jPBj%iq!ExvCKuEC zFb~AO58pui9u^(^9GcwiZ0muZV$oT7YsY6}qAm@%f&&t-&JA9J63OvF*RPOZH#!b2 z|KxnzU^<~wKj8Xz6Q=liI9#>$E!`Z_Oo_Tv|BbPJ5y!i`s)Dn zk`;}vG0Sc7q$xLi@wOXOZd^s{`L($ia}x~h*rB$3gabTkRbcsttLteh>lq?N&E@ME z6ipVH=}6l!8wP5!F3XZVY9^XUx&rE~Y9tE)0dBOMSNxU4-X>*>hSdD!=n+(NcFlqgPO zD3E3&OvSO7$d|R|)t)Y8hq^|V*`*e)Szp`V*PO23w*f_(w;ElmZ|~q~bf1{LO1vMy z4Ox}HcP3;OWB-Vx`VPb2QPzdysu{^P*i%#at1ZvX}(ewQ}wCfPB%w78jo$bhzj=`EEfaTcEe)h;4A`Uhoe+e1YfvmN=T{%l`|rmC>F?XmFZ}NPpDA3p?{!m~jPx6me9aXA`>{&9qit*v zGbU5cblOa+AllN2_eSr5^7U{IVo%{u25PA0%_+R}i>!^S{T!gOgtsr;YeqV2)7)3` z79FU*Q5$zoXUX!PAYsfiSUmzr{K1gp-Im>kO#n~hQ{Y@2k~}g>vM!QE?(5T zn>y|haAUod=H`1-{Hr@$;VU2huvCuLd0FU}lw&W}#nWdt#c!Npi-CaWyqnj;-Cs@u zc<_iXZdli%lIf32Y4SuUJ4g?dpuM!UNH?^Qv#cM4EeB_aih1Awgn_K=2Bwcj>Of$_ z%I$7f=(3Z*$KQ~3(XYZkN{Ie#x$r>Y9g`z1GcdlIvV&ibY9NY|hG4vxG_)pM{ydw` zmxWm7YR$dI5^ouOAN94Bm&_0^Nq2vUazao7Be~ehavUfXIc%!3tX?w?8@jvK?e$j^H~7VCWTkoF&Qet% z8IDjvwaNV^&$!7@_xkO5^}@=#m0$qkg zR~o^xN~z$?nt?DbCNHwg7c7c@1wLq{*VKmBm9?j`VvZvq)fOtKyxZ^pBIczAno zA{^KmFvDyGq#6Bb`8l|~TWNXm-1y#XooONv0Cd9b{B**m-EGA_GjDd=OYD5+vjWshZS8)T-+<8Xy=$H)JIwDJ@9;RI*VutXVGBGwmimP~ zuHT>VHNFTUt@WBRz9p@1^|3IC2(FR9bEX~jA;^?a6i4_RdDCIGIm zP1!C7?snFN|8Bz**^4qNIj+`#^oa&kv=*evb!+%7_)F_5D#xBO^$#9x|6y>y)5HKI zI>(0#@{1zH!0cVl6eA>bB3K})g{~%iq7)X*dLm>Eh0z=iM_<8`Ouz5ubZ@eLpR6J7 zpDm$3uIR8$b|Lk34Td4YAFh@WCx8$^(e($)c=f+xW5i?STr!Noh^Os%sINI7{Dr2`a={sm1r@S@3q4T^VZV$k zr?rr0Nsd+0xkt}#rj7O=kY=VF@)WluHJ0=0mDHA?`E0oE6w@GS*Xu4OssXUJL_VD% z5mu9=vgwGAuiJ2?Dy(_ZC&f!9_K}H+aFQjN#levNzz6emD@McB2&oS$-ytHF%Zu@% z06F2I8;_d?o+l6+)@Af3W>xL({gHe+MxpNYi_Qz~CkhXjV%hxAK< zD$|Xz$hm@5;0|H?*O`YS^{pvt+W>jA^pokPs<;}l4Qc3s%uVqo`x@c%KD%@6kn$;r zS=1CBGm^3*m%H%+7pqN@U1p)RDff1sf)TrX$ zo&soFvd#|2+J^3RQ8mtr=?imyiMs!qQzOr@lAUD9hn*x+nJ~UsTW?N-3T>n~Ri zQl3g-;yBlh$gdiCywFl?L-#k4?B`7g!-y72kgcdbxzLC{&dArCFKTZ>T&@zU7Ub3g zS@AX_jA!C?7;@Q)69;fwd5uWsSJfhb)u%qu)TpfBY0ac6u49PuiuF7*I-!PAE9}3E zLlcEQB+e@4bFm@sV~{sBTv zGQeMfI4|DqWb}xu71$cz8PjF`Q8dq(1eZZ@STXbD7`|MzBqMD8dFlZCN?Aae`|n-z zv7P&JKbP1E`whl_oZ}btm!Ps1DjhOq!+&uyHU92m7^d^Tm&ASmDoTrZ@Mk5A z5@mygM5cU}G9wRdxbs^#*NaaNJLQ?7Z#_3TepB9YE+2neBw7es2^h9MW>nnIZ|oct zwvNpBy)d4euIOiW_#e9f*zfvrE<8_PyC+|1o*$2nr4V@2>wn9;b8T%0Sf;ab_Mc=5 zF9z7wCw*lv-Q7B8EcZ{c!d7cOvPV`=E+_m>z8zLBjIIt=yqoI*uRgdiCuuHw>E^Wq zPEWUA_htbF*H0ci$#-G6(%UBjLUVWcK*U)({<~klH|1&a9O>QxR3rh$$nmN9d327p z)~~{2+Nur7Ly3H01rFxO{H>nk^!xyiE?k#8>`lNb!bZp+<0)u3F!VKB$+@>65M@S4 z+)~DUL}_&d6=h0$0Og*MeTZ-_TTLxoRI?4FR@G?z@k&XEVzM26hl7Afc$Fd!X;>rtFf|6$81F?Ayw5NhJMDKhhOp8zoThZXBBF2`gi^ne z|6poEN9@zxaV720kz-9!gtRaLvKdM;9k;qE6)H$Ow&Z8|PWJmYk13{K<> z3<)IJAyq8k%m>|?7~{YHX0~hwL&PjwP${cfs{05e2b!|uLK6rznBN``g+?2NwL)9U zFdE`iBO)Z7l&3)$nnAy#$b?D+hbVcA?eZL44She6NAYO;2t6NFJ6;Dyas-A1Yvz=pD?n=qT(mq6as0Y@RhrU=RHRs3-6dPo zDWnQa#FQe$c)bNRgNRNyEYzD;Wx+CBI$8N;&f!<2yc7&V+=>r8WFcPGzebt|mu&(` zb=5|~YU~3gqjouA9@F`^#4(LJzc2L_5sAkG+4`|_r*>KJ_TwAGMx+mH*-&#MsScTF zCjq7^=NZ*>h*Pl@CXcps$;j{q0=**BrVoe7Vafjjz(8LX9B7&}+~Fyk4gFVmVMtz` z!T+OqI#}Qz8I`>wK!u*YUvFy`%Pl^$EKD?ay?TyQ;>7gDVx}S}On4sdSU?j{SIjlq z&x$f`5w5jlhm#Cq-O#K^Da&ZpZzjcc14wzST6F#Wk=n13)f+XJH)y^?ob(mgc^wrb-0Yb9LERi&) z#|dCz{*NY-Q^`CEgN(2v?)+liy=W(UyrjQDkh2Cu7vY##`C=miZh8zyqzE8DR`z~d z|2QYDI^EmgK2MI^#G{YCoR16f*J=bfz%m`VXB9m6XqdZ-=g-AV(v_kx{ z`8Aus>OZ!~pMdwm!zx*L4FKsS5tJl$u_f_B{V5pB^<|u=rIo24U>+75JBT@T&niuM zrM#v4YyMMfV7Yo+owZr7KZG_FU&t&Etn!?Iu zFGRL6s7kWi2hW$;2kzH^lFgL*6QLr)a8?0NP4UAz`zqh=$YQVRD48oipOVJ*$P~l& z7vZfMyO<@}Ji+3?LsM7^*1(hNfgj^40x8uds0)Fq<_Jk0p)o3ay%R6DbwIcH6PaoEg z%^+S}b>0g>Vw4XHxs>NA;Qi4=d=ldZ{W(cY5qo>r(lwJ1Z8I>a$0GoU*50(zMh6%j zEVfD!iNc(eTAxE)I?qW;A|v$XZg>csLa9TL+xRg3b(m6gnd?9ItFkS@PVq7fGxqP5 zQSsJ29WG2Xs)ZDj6OGbGo=gx$AJ177VeNC%8HS}KkQj?^ZUgs{1p)s5m^4F z{ex?a+>GCYnj!Z#W?qN{_xJUPCyBkLo(5_GzeBAeoh1>fO&sp;wO8-l-E~6GSa4w~_et=tNny8J=TY zGK?2pkRsxer+`z80C+N|sj>Z-4UbE)j}=mqmy>1i6#OY%iYi}SbmCOMF_H@5 z(xosD`db~CfBM8gpnSMqGBfR0bo5^OznJa*s4z@7*T8PAcTh=XohsXmAc(4^@g;qw z%FS`vx6`!86%g&23+#NfE?s=s@34nF2c1=PsV^K5JrtLn!Y&OIk{~fU09M`(mQ@;; zPSk19V-c_YZhbDML}XLdybIFqc1p`f7vG7=M_5|{_uead<@r}2k6i89F9>`(8pA_&z7$e zdGH$VU>e}wz-bI)zW!Z(<{7HD5u^V61+o%w6!$;l#-5b$k4<9Y{J+vuFImB1oe?wa z<_(pvY8Oy5ZPl(&G*HhH?=-A$&7vcPefmI$qh?cLdR(HdG$!*b9 zJ|kY2y7#7=GRUx^ zIVi+4lKmOHSPvn7Tuu1|rp)WKMOG;N3%tS}*t*dtG7v^~MFh*@$b;)5#&o3gq=(96 z$j(llfw-c2&hswEX-U_?q{vkYK!F`0*YmsY$$bWEZ2^qFv-g^q0I6Rpy^*w5sE9dF!0ve(DbyUP{@uO zvEt5iMlLPn)?+f@koCO!xa%E3_omb*TPUy1ymeJ`wc~h->;Z85;Ta@J`V4^(Y%*2fm1eRj|m4(Vil+yEV`@PCk>*_*N1(*2=_!GBt{>ubUc4} zjnLJct6jA#4LWqU>R=CbyHuB|0G?9^W8t%-l&RLIV`ca3;-(OXmP)YnGe{y{%YA%4 zz5aS`+j^~(j@beD*afn?zCJZewVRqGu1CxWiT+$HJv(EcX?FNBq)teNzxr~&v%W{Q zQE&K3tURn&I{Z6TA*|uAw9{P63EK;cJrqw66+?=ZF`9m+%MKk*bl40iFk|Km(X1&g z4Igd(*{2?9+1mEw@RL`lq~eqMtvyL@1MqG5+Br+fz9`y1dw@wxC zcXXcuWk7aSx@D#V55$-XK9{EoxEBnS+C$xx>?;kn0kc5%f}VzbFIDu6Xs7g0I?WBm z7yxB}=4Z1!H%`yX$w$ao+S3-%%csHSW0pNt8ZPbC8ulvtBfP_AZJxfXE!0=1P=@my zfrIrtHpnKD0_%gVI+mAN0ovD0HK2IAA`myiy95xio7q+nK^GT@zTvgXF6;q>Z8O0RXd z+sbg6=|KraDqwy*b$NUHaAaG4q5f*HoUCI#RKJ{3T={n`aWFu>`MFeuZ1TS z_RP$RSR)lZ>E}!9%GAirgLN1Cdey(RM;Fhy*X&Oc43n-EbvwJ5 zRLd;I{jiX`nQ`Z7%_NE5SJVi!S(wS2&MH4&mfM+7eDI&b%>+;}CQ4u_32SKOJS>1E za0(`<+!*MT2XF=n3lLpARHVqDvSKhM%DWWQz9019mmt(im%n1#VIGpPtZo9MM;qEvl^y$6O#Ke&pmu(AA5uy0Aa ztdoGP)Lj+{B0%6$EI&7cGqaAsYk=U9AF$flskN~-{RKNc+x22C7xz=lBxi7REikvT znEStTv(JGHz{d9f&CQ*t!vEGTH1};I^^2t9aLIvcAR!l9^;;nw-i;Y%OgmX8f=dd|@r>)L9?q*Uyuy}a>74C*6v|SG9l&WOI|x+J7JR=F zZ3Dhc+lyMYomma{YUn{Di4GfohhXM#Ph_oaaE_E?Q2he4=kKIRS|5cWgbJCw?vOUu2pY zNiLlPXQe}4VKOb$Jyf9V0%Wr?)F|LDn2hPjaHWK(#JmOr)QE9p3B#=S2RFLgn<$D*JXJ|N_ve0>%D{Ot|DaHYB_p3WZ$khs7~MrQfe23 z2=+S#DM%3NPqrq4K9O z7I@WAN!SQtCDKq-5+~9CR3f1nEO&uyCTOJw$a2~QWzvvjAS$lWc&LyQT|n~igJTLc zgjuZ*nMG8=vc`#ez=r7qaV5cWEDYE(UKMq#ns(ty%&L1A57>=?FN^}>e=7i1#{W`o z0M7qFHuREZYz`PvLtehowB0bJKNA-P8H?oM0<0)V=^EW^$xO-!D5r^k{EPwa?UsZA zGwt(S+#D-WJ{@arQ)TEpI~oui6)Fo$+j{SnwOQQlE(>5vRp%wxA;&uvbYX@GJ*fja zYuwGRjouUPAGtmHFTMaBH>KA$_mc~YGXjC_sL`_Z>@vR|IQgOejj261EWM5^`7Z5h z;9~R_yD76rj;FeskHQZozK<)Llg#TXeE$1^&qGJNP=%FtdsU-&vj-=n=ZbE(wEU_p zhK@!U^}Vm8ay6$P(}0U}hl};PE02orR6pvtdv8z749ZMOcnd(}iekSuET0HeQ$BN; zzPYqYSst=9@iHrFBYCj!J!tVPIP)b-yjkunN{aJ1GBV$t6_F1)Lp-MlL&`=$T4WoDpIP1C^8Hgdi8H2Qp;k5V?b-6aYgKaB^N@B>-?kIdj^#OCZ!Q-EC?I(C`PE_kS66Goz%8XlP(o<|M~m?nbk6Sul`)w(EcU zqNLRtrhkv@|Mecg#LoF2GSN=WR~1DAJr}WmHeJ z7yE9p0UDeclyoxKr%>i?8X`{6pE?i`Q5F-l7eF$eWd6}3lnk|73NRr&5?{`AeY|CTu7E`m@invi&8hGhK*hBApB?#7ru zkv{6z);vbSDy3*JtzeE&js!Z1%cy`D6Y*Ibg57e%+oL+&RC?6Z(HNAJRBf522;gtM zriN)rVp5`d7w()*c`_)qo{D1sx-8jHa_F4$8I-CyaLUknza>+1mRXrkyJR6l(W)?Y z=C+)cY}1_RS#;Hqp>~3x83tM=C5{bket%`c@&YzWbun#pD!cMAafb9pGfU=rKeIV? zVVl3XEGrs;f)aT~N#vyYK>MI*BS0*z7uz)=#0W>F;2VNO#dwgxZ{oS?@-@1;f}_E$ zJHfCevs`#t(#oo_MAJd~RlxcO!nA%^IR+nUf|wLvE2u19Ry5{PL^@u1Ffuu6peaNw zQj!EzoFryc4hfMjd|nJLipr_-b;FVn3FhaYhrqq6Zp{62g2DDasnziNH{cX=d4L50 zHI0r(skyu*5F8~NR5d?|Fr7mh7VXIGaLxc74B`L`(aDyGbrvX z#T1D8($&t#E(m#}>04o#riTZI_lBe{fNzT8x5e#tc`SqEUbE9sWb)UC^S#9Mz{y_d zGZF$(rbLLGdYg?rs2ZQ~D@t-(Xoh*!vu$ZZn3c`**RLGtFlU{$=gzL+ zg{56LYCQe0m#x_oL^9K5|Gn=+-?Hm~!^!6l+q^aMANJeU-ha2bg`oI=Z&v!l=mVNm zvn@;~AT|As6^PI*V$EAya|R0#cR|_Y`E z{UQRKGPaLX2=Jy7fZziOe%DL4dkq5-yR>Fp_lWwkhlE@sdJ+ieTsQ`Tn-Z;lFzgZ< zdsM@_I^#knG~Gq_xkr@ddHXei2fs?QyZ}1zelhZ07ZeZQjcHF~$w~8qf*wiz6?WpsAM!2=UJRz>7v3>?V6QIxU2EQ z2L+_TOf{JQN06f}m-2$3vNPBn4(OV6>F1%0u)!O(RG7kdIjj^)P0R$z_maF`{5todjc>-j^Ceow9>EwCY z&n5uH@qk0R0&;@w+XkyiP`WB=LbQsG{IGW00x({7QotZKe^487zQrq_%=lNCE1O0| zG}E;im!z(DdMvTR{dwPiMU{x1^!Vm3-dwGDMO)QGyY8p-4()8 z$MX9(JB=^2B!;7c0>dqvS3jV;`0pIbWwLY}7&kB7HhEm>-(szImhbG4xLlP_oCHr$ z^8ZmdoC!!|X|)hQh$y>;hDMf%hDwH(h(^x8ej)t=#`v#KlI(YR0ss1p{D%fg(S*`P zSy|4-Bu7CiBo}8YWa)#x-hEywZgw3?K?kd>)(IpOO@%aaHA@*Z0v#zp8_{3?Dn5pwKI zU&V_}a;vCso?w%Hp5+wuolmQ?O6wV`QL83-%Bp`&od~aNXZT+TAPxh04*Z#l)qHb28ud7geyosH-f%wo+Zg ztl(->nWmy_P7w9MaQb7F2D0Nsc8Q~Ok%L{eSGtqR zlN;zzN<+Y_@!*tSzuB5({;M}TQ4KBM8oxGp`}|Htg?IKMF)Db54j(r$c%Fi?Hk9owYV_48eK*kJ}s-+GoK);3o#nNp%7K84QDz~HKrCBD;8TvK7U648-fU)%frP>;*vaoXy~h)m8N&>63?DIWN$7~c!2 z7$IU*)^q{sQywbH3;(_+IL)Q^videFLm{6g#26juR~IoNQf09g+30jXqz=8YJXN8G}z6Q0dhZs)B_`bCc}fJ4d%q;<;35E(7q z+vq;aef%YiI)30PYu5h`4nRkE`4ipz`u8R`jL1S>l^gzn25YH`!oK2NmA z;CvqHF?Y%CbFJjz8N21na@-3dppv`;Ef9N`EUY2o^5o$cpQ0~-lpJw#yfy!_I<<4r z*52u|U~q}{x#gF4YFMjT1z>qqNzY`tx|!k?-gmafj5g=ClSAG3N&&@?Hk$9*1K|cj zWRX+F+@=-3#yW*V1HTG=m11c1z?G)fkzNRx7p?a&{@2zU*l6)S?45xC~RFoz*{Ys!z&t9}(>Ua)aE! zOIeED8~&dQPvLSl>u6K|T@md&5TZwGAysJUe)Cm?V9p!>b^QgjtrAT1T9H@+Eqqe~ zkEbU~#Aq8)tc+ab_-x~Z#8o#^Zx}vH7X*Gr?z1kB+dIRWT;8X^-N*ga{lb-x=>(Sw zzfPz$wZ|+IbY}||IbYintWHJ@-FQjI1i?Vo$a4oZ7nxf)$zn6rl$V~Qu{3e1vT`#~ z{!pdnBv=oCRiJ{jXrR|YN!)nUtE8Bt?+ zsYXALywT|*fh}wWt)yAaIX7f*6O?IS}9L*jMwKdH1>Y*o>=il;- z!L6~6mz$cZeaBVIC+$Ol5EV2fmD#0(k-4QA0f}#bji;l_vx8l9XgWYImTiOiHLqV$x`m6kN zN0npu!c^8pFc*Y9p$nXXIwocwKfmvtV#=8r{K}7!?p{$&@uSNw6&eHravjc?0tfQm zZGt3V=3IhUhqIPMU0~4O)ars{vCoZH_*lK+H%h%Kskk3FN%BiC&IFH|>nyJ%B;p8j zh07~hn$}52(v$W}i-r=P_ijPdEcMLF5TIfMPR@uZAyZgHyI_4i6EBPlOg}U@US-z6 zUyS}9>4%3$t$5#68LWkz$cDRzPk=+fm%|SLRKP3rMxv*zuI-aTHatw8A7g8a*f8|K zT~fvFAYt!Tvo?7!v>Jx)8a;k55wF_lb*ZGbWZ(;)99m9Ev!m99@xBk0Q=t{YqliCW zHqVk6?*E&itm{xa-p`tY+5Hy!s>de3v`Ge@&AMw+z=mcnHl!+5kW(~%kC3GcX(Ap# z7Yb0v+%albBDv7bFXmAT+hvTf#+WIq`R0lyNW3~1xP(W+^=jD|MKPu*zfzC^htl64 z7y)KyjQB&zlr*9>9mFZaE2&|Y_|LH&PscEUOq;r@JDrtiDrI!w%B(ZBqnF>ab>+m{ zxE6gc=OY_1q+N|Pqls1K!k@q3zRm)8uJgVmE9UXEygV1v(JPs-eabEiu{-;X1`YO_ zFuTIT4iy%{DS;rd^74sbs;ZIW^}_fhbgHvDWib^vTD@7SgOSW&G*SbrnLVnLmnQ3q zI`i3_YVbj`Tv6kHIIV*LuSPNX$>QMc@wF32ga3`3jbi)^Wwo+Z(^pf0CV&FK(mIIR zVl6j1x1+30br@~dMrNHp!8&oA6HPPMtdgRQ{rV|5F&xC3u03l~t~A5YiMnL#^av|& z(SE^jZ0Y)_@?Ogstoea6uBoE`)>EH>3r{!OZNGba2aKEy|0J*6ls3CQIb7L`q6eil zRf2b`i~QRGK#Uln;D<`=JK_T1wJ*Uk$8uAXUmfASuhiqDbD!Gh9am!YYk6k2JvCZu0tA|>Xbp?Rg)-n{MD`PBnQsXTK*ZVV{B{l^ zJkFC(ZV7o``#jPiRRn(UlX0IZl@vsZY84NJAiZ|T8g$pj@yhvIGz5c#B&W-f-*y-%Bi79yf~ATi8{2Txk*J3wrdsMY?s)JF_Ayjdr(SU%M_!Vy)gQU@C;11{VTcOgC?~? zQM0O-BS!f0APgj;x!r(Xwd6Y+U=JSAzlOGIZi%~VeIiQ+pP;hVyFNX=w!0NYqi~x# z(dT;(4{fJ20Wjvs=dr_G6a94*x&8=oeBeeme=Ac@e=kuKa8jD8f+*l~>p7)=?fMWt zU(z=)3Ogf+OA@`cxBIH7Se}-Y#Un~A?hVA2xXC-QPbE<<8n0%BY(5omqlXuC}M)QxrI$OXgptU-`Y{E8(K& z!WW&TLMc`_9k%Q5)tQ!#GAmImovUjs-=Oa&;RaBKIOg)JP+`#(v~3qLngmEKKh}lh z`b0<*<|U2vp^UTv9bfs|E{rOe^2aA9PMS3xF`gvacr9R0z=T?9uf>*Sd~^8dxY*+8 zaEvAz=W5TE6+h@UZ*Nv0pDJvo+}#wTPJ%gZ@QjT?PB5Y}Rh^gB`J;huna-AYJnDpL zfV8mcUKpohb>ynHh$OZv=DESbP!J1t4-H-TV>BnrkSDx4CHxyu8F172xJ}w=;z#unA`i`?EkRm8#pw%Y$Hc zWZkkl$A4V}yE*GpxkAJsOGZELpfv?EbR-v(@thutD4)V6Lx58F;@uROg;TQgeU4Ns zxC?;CFdC&2az+moB!o?lWDh6h&P+>3Yoim%X*MBWA2t@iBJv}^F3Q#AdP0C}vk@4~ zNyl$A-L&a%$tYQ_IKWd_o3*Syq&~x8H;&|cF62C^NlMk~~|zVjLwob zcU@TAT^9*~;J&yfNN^`W77LyL%c8+q2oNM_!a^XI@4I!Y?*H^x%|mz1RP|K#OV>GP z&VY$q^DVVkBH9_t`z+^b`wlEqsT1_Bx^_=9VBE3c$G-qAe5q8DI%%2&g;%c}e*W5dF^Z4*S9 zA_EH)xw@gt((t6t{I)(PIO0`dxZ~-GBggP_h_D^LKb~48zjK0zF&4rF8JnJu(F%zS ze9NPnRrpKL`VCLbaS3HVBNjCqhfgki7fLi^3+v)r3cX+GG+uPikt zwE|!ffR;)6lDRN~%O{(5y}g#c1lb8XIuwm8uFhJxJ)MN)vzGoAhY>UK-1_Ck2XpgKi8vxH0d(1z3lV;wha1lI5T2ec%EBRx?#6 zB{$yUDuTo~wVtTye8vhi*kQv&*ZEo0egc$mQCj3wxt($HL!`?TcbU;oeG?`&F!Iml zfj57zd06lD+LZ1|gJ8ziR0pj-wvn8}T(539-k)U0zqXrTgXQB%2Rkbs?+U)4GliqD z!Yp{n5M1U!{1?i{IK$R)VG(w%m)>eK>7A8TE`-RC(9AN_`jND(x`=R;q4f7!M5dovS~5yXF|QxpF)^W-#v%rcMj~Osch}nVQ{M@lJ&w@|=BvqE7F6)6~m#+t7po z*Y(?Ueb`uhQN|dpx6Q`>k^!noTY~JeTA;MH&4MjB{3wmF_sGr>l5u@TtD-lb)L5Oy z%!8QxJYa9y-}Fvv^kXkJy-X;3nrw`e+O6=e#pPv#sGFYM;R}+q)EK0rl*dRMs&fY; z4Yt#Z+3UK1^B03|tY|)lN22^c^mZo}9)1Wl@`uQNdzh7_Md$wgUSb7*I+6Diz9+}6TLD(I7Wj>R z$Jq_XL-)5~p&VxHs)D~-qXDWJbm~Iuk4}a{4NQ(~lt?W}u6n0Lq#WHyy86Uv>>#GwJUmN5 zr@U-^A?Zv(=kM5AlNBR=4bF;ODf0BQLljyp4aS5xYGasuc5{eq27WBe1_5;+GH<(l z@*;i=gG_#`9>cy_twUQMyF9^Jm+g0Ph-P`SC&-B)U3P2(9aAcnAMIc&jA_WA<8q@V z%L(cEYbI5wn0J;^{_jMIaV&0c8Zta38VUh}mlVX^1H#b6L94Iml}KlFym>d@VWc$+ zi*<#vaGCxSv>l-J5VkIBd0B8fDKQNo)(QPWV@L{1pZD{KJ=?O{xjo$nTFa&2Aq-*X zZYd_pG)#==$KfLET`>Z5GdhSilCMwT4x{TI^J-*wGu7GZo{0QKq zv5oJ^Q$^v9qsu-K zpMS|5mrkIqE6k?8UFoPqcX1lSm}T)s}f5(7Ocjt6g?zDg}ag#xhL|%F~Qe7_H`Ub@m2xa z7(Y5hv~XpVoAw4UJsscFmWG0yO)TsRhM5YJBz~W` zdARRq1UY#qco@0L+b)qIFeqbZ+Dud2H9Dpi~f|LqAQB zzC{`Dge&(%e*O5(WU#AT-mTkW(b;|!zTV)RD6BEsfXyj+;>#`gXF5E z6QZxQ>GpE(S{rt_GX5@0S=0=mzD>7mmwLEsKw?B1Uxcuo5S>9u$ccQn;{t+KKCd?VG0pD4T{bUMHl{+{Y%-OJyWH96xY z`&N>MwL_5OnpZLaH=>^Xr%PHhM>9K01dn1(QFiLLieZwjecW`2_XD{Xo?|SlmyM!> z*L^?dwlszw(9<?#G4vG7L&CN@J+73Up5tf73jsNb2i% z8R;fp;P8XBy*L>xqB_UbqwFWCc3PvW=bT!Za?>DTm?kPznqPSs^hB&;TZQPfIF4CBOe1c*WNCR}}{fp5hk75DPpzwmf?aVCTE zYQM?{<@2<3O8T2S{-^=LOIc&xcd8dnu@IF@m*E-fz2p4+;$lU90tptD;z^=g5ZoEY zb>3M41By`HyuY>n%98YJ#Xg1=Y%FLn4sOwI9!6g4!<70ic=1+h|(C0>rUVf3yhFTle zbInBJe(*H|TrLI<{bjtZuGE2qEP31!tOR>#LST5k5f7785Sy}tM4Q^D`+KENTt~X* z^AG9)qsqh8j?Lte`Yc`HYuE;qQKKP-MDh$_dO!H^$)m+`Ufe~!T9;3tz*|6nIoUl2 zMC`n7qyphnL_6_kV&^Xqz8zRMAGj-;UDp{0sIf`GJYt_u8rA&$@?q5&vLS#9hi&Xy zOv)6ycdC&vH+pVq+5F@x9BQ}-dqqpXz74Yfy;#K&)=oZ`EjunLQ6;FtC0kJZ?!yO~ zpfL>*A#Lbkkke!!X4Zh0{*5+4|3OrL2Fr;J)?e0%(+bN~qrdf*u34dG0ecyF%>a#j zVH5auy_7@fGl#&LyE20a$VXqr;{6nzHKG&N4cMlZU|BnXpNY8ZHgJ2yQ(feh=>Lv&F{L^Ogkf54A zj~tUfuTcJXFh&^9y?%$~cGy}aW#LErbAiKEq63Vwc6YZZS6ulOh|->RZ@9Q2Y(4S? zN%oRGb5sxGz)q!!9&KAiGq|9O-oA&BOzO=+U>$ z^l7X7scG4_z4h{o6Ss*ArM=aYt{?ItylK_?EnQ9{q9v44f#Bc1`Ue+v$$b4oH5crf z>Qi%#=VxZ3g}O}j5ZTn4O#;ai7R~o0ODh~U4Z0;d)=vF`w>_U_ZA-2tdk^3ML9iIY zJ>tUW*WI7cK%eW3`#`pj6l)h}7ELBFu7stED%m;!DoZKm|U^A%A^L`VpD`Y^Y zbPB3(4e_XB_VM3eam^)yY&ghm>?WbeH%3St&iM#Z>X?=wF2yX%G1Ph?;uFu37C0k} z1R27@qyjx1UekmD=eUw%RYUPXgI^PLCb!7$ljFA7>F32gt6GTmK^`h|6D=+0(~9^e zdeB7>06A=%;fm$|F_=&9qy7EyR zDur*h1hZ3+u&=YJ4RmlCxAdGxGr4^MX=m`EKiT>ZvY0JX`Nhi;)s&L7g9FOLa%Lx` za~hZuTV94L|3dClx&prMp}Pn3Wd4SV8+7FM8v{bD$Ra$f;&{Da+e3xz=N;rKgS^=` za7$m>$XQmFH6Dq?loeTcU(fL^?NPJTYf@Kw{De*jRCIdRvzA%P;A-9&HCooPWW|-D zcoSrRXF{4-gsYAOic9qIbC`fItn{!gI#VS!!MGLo2R_o{j(WL$hb(7mPmjrna?+K zuhZE@p@L{P=Y8KZLmyVMp!vNodcU$1QZYUd0c6(G30u*uxy&m}2A?RaO2oaRk0(GN zu&XLWnry`x^M%ioBvkFApC*zTjS~`NUuJVcC~I~z;8+Ba@A#@SS{c@Boj>L5CcpGy z%fpXS0||afU`Q1{Q6t{qq>*CXn}V9f*7q7nYub zB=V#6;4QFKsZCTwC$xZ)D-V%{IIb@QWo1&xIEL+`K!!kQIjAv^-cL&ZcUp{)Fj-KC@k8!)25>XUDn{Tj2{ae`;yI|ZX-j-HMlY(Jcr z#)fJ6GTmWh(v{fclk|Z&E7YIRH1{>+H|KUlxUMmP1OtsPfj_zHJuzb|4Xh`mbWH3| zmQYVMv23`&ibKoq6Wx74$)}ss!!~GL-GqMk!_@Hw7-)t#Gbt4f3A}a)X-QOG+Y8TX zk}EEnHvZYRaW;0%XE6hXKGOx)eEyEAW&-!XtR1eBL`Gfu@$a2hz&#~4m9mgTRngit z@vztwa%R}uc*Ua^0ghfTfYK>rw0Zk)U@2oqFg)-Lf${y&YgM@Ky?CKbB=>01LnI0; ztAujwPWr&~YH)F^)h~eAVn7{_1TQcUo?p zc3_&xUqY`xy*9Th%En@m!aI;~*f<5_4PT-KOM%|kT8p)fH-`f^SG>!2Y4MwY>N;9Q zpjHWG)K@M6du8l^mP%Tf^?hFv$X~_zSg5l2McT%v`-kI|MY<5EJFwT?2e);k${o&W zQSZ_`!sV_<)M=|6^P*__z+jte-DVs@Iu3OzOWy4Q>@l#DAd5`fAnKA1K@;qqNkK^zT~! z;LWwW$rT8F7j;@YFG4LLRc9c<26UIg@e)B-{H?vpjWYG)0oD-CBC^I^!V<2cK`W;~ z_!i-q3ahV5oQxM+%f9x!b}lj2)7E8hzFQiP-)aZ5c5d$TC1%}Rs8{KI=M@<2r$)m1OwDxT1bn61qAHD!a@!pM-W&@!cpKc0)fDeqJjbr6+qw1 z%bx)FSVAA5?g{o{5&Snaa0zu}0g4F&OmHoLEJ7@TR{!D)Lc;(5ZH@>+-7Ce8;m!G- zXNr-zlEf&LGMvuUwc|r}CwR=z&y6WHe{OLW7?WVS(vq%re1qrSoMEcLaZ9wkIv>*X zrx&$1S^O-7`sbzE-0Z8=>^U>(*svC~XIlY%iCt$nok)-4f!<(dsdF;f0qf&Y^y4;& zAq%{9YD1;Eq3^phQns#!HJ3r!Hm7|wGYPlhBGpu7i?xoQa!Sp;r66)dkRZ+Uu)tIW zsaN-2I2RcF#g+cy)>PWgkq$I5oMjvqz=v#J_;JgiDcfnbE(fAb*yxu zZ6Sr)OXz$iuzZu{EzvkM(M`9sWnKh{QSK(L^L}oO3@v)|#G9$AnOMwTMrVw+o0fW8 zi*I;D{0Y>SXnK^$G_lZ|_;G0T9F*y%>;kci+6j3M7k6U@A^D|K?#>06#te+Teg&FE zl#=51pHmu;LNC*HMlluhz^oKfPI&^_KLTGik>$LCsR`rVUNLyLq3j1i- z_Dbdmh&{p{&_)`=IKwzo{EXOsU+8Z0v+=z8L) eE);N&TfJlK0X;KAXM7?cAS^(@$*HWXLhwJwZ1P9| delta 46019 zcmZs?Wl$YK*ENazW(R)Hm*i)Gej!$^0Wbl`2%WTC=8Q6OHRwhHSdo@s?mlEZ=1l zp10Y9dv15nfK%)C>~!#WTsket;c|BCwF3^Na9g9O-Dc;CVZ4>OxcU3f=xzkc27>Hgfi5B^&EVDw1`Pc`LO zl)SZDpz-4u-Na2?Hs>8%Q-6eGX8I5TIhI?eIsXz1eU$O)1p@*O=ekR)+j z9je0EGa_?oz^)wO>}UrlM@x{(;l#?(ag;t`PVnpZJa2{a5YK`b<%5`qK;B$BOq%)Q zO5@Bu{pZN{+1oFCvA8~vk$l0Ow8o3v;c~*R94X_|N`d{t@fOemc|-H8mnI}Z!`8PWWo~Jh< zm|SU{x9b2xE{I7Rl#wc|Y+D&N&h{^MZ|`G643|xVNxyP7(ot2UA2tCMIVBL`ix0l0 zMQRSwaxR)s&c-#@k{Aev6>1L9Q_)`;??z21?E`qOxPc(4-*o|lHbHF*F%F_(Gdgy$ zQkZgl=WMBKK`a>F3)`mw*c7tmhF6Nhc@PQ|p2lxoo^A5{k9{tJqW-l7Bo6 zOlM!FyLjwvu5bmy5zjV;zvbDdf=?Y?*upHp8niq&b49Cty`d&Bn12~BoVYP{s%tYB zM|S!rJ*+@RySO>`ce&;y-lgUc?P1V~0oDhJbVkpag@D;x%J-+OPloFO3xC}BykKaY z;|qK<`0dSyQfdD}qJ#j|BMU_uS=B?cA-BnM z2o=a8KCFC-KkAnUeXj~e<3GXh;YU;0K3y1mjvN&TIU=F4! z6a)D`yoVX(DrlF8rE*IUh`QAUR?SoAW>dSugoGg$p8Vexw$?BbRPG+t-neEu?<6OD zU%n>G`PpL&1`|yE?cZ6(jEuv$GPqdRA%_Kxm+us-)dfo(v?6JALaZp}YlTqFQtVfq zn+Ko}T&M|+74!G#NJTHlp(=9AOAo*aZVlx&ENZigILT?TpT4k4ul4mH!j=5##M^ER zn*idPZbdA@uFuyQA_srz=uD|aHCNs`8TRQ~OjSyDs2KUqmYkC_8(K#Qc~W=cQEdo% zDCDd;GBZ(u87+A5WHiQD+kA0C5FmMXx;SYcNyvLp z+Lo<(eb;;EaZuMbxjY?>i>6hIwldWDW1fZPtR+1OptKei(uH(3I+v89nJDec5HOxP z*)5NgQOEtd9w_E**U~V|byKxT@Ms2-+9!wYI9qr4Fpc*^Iyd!pq=6h^^Le51k{UQ2 zW>>DOfPda-ZA#)FbtAvlp5l0BQiBe3mFy4?JUEd?WQ?RxyjmdNcl*n3@o@c9dUY-R zv|})5+_%%(*1jQ}GSHprgVJZsEUo6-%s!p){l>W-2;bffD%Rrh{!w})u}2ETIkLQY zaTynX{^2{^{KqhE9-Z!Dl~#I4K4&|855d_wt8aTqhPLZVBOdRL{G3BpcIY8G7QFIGpC$bC^Em(yJVzYrQ=4G)@3Nr>&A$sx>uUkfew>wx70 z?-&bg`cTV+FCIM{Ku;rU1(uSPy2RvP6li86Srg72>^6``9wC`&jLGf7f>8%&ib`%# z3@QpoLu5Jn4L|oD^R!Me%u^ zdU=4)JYP_sJ#v!YCimEIrZ|;(dV5(p4 z*Z2%5fpSy4S(pC2vUCiIi&`|zBasQdw=-9#X6~?pAmQku3;sG#Ty$Ajl$9q8>ZanK zNId#guDX>{mlnmd#FVtq%9kLGm}Jb|lN*};FawHlwgxZBBikcIBu)C?6Crb*`e`aD zocu7B?^u?C(-VV%Eu?UL^VjHpVFk@L{&|JKcG2doP-z*gjs`SQzG83qa-H_ORxF=u zGU=I2G*OM18?rf2tOjX)f8`{S(KT5^+m5%@LH0mtd%mS|euL;jv_%RfW9NX7MabOW zi&c-}ZzC-EX|2=8S`3+4PQ8iI7_;doqMu@CfvJgGx76nMr$3r2tfxQrYRBmMIfOa3 zew&fh5yntnY(L3B9m-3Q!?vq>$N!c zf=*u_5EohJ=nW=cZE#E}GrVB6_Tv{FH<(hl=ky88IQq1@%hxe+#a|_F`V&~_e z^Lnh(&%E^xi+~;Z;+epTjoBcBs7(%H|DQhJhJl>@_m;2SV68h+ymW?cNO5?m??_M0 zQ=5@Qngo`>Wt94y%Tz0A1}Jb(YW5 zNJ&m|kFRbwv%CyN$`(!^jF3CzbZ6aoiv;=?6}WCDkdYATq0G-&)-`#xHo0nSrzPa? z;S{{LW1Q=ABKzGwM5$uUlY5Q;DtRjUeg|6vbSeQF2exP%X`4TVfkP~IvkCeFq13?u zT^`TgvZpW*&JGB(TVDWk{XivKicxKqlt@B>W0fD($f(>QN(LL642Ln5+W65R^o79A zQW6!Tst5(Bxc|s*YDPD&j?8J|Uw+1J1A5Ku*_89b~mJ>2A5grwdenYpI zW%1)YqdK8apn(!?WfRe{A-qZQjqMB15r2mI%(M@D z^XWQ!G+n~wk1==vRm7AoZYCI&XF5Nn(w^fXU53$(lgatfS|DsOx6Y1p6HUvFNga&6*0K~10n!*j0fnzwZs8U~C6#`tV3iV)-935C>F7x6`NzI@k|794I zT^8?)fg8zp=<}}fM)Z~F%YiHI%Qimpa-GNwsVD+jkJ1HjI@@l|L2!oC*Phmp2bak-!0Sjt7BWNO_Fu|d6+)m z(lY4VVUi={SZ*Cy9%vwWNM|8SNY`g2L*l${!)YQm5|qya^lc=_cuE=ge=ntquWgu%QcSf{LmQEHv& z$m^$wHFb&p;!|=-Hu6~qA4!~_W`by!_;R3m35)aqp?uq({19igx}KiO(yz^AJ6N~* zC z&Z{jo{X-wkIA42i&1|ZU2xNv!E528G!9oC!KfYMvc*=7nS9@0@BsaW<&v$2wL zFX{xt^NI2!D7NTTW9l!+-HR7jOz4rr#Ax^U3ZpREch^acEyQH@U%9IA3Gn7&=0WuX;mkGT&qgp zW}Qc}hazVCYu1I(S{LNac#fC109AC8c=y;in)qnRdOCg*tA^U=#&loI%)q@Y+UB>Kai;Hp)j4+v!={`nLo|Rx6>l z)D3{*v@m&x>3SR_9(rP8OE_ z_*J!7?rS_5n{nH^{^SmnCrYYF#jvF2&^NgPFOoTY#rwXLZi$@oiou}tY&DY z71y1OWWG`#1=VZDT6xv({0Oc@2ofe)7y?|2Q5M#f{2frbk9<5VeCLBlYTYV3I6fb` z@csr5wEBb~P$R+Jc4O~;m#%U?za&VMz0%@KP_N~!WqdPlf=^WA*H~p;33YxFmRL*v z+X&LG9&IY_x%k(F;UpLSpN<7r;&HE4XFc3e)o72D^WQaXc{i6%-bA+brcZ<1gb`;e zi+(E_g)pEZ!t?qVR%^M@Z%iXcjCz#@uPB@=>J`#m2@!k^D8)qCy~bsl@ZKKf&%1mK zKVjCY#+6VS{)V*@J+K^IRm2UT9MsBt>J8zA*m3=A_!ZJcx_%3PGP_$0STg|~P$h}< zyITbwg%K!N0>D(*vi*etoOlVQEQeYA^Do!(1&^Q;$P*jVunatw44-}PaR@x@<{1sz zL$SFB7X$WGS>VM1$|C4hR~os1u{dM88Toj*(Pdm|V)pU=wh|10Y7rELUin`Xi|Dbk#VCd&Qi(Dyq%63?O}Z8k;P@ohw%jEl3Hk zuK)4ZrJatOk03i~umjQ@j^;+*kLf%~r@V3o@))trpE)r%aQ%cMbt%xhv722)H6d&igx4I?EB_aWhH@-i8Obp$Z zB<^(1R96DC9@mXpyex^i#gf>iUFLKKc)#3@VDjVE^dtO;mq^Q)SgUke$DmcyY{QI9 ziLktHH7vpbwWT!kl~+rhdd#Ln->8O{Seq?OFIJVe_zoV69)ktm+<`a3U%Z11l(h^Rluc+TP1 zst+7obB4q{s?a3b35suZXR{^t(8V!B-V-wssu(b`#KTA8+bSOz<5`v6jn1E%jW$z( zS`x^leqTj3UjdykeD}f*-Xu$rm$MJ_jSJr#&Uq0M4p;xwPfmf|AgVfz8ratUB+*^+ zg06o1&r?3q+kckAVn+Y?B9Y9RTIl=1g7?Pxo`$q$v^#?Q!}5n7mBetm z$g>|?{_s*N_71IuA06{W$8`^fnrGTGh214PIQ$V$@u5>?Wfb2i@>x}Hrj!#nsb=}` z{%&{bSqhSoh^`*!yY;-O3Omxx!n9*gigvTnT_b8l;1ntC^TQ_LCVPGnXHuUf;gF^- znw1%iw)lJB)^mcRtctg_w1GtT-MU(rRJjdon$Y5TeB*-3J?puRP}%4ATGOBgitZz&#G@TSzA)@&fe)I)aUdjGEZN9e7pxbHTtBhLkk^9rt;q6FOCxklXBR;O^j6^k^s`+Jdh-etVqo z@dkb02F#E;OHF!Q9-+lr?cY(LLI(F&MD=r5*Ceo(50J+Axkb(RMzgc7s&yL})8 zY%6R2q1mamu@HnxCkya4ig5;J-bhR7ym0;8JTRKOUm{A@{`;&7W4oYkCz;an`=}8Bke@Usom_zr!AGSShtBytUXs$bw9U z4TAR`3^5FwaPIRxUKF1ZL_d99gRP8Z&520VUL>F!{``=)jf~=)wUZ-|~%6#^dDRv55@ry!Q z2G(S4^p&mS(w~NI{x?*{($UfsDCG@%vw|klQ zDlCIA?^TV_mxNy+nG~3~fJEVt0L{Mi8@yg>8dqS(s`RSvZ|>j;!D%=DQ4hjpB^T%) z#d~u}ofbQek}LFThU{nKU&wNtx_g6)^*TlGW8i!~a_(;*i5;Bp$mFuQgTDNN?qf*3A>FVB=v1V6zodgp;Qa|!QpG||f zd!O6^7c(u=ZP?=dZS~PyhzP!Kb0A1<2x__nI3ZpTI~(u+jK?exj@#`IUuV~Bqy4^k zgw}?a^!o|9#r1dcO0*#EY=Cdn4)w!F>%L?3@QO8YS0taO(pXDNYpFwbpv4~z+4l(m z`&Pem;O72t^T`x`d&bf3{OkinZ1}b# z+|&L3bTc7^_h96WeHFclR290lSAbp69DclG?U`IqCoA`354J)4dtajC^RpRVA}ht7 z>NC_^xCp-!lMiuOBq2V#yv9BX%{1wi*{|#~YorOR1^n1;Sr{S6WC0w~f&5(>??^45 zsha-Mt&kmCyzcMOo_mp*8y!TT`%QSMHUiOFZQHcOxZf~p6u9yf+({TW5I_{~WKGVA zrd|?z+o26VHUwsIXv&GIUJ>Q2Zwc!+++y%C6i27AUw=xQAwx1dc=s>ZKfR*gf7=K^ z^wyb+&oEuSn!CDXcIfD@op!iSRE#dw`R3HFvDhPwBK(G#3NsGKMpLo}1C8GjkUb%BXdYf^wj*y5oG*Q=xiMtJYh@DyZrEITVCFe zrTg&qUUzl(Niv80Y;M`zGO?IX-hN<=@BO1?`{6AbReH%hQwVcE3g!L36RvGbc@tYA znKPIKqixyaGY6>Bx4M7ljDQPA>33{zf4}ArvWV;JX#ab_h*;(X%7V4*@@K`0(Tux> znYPt9D~lv{&7VCg!^3BCl*sYJ_;_(zZ44E;;0S%3Wa3 zC&_Lot!jNB!6Kim8BBa=94YiE6p;e`Lkxs5@;^23e5M`c`&6&ecX5tEn^n|c7EA0p z_;#!3nLqlK&QNc`BT-sbgR~zOI-|I^y3@3|q>Z@hHX7^5Iu^kqgahW*kOFDdy(_Ctz&m93qdG)%P8` zayxx!K}HwP*cpzyObdtP2gku|nTJsUiv^`G+|R)THk4%*+&4j| zk>5Y5OM0uDK;G!S&Zq&J)q|tPYN}_4;YRo0Cur125V)6}(+RA8g9)Cd#iO_&$nwgK z+|qc;77J$B6s*V&v6nCuQU}rkB78$ka`|Sb&0>=G%s_9}I7j}gb?%~Pr|W(cmc)kiogwcE2r>Kif;<2nW0ulIh#!W=KnWsy zY2{ntNu6lMD+Z)G2DN0xkm01aZM+Lcu^Yx#24rVjVjKb~q6_?|PX%Q zIh>wOp|8p`6FWyNATAu~me;1jbIC*2dwj~BU3QrI+1?BVLELRjz7GFqG~=S=ZK~jG z93IOo=v*fGut~hBy8~5I<*8>wTK|fSs(bIsaYu}Q^V+LKE zggea(aWcjO zP-Y6dnR)2fv9;T8Z^sA}>1{n!T|{Xpb@-6Wq8J;rx-||EA6C~Lf~Vcu`a)KuhV8X{ zr=_azP_$N`Vgsuar|L$MX5&cDWRLeV3{v<1bgE2|xRn)6{y6s7+)+IKi;0}Qk7~FM z$}l9z=48_W>x*l)#3NDe!c!6~J*-;Eu;!+$TB>|^q9BhuxVDT&`rl%z~z@;u}wTXu2^k~DK`(b@?rEg4Q75w?d} z%iBX3gaNblwoD36g5r(=ieAE2ywj*r*0MqQlP6S4f9j=z*Jsd+&V6!!IM#Mdfy|nh zxX#;S_ZuLw0!uf%6H!MI5gU*-E8FCyVA4l~Tvn+5B4ALUKKKE1l6DywlysSfP(vkD zHoksgOHq_m4k}6}VD1WT*2hlE<=gowy5eHoyD-hJ%$Sil#5HxNI*|38=a)KXR-h+d z;4~FTMa%JL>=@CSU94Q9(PH&f5G0Q#j3t#Lv1|JhfFJmF z7hKpZ0;FZJrfhAFMY>Bqz?nERXLAF#F2A1NLX zD-So@|MZ?F_2lf=B+_x8#123!e>C<2cR(^{~HwbEA?>)e<9Ye$P{h2C*O#_ z_*&0i=eBDUm)v1khOP*j+S(a*1A%xyIzcMLvG0K`PkKUBT3az6a#Adg6PQjvbl8Q1zhwsHA^Ja8_3 zp&9*&*yygkv2;OuQtmt_JmKJhl1Ej(2u1p$8P$`5M4V}U?TF_FH3y;IrjM(UTdC_) zcA8<-k;|CgXFfmKtiV&Dis@4)?M<=gx$w`Qe`Llie4ehh z8@+_btnL{vAfuOHiK|~mJiEIarWA{2mt2e^FVsQ#Xu_&vs2DP0DngntvXiix{_q+; z_eOt{5vXUgz%sQuY2pF(XTUeEXrLw!Zha^^+)-5@kP$wZ89%mOf7`h)Ab33ponnZn z!7Xnq5AbjNi7xl?hwXNZ$4c@0-t9!0SdAr}x|9;pXpN!t%pg2ngu6Y;hZqIsLb0D) zR+G>-lYPC?C?ab0cj>yu^M1fHQ*LdyC}|)PB1Lx7m+Mb*(;S7pmg(XY(j65vc&Uw@ z(W8y6=zE4yFJL5&{Ux)mt-kK69Ig+ImVU-I)C4ig!Q0=~M)gGH|^yDTzQf5 z;xSmAfXNA%u**ShT-I;8lZx{cv>^!*-*H=I{0l{;^>(O8vfMHZ8x;r>Rs zfeO+#c;jYb-osS1Q}<$7L71&6B(Ha>Q&+AAl8o#c)=$CQf^A!u7^h)g!%S5>P`(Cr zn>tiji6i8@nh8aRmuYf49iXSOHLsUSiR%JpvCetZ>s-`m#k`A_@Pb)2 zMH|En`A?_f!-he;x9sq`I1AMGxuAE+V-fnM`CF|ApVj?p(|-3;=Uihq@V#CU^uJb} zMOu|cYzT{k5-b?$OPVl@{5-4?@myW8HZ^H`$9DtgqcckU$QG1Y`xhx+!}+J;K-r#- z2`w&&+d`kRzeZ&xFu|@WW1`@54VMVhER|n^i||OW6i1Hzd7&M(6mk{wQPrsLNikU( z7Gj6z)N=#bq1)Px)HRp_Vr7$*X79Dn{E9PLa-{||2!YCmSv}kx&tGh9X6*S-L={ro z6Xu?Q>^VPCG4gz!XR4lE`*M`uK}zS`cgPhI^Rp@avIhduUdM~!hTNv@WD1i}$4qjv z!$H=ebxqn*TfA-N4#HR_Vr_%PYo7bs!kQ5*bXjAygNEWRFkWp)dBfZaD4ZzP-=@Zt z3)JN%ZXTw%QmwkA1IwtK47e92n{+e&(t?$&nwX6gkM~Nc3ql9=&vGOSL1upw;+nRk zzFpK+7bI1ZSa;~?Y&8W&OSc!hf6u~t-GS#{Y4TP(I;aAE{JBl`4pws|!5lM>jYilc zJ>!)R~vRxT= z&+GM+>gI=WX|U>Qv1oY4AV?|D#2e}OtwQQfqqyV-4fKKWJaG5kcgqIFT##D_zjM>s zNwt#Trjnip08ZChGJ@mG%^j4vb@=AUy51wDRHHN9$ISfpaQVjl2^z_@2iERI;zD4t zC<378@6^(RMH2joa+&3Gr##W~C|^))F|3+#d5MtRevDYljZG7c zv~9dNk~sH9;DWiBU8Zf9W#T2;*@v{;G{z>^i`4K!T|42a(LjuJ7JUkLF$Hj&!|EXK zPIaj;w?JL&ovdG91qB0m|9lHYz+`Cgb?iI5I2g9=rK_F%Fhy9FP2tv@i?4-ZsPMg~ z*wC##a&264im0nU(K7nQh(=G}Z8b0fX-%$FNAs=1mu%VVtf?5#l|j5hBSqC9d2k#* zqN6GN_UV_<;+Et4f_616QH^M3_w)bVD*P{JZ(@bWkszQ#3F2b^UjdAv^p5B?7IcYM z?_l3vR0*d+GIX1--*mniK)8q^Rz0c|jec^INwCwe&=!h|_$$EI|G53-V8J9)`nqwS zC6m*Q#O0&l_M9|DqUP1nJTm0L>r2uRV|wU6dR}Vs zL;n-v&VR598Sy4JF0Q)lXIuieoW(XeON+Vs{@j!aJ~v}rz^GSbsU7V}oa|%=*+3T4 z#qJp|(Je!t5&{H+eDc{GM*0thPR;rs-o!o@#>&6fBEHM5#25FoHE3KI{jAe=LSn+P zZWG|BpZx}XVx%Dk+G+mP2qYunr(cblkbDZJs0+}Jc-D0$w<-1(h z?8f+eTh(KU*Y`moRiTm((Cd?OHv3&s*9`OKC8%(3?(tbJ#6|0R#CT&6P{b^p*xMe~ z4+qN-bygpov`2?N%*XKgz&m`RZ^cqu#u)2axQ{GxU4MxcpBm-|uVFOgPJD1a(A(JcLYOk`D(3)X@; z5OWXdMN1V8!N%l(&H+(-8c-BkJKw`{22;YS_cyurmw7Cp3}gv-WQXSchn9%?!ysyw ze-*4;C2Ns_iavskaep=Ab8#g1L4oFMlKmTF7-(_io~LxeKt=0;4Ga%8z3Q>X)bJ6f z%*J%mwWBsIu75Gy3q$n$9f_&rJMi)5jgv5!OtQad7s2btc(801Mmxf?JaBv=*Y^*E z<8j!Y!$@vf#GaNFTZKxI6=6N}){{tl(I|jFa$K-Ozfypd_fdq7T*QwMbpquS^imSi z`ISvCJQG8@ViE#91_GZlL)Vm5Eri1%D01c?BKpB&g^7>_UF<5Ag4Mlwf-DbhZ^|kV z&@%fjf+cvs?Cbn0T9r!i>TrioNNrQXQLws@TW+G z>8pvJ7;$Va2B9j~;5OFSVDG0;W1lGcvAd{vXFk1-$*uVvdo zX(77jU&4!3F`N_Q&%fMCdDZlt#;m{ZcX}@hEj*1m<{4ISbNc>CkpC)omK_?3uxvL} z1nF00>34>+HS)s)K_e;ONh20ZXlvlFgK7!VpN1 zXNCxXFHkYL(i1RM*!FD~s`H>lES?=93()=>F`oTS(5pVAq;9J|u2xgfD-m=gA+SFR z)i&&=rRixRPiW+7H$NFZ2U?>QXL1O&1H?J~9gkE49zy`+Q205ojqnXj&E~E67J)IJ}-)gx2Z4Ua@k56X2YJZNFd2 zctKKc=`?GvoQB>tvZ;haP61+M`06vQ7I0e{qCjY5g6IQT;-e27w8x3C?D6Bs%j|6`G$clZ#9p2S*78_Qgt_(M#bv1Hqj(YHDJhw3%9uZv^ z)|>g$l+ukkK=#ot%YJ3D+#@JmmXrv{cTp{8ok9aSuG?zneiI6QYWzYw8Jf6=tM+uV|Sil?=BqEWZy>&e#RL2AfWYZzDt!DXSXxGjY4H#dqZFy)^Y z-a`89A^H_+{fjR^PR4jLt6%6@)sI-Xb}=y#!gqU9*-rr)&})yO%RLG&1 zkeL7#L_iw8sQ>)&4I;rA6@RB_N&_k(=z^|Sd4%xEW2QgTPlhiPra3X4;+P1E=Q%f+ zw*=-2x*(VrnsC#A5t2E$t@==!FBJJoHH98}q6)}MQQI#N+Jz6W%Np;^hWO8uQ0VHz zV$Ky0sehs1KrBflAe<{V+`1T;q3jTu9dzZsA~U~0)G-wmBuUOVRt?nNe!dt-iR6n< z^=q4EWM@J5YeQoK%C^3w^IwN7b|yg{3bjo1W<#A36guPMY<$3*GBr(dX#~+RcnS*?$qk=kgCRtla+_F+FyH zEa=1k2QhWLJaQT#vbKeGNz3J^RG-DDKA%Od@Qf<>C$Fk{G`wG-hlNpdgpDF!)RSg+ z)XElvYBW1ylvm;S4Q%*n(bh&3H-uZtXRS*NirP(=JHjcQ%GY`Xf7wPYs(-0yWX=S!>vE^A#7I?aD1&XbTy}W}cVu0w6(_~@qVR2sNsLwtC%qn> z;F_(==lsBgpGJUj2nQb;6nW5L6jyUb(h>r(03R2jpBC!!cn~UVvnC$Uuo(E; z=Iv0yx%LNG`3pBE)EO?E4U)6*_1Qb}M3MiTiDEv`VS;rw*x#FJaT+ypA7}#j0s)=^ zf7?EWh?^T&I)MPzNxfWa0pSPsw0=L@RNGK1DBG=?cKgT)ls4ifn|6H7bOPxTPCT?Zh$8|m6SiEn`fOv)K z6@;)cVBIdccp(lCFJ@N38`!}ym##{0HekO!sHQsgN_eSpGXfRJ*ZmhB{hpK{4zB-w zP%wQ`aexJRH{~^EEm;PMqnjemtI!4rAPPS zqeA1pC~~$HpHpYOY1L=fCh)%9Ql-fm)d`4)(f_DF{}$ZT@5-UFyE2gb<2J0KkNcLT1q`jCciX#yz<(h#nN5P*;c z^oL-`19EJCM?u!;BMMXU{)n(-67?0Hn>N=%IX+cs|qT)zYsLNFWa__x3S z!l%L@)SMOJc&?#axO~`ZB4+wH>#9*L2Dn|?kP|G8q%MxB$cE5%6_ufz-T0;vVmoD%qh{Nx^sZ$@`Eq&f!5R#fo!6XcWct9K+ zJpWq+y8ewax}RHiKl9)1o(5)&6F-=w<=A?%kj5f;%-dI)4IxRhJC>7~(qoaHYE=YD&U2%SMpsSi ztHj7fSGfb|gAUL=B7?GRYT59iKs4{lyIa-D0oTag92GINw1v~ss{4cwY{p!YYvLDj zr&iyr?C4%Q2qU<&;Y~jk2zPxb@s-~YmYe+$C78~z-d(q!u&C_N%RMeIM9n?KZqS+e z3`oclo#WrQ_cqv|7#FhwMUZq|GQ%6;*9(H{U4kEFZW>5>c4FELhKP? zA8E!Qltc@xs$KimQWyBzG>7W-Ilf=W$aI?591{F1*b#H32mTL;F5g$ZhQrsBli2-Jo?)>y;hD~?mbb_Zhu|`ctu6Ru`#ecj5c&k;AfaO1}-~Q`aHHib3A_BRcV&nBz zi2s1@40nq%W^E8QI(Lhhw!zJDMswHyvw7p_yJF1}v2le27F^1iTSK7*W1K?1uPkY&(4$}s%P5v zQ`l&u&ixB&;q^j@Nc4Rsn?Q?gJ)K4Pkh6Zo&@9TT%ZSj2f7`6LU#*$8T8{G$>7g^Z z%>Oaj;q(*e3_#67g87$pz%fGF5Fua@1l@Di14hlJx zH(c=%5fY08xZF?N%;E`(C>V0B7e!wah?P~r7yJkRAJQuLR&W8-$fSXXNCdh%GBa^x%e?mYH!f7d* zSzlLt)>>uG>Y9wf7TVa+eW_GIvG|*7)7>xix$(KE#VIQVp^%x? z3zJWD(AvAXL~?RL!J;_wXZkAT+4ytmN=c@v_W?r>S#%qd5311gb>?P+DN@dffSnVoSw1kj?_|HN2vv(cep$hEKOM;@$b@(ki z0~S`&y_&=wt_u9UH}*ag2CqIhgMMTkS$Fyz1~#M;{NtYiOFxhe5Cg|V2!x-G!^Xnf zX|vk)4mg6t+u{BCuPmikeHiI#odppw@+uQKl^syX0kcu$~U{w-$T^wZJ4@Pn~;Iw4@R*d7*xp$bxLy%o+E)EFQO@K|VlJ(H_4x zk*?}zsZ4uz*05B%w)ogvpdQZ_QvLsqH8<;jS{`KLQEyJ|+pbQ#5|~BV9cM zSbL2V;WC3E`J+kT5dR55wU_h+DHb{6#15~n6pwK)Q8us+d-?YF1;)S0kyk*94#YA9 zJh(dWZ(sq0gWdfopdGAoc*DVV1A1%yhIs!98G5m%lino!!3zc$Jp0XQ8L#0nE%pBz zDuA>w#&(tYCpUIs?P(Mq)lIz=MZsMxfOs&EAFsd?vOwqobO@ax&a?`m{Ea>Zi#DPI z4&1+v{70awGk8&_XLOh4j8Ea=rtiZbCU)Amw%w0;gaDZ0x8T9u-GaNj4lcnh+{yEP>;C`q z-tVpjtnM?-nOU>D>eSw+sv3OE3*kq|vX#%q2Ih9F;k;m1{@fMCo^xSCn0@-X)b1F) zEy?)5fXwv|$Q=Jt6&5naZ2HAeq#oWPpCH>a!p_)}n2_xA+5_E-qMS($TxZ_5T!`sn zKC}@a`(Q@k(>7hze)nk|I2_iNcrsHu{hc~~A$Zxu-!)mAeO_|c${SZ*S^L?2UT2Q% zrh1`eZ}Rpfj*jv&oNvEI(&5*OnIX_UGm<1uhDvSCV2OSG&$s!D;w+8VgJ_VG$d>mH zis5UrBb7arlTA;qUz@*xEl<5N`7~ zAZ?F@+(FQBvEf~N*)vB^x6;ZA1U5p#f9*lYZ--^cO$oKL|Ah9!*in)kh8@6l%Y_r0a(W2xgQ|uq&QQ> z{Hi`+gJ+;L4xSwG4{A!evf8do4?0?(g_h=``D2`cx8NewX->3^36w?ep0+=TR_prX53u32$Sk?IOZYFBV zGMXyQqCxE$e^^?kfWSvsXdLAq{|lbH|KQ2V{_hZM>TjSu^f%DXqj_Ja#ae|#q;-}! zL3PgC7>aN_Nu%Hi|sVb^LVynxvemR-i;ynbO>_TH;|~W>h)$ z$4!0dp@5?;PJoa*dv=DUn*8VSl}kZmy>5_-NkbDrH+R+#xZ-L|k&8+bidQ+T-A5`i zl~J^xwfB(@2>&fd!4oj@HhG<|Z(Bq2=U0^NW`}nU>10n8{1)69r7)W|iqke+D36qX z4HUXz24xnTF%TO@|MUwAo3__ByMwo^`DVh@$eOk>TCM~n;E@=OPc{(-;gp|M33EUs zmF&l#aEPHt;QCiI*qY%d@On4|`U8)j;Q=lqP&baAs)Z1=#u|ED*D?mNB|tcA59QqS zAw^&FgB zb&Q34bZGMl>l1s!3TdR+Kq+;=9~KF&+0^9b`uUrzS_Iluujjwr# zW~>eO@~wT%%qBp8JTIOk&vbtOW@&%5C@u+Dd`1Mp6N7-~a6;QQ=!<9sUwnzmpWYsR zW@b*!`1!`EmBdmuioT+>A!3+l3iQ6K1LuG2tV~d9cuGOCV5;BLqaL{ zkVrW+AlUv62Sq(zj>B z-tJ=a=NAXMlWem zjG3Kz#7*}DP!SDb-=n671h#%k0h!8H$^yW486*n_c>0uSeo`kjIN||-hrvX=Yt{67 z;qF%$E9lr!{}F*<2vQE{vRE+_nQg433O@U{{`Sg zlK?P5=KL@HWa@87J#_guq!w67!GiyPLu!ePKm6x458aA#Z3`sre1w=kaC+rJfKT6l zoA6%DdBVS5S0TQ_^7>p^4Lf%Yun_hb*>V3|N*sOIDtuIif?bLtuRK z@UH<~N5u65n|_G-G}4O9L2YB^S%VP5F3bJW0L70Z1VD+18%odvKp((10a!4J75&X} zv+ew{*{Rs$ETIjNRaIl*&tL_uf$7-EOMAsv#;|Qw8GO_A5l%WLA@450ED553$7w~v zG;1xXhcZ`>h>z8X6MQfSvB=g|B+M&SJ)uWm6AGLf@dt|q@l(t&RccVtt36+Z>3;L0 z25>I|#sbt10nPj$A#MLgPguDxWFckI&AeKeGrEWhJ`U_`a`I^vkl;7bnUeESmNJf* zszl2ww}sQ*1u*t7iZs`NetS+JRMh4q{+%JAFc`;287_JiDfKREw5v$cIN5iss7IT> zf&hG^fLLR-g|&3v8qv&4TLXR>L3`)Dby;3MZQeh(ZOxrt=zvu%`1_;9FHP4uJ9YWg zc0#?_zDw{T%etx}+5W$QOA!hN_rJxl+P}fFn0r^B=(O>RPc3sknyhLBDg%+JlkQ*X zo>l(@f`JVqcYdx+SjXYpU6kv!#(tXnUUX3$=gizv#v9R{8+J9|4EJqy6YcO>gwV3x zuAZ|$-fu%uc-(?dXhR!$Qx^q$Qdd;{UH04$*#0Tbr9sBFAGy&*dYt=p=caURDNDoW zKoMm80|+(xk^4cAu24Yh()Kl;HxAnKMMx;iKfk{Q&4W-M=L);7=TUT1ismS7n*hI) zW7}^YPJh*Z=gVY>KZPx{@@&s3ZiKaSJC~yo9}RI5WV4qt3OFXA>`}1%1-Zw#sWlY= zRB|c@?fgybO`>TMb%N1>3mLtCO*8aArny!}{C`X{D6>lVAJeQ_V+NZhBhC-Z-bNfC zX7^+7fDeHs(_7ZDu)(5YF4w9mv2tyC%;}pm*TYn&vm5tvZ@RO}yW7{`AOd$s^u@qt zTkT1Iyb8h&RQ7a0Q4NamG%9^aAxU| z&XMo3(T~h1wx_0V8jd&~GHcC$?D&?k{Zk=DZW267{&T%u-a`eOt%AwLKaG`CNxIOHG@=mBu3{0o*uXM zA+J(1?Er+N5FYhWh_~}lwEI9T0c&2$eV`OPzF&`jMUcNzgUB%SWTY1hc%c06i3W@f z7#aS3S*XIl7Y~3^a6Jt5xqB!@h9PmmOR>}99N??bG_46{t{DRqIA*g?fG`lI`{t02 zi|c__4yV1po&YV#^vyvhAa`iYk)C)`^v9LR%vj#B=~gw^(Ka)7u)j|LJ73^jp;Sx$ zzZmi-?o@+~k?p^h(tnIG^dII5HpU35|6`2mU}Jn?&Dw*2EFQorF*+alwxQ6a?f%5b zeYnAAb^x}Y1N(_v+WN$SQ>>DOj_Br~)~?IwdL>71%g`tvuV0)O6Hb0@wJ9Bq#MD`h zeo0SKL&rc5ZKzZT{{LmNGE zBLMCm#+A4JTg$8sNuumym1oC9X)Gn+qemSWxIV&3-R`ApxraSq6DF&53fGUx6E%aZ zJ~BVc4ZZ{YT~<00Dw9!rc+?z_Q(D!+GsDG=IvUv=cXRIB8f+R*-+!R=4F?W zaa1C~yIF5f(@#;oCkg~bH^Zfyhf3wgoKtT6%AjTL8v&8nlRvFrq}_u^qY2}l)|AqF z^M?A@g1~AKsz?}EExLJ48wOgpLn42zF5eMtxCkS>zp7tSVWGqNyV3V0WELjDMc&&12dw2SI(0?PugL&WC+RYOz=DPz(DSqSOv zu>F-cf+LcVEeuQ+mSR6ZgDE@~Ti6ksu=|b@m|fguwU6i^^C3YG0li|;m-hmYdscrbkKJ^NfW%YYj-fBL=Gyu*Dt@zT1*F zU<&kCqaea5X6RCO{K4hc`_BV7vFMcAG*=h^A?eWyY z?E>yGtrOf5nj{hdcW5(VL!Xw%HZ@yqh{OIKbGCttEZsWr*jIQPWF}99S7){P<}nJIHpWQak9O`&&)6yGX1p<{0TtxAwS#=ae*f= z)>SC=IUyFoxQ8%j4W;AENZ43+uzaWW2khdNKEdDlbh(U&Rsn=LA8mKS7cn7G@W%j{ z;{eQw|M^iH0Z1US*+N&Z0T^>W>KNlp;kZgdKxFu`azn>uM@A8vu zlcKZ?0781uH>SWV9JA;T!mFHvufIN*S=(5MbZTCNV*^2=TRxDw)QZ#pK?$tm(15D| z|E$z6{LN!OLQkUzb0pWsh9!hOGw(hA*+2m3a3HbBq~iq&at34`EjL z!BrjCzk@Pp{>T#^uos8Vw9QJE)3v)HOltJoS+q?BWJDV3PIgAlce5U~vlCwFzN7rI zvvc+0JxSwKg##oF1T62pVNR*E3!)O{{+wE-ymrdc%SGc$D%oY287P1dPzG7!&O4mBY=6PPk)^e<@F$7`$9)dsvWvY2poK4VI>2zbWb8aay7Yxp5>nCefS7j=`3AQG5gD-a6U6qP8LaX-|LDPx*WYe-#^_qta0g{^2 zD{0dPSU`D&BiZ;ZnHi~cacdc*vzts2;W%SW{3C0bySfCHDzMUG zSgTuDs{sugKB{b|VOGShd5PYQ{#u|6IY1rD(*b_h^5WNJ+VQn!=E5}daZs|R;G@1WK zY%M{TkGt*N-NZmYFKm>NKZw<8Bx<9XS10~Im9+z%|4QE z3=KOxtEr`Z_5!QEo;YQHh(q_-d`6GF_MDW{bJ~x*9D5dSmKM1_wQ>^{g@~=l@06nE zZJX21BOJ|gK(=v-mRQ)c^~GjDGi7<-2GRyM#rU&KNL5>|mO4!)Q!yy`Pr}heBQpXc zX0VG8H$NzLhitWTQU7K@3srH&xtjO)j1NaA3)_0{nS(sOxPF}7q0_-Phw~jsJdmZ; zS0(!_Hk{Y6bs-qD<&B)=uFgqy6>ND$^A5E%>UzaqIX07bJVp>mL~}6KC)$E87f9Rj zkoX;mWYfxSHyY2ki^E*YW|a7}K1~3aY%Y7iHPkj!h;u?2eaB&5ms`>Y!a%%W^KVV3 zPbxwkw4k7c*uf$&ojv_5$FiN~lH^7~E~!vdpAc->aK|~}XEutlX4JfaY*Cp0ZNV#G zBq?s9Q>yV}V(h8T#xbHIZO_Pod6IwUvYsdHpgv^8=<8a5>5lC$p%Wh z-_RWrzs)L0F}ZL}BbU~2CM@2yrePo0cm9YnWP}J;ZW!4649n)>K@f^TY-KGsIvVqR zn|S`_?uH^>_u)(jI-+5kNGz%!+!+bK=S+o}aL(3#=yo(PdvqEZ4)Qs!?-(e~9}`sc zin40R!iONq%g>4_;vRrjwzC5CF5O&%UR&L4$nd18Udxmm`ngG+^rl`i;UxM^U|Seg zFv!ouy}^1r@(nUg(pA7$F3ey2U}1baX3w@VPGBmjyDkokpxh@()zG2J-jg1)hW_XL zfOVO}M{h~zk(&DIRjSWJ8Kn71gS-mDY(_#fQX{K|2~BjjugBfWj7ir(pH-s`cPQlJ zX_tvEDh{nXlKc+kQQ96wSY|@a#`5N)mL~1b@*I1wQc+T)oVb&ogfYF(-&c>?zIthO zgR4DgUkqusZcNjWLL-hAaJ~|u1<#x$%to?cb5U7fbPgAZ&J@Wi%uKs~T1*LG%37%P zp+L?JI=oseDUf=c{}y5gEOAO1Okaf$i>D>w#b;FjT}DL_o7OIUxnit$7@r)cmd-KJ8{-<@b^#A_}X8;JJ=){-en?$f8_*P zQs2{nx-5VYp;^i$tnz0u19J8jS8)tkYKElZ*u?e5)&)%Mii(SwKv#w_<`1Dx_X5g~ zPMGo~h{Efx2sW^J~i2d z0%t9Y!_1H> z4ISD;GRp>ZJ5-d5``+{Oc&9@*r|}60Hp~RgH(rt%P4KkMVrvvsajdSM00iW{)qK0l zh}u7VRrY-;`C3N!^04+d_{lo=+xv@OFiz*mKonrfV$n4os8PVIrqo}DUO()WQjvZ@6ccZDwz;RA^y2x3_=frHGz@KY zv(JTYy;OS*$R^KQrq%V(W#Jh&f8XZI6~D27{>OaE)iTM!FlfBKYX_HgasP{RPKi0* zNnGV1ho;4dt_^b(Lz(;wUGqs$*{ihTr+$>V8PXqZy?m_F1HbuR#DM0GTgv1bRW+NT zx>&5YF)4OiMX)dr&>bYK#77I8%D;+LM}FKvW92VopzpYAvcch30`6gy9cNRXV;hz7NW4~ zJt2Tfpm1Y060%fhPkk0_XL1psyQrVns&epdT%Bm)0~hcYxxx~4S-D%~6>usJdv3mwMEXL3NI0D~y(choXpXXkPsQi|J+nRA{e8b5ZesOX7x~6G$ zZqEsjlT_2<(3Hbv>6AEHqa01*n5z#mK+Nx<$iF_ZqgmUhyt#=Dg&bU zi9glo_w&ugN5b6t>|`K=WMoGMEu?LKG_YZ@uX;%Hn`OumYUdyN+Vq6a;NZ)aCz_w= z8Ok8+y2JRUa|EK4DT;3stzYuJ;`phkeVS!19Rh`&0NgRM{c`-C7PeUY*6D!EWd-DU z-wgj?YnjQvek%d`Fp_nezXI=EpLZ^E-2GhVYme^)*tb*OJ$4P<8#4`AyI+rJ(lr=g zX)3!jQ;m#(Ie+eToj>TX^lS;SOqm#j8Lg0l`snE*cEGpdR8AZ6k!}Z6^3~bvT|E6g z+dXM_*rmCbOxB(t1p1bsN3&|(H%-43n&+dvWW-*?`VJz*Bxp_PVIYoGTRI17@uI_a zD};T*Poud7jbxfY8IefaUC;fk2*D2^^FA{^KPA%6>2CsvtQUS6IHrx$AT3g6R-MT9_Er>W`Wk2zpokj%&t5K8U1x>_6{{os)>dQWyOz}e~#g|hA;Oa zhy+_hlZ~#g5~(_#I|8pwmctq3z#IHN@I>rpKWy^&L#d~c`eKfxH&g2-uxQOcOK>fP ztL>_@s(GeC0jLUKst?JUT}<3lsqZ{9^iZe{C&O!b>o=_SeTsa%Ng3bFpQHg5365O9 z1;5SDGtcnWxvvZs#KQ~NkX^!6H)Vjt%Z?VVGO7TbR`w#z0>H@!b2|IVDc^Zqx)^Cc zJsfm3#th&ucFWX1-+yEBjzatL{6UnYk}Y0q#FkBlVl76nq;x#FJ1A@kRJn{dTm34l zs<8P}&(nRJLD+37nmOT@FJ+~tg=3w8nmk$kDror=0Xyy=Qt4Uxc^ja4gOp31uSLR# z-p6x)3+COdY!l!Bpqukd4n;vP0E6^Q{@YC;D?vcBm7c zo1P92-&yVe={tl}On`7~0d z9tpMNlXv}HR7GYs((9^^DXuthLIiX5|nhOY(TGHBmC%ge@!i>QF^-DMwLeS;r+zPA*lO@&Zyp7I}_ z_bd3|Kl8u!T#{+LPMb-smA-z9_^yi50;i7)#gTrX(3$lrt@dbTl!d?@hIenq?O_iq znPv!0CE2m?1|RcX$A*=aGSBU*7)EQlbPe`~e=~;JL2&2nj#>Ob&&YR#^A-nxKupR2cjD^fV0hn} zz<=80GIP!2ox#r7!lN+{Zl_@iZl^(l&8#Y~ zy{F0;^33F69!8nW0ZIuEx&N?W9MZio4kWE;pg@Uy-!Mx>P88of_}01ln3iEAFZ6?# zjC1vEk8F#_5I8*J=G)ldrwl$T`>?O==5&DzyKSFmldZBLsC?qv3PT@Jw^>)y znDI38`BXKpTUD+l6Y9u|E9oi(0yIlX_bG5@Hr_VWI(JU*69Y>g7i{wvJhhr&Y#L2G z5FkzKj^$?lIVYQaT@YunXeODPKG*=rz&U7<$1hzGW6BOaNL6&)OsA6(2m5p3f&)26 zKNlXG6p4XEhP54#7aL2Dx2Fj!^omdkh^61rvdJF(@s)hS@j&GBtcJQVuEt291P!Y3 zSLJ|F$QX#vN^S97)iqGz_74qm9G^-jl51)Q&u40~7WO=4xR~&m1S6-+u*1Fq@m8l> z_~B(!lf=?>7!1BHO(o0v`34TkXn9sIz^6!UZB0fZ8}?S&7RrrvZI?QIVf+-GwsdH$ zYciTxfeghkGMlgFdd}lL*vR;$X6ZThqtDs_9Fc`1foRpM$j+YDhI7$#$5Qex|6+@} z&~Ku4b)lykk_Indz|tm} zrKKrOOTlAqFG=Owevx~be=~8+I-)83#!I*xgH8B{AN)7vFp?_<{41*$!nkvv6o;qz zbsV<(C6*pn*V?l)P{*@K$VJaFllef2vEib*dJrG$D;oJ!B_y8F>}b=Xfk8fR%*IWydkOqQa z9a+ST{jNNYc-3PWgkKRPDu#Z&zo0KT&KAeWd*`ty7FTXRNQnStomo63@Y^R%AlBm_l;S_2%} zRZU5DI@mgL+(ot0=51z7wi~$v!~SHCGNC4W)2jDN-Lxng)W%o^4ur>q9Y3`Y?twdP z+&dH&v_0Dg+sN=K+A{^aAt=c4kFXfcc+cevv>jbhqR+op6{*^CAXy1ce9zWmj0lQknPv?6BeesBj}43 zjbV#{&F^d>Q7Jq9O&L0s` zBCLr9VmH3ylXR~Zc)vVj89fX-KaO+YZ#b9HI+P~9guc@JAq=LoXb^j+*b=f9Ptx@T z`ZeSEJ%Yxbmi9iRS1fm5sF! z*gAeYx6RAHg-jFVhgs=SH8#V(R;(@p@#VN19Id|fz!X#a%i7!Y1#<9;Pcjhk)N3-Q z@Aiyqar4OIWGC5Uj$`6f_Kdf(Q!4l-LCdG0G54(IAH&kXmeFJ)a2eIS^>uBnzZ=NL zKJLuznsB;SA(`A_`aau@S@sfnQL1EWJ-v54H-&$5!|EQ0^rA!MGLW=zd}~rt-w~T5q4J77Y5t}U&%|ibTi*n*xn2VLy@vY%vrzZoD{X=- z784|gKM5|}Vr&aq;pRIk3_Cyo(UN|VDEjz$0zmzjjSh{aE+21s3?*ns_Qj{ur!&`g z8|+AHc8$d-Y(hI=1=R+yT1g=sM}S!ROf==arrl%FS1BGTJZcg8gZ0Kzh1hWWVJ@)z zDeSu7rxe8uFWOkN*v_%7^fpd>k#uiyQZPwwoCU?AXs5mN6fiiXYr1_0_Rz5Qef+lC ze4u>{Q8JSWaytI0W!kDp)*Eu;I&&L!&6OCr*o1&F6W&Oj6gESRZSy0Nxy@?iQ!Dl= zW!w^7f4XWLJm&|bB%iP1wZckCHDt+ZTJ_H5@I@$PJ9mMGyt9s94kU2Q3x4qXIjx9uPAX z6QZ1b*mr(il#DvR{X=U+O;ny_y=8g@!^e$eJVaam`RSGT$M|6GmTEjNNaQT|T9`71 zO`XUad}Gk!7TW}s$gp~W2<8vq>W}8Axh@J;E|Zk7%5$DMfS!1=n?u8dVbnnadX%*G z=buJw?aodrI@6lb>pY}=eT+!S0 zTV`SViZnhY&?N7vrphPhAn&(Eg`(nG2EP0Ye=s~M*V)lJp|V)FDsPF%BKuvw=@%x# z2b(g@19c%~x+Py^TNF$aZIPKY))W-$xu!v6PE;`^N4}``C5Fi->luPu z5*2Q;kQQ>?)T?!vDtRP|6ERS#J(bh4DTNQ>QRQiuZ4WS~ySzyu&%d(Q8~rJ~6Fn(iC$47CaTz zTMX-i1~P7@qv0Pm#dKkjh@E@SLC$xrhpxqAY56uPl{nLO!QE&f$(DD(c68E1ygw4#{y3%IJf=l6J6)?wI(+Kg$gpoq-}s}QQh68R(cKG{)J z>Pg5S3U#KvLZo5OuG=e(T7y*jpKS&YaB<3vHW6M&D{|y3yK}x&98)A462ESsh$W)B zXUAL8ZTGs;0ZMB%96x!N{69w-ZSJl#?3Z6qd^PHIwtqP}jL6U<)6nqr;oPkL0Xpf2 zm;t`~C30Q-?vdeJiWA^tbIXjBCe{fYXcdjT5w zIxJqdtmKZus=_@M3IJRuk~Z;4I!0w>GXTWCbp`OZybPj|vBQ5SCh^Pm+uULbR0^%f ztcMs8o}fg(bWC}iONOi$lQT?`wMd0wR0ZzmPV4{KKzSvZlz0Wa0}HOcKHW5rnYpwz z)VWr4bm`@2yWM5U*HUTgC#F=ekWjOJWn6=^PL~SGN(4<^9K;0S#eX9)?#i7*OG<}u zgDFk*W(XShNwn2E_SWkhovda=XJ>lE`!zx~0@yzIX}z#`)+CFnoy&y4ZNiGg%Q zT$fkooR&x+fjAL3gU!$`7Cc6$m`!Vqzi8hO6)Qrkco>=^UygqoareN7KHby&Y)eA9 zogp38_`{`H;_kI{L*slUJS_JQ{5-ToXz|}@1QNUz3(gYc{_g^-e^Ma+R#_U7msQ-X zVEv~qce~Q%dt$LhSl7GyTSQPIimOO(#CHU}6!bTru67k3t?r*{0DW~+`5_Uw?K50_O*&-C;S7p z)sTYDv5B*ic#G62m2($Ap|Hxu^Dj-OtF215H$vGX=c_C$)a?_Xs}y7FiF4E%(x8}^ znR|<6V+okPgv2|$jJZh@PyX`cuQ4zC)AYs|i$<%j=tQ8XCMRskOIgo*7~7feG@jZ# zcE<7Wy(QgM(Wv$YV~(^HAK(=V83;nI*h zHCF6Bw_|(Hn!7p(ZiWcdCf?-TyFR6U`4#59CBfKkd&0e5p4vwfnEd+PCq25J%Qs)g zf4?RyR@*M>i8ob3j9FnLfaj`c@GE*1Uktlh{|P@9rDJ=9ck35qNAlKZ@>BJ#Fxmad%Ht1aAUlsfLb>MBgRSlPb>#2z zR)v>`Fj03H8mIrcfBq&Sa`OMXKrektegIshdnp1g!#t%mo0~ z+mpB8fmGS9UGOPpI8_jW3&LE&3W782nIUP+4=5AbHqQ)*r9bv|`q`_s@R$M?5mO1g(@h9W~0Y5e#+_ zK+Uk^D^`&Tdj?chLY{Y@h(|n`W$Xq-ZNzu+sbJ>o>gSq7E~c;b;D6KukL_AQHp1BI z$wS+LeB>VS6qe`FenLBvS&|Knb$gO#%x-xOJ=PGYw{NyidIL8#_3LxrV}V^yPv<^vMTe9v7Kv5a5R4)8^ z`rVHjc)tC^T-#T*IcsK-4x`&ShuTJnh;`i&B z&=IiqV=hRfSQAyNU1j559vfEtD5zy6_xsQa+F8kT*3&Kp_L2Vm4)qRl_4Rr+k z_LRL3{vC-HJS~wAdn?(5DqyDf=Oklh8EuinasfWY)$BYeUfXr4IUl{a+J~9^R2V?v zcWgi*DboFCAizLL;XRz^M2V|B$QZ9Av`I#$wTREHgN_kgZzp%(>q90j&nFWtl9hRHAhYcQaADd$KUV$=CmxM z24$KmyY-wu0(0i&72vZ8vLL>NNgs#PNHWf-1UFl?#_aC?I*(`;+x_hg6tLSuAC9ta zwHSGYnz-6_04LdA=*UygfAaO!B_A0$sPb|XZYZik9jOcd?$Jm`1R$P)yf{l3CdCBHwK5f+2oHait9HLuI?o)vM1JR7XvjKGsE6qpg>cr45-aO@U|& z;OU>{bP|y$)yPm~zGqJ&S;_G=CC~QgdHcIEys;v>;CB-$BWb_~^zvfS*}dD#4K8zC zUtILC4j16)Dch^Mj2Cu5>Q@&1@b#0f{4v!9b4FiUM}lR#4`Y2I=`w9kmMZEkBr10* zUyJul9OB>uQ0gOoBo|P_RhpQ2g)&3p0BZtv5t0J2WdjdmO`Dj-&W62^2D}t2*xZwX z`C1`rNfTizKdGvG!C13Pu9j=sXQ%PGP=XKd zh~aalDPtt3+3qtNRR}-5q7;QnBb2i!O|eBa?N;w`f^xa*R>iYTbmxh~`#P4=L+65v z(M&!WE$hP{$Z4;W>;ItnPrBWIbz?9{GO%4^!^Sc&d4(LJ*ZSc-fQ$#eKz}hA+lUf zw4A{@+z*ye7zMAkkkY6ReSsW)B++bLW})L3O+_ zH0Lx=igBjX3$`2aI_cDilJ(IJX{N_l>EI*&nOV_siz4`IO*4EAP5|F8O$P8p`14D@ywK zgiD%Hop3zAr1aSKv#lx9YoZ${p+(y9Kh6@I$PRvl%)gSL4U%STg4nSCx=VeB=pWLF zE~+q=dKioL6>HGT9;=APgTt@O!1&d+{fZ{E+MsOg_)>{PaX58pbeTc1~{Za27m=F*u5ngK4(c3*5W zZDwXMTa6w9g3YUzUS3=a70zQjwJRwpcBhVyPLD2k^X*=sg(qzEk08qS^^{xuH|IFPm&U=F)V&IxZHk2so_ha*d+5rkr@I2FLLIeY>)aH3dI zgOWsdRR>qJ{D5NWN?Ur0hcORWr!b-%8u0R=wMlUlYiYL##EWR7_ejTu?v1h#cP&X0 z^Ca&tbV70{+}7x84CF)ah(UQV-_6LEhZ8~lX+Eu_8MwWg zWJ3RVnmcTkbJCsP#Cusnfm}c~KaGAa6QdSWvKwnIb$G#rSiT{mdk#Fv(?hKTwsc|Nwc}!f%?^;RrHlflxMAS z#$qn4*W3YEHL>rZ9iqU4cQ0EtKaJ*A^251s2lA+Y4whgJ$2ahTYqWwCwmuW0+Pw_xq`%eRwES@2 zZ(?jSl6rskoMj?UZZ#f1ULokC+q4`8`LZ?p0<7f{Jc8|pYMw=IRet~Y zd2i>Zecnk{zc`WH;ZrQ58XH&j!oxd<6703cTxhmz&fny88993^1>iOmAD&8~_IkAw zp8Fgt+*5i}^vdXU6)WQ?@7VyWF3YYCjBJpYmF3463y=&A=82b>U=V!yV(SpY?qmkl zdAuLv#*($kms`i77c^Ez36HdrMU_v9s+A)?mRwR@$s83tuD(-%#S+;N87!xEjl;O| zqvth2s-6=1^R5)4+6uFL^k{;NDXFOYjqJ`dOy4zF=iJ>tSbr@c}j~waa@;w>~rYn^xTB!CPyjnwy6DC;%^d4)t zf{;;*L&e^$4+^Cc&V|~$j0*HPDr1~0;$2N&-6&a-AVwLmWK3mIBB)?xq2wr^WZCMm z?o~ippKuST4*1oB3*)iBha$<_@I=?&u;Y%8C7)s^WxFLOu|^t-W(&jKCrpG@_}spE zsBW+KmeqYJav1l)Y;USpyLv+&?Nt7UJV0{(r^1ErUo6icNx^oH4LjuF4~}jV%AkrI zDvU%1Jbv+9zO{+P@v^*)m0bZ})ay&_+s5&a02q-|dC<|ie%Pt0M#Ds2-&R-oy@Tp- zab;(_%NHYlSErI<)LqF1C2oX??uFK~Ag4F|`|iwacx` zTIJ-zw=*~QtlIbec(RJ*Gxq+IkA8IL&+?5T){mgE`~ASLTz=mg*JE;pob2E14TzdZ zVi#nNOMZI@r^v)d+(aY*t%}WIiz;ZQKO~70F)mHJ6*C37eHOPK&j`g@3|5sT%STxi z3-yJoJNw{OsIIId_JdA@Ie5wiIfC<+qm|h!olj4!G?NbDE_%(Plqes^_FH5ra=b&R zS12Bpqy>XlM$$N1pMv}suGVFQ*_J~z!rX|gz2-rSpi=)vCB{j{CgFVL zcb5<$)Ds3@wj=8V-Xk4~Z*m9N-{hp?^ts+R_f127NhJp3R=aOH zGB_$43{!dA`+Ditk@+)`-f&!kYK}c-itEA1%*ddVEX>J|e|%3^$#kx*`ax#vZm#O- z1wFnHiR*ugFB181P@p)u64`PD>UrZ8VZr=}Jd`%^X9_zs2Pfa(VIVutzthF-HN90e zz7Td(Af~2ptsQ1Kcvo+FyROF0{{C)XsXGjxUq@*i!`Jw^SB!-M>LD}}{!_|tWrVQr z5O=*F$>T8!AP$hP=z1h+1I zww@+{?uUuw90Zi{vY!dkb*nxPv#7)_alqL|#Kz?jg@p7`=8?lzCe{zY1eyQN1DB8&Vjg+S^2O1t}}@~OyF&I8Y&%!38WeW2Z?{p=v&_;Y_oDoDhIaXUfpa!fX7q-sPeqrHOC-!yuQ4)-R!57u> zqB(r0mV%ab1vVag5p8BAs41!HXB9BQM4pxP{`lx7& zwyeFV#BkPalmL99>1Jl-P&0pt;_!q@3@saTt$EUBax$G_Q+kTm{S1Kzu z0JTvOi5+E`P=?b!DEOgq&`En-6E&NE9k#Ta?Z|07zk`X|5Xbu_RB6d1U>3o@^IrHy z_I>cWs11wS4msY#oaVMH3`<{bc`1KyGw>eO(1}IBKUmZ)B~Se#QH2ou|Wf+Q$vF&v(qJ?G2s1h;$Dd_ zHBZAFV8AKxDEGKSE}~lbXKK;eN_L{C7oiCB5-GlG;;2k%%O+g&R-%|vE_T=PXP{9{ z%&|^;F@E?U3ax_{Gdj=(=KmY zI%2SKLm=o%0SSlpj=ag3X(%fV^}yY z8QyhFZP+Xw;TH-VM@*dHtXlQ4PPgo9PM(BJHJ(RoagxT?7SCwBpk89pR@$E;>jd5+ z$DCD>4j#J0@Qbjmjoa^nR7sMu+R=?VC~!7?Y9vV{4Laaz269pZ)+B3Xrh$T$3vefD zq!}GIG3QOZ0v7TkCc{pb^cU4ab`43IIK`6|em8gSxqbBN)A?qkmo)0F*f2ZZz5h>P zZxxhR&~6E%!QCxDaCdhI65QQ_CAhA%tM2cDrsoL(cIKgDOdR+0{K!FjGOY={qLRtU>4=r!JZZzH z7%F7)F&_@iZ3Jqp%|2eIjjil>&GJ`PLTkUM%IE;#<&x~He)qzJQW#Cb54C{mzxrG^A)?|F zb4&o)TcGZc`^?*uS|F=JW!FT4hM*^5qkQlS+PTIDRSf}kCs-!6Mk6;(`vTSIzU$0D z4%YCk9vvqCcpPxCg@#qfb7xV4n;&RVZRi4_P0W zdio&;&PdbGAJ{~qb(nBQ2Ct!M#%V7Z>M@OX81cIL?oJGfA=;rYj-lEH!DWxpTy@-xuzO2b+rOL6WyB z+iYAgC{^5%cIL$HwyA9+n>%(FVJ8a~&A0rGCx&$Q|B@?76v2Uo;`~p3MdCvi9`bkN zpXlG6ARy31A;JH359Vb5IHd#t9!}Q(Q_Cb-8$n-9YY||hM1w52K^_$JVWHhdwR$aU zYIH4anawwD9HbI!8o?w#r~hR$svznHD~T)FV+^yiGpnLsW@~Gp>snN~sQ8g0j9NLz z?)iE1?%9XOxOjY`Iha&uvlG1G{C4y*{kTPwmzhZ${*@hYa^2ST$>jE3JST{}uic4} z8y|4o1?>-o6I0z_stC#7B}ABJUSaZ_z}YGOyJ+#s^d93ahFJkcBos!ubO3^c>l@T@ z_MiJVI!N&B+8&kfSY^fe)F^1Fe&@f*`UiQojpjS}Xb(hOg*{Ljs6N?rr*j-J{;^k5 znwONMDg6OVXFi_03_p9{n~2=YUM%AKl#xpnFAjy4Ydw`hzs^^ms*P%7dDaOVlHg)g zMn|ikQ7n*S5Y3Na|pR{6&)ilMo|`6ee383RGpG?gxSVK5+>LlhEWyR z9|RUB;XxM*aRlXBj5<$NSE}8G4V12h!YU-6;|muC31568PecHfeXf`cS%rcZTEL2% zD>XxUvN*fgFxRjMS0%C(!iu0VQ}t{kiy`TQplwWBcOS4lrOZAnY z6y!@A5O&(#ZLKG6kg|=Z5xEuitUNAWOzr}paxWwqa%9ShZ?y$tIysSUj^&djO)9Xp zcOUj@Ce_C0s;pXGoSm(Wzq<;U|B6cCXnnU8cRjza3VVFfB%B_aw(Ffu-_G>hQ?hpj z6Yr;^#LL{eUw*Kb#9UV(A!Mzov|pcZEz<7imqZ(VnnTqZED+EW5ks(La47!p+?Q@ zYUw`e%fXge5XJuv4+XVh;LtcDrH)L&u-NMk>~1WhJ#Whpe^R@`W)+8+$%Vi(^tXNF zz9#rp2)%Ah&>}PV)5OA+du-)9yEzWvVCk`z`dq(^)O(khI(daGsOGk!?I$l)5Y9qB zww_)x+w_CD?C-MI)8`_&r5(TD%LH?v_2xHL1dB$0XM*Af7zC3XGDv&l0Y_Fveg~+x z{2**W5~*FXDPg1AkQp(i;}L z(?!xFy}r}U0oi1zY3f?6uXqdD zCIe<$vv$wf=8vCD^dmRCoqsA&acOHNvITbxc}v5VZme04b?}!J$NbQl0ZX$msBU=} z=3Mgg)Ob3;nUXJ2>EX_ub=cn6>$Ow^I&>CZCMgFp5$23nN2Q-V0Q~ZSRwudC*80su z?}kh5z#ieM9y|Lqk6<4h%fVMg-&SxIUdQAaqkDj<%;_(awl9>T z5_*Jxb17`b4a{tIgZZvt_MzNf<1gQH7j@0q2;^F`_*x0*$||Mj(0G=uJb5J-!_mIq zonqB&4{)DtiE;O?=A5F=)6E3Z>}50Q8Yg06-*)xvHHTcr{Bd9nwty#J`=x?1E>l`0 zS8_(UrRjcYR$zrzHnj^VV~o*lr=fYe+jw#aq!|$DHUbp(qYQP}wK}vkfx|*Bgk!?m zmTX!n&d+7GLOJzlGf{Y{x#1wQ{ZpS*7r}xePjL3n?R3Mjlo?`zW+tD#Or^sBhT#&t z0HZA})(9=J5%q*LyRU#iJ@5JtK%vNgPdkPR z=2yz!_-Nbr&ji0b`U0b6qxSu_CDM2Q1_kE@95c6IA@+SITe%z!gI*$+pqr5jBpj^a z9TC6WKaeOUdtLm>%)!MiJzka7@mG_K)OYp@wWZW|9k@07+x?p|{QW&VX-T;B^t1(%xL{Nu`pSkxD4o zbiK5w<|+b(@lf1Qmr##jv2u@NcDA5EJ!5F(iG=vppov_B)v}WQ`1PQMFuNOgn@Y9T z#Vl_vn$bAc;dVq~$Ggg?tBB#(Xl*UBVo7y4{1VRYfcFNt+}d<7l_%Jj+S0GsJ2KPL z(od-WK9>{l1{oO$g;EsJ^W^#Q5vwUqit#9BiSu+lK*8GV_Kpgx5|DI$@^YGBSXt&c zH5pz2E2E$}Jy!loKSM~hNEBT}T*|pgoi{?Tp@w#rV#)*4MvkD$o|AE5HY97#LJj z&noMO6SGh81_pCz>`E89T*ycs+W2ZYes}yQ`~o%L8Sso&H+EVULBz<$O^zd3htNBc zFkpRrT9m&L{1_}n5vyfllwOiYF!xu?1Q&Isf1YOxgqRdjRav#N<^UnJot{0>SGN(= zOb<{C&?LKb_QEQV^5UB1Y=lN_bcnxBLtWC5ID|<3*8CjT9LU*mP>~_Zs~NXvT)>Dq z9#54G=uU1G!KqN35$7e}a}4Jtes%RLPA7UN96}p)nTYH6hRp9<*D>#OSlI3BkgJkUN-BPvi26uSUzU6I?kOpD`nUr z$CEZ0D#ttSA9x6Tj$Zo<2u=%J0C2j$-zW4O2lLyZ=h(lwd)vJ#b@=j>n;IwCDhpM@ zu$+iecoziyobYE+9QY*oS*cHj7jb?WsVPx}i|$&5_}^-AW~R{JXb8nr{Cu6BD*X*N zx&X9JjjJWSGWUqT?gmG8|GZs}*}cFK7S2e3w}R_7WpV&5RkFtJ@$>oZRyS z#Qylu-K91`dzlC%U`0Cgy%uV5UanN-L&IPz9QX_$vPGerKQ)x~lh;Gkm}RBu415b$ zF!Q90V`*Vh>yUg`XF!%1QCi3tO#^DeS3z#J!br(C<5p0k5n0r01H;(c_XOQtwMiB& zOOjQc_-&FKz!sBacQYT-0{$OLU)j?u#c0>cc9tIUZP^!W>`hkzc=J7SoE=w_M$Bf6em#>td-`hh`m?F@Dn7rN9tV9juTow%O zF&0n0lY2J+iavancL~ikBGMiy@Tp7C?Y2i)j=N&^C!yl7ld=>CU^+$oxAMYltD87T z2riNlBzGCuj6Ex?>Sk+=gtmoCgliFfao*76>KGBJvxp5Dqt{E_g7zdx+>y~6SxWS< z2Yv9iF_>;y)FF=rmJ+jx$TmoJzv#6S?&kT+T~smEvsyu#r6rWj$If7RlykHNU}bo(AU}tFfD^8} zXJ^M4T8nRjDVy}G-}9bvEE(O_);k3+ht;)tyk@omKzS!=;P;VQQ0S!kboeT~1W+t% z&Q6$F69^*pw;HT?5S)~;S`Xz(EWQr-ypU}nmTBT|J^2FVg?s$xEzCH@5Y>W-#tK3` z!Hs9KOB(#uWuBF%QKzrf%0e=7snohH^3HrJWcMX4@$Vc4p<`}`L@JkxRW-B%)wv_B znxr~}gF`!|@LuwMDx9Kl%B8a!hcuBe0XY#19MG-hd=%VPgtWb{Y)vC1A+r2%?)wE)jk z+w+VzM4{$n>!K|>U+{=WRr`+3(X&lfC2ehZEbV&UR#k+RBERQyTldk}qC55WA5bq; zhdo~(S6n}vec+KhX<){1lnx(3C#$2z?&FQPT`GX%qnj5H-+9Cg#2Tgz(8&jB8e!Ub zBWHg#No-1{uF@?cB7t2ah8I0p#8=Ns&~II+O;I)8_*$}qQ$!6ydGDwykgFY z4AqD$@s*ZQLPF0xeS?;1*kH~7b%JH^;n}Cvo^6PMNBT_Iwm2dt076Y*Y*v;3Bqj~&eu&6R_;)6m&2F+X?Ae@h9G?8HE6>u)+p~=69;awpXoQ1 z1NOB52iW{p;ID6fTm8P!glX}k*mU=w);)*u+BUUlz6+x9ma5CMlB8Ee*nvSDy9k6m zEEj|!c+(WR1@)gUQ!L^h!DUgZE^z`BZ~zS_n+wNHpZZwD?muP2k`ha64cEcuAV>L4 zaa-?>#rGY4r{j$G(IvF-x=(m5h^Ets36ZS00me~D#E>R3-7K=BNUQ04nay61tD`0W zA9-#$a$$~v$TJU4ONq!pwl=rx!rfAER_iKj``J5Mk@?U@f%##Ynv<6gL|blXEe=IRWCE>8~KMa&VWbk z$P0)o!lYncX>gd4&`bD>pIt!D^M2gfRJf_{Ru=K_kQg6W&K=RwrMNg$iYB9W*Ut#l z=I`rqTaJW_`?;-$E&GZm!}9U|Cx9w+qWiK}w02z8lb(~xfrPVEW8ZAHZGB#T{&r%` z6n|xHr8ZC+@byLX&hY`y{Ok8~Anq1sV_i8eG_IB`bhzJjZ3Jb;Xcnn`kP<$-G? zp~rf&U}#yskDyn*B#e+l zLofH<4z~ex99P!Pb62UQe};p`CA4MtU~?gV;4`f0X{fvrU@W}B0Gudw$Klbg_mq^d ztzT89)7c56M%sW)0lSUuAT-Og@TCL`k#9{53@rAa%k#QxM|9ibui;r{Ea|+)(CdND< zPMwqqL*Lc1qwx}SA8joq&}3G+M;w1;r5spLm}b*A2F&CA0W^dQ|7F8xI5+o9N!CsujXYt{_f{l_dyrCgqr1a91W$0YKo09{{Nx@vf^Xw*~9PqmVy_o0im^9wV;l8vgiO3O>7io3wBi}40C|^*V8{Vf@Lg=Mnw!E%8G-^PjHN? zm=q-Hcbrtje5Zm8`{lq-uh&;`&I13a;oBde7om?ikRg-Ymwp>2GcGrjBZf@0)SpS! zv;{T;b8?C>AbJYHv}Sl^g!-p@L|EP^Bqjn4$xiAgCqW^N<8w>$8n;ah`LrEN^4%RM zCYS^L+~5aie_Y*>@CH_lNZdXE&ypN|+k_IqU?;c`F-{?_sb@9O2@@NJEyMZ)wI>H$JiR+Ko%T#YuMNI;(GYy>6p z1(F~cT{7$gg&4u_tBzw5PO3*gsLb)wLBxxc+_rjn-=E@wFOU_^i(h&2GRbYi#^+F(NB5LC7v-SkS z@j@oil^0l)Dl>{W^cC^%r@_d(Xji7$C#3JD-9$9_YlN}*#dZANu^U4wWTpqulq$@Q zK?$pAUnVUSb~ zEdneizMndT1gmf9&qTZlGb?l83pDzmQxVE=ZIe}G8qJM$XbZij&xv+9O$ZP_!CsVCQ?m%$b^EphS@v@BF!>MP zZ){xKb-O!%8KtC=f%XopS|z(vIRy+%v&U$V)rC#rZwWT0e~bp7COws@DS8O{ynt>j zI@`SU?1~e4ri~OglS!KswMY+n4(h!@8EDlOv83+@!~#S}gypNB8YZ63n_fOKqxcD^ z={EUtmp}FW!M~*`-9XX;pjD#RNYV2@S9X#=)yqFhpgiQCMOLx2bsazb?_i?sNvp53 zJa@WFP#F00HY_~;!iUEC&f>DMp}$pW?<;=v=%!bdns{`;5;=YoTAjm9^7iV)%mOpT zaun?S!n*}@KGZDR6~$&(GY_-6b&?3R=}cGS9yqlV0M%8)YdPcB7gqHI?`_L(f`+NT z+Haf$Hmimr4>n%iI5HHvegU*#m=XE8tjy~Vq-Ouxw_bZQD)pdx?*OrdQF9!lnR3lX zX_k&ds@@09Zv4^n8?3b`YF6s4Qh_w7CMntd8+1ra^Vy?GI#Y|$w7uiwDZR5h3!qspEWyg`16r^;zTdNwJfNtg&1-9oymAA#ZSfwpj@sZ-7#UDg)-t1p_EYggFHxl(%l zi(S=Zn80ADr*5G&=c}t@W4e>t=hwMxZ{Z@g+)u@}E zzy+`CHUI^^)+Y*Wz19cu+FC6_si9$Ft3@xqs~ZGYr^u5Ev4}FesKMS20pYW>%ed{p zGzNvTbOUYN^6-mn*r3n`hd|3by}WJ${M0q1U)Q#^9}&Rr9<4fb8mikMW5Y|aQ`KJMWIJ?wpkXa_MbIk6Zd6PR=ze6a4TWiHXJ~c>T8=|QcWku>{(P))hl9Mm z4pSVkaekjulx}C|G|d z`-Vm-=$+f~>uuP6AAClAK!Wt0GB>|TK(BHX5_Ztwsiuf;e^8WGA#O8+sb3djrW;C zJLG?rAqwpwinKwBwCjcacqyNXKUvtTc8UDEogXEoSn?$IA`E;Oc6%6xHbl~Y0MAsY zK1_HKUMm`mantt;D?+JwG2PGAQKYpo5s&|_Hw4P`92Vll9++#=XyA9NTSeee7qhkb ztyWwpc3OoJ{LW_)ukaKk?-_Gg=^lAu`?%oh;lmV!Bc2-sABy$YA?l&uXWE(xCL^f# zj{&0ZSMrUu4IYoo^5NviV+D#(aEGs;2UMp$jZyVxCH1skB!Teq^itZlIcb*TA$L%N z66xwO*V4n7K9II+#nq&0`foBnU6HhDt*w%KMQd-&O8@z(%}LsAe9iX} zvb&0qWK^UNpWf^2szGAm`fR34kUOh=EN3ftgALG#?Y6I}`$jrr$w&GbcRs`3_D4cJ zIan^lNVC`KI?;dqx)!Nu%8y#84U+7>v&u{p=3CF&Dv)9-diDHy& zNz{|k`URm(f?syErR(epHc9BvwXGF3$;s6R-(hRZ%HycA0IK2X5<>-KMBTkH&HvOu*8X9{ySaOx2Cx@uK24I zaoBa?sF(sPjpUz7lh6EnVOo}IZ*_TqMzfK|z3JwBMBXll_!bg*o%K`Y%;KnJ#(sry zX~NuruSn9bh1aLLb!?9{(9PwX0m3zpd8LGgHIVnC{2)bdUu|Ji$vPTb9Ap3Y*F~E> zeY-jbbMbf2gsJIik8i%YeD}7#}3USz3ipN;FQ<0jkwbp6`^fUL04}O z#Ic7dMRh)!`qAWp@PeaJMK07 zFCepWT|heAG3wI?Vo>bQ^Hj)i9O+8)ox4SK&#_uiseB%ORvT`=RhVuU48EZ0A{z53 z6)1Z)aDN)ORut;Fo4{XE^5#jpopn)1Yx%51LOR7G^w$z+N`%aj{rlPRlJcS($-_=4 zV%WJneix~k0Q+*0xu0n%=>2>QWC-f=WBXuFU_N15Q)0klBAt#mf= z?z7$;{>?0V&&_B?agpmi<0*HGsF}Z>^E5HBr~ermdqW zf$s=03_5+lLJG6OiR;*9G!eP}jW6rDkj`Eh2aS0?iaGIksP#M{*(kJJ;7beKCtsG4 z`iYLIQOR7R!|5N05-W?SLnNL2r|~I~ss)Z$S%Gwc`8K^&-N&Kei$t`g?P?rr(mf(_8e23+Jy|-A`No8i#;jNew=01@$UWm4_3b6oMeD= zySFgqz+j?zhfC^fqz-_ADj-ITtbpqzYhJ>oM*6VJ#2I4zI#iwe2?_7FY#WtG38a=o z|FzV=bZOYV?QC`KbcHR*lwCZG3;A+O{^bo4iqpBg?)2~t=Ag1_n%vc_w!%@;upEzH zdW7YDhQ&Ip64a9>_E^emsiN4i9!bjb()A)SWH%*W_rgYe7%%}0p}E}Rj5#40%wo{b zCtZu`7}6}>*8T} zV%DkVB88LE9fkhr>Ps6arS6DrHSKuy`4&M*Fx!DxF$u^fob>&8)npXLLu(9jsrgh1 zxnXP=1j+W4J{vzc-;biN{0Z3hGgsER8sYOjdfT=i%2AeCW+*;Liq!i{!ZIM+6At_ zO9goU;#FylbGX}c%9=ZDbndZ8zEY&?{b?|g=N))@TKl-99?G2zizM-Yn{T<=yI&ht zQb-M%sPtHUIsMDABx~_ za#DYd-|wO_6y62fF!N@2v#6sdf=_;JO@{c`+H;uPQ2I2ai}3Y;&rZA<3$i+bHQseb z`~dUu4W5JD!=v@1Q5K%Rcv{+iIy;Fa#rk#8@!W&j*Lau>hdd7|U#P2V)^+rSbvD;i zg3~%WvbJmx;aYR-(1ONK=fOZVB;uncN?|adHh-me&Qd8v|O-__eZnm>yKmcK3Tu{+m>|&HvH}UTr+? zmr}PnecS4#d=#4=nDutUZ5(Zz(ju6qE^7xHm-E5P0Z}k|5ACp)J4G)diA;eHjVqY{ zmQ4l-YzPv7kMZPs>yI`bk9WVo05Df~cyjvhltY=S+v=yv;PI9Oj$j(H<6zz~(9jYf zjpz_8(zQ4DK>|8R@(HR$i1h_g1f7J|45pm=OvEr9A~)S3ilowe7@6iUym?o3fc_n} zzk|^9x(t}KK>uF7gMWKMFPKD%gfpeh5;f8OKJA;OMtW{2BL71jnLMa%+In;wyV~Mg z^+4fDg*s~n-^s1AF09f=@Ra}^0Jt^o$Tj*N)gLbx5@}crxfadvMc~v-Lg_T$$mPRv z)Mo9ONm$JpCxd%Dg-?yGXGh>_Eg(j4EpNt$Nh~Y)136!3rw^lq_T{}Her)=_U%p0; zr7h{D-KHOYSu@yQox^PvKF0k~Sj`*Q&NQ`cRnvy=Ahj(wiO7y#GT2 zyD`!JtFDak{?;zHj-Z|}f+Q~ODx$KX4BqBw-cL(ZrD6yrL9$ND)@L5qQ%V>RLa zHqq68T$w$-@)VeJKRdASa^^B(jN*14FX3U+xHNB_ZEDDJm$SQ@+W*Zw^3ASnHEXtY zelffO%_eA5GuF&h_yltt$04UpJ;58aH0a9hdd&dSENIKTizDLRNd{6FmQ$mw6C;@pnDH+!Lf zIsQ$Jb^F?D%DKJ$sIF`i&ijy;8%tSVNAF(BOb?qhL9f$5rrcVN-S$Y^Xt|KqHh7Ix zs-&dpBnWI|36^wE{#lqDlO<>SK9AO;ps53M9Gl90-1Lb8yIV30MfzvD&!eELVD*HU z)xhI>FU2wgDCnK#Bma$9@H=`=57zeh4JgW9u;<4xJ}HV==)dEc`s2*3Ed)0~W* zn}sfH4W9q158ocX%Ef5UVO61_6#kiMjRc1X$P0~lSSC_Isch^t7_`o?9n7M)m8fjl zVa9paX1HvTrsCa*fVaYwVpx>FS;2a91zZKE33K;#9#o*iRn` zT>;v`NG_3Q*aXQ_JryKdq*Y=*Ps%zA=3#>SWsxRS!mFEi1B*2<31#I33ld5+z1g$? zs*}`YO^~Abq!vTvb%*Q?9)m5b5!FF`auZ05>OMHxO}nJ7b|Qk*P<-ymZQCfx>kbSC*G00MSPUBugfabQ=l&Fq zoYcljnbDDK*j_u7nNeyE^#6t+ko!*nUg+m-`OIKBR`jE7edJ-X%+Om$NalgAWi$#0 z3@H;DbeD)x&DW0Vu4;kLHwzP9cKh-&zH)#%xtw0s@8!5~l7clw`Nazw~4 z53S#ILvEF_8q#ZE_09MCSJi`l%r&q^7ii6QZq6SpWb4 diff --git a/lecture23_24/notes_23.py b/lecture23_24/notes_23.py index 6c39e3e..e37ecb1 100644 --- a/lecture23_24/notes_23.py +++ b/lecture23_24/notes_23.py @@ -221,7 +221,7 @@ class Optimizer_SGD(): self.iterations += 1 # %% [markdown] -# # Testing the Learning Rate Decay +# ## Testing the Learning Rate Decay # %% # Create dataset @@ -270,7 +270,8 @@ for epoch in range(10001): if not epoch % 100: print(f'epoch: {epoch}, ' + f'acc: {accuracy:.3f}, ' + - f'loss: {loss:.3f}') + f'loss: {loss:.3f}, ' + + f'lr: {optimizer.current_learning_rate}') # Backward pass loss_activation.backward(loss_activation.output, y) @@ -335,7 +336,7 @@ class Optimizer_SGD(): self.iterations += 1 # %% [markdown] -# # Testing the Gradient Optimizer with Momentum +# ## Testing the Gradient Optimizer with Momentum # %% # Create dataset @@ -384,7 +385,8 @@ for epoch in range(10001): if not epoch % 100: print(f'epoch: {epoch}, ' + f'acc: {accuracy:.3f}, ' + - f'loss: {loss:.3f}') + f'loss: {loss:.3f}, ' + + f'lr: {optimizer.current_learning_rate}') # Backward pass loss_activation.backward(loss_activation.output, y) diff --git a/lecture25_27/notes_25.ipynb b/lecture25_27/notes_25.ipynb index cefffce..d73463f 100644 --- a/lecture25_27/notes_25.ipynb +++ b/lecture25_27/notes_25.ipynb @@ -254,12 +254,14 @@ "metadata": {}, "source": [ "# AdaGrad Optimizer\n", - "Different weights should have different learning rates. If one weight affects the loss much more strongly than the other, then consider using smaller learning rates with it. We can do this by maintaining a \"cache\" of the last gradients and normalizing based on this." + "Different weights should have different learning rates. If one weight affects the loss much more strongly than the other, then consider using smaller learning rates with it. We can do this by maintaining a \"cache\" of the last gradients and normalizing based on this.\n", + "\n", + "A downside is that as the cache keeps accumulating, some neurons will have such a small learning rate that the neuron basically becomes fixed." ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -269,7 +271,7 @@ " self.current_learning_rate = self.initial_learning_rate\n", " self.decay = decay\n", " self.iterations = 0\n", - " self.epsilon = 0\n", + " self.epsilon = epsilon\n", "\n", " def pre_update_params(self):\n", " if self.decay:\n", @@ -299,114 +301,114 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "epoch: 0, acc: 0.360, loss: 1.099\n", - "epoch: 100, acc: 0.477, loss: 0.992\n", - "epoch: 200, acc: 0.560, loss: 0.934\n", - "epoch: 300, acc: 0.600, loss: 0.881\n", - "epoch: 400, acc: 0.637, loss: 0.826\n", - "epoch: 500, acc: 0.617, loss: 0.793\n", - "epoch: 600, acc: 0.643, loss: 0.748\n", - "epoch: 700, acc: 0.657, loss: 0.726\n", - "epoch: 800, acc: 0.663, loss: 0.698\n", - "epoch: 900, acc: 0.680, loss: 0.681\n", - "epoch: 1000, acc: 0.683, loss: 0.664\n", - "epoch: 1100, acc: 0.693, loss: 0.653\n", - "epoch: 1200, acc: 0.693, loss: 0.642\n", - "epoch: 1300, acc: 0.707, loss: 0.632\n", - "epoch: 1400, acc: 0.720, loss: 0.625\n", - "epoch: 1500, acc: 0.717, loss: 0.615\n", - "epoch: 1600, acc: 0.723, loss: 0.608\n", - "epoch: 1700, acc: 0.730, loss: 0.600\n", - "epoch: 1800, acc: 0.740, loss: 0.588\n", - "epoch: 1900, acc: 0.740, loss: 0.581\n", - "epoch: 2000, acc: 0.743, loss: 0.576\n", - "epoch: 2100, acc: 0.740, loss: 0.570\n", - "epoch: 2200, acc: 0.750, loss: 0.563\n", - "epoch: 2300, acc: 0.747, loss: 0.562\n", - "epoch: 2400, acc: 0.757, loss: 0.557\n", - "epoch: 2500, acc: 0.760, loss: 0.554\n", - "epoch: 2600, acc: 0.770, loss: 0.550\n", - "epoch: 2700, acc: 0.777, loss: 0.546\n", - "epoch: 2800, acc: 0.777, loss: 0.543\n", - "epoch: 2900, acc: 0.780, loss: 0.540\n", - "epoch: 3000, acc: 0.777, loss: 0.537\n", - "epoch: 3100, acc: 0.773, loss: 0.533\n", - "epoch: 3200, acc: 0.783, loss: 0.531\n", - "epoch: 3300, acc: 0.777, loss: 0.528\n", - "epoch: 3400, acc: 0.777, loss: 0.526\n", - "epoch: 3500, acc: 0.773, loss: 0.523\n", - "epoch: 3600, acc: 0.780, loss: 0.522\n", - "epoch: 3700, acc: 0.773, loss: 0.520\n", - "epoch: 3800, acc: 0.777, loss: 0.518\n", - "epoch: 3900, acc: 0.780, loss: 0.516\n", - "epoch: 4000, acc: 0.780, loss: 0.515\n", - "epoch: 4100, acc: 0.773, loss: 0.513\n", - "epoch: 4200, acc: 0.770, loss: 0.511\n", - "epoch: 4300, acc: 0.773, loss: 0.510\n", - "epoch: 4400, acc: 0.777, loss: 0.509\n", - "epoch: 4500, acc: 0.777, loss: 0.508\n", - "epoch: 4600, acc: 0.780, loss: 0.507\n", - "epoch: 4700, acc: 0.777, loss: 0.506\n", - "epoch: 4800, acc: 0.777, loss: 0.504\n", - "epoch: 4900, acc: 0.783, loss: 0.503\n", - "epoch: 5000, acc: 0.783, loss: 0.503\n", - "epoch: 5100, acc: 0.787, loss: 0.502\n", - "epoch: 5200, acc: 0.790, loss: 0.501\n", - "epoch: 5300, acc: 0.787, loss: 0.500\n", - "epoch: 5400, acc: 0.787, loss: 0.498\n", - "epoch: 5500, acc: 0.783, loss: 0.497\n", - "epoch: 5600, acc: 0.787, loss: 0.496\n", - "epoch: 5700, acc: 0.780, loss: 0.495\n", - "epoch: 5800, acc: 0.780, loss: 0.495\n", - "epoch: 5900, acc: 0.783, loss: 0.494\n", - "epoch: 6000, acc: 0.790, loss: 0.494\n", - "epoch: 6100, acc: 0.777, loss: 0.493\n", - "epoch: 6200, acc: 0.783, loss: 0.492\n", - "epoch: 6300, acc: 0.783, loss: 0.491\n", - "epoch: 6400, acc: 0.790, loss: 0.490\n", - "epoch: 6500, acc: 0.780, loss: 0.488\n", - "epoch: 6600, acc: 0.780, loss: 0.487\n", - "epoch: 6700, acc: 0.777, loss: 0.485\n", - "epoch: 6800, acc: 0.780, loss: 0.483\n", - "epoch: 6900, acc: 0.783, loss: 0.482\n", - "epoch: 7000, acc: 0.790, loss: 0.480\n", - "epoch: 7100, acc: 0.790, loss: 0.479\n", - "epoch: 7200, acc: 0.797, loss: 0.477\n", - "epoch: 7300, acc: 0.803, loss: 0.476\n", - "epoch: 7400, acc: 0.813, loss: 0.475\n", - "epoch: 7500, acc: 0.813, loss: 0.474\n", - "epoch: 7600, acc: 0.813, loss: 0.472\n", - "epoch: 7700, acc: 0.813, loss: 0.471\n", - "epoch: 7800, acc: 0.810, loss: 0.470\n", - "epoch: 7900, acc: 0.810, loss: 0.469\n", - "epoch: 8000, acc: 0.810, loss: 0.468\n", - "epoch: 8100, acc: 0.810, loss: 0.465\n", - "epoch: 8200, acc: 0.807, loss: 0.463\n", - "epoch: 8300, acc: 0.803, loss: 0.462\n", - "epoch: 8400, acc: 0.803, loss: 0.461\n", - "epoch: 8500, acc: 0.807, loss: 0.459\n", - "epoch: 8600, acc: 0.810, loss: 0.458\n", - "epoch: 8700, acc: 0.813, loss: 0.458\n", - "epoch: 8800, acc: 0.810, loss: 0.456\n", - "epoch: 8900, acc: 0.810, loss: 0.455\n", - "epoch: 9000, acc: 0.813, loss: 0.452\n", - "epoch: 9100, acc: 0.813, loss: 0.450\n", - "epoch: 9200, acc: 0.817, loss: 0.448\n", - "epoch: 9300, acc: 0.810, loss: 0.447\n", - "epoch: 9400, acc: 0.810, loss: 0.446\n", - "epoch: 9500, acc: 0.813, loss: 0.444\n", - "epoch: 9600, acc: 0.813, loss: 0.441\n", - "epoch: 9700, acc: 0.817, loss: 0.440\n", - "epoch: 9800, acc: 0.817, loss: 0.438\n", - "epoch: 9900, acc: 0.813, loss: 0.436\n", - "epoch: 10000, acc: 0.813, loss: 0.435\n" + "epoch: 0, acc: 0.353, loss: 1.099, lr: 1.0\n", + "epoch: 100, acc: 0.497, loss: 0.986, lr: 0.9901970492127933\n", + "epoch: 200, acc: 0.527, loss: 0.936, lr: 0.9804882831650161\n", + "epoch: 300, acc: 0.513, loss: 0.918, lr: 0.9709680551509855\n", + "epoch: 400, acc: 0.580, loss: 0.904, lr: 0.9616309260505818\n", + "epoch: 500, acc: 0.550, loss: 0.910, lr: 0.9524716639679969\n", + "epoch: 600, acc: 0.563, loss: 0.860, lr: 0.9434852344560807\n", + "epoch: 700, acc: 0.600, loss: 0.838, lr: 0.9346667912889054\n", + "epoch: 800, acc: 0.603, loss: 0.815, lr: 0.9260116677470135\n", + "epoch: 900, acc: 0.643, loss: 0.787, lr: 0.9175153683824203\n", + "epoch: 1000, acc: 0.617, loss: 0.810, lr: 0.9091735612328392\n", + "epoch: 1100, acc: 0.637, loss: 0.750, lr: 0.9009820704567978\n", + "epoch: 1200, acc: 0.677, loss: 0.744, lr: 0.892936869363336\n", + "epoch: 1300, acc: 0.670, loss: 0.722, lr: 0.8850340738118416\n", + "epoch: 1400, acc: 0.693, loss: 0.704, lr: 0.8772699359592947\n", + "epoch: 1500, acc: 0.680, loss: 0.708, lr: 0.8696408383337683\n", + "epoch: 1600, acc: 0.697, loss: 0.664, lr: 0.8621432882145013\n", + "epoch: 1700, acc: 0.650, loss: 0.699, lr: 0.8547739123001966\n", + "epoch: 1800, acc: 0.707, loss: 0.646, lr: 0.8475294516484448\n", + "epoch: 1900, acc: 0.693, loss: 0.633, lr: 0.8404067568703253\n", + "epoch: 2000, acc: 0.713, loss: 0.620, lr: 0.8334027835652972\n", + "epoch: 2100, acc: 0.710, loss: 0.610, lr: 0.8265145879824779\n", + "epoch: 2200, acc: 0.710, loss: 0.599, lr: 0.8197393228953193\n", + "epoch: 2300, acc: 0.713, loss: 0.588, lr: 0.8130742336775347\n", + "epoch: 2400, acc: 0.733, loss: 0.576, lr: 0.8065166545689169\n", + "epoch: 2500, acc: 0.763, loss: 0.579, lr: 0.8000640051204096\n", + "epoch: 2600, acc: 0.800, loss: 0.558, lr: 0.7937137868084768\n", + "epoch: 2700, acc: 0.803, loss: 0.551, lr: 0.7874635798094338\n", + "epoch: 2800, acc: 0.800, loss: 0.545, lr: 0.7813110399249941\n", + "epoch: 2900, acc: 0.807, loss: 0.541, lr: 0.7752538956508256\n", + "epoch: 3000, acc: 0.757, loss: 0.538, lr: 0.7692899453804138\n", + "epoch: 3100, acc: 0.757, loss: 0.532, lr: 0.7634170547370028\n", + "epoch: 3200, acc: 0.740, loss: 0.532, lr: 0.7576331540268202\n", + "epoch: 3300, acc: 0.763, loss: 0.514, lr: 0.7519362358072035\n", + "epoch: 3400, acc: 0.770, loss: 0.512, lr: 0.7463243525636241\n", + "epoch: 3500, acc: 0.757, loss: 0.507, lr: 0.7407956144899621\n", + "epoch: 3600, acc: 0.780, loss: 0.496, lr: 0.735348187366718\n", + "epoch: 3700, acc: 0.787, loss: 0.497, lr: 0.7299802905321557\n", + "epoch: 3800, acc: 0.767, loss: 0.491, lr: 0.7246901949416624\n", + "epoch: 3900, acc: 0.783, loss: 0.483, lr: 0.7194762213108857\n", + "epoch: 4000, acc: 0.790, loss: 0.484, lr: 0.7143367383384527\n", + "epoch: 4100, acc: 0.783, loss: 0.477, lr: 0.7092701610043266\n", + "epoch: 4200, acc: 0.780, loss: 0.487, lr: 0.7042749489400663\n", + "epoch: 4300, acc: 0.787, loss: 0.467, lr: 0.6993496048674733\n", + "epoch: 4400, acc: 0.793, loss: 0.465, lr: 0.6944926731022988\n", + "epoch: 4500, acc: 0.787, loss: 0.460, lr: 0.6897027381198704\n", + "epoch: 4600, acc: 0.777, loss: 0.477, lr: 0.6849784231796698\n", + "epoch: 4700, acc: 0.783, loss: 0.454, lr: 0.6803183890060548\n", + "epoch: 4800, acc: 0.800, loss: 0.448, lr: 0.6757213325224677\n", + "epoch: 4900, acc: 0.800, loss: 0.441, lr: 0.6711859856366199\n", + "epoch: 5000, acc: 0.797, loss: 0.437, lr: 0.6667111140742716\n", + "epoch: 5100, acc: 0.800, loss: 0.433, lr: 0.6622955162593549\n", + "epoch: 5200, acc: 0.813, loss: 0.429, lr: 0.6579380222383051\n", + "epoch: 5300, acc: 0.810, loss: 0.426, lr: 0.6536374926465782\n", + "epoch: 5400, acc: 0.813, loss: 0.423, lr: 0.649392817715436\n", + "epoch: 5500, acc: 0.823, loss: 0.420, lr: 0.6452029163171817\n", + "epoch: 5600, acc: 0.820, loss: 0.416, lr: 0.6410667350471184\n", + "epoch: 5700, acc: 0.820, loss: 0.413, lr: 0.6369832473405949\n", + "epoch: 5800, acc: 0.820, loss: 0.410, lr: 0.6329514526235838\n", + "epoch: 5900, acc: 0.820, loss: 0.407, lr: 0.6289703754953141\n", + "epoch: 6000, acc: 0.823, loss: 0.404, lr: 0.6250390649415589\n", + "epoch: 6100, acc: 0.823, loss: 0.401, lr: 0.6211565935772407\n", + "epoch: 6200, acc: 0.827, loss: 0.398, lr: 0.6173220569170937\n", + "epoch: 6300, acc: 0.833, loss: 0.395, lr: 0.6135345726731701\n", + "epoch: 6400, acc: 0.830, loss: 0.390, lr: 0.6097932800780536\n", + "epoch: 6500, acc: 0.827, loss: 0.387, lr: 0.6060973392326807\n", + "epoch: 6600, acc: 0.827, loss: 0.384, lr: 0.6024459304777396\n", + "epoch: 6700, acc: 0.830, loss: 0.381, lr: 0.5988382537876519\n", + "epoch: 6800, acc: 0.830, loss: 0.378, lr: 0.5952735281862016\n", + "epoch: 6900, acc: 0.830, loss: 0.375, lr: 0.5917509911829102\n", + "epoch: 7000, acc: 0.833, loss: 0.373, lr: 0.5882698982293076\n", + "epoch: 7100, acc: 0.837, loss: 0.370, lr: 0.5848295221942803\n", + "epoch: 7200, acc: 0.833, loss: 0.368, lr: 0.5814291528577243\n", + "epoch: 7300, acc: 0.833, loss: 0.366, lr: 0.5780680964217585\n", + "epoch: 7400, acc: 0.837, loss: 0.364, lr: 0.5747456750387954\n", + "epoch: 7500, acc: 0.833, loss: 0.362, lr: 0.5714612263557918\n", + "epoch: 7600, acc: 0.833, loss: 0.360, lr: 0.5682141030740383\n", + "epoch: 7700, acc: 0.837, loss: 0.358, lr: 0.5650036725238714\n", + "epoch: 7800, acc: 0.840, loss: 0.357, lr: 0.5618293162537221\n", + "epoch: 7900, acc: 0.840, loss: 0.355, lr: 0.5586904296329404\n", + "epoch: 8000, acc: 0.840, loss: 0.353, lr: 0.5555864214678593\n", + "epoch: 8100, acc: 0.843, loss: 0.351, lr: 0.5525167136305873\n", + "epoch: 8200, acc: 0.843, loss: 0.350, lr: 0.5494807407000385\n", + "epoch: 8300, acc: 0.843, loss: 0.348, lr: 0.5464779496147331\n", + "epoch: 8400, acc: 0.843, loss: 0.346, lr: 0.5435077993369205\n", + "epoch: 8500, acc: 0.847, loss: 0.345, lr: 0.5405697605275961\n", + "epoch: 8600, acc: 0.847, loss: 0.343, lr: 0.5376633152320017\n", + "epoch: 8700, acc: 0.850, loss: 0.342, lr: 0.5347879565752179\n", + "epoch: 8800, acc: 0.847, loss: 0.340, lr: 0.5319431884674717\n", + "epoch: 8900, acc: 0.847, loss: 0.338, lr: 0.5291285253188\n", + "epoch: 9000, acc: 0.847, loss: 0.337, lr: 0.5263434917627243\n", + "epoch: 9100, acc: 0.850, loss: 0.336, lr: 0.5235876223886068\n", + "epoch: 9200, acc: 0.850, loss: 0.335, lr: 0.5208604614823689\n", + "epoch: 9300, acc: 0.847, loss: 0.334, lr: 0.5181615627752734\n", + "epoch: 9400, acc: 0.847, loss: 0.333, lr: 0.5154904892004742\n", + "epoch: 9500, acc: 0.850, loss: 0.331, lr: 0.5128468126570593\n", + "epoch: 9600, acc: 0.850, loss: 0.330, lr: 0.5102301137813153\n", + "epoch: 9700, acc: 0.850, loss: 0.329, lr: 0.5076399817249606\n", + "epoch: 9800, acc: 0.850, loss: 0.328, lr: 0.5050760139400979\n", + "epoch: 9900, acc: 0.850, loss: 0.326, lr: 0.5025378159706518\n", + "epoch: 10000, acc: 0.850, loss: 0.325, lr: 0.5000250012500626\n" ] } ], @@ -457,7 +459,239 @@ " if not epoch % 100:\n", " print(f'epoch: {epoch}, ' +\n", " f'acc: {accuracy:.3f}, ' +\n", - " f'loss: {loss:.3f}')\n", + " f'loss: {loss:.3f}, ' +\n", + " f'lr: {optimizer.current_learning_rate}')\n", + " \n", + " # Backward pass\n", + " loss_activation.backward(loss_activation.output, y)\n", + " dense2.backward(loss_activation.dinputs)\n", + " activation1.backward(dense2.dinputs)\n", + " dense1.backward(activation1.dinputs)\n", + " \n", + " # Update weights and biases\n", + " optimizer.pre_update_params()\n", + " optimizer.update_params(dense1)\n", + " optimizer.update_params(dense2)\n", + " optimizer.post_update_params()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# RMSProp Optimizer\n", + "Root Meas Square Propagation optimizer. It is similar to AdaGrad in that you apply different learning rates to different weights. However, the way you change the learning rate focuses more on the past cache rather than the current gradient." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "class Optimizer_RMSProp():\n", + " def __init__(self, learning_rate=1e-3, decay=0.0, epsilon=1e-7, rho=0.9):\n", + " self.initial_learning_rate = learning_rate\n", + " self.current_learning_rate = self.initial_learning_rate\n", + " self.decay = decay\n", + " self.iterations = 0\n", + " self.epsilon = epsilon\n", + " self.rho = rho\n", + "\n", + " def pre_update_params(self):\n", + " if self.decay:\n", + " self.current_learning_rate = self.initial_learning_rate / (1 + self.decay * self.iterations)\n", + "\n", + " def update_params(self, layer):\n", + " if not hasattr(layer, 'weight_cache'):\n", + " layer.weight_cache = np.zeros_like(layer.weights)\n", + " layer.bias_cache = np.zeros_like(layer.biases)\n", + "\n", + " layer.weight_cache = self.rho * layer.weight_cache + (1 - self.rho) * layer.dweights**2\n", + " layer.bias_cache = self.rho * layer.bias_cache + (1 - self.rho) * layer.dbiases**2\n", + "\n", + " layer.weights += -self.current_learning_rate * layer.dweights / (np.sqrt(layer.weight_cache) + self.epsilon)\n", + " layer.biases += -self.current_learning_rate * layer.dbiases / (np.sqrt(layer.bias_cache) + self.epsilon)\n", + "\n", + " def post_update_params(self):\n", + " self.iterations += 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Testing the RMSProp Optimizer" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch: 0, acc: 0.413, loss: 1.099, lr: 0.02\n", + "epoch: 100, acc: 0.467, loss: 0.980, lr: 0.01998021958261321\n", + "epoch: 200, acc: 0.480, loss: 0.904, lr: 0.019960279044701046\n", + "epoch: 300, acc: 0.507, loss: 0.864, lr: 0.019940378268975763\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch: 400, acc: 0.490, loss: 0.861, lr: 0.01992051713662487\n", + "epoch: 500, acc: 0.510, loss: 0.843, lr: 0.01990069552930875\n", + "epoch: 600, acc: 0.613, loss: 0.779, lr: 0.019880913329158343\n", + "epoch: 700, acc: 0.633, loss: 0.741, lr: 0.019861170418772778\n", + "epoch: 800, acc: 0.577, loss: 0.722, lr: 0.019841466681217078\n", + "epoch: 900, acc: 0.627, loss: 0.698, lr: 0.01982180200001982\n", + "epoch: 1000, acc: 0.630, loss: 0.675, lr: 0.019802176259170884\n", + "epoch: 1100, acc: 0.657, loss: 0.661, lr: 0.01978258934311912\n", + "epoch: 1200, acc: 0.623, loss: 0.654, lr: 0.01976304113677013\n", + "epoch: 1300, acc: 0.663, loss: 0.640, lr: 0.019743531525483964\n", + "epoch: 1400, acc: 0.627, loss: 0.672, lr: 0.01972406039507293\n", + "epoch: 1500, acc: 0.663, loss: 0.618, lr: 0.019704627631799327\n", + "epoch: 1600, acc: 0.693, loss: 0.607, lr: 0.019685233122373254\n", + "epoch: 1700, acc: 0.657, loss: 0.658, lr: 0.019665876753950384\n", + "epoch: 1800, acc: 0.730, loss: 0.587, lr: 0.019646558414129805\n", + "epoch: 1900, acc: 0.690, loss: 0.623, lr: 0.019627277990951823\n", + "epoch: 2000, acc: 0.730, loss: 0.573, lr: 0.019608035372895814\n", + "epoch: 2100, acc: 0.743, loss: 0.576, lr: 0.019588830448878047\n", + "epoch: 2200, acc: 0.740, loss: 0.560, lr: 0.019569663108249594\n", + "epoch: 2300, acc: 0.710, loss: 0.567, lr: 0.019550533240794143\n", + "epoch: 2400, acc: 0.740, loss: 0.548, lr: 0.019531440736725945\n", + "epoch: 2500, acc: 0.687, loss: 0.576, lr: 0.019512385486687673\n", + "epoch: 2600, acc: 0.710, loss: 0.550, lr: 0.01949336738174836\n", + "epoch: 2700, acc: 0.707, loss: 0.573, lr: 0.019474386313401298\n", + "epoch: 2800, acc: 0.760, loss: 0.523, lr: 0.019455442173562\n", + "epoch: 2900, acc: 0.770, loss: 0.525, lr: 0.019436534854566128\n", + "epoch: 3000, acc: 0.787, loss: 0.512, lr: 0.01941766424916747\n", + "epoch: 3100, acc: 0.763, loss: 0.517, lr: 0.019398830250535893\n", + "epoch: 3200, acc: 0.750, loss: 0.551, lr: 0.019380032752255354\n", + "epoch: 3300, acc: 0.803, loss: 0.498, lr: 0.01936127164832186\n", + "epoch: 3400, acc: 0.780, loss: 0.493, lr: 0.01934254683314152\n", + "epoch: 3500, acc: 0.780, loss: 0.514, lr: 0.019323858201528515\n", + "epoch: 3600, acc: 0.793, loss: 0.522, lr: 0.019305205648703173\n", + "epoch: 3700, acc: 0.780, loss: 0.516, lr: 0.01928658907028997\n", + "epoch: 3800, acc: 0.790, loss: 0.508, lr: 0.01926800836231563\n", + "epoch: 3900, acc: 0.750, loss: 0.523, lr: 0.019249463421207133\n", + "epoch: 4000, acc: 0.763, loss: 0.516, lr: 0.019230954143789846\n", + "epoch: 4100, acc: 0.787, loss: 0.499, lr: 0.019212480427285565\n", + "epoch: 4200, acc: 0.770, loss: 0.512, lr: 0.019194042169310647\n", + "epoch: 4300, acc: 0.790, loss: 0.496, lr: 0.019175639267874092\n", + "epoch: 4400, acc: 0.777, loss: 0.501, lr: 0.019157271621375684\n", + "epoch: 4500, acc: 0.810, loss: 0.479, lr: 0.0191389391286041\n", + "epoch: 4600, acc: 0.783, loss: 0.482, lr: 0.019120641688735073\n", + "epoch: 4700, acc: 0.797, loss: 0.463, lr: 0.019102379201329525\n", + "epoch: 4800, acc: 0.787, loss: 0.469, lr: 0.01908415156633174\n", + "epoch: 4900, acc: 0.777, loss: 0.497, lr: 0.01906595868406753\n", + "epoch: 5000, acc: 0.810, loss: 0.445, lr: 0.01904780045524243\n", + "epoch: 5100, acc: 0.793, loss: 0.450, lr: 0.019029676780939874\n", + "epoch: 5200, acc: 0.810, loss: 0.438, lr: 0.019011587562619416\n", + "epoch: 5300, acc: 0.797, loss: 0.452, lr: 0.01899353270211493\n", + "epoch: 5400, acc: 0.800, loss: 0.453, lr: 0.018975512101632844\n", + "epoch: 5500, acc: 0.820, loss: 0.419, lr: 0.018957525663750367\n", + "epoch: 5600, acc: 0.817, loss: 0.433, lr: 0.018939573291413745\n", + "epoch: 5700, acc: 0.763, loss: 0.533, lr: 0.018921654887936498\n", + "epoch: 5800, acc: 0.820, loss: 0.411, lr: 0.018903770356997703\n", + "epoch: 5900, acc: 0.817, loss: 0.424, lr: 0.01888591960264025\n", + "epoch: 6000, acc: 0.810, loss: 0.419, lr: 0.018868102529269144\n", + "epoch: 6100, acc: 0.827, loss: 0.403, lr: 0.018850319041649778\n", + "epoch: 6200, acc: 0.820, loss: 0.413, lr: 0.018832569044906263\n", + "epoch: 6300, acc: 0.820, loss: 0.414, lr: 0.018814852444519702\n", + "epoch: 6400, acc: 0.810, loss: 0.410, lr: 0.018797169146326564\n", + "epoch: 6500, acc: 0.830, loss: 0.389, lr: 0.018779519056516963\n", + "epoch: 6600, acc: 0.823, loss: 0.407, lr: 0.018761902081633038\n", + "epoch: 6700, acc: 0.823, loss: 0.403, lr: 0.018744318128567278\n", + "epoch: 6800, acc: 0.820, loss: 0.405, lr: 0.018726767104560903\n", + "epoch: 6900, acc: 0.823, loss: 0.386, lr: 0.018709248917202218\n", + "epoch: 7000, acc: 0.827, loss: 0.393, lr: 0.018691763474424996\n", + "epoch: 7100, acc: 0.800, loss: 0.428, lr: 0.018674310684506857\n", + "epoch: 7200, acc: 0.833, loss: 0.378, lr: 0.018656890456067686\n", + "epoch: 7300, acc: 0.820, loss: 0.382, lr: 0.01863950269806802\n", + "epoch: 7400, acc: 0.810, loss: 0.438, lr: 0.018622147319807447\n", + "epoch: 7500, acc: 0.833, loss: 0.379, lr: 0.018604824230923078\n", + "epoch: 7600, acc: 0.837, loss: 0.351, lr: 0.01858753334138793\n", + "epoch: 7700, acc: 0.827, loss: 0.397, lr: 0.018570274561509396\n", + "epoch: 7800, acc: 0.837, loss: 0.369, lr: 0.018553047801927663\n", + "epoch: 7900, acc: 0.787, loss: 0.458, lr: 0.018535852973614212\n", + "epoch: 8000, acc: 0.840, loss: 0.369, lr: 0.01851868998787026\n", + "epoch: 8100, acc: 0.863, loss: 0.336, lr: 0.018501558756325222\n", + "epoch: 8200, acc: 0.840, loss: 0.366, lr: 0.01848445919093522\n", + "epoch: 8300, acc: 0.833, loss: 0.369, lr: 0.018467391203981567\n", + "epoch: 8400, acc: 0.837, loss: 0.355, lr: 0.01845035470806926\n", + "epoch: 8500, acc: 0.840, loss: 0.357, lr: 0.018433349616125496\n", + "epoch: 8600, acc: 0.857, loss: 0.329, lr: 0.018416375841398172\n", + "epoch: 8700, acc: 0.843, loss: 0.352, lr: 0.018399433297454436\n", + "epoch: 8800, acc: 0.843, loss: 0.356, lr: 0.01838252189817921\n", + "epoch: 8900, acc: 0.797, loss: 0.447, lr: 0.018365641557773718\n", + "epoch: 9000, acc: 0.847, loss: 0.354, lr: 0.018348792190754044\n", + "epoch: 9100, acc: 0.840, loss: 0.349, lr: 0.0183319737119497\n", + "epoch: 9200, acc: 0.853, loss: 0.337, lr: 0.018315186036502167\n", + "epoch: 9300, acc: 0.847, loss: 0.350, lr: 0.018298429079863496\n", + "epoch: 9400, acc: 0.823, loss: 0.383, lr: 0.018281702757794862\n", + "epoch: 9500, acc: 0.853, loss: 0.338, lr: 0.018265006986365174\n", + "epoch: 9600, acc: 0.780, loss: 0.522, lr: 0.018248341681949654\n", + "epoch: 9700, acc: 0.840, loss: 0.335, lr: 0.018231706761228456\n", + "epoch: 9800, acc: 0.853, loss: 0.334, lr: 0.01821510214118526\n", + "epoch: 9900, acc: 0.723, loss: 0.696, lr: 0.018198527739105907\n", + "epoch: 10000, acc: 0.860, loss: 0.314, lr: 0.018181983472577025\n" + ] + } + ], + "source": [ + "# Create dataset\n", + "X, y = spiral_data(samples=100, classes=3)\n", + "\n", + "# Create Dense layer with 2 input features and 64 output values\n", + "dense1 = Layer_Dense(2, 64)\n", + "\n", + "# Create ReLU activation (to be used with Dense layer)\n", + "activation1 = Activation_ReLU()\n", + "\n", + "# Create second Dense layer with 64 input features (as we take output\n", + "# of previous layer here) and 3 output values (output values)\n", + "dense2 = Layer_Dense(64, 3)\n", + "\n", + "# Create Softmax classifier's combined loss and activation\n", + "loss_activation = Activation_Softmax_Loss_CategoricalCrossentropy()\n", + "\n", + "# Create optimizer\n", + "optimizer = Optimizer_RMSProp(learning_rate=0.02, decay=1e-5, rho=0.999)\n", + "\n", + "# Train in loop\n", + "for epoch in range(10001):\n", + " # Perform a forward pass of our training data through this layer\n", + " dense1.forward(X)\n", + " \n", + " # Perform a forward pass through activation function\n", + " # takes the output of first dense layer here\n", + " activation1.forward(dense1.output)\n", + " \n", + " # Perform a forward pass through second Dense layer\n", + " # takes outputs of activation function of first layer as inputs\n", + " dense2.forward(activation1.output)\n", + " \n", + " # Perform a forward pass through the activation/loss function\n", + " # takes the output of second dense layer here and returns loss\n", + " loss = loss_activation.forward(dense2.output, y)\n", + " \n", + " # Calculate accuracy from output of activation2 and targets\n", + " # calculate values along first axis\n", + " predictions = np.argmax(loss_activation.output, axis=1)\n", + " if len(y.shape) == 2:\n", + " y = np.argmax(y, axis=1)\n", + " accuracy = np.mean(predictions == y)\n", + " \n", + " if not epoch % 100:\n", + " print(f'epoch: {epoch}, ' +\n", + " f'acc: {accuracy:.3f}, ' +\n", + " f'loss: {loss:.3f}, ' +\n", + " f'lr: {optimizer.current_learning_rate}')\n", " \n", " # Backward pass\n", " loss_activation.backward(loss_activation.output, y)\n",