Logistic Regression From Scratch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 326,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare Data and Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ones</th>\n",
       "      <th>E1</th>\n",
       "      <th>E2</th>\n",
       "      <th>Allowed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>34.623660</td>\n",
       "      <td>78.024693</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>30.286711</td>\n",
       "      <td>43.894998</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>35.847409</td>\n",
       "      <td>72.902198</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>60.182599</td>\n",
       "      <td>86.308552</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>79.032736</td>\n",
       "      <td>75.344376</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Ones         E1         E2  Allowed\n",
       "0     1  34.623660  78.024693        0\n",
       "1     1  30.286711  43.894998        0\n",
       "2     1  35.847409  72.902198        0\n",
       "3     1  60.182599  86.308552        1\n",
       "4     1  79.032736  75.344376        1"
      ]
     },
     "execution_count": 327,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"data/ex2data1.txt\",header= None, names = ['E1','E2', 'Allowed'])\n",
    "data.insert(0, 'Ones', 1)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x1af550af1d0>"
      ]
     },
     "execution_count": 328,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtcAAAHjCAYAAADojTN7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X90nFed5/nPdxRBKgFUieNxR7KzcVhaA9iyRdQBotgE\ni6B0DyZqc9ZjfoTQ0JM5vWkMgVXH3t5jPN4Ga9c5J405DKdpMpmkJzgI4ygJ6cHJWECckIXILSPn\nlzA/nFhlJ3YcJCBd0EL93T+qyvoR/SrVU/XU8zzv1zk+VXVVkq+vy67Pc+t77zV3FwAAAIDS/Zuw\nOwAAAADEBeEaAAAACAjhGgAAAAgI4RoAAAAICOEaAAAACAjhGgAAAAgI4RoAAAAICOEaAAAACAjh\nGgAAAAjIOWF3oBQXXXSRX3rppWF3AwAAADF36NChl9x98VzPi3S4vvTSS9XX1xd2NwAAABBzZvbc\nfJ5HWQgAAAAQEMI1AAAAEBDCNQAAABCQSNdcAwAAxNHo6KiGhob029/+NuyuJM65556rpUuXqra2\ndkHfT7gGAACoMkNDQ3r961+vSy+9VGYWdncSw9115swZDQ0Nafny5Qv6GZSFAAAAVJnf/va3WrRo\nEcG6wsxMixYtKukTA8I1AABAFSJYh6PUcSdcAwAAAAEhXAMAAGBaPT09MjM9++yzkqRjx45pxYoV\nkqTvfe97et/73leRfnzsYx/T3r17K/J7lYpwDQAAEHE9/Rm1dvVq+ZYH1drVq57+TCA/d8+ePbrq\nqqu0Z8+eQH5eEpQtXJvZfzWzU2b25IS2C83sYTM7mr+9YMLXtprZT81s0Mzay9UvAACAOOnpz2jr\nviPKDGflkjLDWW3dd6TkgP2b3/xGjz76qG6//Xbdc889sz735ZdfVkdHh5qamvSOd7xDAwMDkqSV\nK1dqeHhY7q5FixbprrvukiR99KMf1cMPP6yxsTF1dnbqj/7oj9TU1KS/+7u/k5TbteMv//Iv1djY\nqPe85z06depUSX+WSirnzPV/k3TtlLYtkg64+5skHcg/lpm9RdImSW/Nf89/MbOaMvYNAAAgFnbt\nH1R2dGxSW3Z0TLv2D5b0c++77z5de+21+sM//EMtWrRIhw4dmvG5n/vc59Tc3KyBgQF94Qtf0Ec/\n+lFJUmtrqx577DE99dRTuuyyy3Tw4EFJ0uOPP64rr7xSt99+u+rq6vTEE0/oiSee0N///d/rF7/4\nhe69914NDg7q6aef1l133aUf/OAHJf1ZKqls4drdH5H08pTm6yTdmb9/p6SOCe33uPvv3P0Xkn4q\n6Ypy9Q0AACAuTgxni2qfrz179mjTpk2SpE2bNs1aGvLoo4/q+uuvlyStW7dOZ86c0a9+9SutWbNG\njzzyiB555BH9xV/8hY4cOaJMJqMLLrhA559/vh566CHdddddWr16td7+9rfrzJkzOnr0qB555BF9\n8IMfVE1Njerr67Vu3bqS/iyVVOlDZJa4+8n8/RckLcnfb5D0/0143lC+DQAAALOoT6eUmSZI16dT\nC/6ZL7/8snp7e3XkyBGZmcbGxmRmuummm4r6OWvXrtWXv/xlPf/88/r85z+ve++9V3v37tWaNWsk\n5co/vvSlL6m9fXJF8D/+4z8uuO9hC21Bo7u7JC/2+8zsRjPrM7O+06dPl6FnAAAA0dHZ3qhU7eRq\n2lRtjTrbGxf8M/fu3avrr79ezz33nI4dO6bjx49r+fLlOn78+LTPX7Nmje6++25JuV1ELrroIr3h\nDW/QsmXL9NJLL+no0aO67LLLdNVVV+nWW2/V2rVrJUnt7e36yle+otHRUUnST37yE73yyitau3at\nvvGNb2hsbEwnT57Ud7/73QX/WSqt0jPXL5rZxe5+0swullSoTs9IWjbheUvzba/i7l+V9FVJamlp\nKTqcAwAAxElHc+7D/l37B3ViOKv6dEqd7Y1n2xdiz549uuWWWya1feADH9DOnTunff727dv18Y9/\nXE1NTTrvvPN05513nv3a29/+do2N5WrC16xZo61bt+qqq66SJP35n/+5jh07pre97W1ydy1evFg9\nPT360z/9U/X29uotb3mLLrnkEr3zne9c8J+l0iw3gVymH252qaRvu/uK/ONdks64e5eZbZF0obv/\nlZm9VdLXlauzrlduseOb3H1s+p+c09LS4n19fWXrPwAAQBieeeYZvfnNbw67G4k13fib2SF3b5nr\ne8s2c21meyRdLekiMxuS9DlJXZK6zewTkp6TtFGS3P0pM+uW9LSk30u6aa5gjbyBbunADmlkSKpb\nKrVtk5o2ht0rAACARCpbuHb3D87wpbYZnv95SZ8vV39iaaBbemCzNJpfxDByPPdYImADAACEgBMa\no+zAjvFgXTCazbUDAACg4gjXUTYyVFw7AAAAyopwHWV1S4trBwAAQFkRrqOsbZtUO2WD+NpUrh0A\nAAAVR7iOsqaN0vrdUt0ySZa7Xb+bxYwAAKBkZqbPfvazZx/feuut2r59+6zf09PTo6effnrW56xe\nvfrsseoFH/vYx7R3715J0tVXX61KbLV87NgxrVixIvCfS7iOuqaN0s1PStuHc7cEawAR1dOfUWtX\nr5ZveVCtXb3q6Z/2LDEA0xnolm5bIW1P524Hukv+ka997Wu1b98+vfTSS/P+nrnC9TPPPKOxsTEd\nPHhQr7zySsl9rEaEa1Q93nCB+Ovpz2jrviPKDGflkjLDWW3dd4R/78B8FLbmHTkuyce35i0xYJ9z\nzjm68cYbddttt73qa8eOHdO6devU1NSktrY2Pf/88/rBD36g+++/X52dnVq9erV+9rOfver79uzZ\no+uvv17vfe97dd99983Zhz179mjlypVasWLF2RMjv/nNb+ozn/mMJOmLX/yiLrvsMknSz3/+c7W2\ntkqSDh06pHe96126/PLL1d7erpMnT55tX7VqlVatWqUvf/nLCxuYORCuUdV4wwWSYdf+QWVHJ58d\nlh0d0679gyH1CIiQMm7Ne9NNN+nuu+/WyMjIpPZPfvKTuuGGGzQwMKAPf/jD2rx5s6688kq9//3v\n165du3T48GG98Y1vfNXP+8Y3vqFNmzbpgx/8oPbs2TPr733ixAndcsst6u3t1eHDh/XEE0+op6dH\na9as0cGDByVJBw8e1KJFi5TJZHTw4EGtXbtWo6Oj+uQnP6m9e/fq0KFD+vjHP66//uu/liT92Z/9\nmb70pS/pxz/+ccljMxPCNaoab7hAMpwYzhbVDmCCMm7N+4Y3vEEf/ehHtXv37kntjz/+uD70oQ9J\nkq6//no9+uijc/6svr4+XXTRRbrkkkvU1tam/v5+vfzyyzM+/4knntDVV1+txYsX65xzztGHP/xh\nPfLII/qDP/gD/eY3v9Gvf/1rHT9+XB/60If0yCOP6ODBg1qzZo0GBwf15JNP6pprrtHq1av1N3/z\nNxoaGtLw8LCGh4e1du3as/0uB8I1qhpvuEAy1KdTRbUDmKDMW/N++tOf1u23315yjfSePXv07LPP\n6tJLL9Ub3/hG/epXv9K3vvWtBf2sK6+8UnfccYcaGxvPzmQ//vjjam1tlbvrrW99qw4fPqzDhw/r\nyJEjeuihh0rqezEI16hqvOEiSZK8vqCzvVGp2ppJbanaGnW2N4bUIyBCyrw174UXXqiNGzfq9ttv\nP9t25ZVX6p577pEk3X333VqzZo0k6fWvf71+/etfv+pn/Ou//qu6u7t15MgRHTt2TMeOHdN99903\na2nIFVdcoe9///t66aWXNDY2pj179uhd73qXJGnNmjW69dZbtXbtWjU3N+u73/2uXvva16qurk6N\njY06ffq0Hn/8cUnS6OionnrqKaXTaaXT6bOz7HfffXcg4zMV4RpVjTdcJEXS1xd0NDdo54aVakin\nZJIa0int3LBSHc0NYXctMpJ8cZZ4Fdia97Of/eykXUO+9KUv6Y477lBTU5P+4R/+QV/84hclSZs2\nbdKuXbvU3Nw8aUHjwYMH1dDQoPr6+rNta9eu1dNPP312seFUF198sbq6uvTud79bq1at0uWXX67r\nrrtOUi5cHz9+XGvXrlVNTY2WLVumq666SpL0mte8Rnv37tUtt9yiVatWafXq1frBD34gSbrjjjt0\n0003afXq1XL3wMZnIivXD66ElpYWr8Q+iAhXT39Gu/YP6sRwVvXplDrbG3nDRSTN9lpu7epVZppy\np4Z0So9tWVfpriJiChdnE9eopGpruECJsGeeeUZvfvObw+5GYk03/mZ2yN1b5vrec8rWKyAgHc0N\nvDkg8qaGn8LMtJR7jbO+oHLieME+2+LvqP/ZgKihLAQAKmCunW9YX1AZcS2/4eIMqB6EawCogLnC\nD+sLKiOu23tycRZPUS7djbJSx51wDQAVMFf4YUFfZcR1hpeLs/g599xzdebMGQJ2hbm7zpw5o3PP\nPXfBP4Oa62oz0J07UWlkKLc/Zdu2QFf7AghHZ3vjtAvOJoYf1heUX306Ne3C0ajP8BZeN3GrJU+y\npUuXamhoSKdPnw67K4lz7rnnaunShe8RTriuJgPd0gObx48wHTmeeywRsIGII/xUh/lc5EQVF2fx\nUltbq+XLl4fdDSwAW/FVk9tW5AL1VHXLpJufrHx/ACCG4rhbCIDyYyu+KBoZKq4dAFA0ZngBlBML\nGqtJ3Qz1PTO1AwAAoKoQrqtJ2zapdsqimtpUrh0AAABVj3BdTZo2Sut352qsZbnb9btZzAgAABAR\n1FxXm6aNhGkAAICIYuYaAAAACAjhGgAAAAgI4RoAAAAICOEaAAAACAjhGgAAAAgI4RoAAAAICOEa\nAAAACAj7XAMAIqenP6Nd+wd1Yjir+nRKne2N6mhuCLtbAEC4BgBES09/Rlv3HVF2dEySlBnOauu+\nI5JEwAYQOspCAACRsmv/4NlgXZAdHdOu/YMh9QgAxhGuAQCRcmI4W1Q7AFQS4RoAECn16VRR7QBQ\nSYRrAECkdLY3KlVbM6ktVVujzvbGkHoUfT39GbV29Wr5lgfV2tWrnv5M2F0CIosFjQCASCksWmS3\nkGCwQBQIFuEaABA5Hc0NBL+AzLZAlDEGikdZCAAACcYCUSBYhGsAABKMBaJAsAjXAAAkGAtEgWAR\nrgEASLCO5gZ94PIG1ZhJkmrM9IHLqWkHFopwDQBAgvX0Z/StQxmNuUuSxtz1rUMZtuMDFohwDQBA\ngnGcPBAstuIDAKBMevozVb8fN7uFAMFi5hoAgDIoHM6SGc7KNX44S7WVW7BbCBAswjUAAGUQlXIL\ndgsBghVKuDazT5nZk2b2lJl9Ot92oZk9bGZH87cXhNE3AACCEJVyi47mBu3csFIN6ZRMUkM6pZ0b\nVlZd+QoQFRWvuTazFZL+o6QrJP2LpO+Y2bcl3SjpgLt3mdkWSVsk3VLp/gEAEIT6dEqZaYJ0NZZb\ncJw8EJwwFjS+WdIP3f2fJcnMvi9pg6TrJF2df86dkr4nwjUAIAKmW7jY2d6orfuOTCoNodwCiL8w\nykKelLTGzBaZ2XmS/kTSMklL3P1k/jkvSFoy3Teb2Y1m1mdmfadPn65MjwEAmMFMCxclUW4BJJB5\nftP4iv6mZp+Q9L9LekXSU5J+J+lj7p6e8JxfuvusddctLS3e19dX1r4CCF4UticD5qu1q3fa8o+G\ndEqPbVkXQo8AlIOZHXL3lrmeF8qCRne/3d0vd/e1kn4p6SeSXjSziyUpf3sqjL4BKK+obE8GzFdU\nFi4CqIywdgv5t/nbS5Srt/66pPsl3ZB/yg2S7gujbwDKKyrbkwHzxT7RACYKa5/rb5nZ05IekHST\nuw9L6pJ0jZkdlfSe/GMAMcMsH+KGfaIRlJ7+jFq7erV8y4Nq7erlE72ICuX4c3dfM03bGUltIXQH\nQAVFaXsyYD4K6wVYR4BSFErmCp/sTVwYy2spWkIJ1wCml4SFfmxPhjhin2iUaraSOV5b0UK4BqpE\nUmYtmOUDgFejZC4+CNdAlUjSrAWzfAAwGSVz8RHWgkYAUzBrAQDJxcLY+CBcA1WC7bwAILk6mhs4\n0TMmKAsBqgQL/QAgnua7WJ2SuXggXANVgoV+QDIkYVcgjEvKYnWMI1wj1qL2JsasBRBvBK3kSdJi\ndeRQc43YKryJZYazco2/iXHiFYCwzBa0EE8sVk8ewjViizcxANWGoJU8LFZPHsI1Yos3MQDVhqCV\nPGyxl/skubWrV8u3PKjWrt7Yf4JMuEZs8SYGoNoQtJIn6VvsJbFEkwWNiC22tkOlRW0BLSqPXYGS\nKcmL1ZO4oJNwjdjiTQyVxC4QmK8kBy0kTxJLNAnXiDXexFApSZydAYC51KdTykwTpONcoknNNQAE\nIImzM5hd0hZxAdNJ4joDZq4BIABJnJ0pVpJq0ikTAnKSWKJJuAaAALCAdnZJC5uUCQHjklaiSVkI\nAAQg6dttzSVphzpRJgQkFzPXABCQpM3OFCNpYZMyISC5mLkGAJRdHA91mm3BYhIXcQHIIVwDAMou\nbmFzrlPnKBMCkouyEABA2cVtx4D5LFikTAhIJsI1AKAi4hQ2k1ZDDmD+KAsBAKBIcawhBxAMwjUA\nAEWKWw05gOBQFgIAQJHiVkMOIDiEawAAFiBONeQAgkNZCAAAABAQwjUAAAAQEMI1AAAAEBDCNQAA\nABAQwjUAAAAQEMI1AAAAEBDCNQAAABAQwjUAAAAQEMI1AAAAEBDCNQAAABAQwjUAAAAQEMI1AAAA\nEBDCNQAAABAQwjUAAAAQEMI1AAAAEJBzwu4AACB4Pf0Z7do/qBPDWdWnU+psb1RHc0PY3QKA2CNc\nA0DM9PRntHXfEWVHxyRJmeGstu47IkkEbAAoM8pCACBmdu0fPBusC7KjY9q1fzCkHgFAchCuASBm\nTgxni2oHAASHcA0AMVOfThXVDgAITijh2sxuNrOnzOxJM9tjZuea2YVm9rCZHc3fXhBG3wAg6jrb\nG5WqrZnUlqqtUWd7Y0g9AoDkqHi4NrMGSZsltbj7Ckk1kjZJ2iLpgLu/SdKB/GMAQJE6mhu0c8NK\nNaRTMkkN6ZR2bljJYkYAqICwdgs5R1LKzEYlnSfphKStkq7Of/1OSd+TdEsYnQOAqOtobiBMA0AI\nKj5z7e4ZSbdKel7SSUkj7v6QpCXufjL/tBckLZnu+83sRjPrM7O+06dPV6TPAAAAwHyEURZygaTr\nJC2XVC/pfDP7yMTnuLtL8um+392/6u4t7t6yePHisvcXAAAAmK8wFjS+R9Iv3P20u49K2ifpSkkv\nmtnFkpS/PRVC3wAAAIAFCyNcPy/pHWZ2npmZpDZJz0i6X9IN+efcIOm+EPoGAAAALFjFFzS6+w/N\nbK+kf5L0e0n9kr4q6XWSus3sE5Kek7Sx0n1DzAx0Swd2SCNDUt1SqW2b1MTLCgAAlE8ou4W4++ck\nfW5K8++Um8UGSjfQLT2wWRrNn0g3cjz3WCJgAwCAsglrKz6gvA7sGA/WBaPZXDvhGkiMnv6Mdu0f\n1InhrOrTKXW2N7JFIYCyIlwjnkaGimsHEDs9/Rlt3XdE2dExSVJmOKut+45IEgEbQNmEcvw5KmCg\nW7pthbQ9nbsd6A67R5VVt7S4diBGevozau3q1fItD6q1q1c9/ZmwuxSKXfsHzwbrguzomHbtHwyp\nRwCSgHAdR4V645Hjkny83jhJAbttm1SbmtxWm8q1AzFWmK3NDGflGp+tTWLAPjGcLaodAIJAuI6j\n2eqNk6Jpo7R+t1S3TJLlbtfvpt4ascds7bj6dKqodgAIAjXXcUS9cU7TRsI0EofZ2nGd7Y2Taq4l\nKVVbo872xhB7BSDumLmOI+qNgcRitnZcR3ODdm5YqYZ0SiapIZ3Szg0rWcwIoKyYuY6jtm2T93iW\nqDcGEoLZ2sk6mhsI00g8tqSsLMJ1HBVKITidEEicwhsmb6QAJLakDIO5e9h9WLCWlhbv6+sLuxsA\nAABVqbWrV5lp1lw0pFN6bMu6EHoUXWZ2yN1b5noeNdcAAAAxxSLnyqMsBAAAlIy63upUn05NO3Od\nxEXOlcLMNQAAKAmHF1WvzvZGpWprJrUleZFzJRCuAQBASTi8qHqxJWXlURaC8A10s7MJAEQYdb3V\njS0pK4twjXANdE/ek3vkeO6xRMCuAGokZ8f4APNDXS8wjrIQhOvAjsmH3Ui5xwd2hNOfBKFGcnaM\nDzB/1PUC4wjXCNfIUHHtCAw1krNjfID5o64XGEdZSFxEtW65bmmuFGS6dpQVNZKzY3wQdZUua6Ku\nF8hh5joOCnXLI8cl+Xjd8kB32D2bW9s2qXZKTV5tKteOspqpFpIayRzGB1FGWRMQHsJ1HES5brlp\no7R+t1S3TJLlbtfvjsase8RRIzk7xgdRRlkTEB7KQuIg6nXLTRsJ0yEofHzLbhjTY3wQZZQ1AeEh\nXMcBdctYIGokZ8f4IKrYGg8ID2UhcUDdMgBgAsqagPAwcx0HhZKKKO4WAgAI3HzKmjgkCSgPc/ew\n+7BgLS0t3tfXF3Y3AACIlMJuIhMXPaZqa9ibGpiFmR1y95a5nkdZCAAACcNuIkD5EK4BAEgYdhMB\nyodwDQBAwnBIElA+hGsAABKG3USA8mG3EAAAEoZDkoDyIVwDQMDY4gxRwCFJQHkQriENdLNHNhCQ\nqVucZYaz2rrviKTx2ULCNwDEFzXXSTfQLT2wOX98uuduH9icawdQtLm2OCuE78xwVq7x8N3Tnwmh\ntwCAoBGuk+7ADml0ytZLo9lcO4CizbXFGfsLA0C8URaSdCNDxbUDmFV9OqXMNAG7sMUZ+wsDQPGi\nVE7HzHXS1S0trh3ArOba4oz9hQGgOFErpyNcJ13bNql2ypt6bSrXDqBoHc0N2rlhpRrSKZmkhnRK\nOzesPDvDwv7CAFCcqJXTURaSdIVdQdgtBAjMbFucsb8wABQnauV0hGvkgjRhGqgY9hcGgPmbay1L\ntaEsBAAAAFUrauV0zFwDAACgakWtnI5wjerFyZEAAEDRKqcjXKM6FU6OLBxwUzg5UiJgAwCAqkXN\nNaoTJ0cCAIAIIlyjOnFyJAAAiCDKQlCd6pbmSkGmaweABIjScc8AxjFzjerEyZEAEixqxz0DGFfx\ncG1mjWZ2eMKvX5nZp83sQjN72MyO5m8vqHTfUEWaNkrrd0t1yyRZ7nb9bhYzAkiEqB33DGBcxctC\n3H1Q0mpJMrMaSRlJ90raIumAu3eZ2Zb841sq3T9UEU6OREJRDlDdKvH3E7XjnpOEf5+YS9hlIW2S\nfubuz0m6TtKd+fY7JXWE1isACAnlANWtUn8/Mx3rXK3HPScF/z4xH2GH602S9uTvL3H3k/n7L0ha\nMt03mNmNZtZnZn2nT5+uRB8BTNDTn1FrV6+Wb3lQrV29vKkEjHKA6lapv5+oHfecFPz7xHyEFq7N\n7DWS3i/pm1O/5u4uyaf7Pnf/qru3uHvL4sWLy9xLABMxa1N+lANUt0r9/XQ0N2jnhpVqSKdkkhrS\nKe3csJLyg5Dx7xPzEeZWfH8s6Z/c/cX84xfN7GJ3P2lmF0s6FWLfAExjtlkb3vSDUZ9OKTPNGzXl\nANWhkn8/UTruOSn494n5CLMs5IMaLwmRpPsl3ZC/f4Ok+yreIwCzYtam/CgHqG7V8PdDaVZ4quHv\nH9UvlJlrMztf0jWS/tOE5i5J3Wb2CUnPSWKbCKDKMGtTfoWZSnYjqE5h//0USrMKnyAVSrMm9g3l\nE/bff9SUsrNKlHdlsVx5czS1tLR4X19f2N0AEmPqG7uUm7WhFhSojNau3mkvcBvSKT22ZV0IPQKm\nV8r7RbW+15jZIXdvmet5Ye8WAiBCWGQFhIvSLERFKTurRH1XljAXNAKIIBZZAeGhNAtRUcqFYNQv\nIpm5BgAgIlhQh6go5SCkqB+iRLgGACAiKM1CVJRyIRj1i0jKQgAAiBBKsxAFpeysEvVdWdgtBAAA\nAJgDu4UAAAAAFUa4BgAAAAJCuAYAAAACQrgGAAAAAkK4BgAAAALCVnwAgJL09Gciu2UWAASNcA0U\na6BbOrBDGhmS6pZKbdukpo1h9woIRU9/Rlv3HVF2dEySlBnOauu+I5JEwAaQSJSFAMUY6JYe2CyN\nHJfkudsHNufagQTatX/wbLAuyI6Oadf+wZB6BADhIlwDxTiwQxrNTm4bzebagQQ6MZwtqh0A4o6y\nEKAYI0PFtQMxV59OKTNNkK5Pp0LoDRAe1h6ggJlroBh1S4trB2Kus71RqdqaSW2p2hp1tjeG1COg\n8gprDzLDWbnG1x709GfC7hpCQLgGitG2TaqdMiNXm8q1AwnU0dygnRtWqiGdkklqSKe0c8NKZuyQ\nKFFae9DTn1FrV6+Wb3lQrV29XACUAWUhQDEKu4KwWwhwVkdzA2EaiRaVtQfs7lMZhGugWE0bCdMA\ngLOisvZgthl2wnVwKAtBZQx0S7etkLanc7dsXYci8DEmgGoWlbUHUZlhj7pZw7WZ1ZjZfzKz/9vM\nWqd87f8qb9cQG+wNjRKwUAhAtYvK2oOZZtKrbYY96szdZ/6i2dcknSfpR5Kul/R9d/9M/mv/5O5v\nq0gvZ9DS0uJ9fX1hdgHzcduKfLCeom6ZdPOTle8PIqW1q3faj1sb0ik9tmVdCD0CgGiaWnMt5WbY\nq/FCoBqZ2SF3b5nreXOVhVzh7h9y97+V9HZJrzOzfWb2WkkWREeRAOwNjRLwMSYABCMqM+xRN9eC\nxtcU7rj77yXdaGbbJPVKel05O4YYqVs6w8w1e0NjblFZKAQAUcDuPuU318x1n5ldO7HB3XdIukPS\npeXqFGKGvaFRgqgsFAIAQJpj5trdPzJD+9ckfa0sPUL8sDc0SlCYYeFYYQBAFMy1oPGv3P3/zd//\n39z9mxO+9gV3/z8r0McZsaARAAAAlRDUgsZNE+5vnfK1awUAAADgrLnCtc1wf7rHAAAAQKLNFa59\nhvvTPQYAAAASba6t+FaZ2a+Um6VO5e8r//jcsvYMQPUa6GaBKgAA05hrt5Ca2b4OIIEKx9mP5vee\nLhxnLxGwAQCJN1dZCABMdmDHeLAuGM3m2gEASDjCNYDicJw9AAAzIlwDKM5Mx9ZznD0AAIRrAEXi\nOHsAAGZz3dBCAAAYxElEQVREuAZQnKaN0vrdUt0ySZa7Xb+bxYwAAGjurfgA4NWaNhKmAQCYBjPX\nAAAAQEAI1wAAAEBACNcAAABAQAjXAAAAQEBY0AgAmLee/ox27R/UieGs6tMpdbY3qqO5IexuAYiZ\nKP9fQ7gGsHAD3bljz0eGcofItG1jF5EY6+nPaOu+I8qOjkmSMsNZbd13RJIi86YHoPpF/f8aykIA\nLMxAt/TAZmnkuCTP3T6wOdeOWNq1f/Dsm11BdnRMu/YPhtQjAHEU9f9rCNdxNNAt3bZC2p7O3RJ2\nUA4Hdkij2clto9lcO2LpxHC2qHYAWIio/19DuI4bZhNRKSNDxbUj8urTqaLaAWAhov5/TSjh2szS\nZrbXzJ41s2fM7J1mdqGZPWxmR/O3F4TRt8hjNhGVUre0uHZEXmd7o1K1NZPaUrU16mxvDKlHAOIo\n6v/XhDVz/UVJ33H3fydplaRnJG2RdMDd3yTpQP4xisVsIiqlbZtUO2UWoTaVa0csdTQ3aOeGlWpI\np2SSGtIp7dywMhILjABER9T/rzF3r+xvaFYn6bCky3zCb25mg5KudveTZnaxpO+5+6yXKC0tLd7X\n11feDkfNbSvyJSFT1C2Tbn6y8v1BuMq9mwe7hQAAEsLMDrl7y1zPC2MrvuWSTku6w8xWSTok6VOS\nlrj7yfxzXpC0ZLpvNrMbJd0oSZdcckn5exs1bdtyNdYTS0OYTUymQv194bVQqL+XggvATRsjEaaj\nvF8qACBawigLOUfS2yR9xd2bJb2iKSUg+RntaafU3f2r7t7i7i2LFy8ue2cjp2mjtH53bqZalrtd\nvzsSAQgBo/5e0vh+qZnhrFzj+6X29GfC7hoAIIbCmLkekjTk7j/MP96rXLh+0cwunlAWciqEvsVD\nRGYTUWbU30uafb9UZq8BAEGr+My1u78g6biZFeqp2yQ9Lel+STfk226QdF+l+wbECrt5SIr+fqkA\ngGgJa7eQT0q628wGJK2W9AVJXZKuMbOjkt6TfwxgodjNQ1L090sFAERLGGUhcvfDkqZbbdlW6b4A\nsVUoDUr4bh6d7Y3auu/IpNKQKO2XCgCIllDCNYAKof7+bF01u4UAACqBcA0g9jqaGwjTAICKCKvm\nGgAAAIgdwjUAAAAQEMI1AAAAEBDCNQAAABAQwjUAAAAQEMI1AAAAEBC24gMAAJHS059h73pULcI1\nAACIjJ7+zKRTVzPDWW3dd0SSCNioCpSFAACAyNi1f/BssC7Ijo5p1/7BkHoETEa4BgAAkXFiOFtU\nO1BphGsAABAZ9elUUe1ApRGugVINdEu3rZC2p3O3A91h9wgAIqOnP6PWrl4t3/KgWrt61dOfmfX5\nne2NStXWTGpL1daos72xnN0E5o0FjUApBrqlBzZLo/mPI0eO5x5LUtPG8PoFABGwkMWJhXZ2C0G1\nMncPuw8L1tLS4n19fWF3A0l224pcoJ6qbpl085OV7w8AREhrV68y09RKN6RTemzLuhB6BMzMzA65\ne8tcz6MsBCjFyFBx7QCAs1iciDgiXAOlqFtaXDsA4CwWJyKOCNdAKdq2SbVT3gRqU7l2AMCsWJyI\nOGJBI1CKwqLFAztypSB1S3PBmsWMADAnFicijljQCAAAAMyBBY0AAABAhRGuAQAAgIAQrgEAAICA\nEK4BAACAgBCuAQAAgIAQrgEAAICAEK4BAACAgBCuAQAAgIAQrgEAAICAEK4BAACAgBCuAQAAgIAQ\nrgEAAICAnBN2BwAAydXTn9Gu/YM6MZxVfTqlzvZGdTQ3hN0tAFgwwjUAIBQ9/Rlt3XdE2dExSVJm\nOKut+45IEgEbQGRRFgIACMWu/YNng3VBdnRMu/YPhtQjACgd4RpA9Rjolm5bIW1P524HusPuEcro\nxHC2qHYAiALCNYDqMNAtPbBZGjkuyXO3D2xOZsBOyEVGfTpVVDsARAHhGoiLqAeyAzuk0SkzlqPZ\nXHuSJOgio7O9UanamkltqdoadbY3htQjACgd4RqIgzgEspGh4trjKkEXGR3NDdq5YaUa0imZpIZ0\nSjs3rGQxI4BIY7cQIA5mC2RNG8PpU7HqluYvDqZpT5KEXWR0NDcQpgHECjPXQBzEIZC1bZNqp9Ta\n1qZy7Uky08VE0i4yACCiCNdAHMQhkDVtlNbvluqWSbLc7frd0Zl5DwoXGQAQaZSFAHHQti1XYz2x\nNCSKgaxpY/LC9FSFP/+BHblPHuqW5v4ekz4uABARhGsgDghk8cJFBgBEFuEaiAsCGQAAoaPmGgAA\nAAgI4RoAwhb1A4AAAGeFUhZiZsck/VrSmKTfu3uLmV0o6RuSLpV0TNJGd/9lGP0DgIopHABUWIxa\nOABIoswHACIozJnrd7v7andvyT/eIumAu79J0oH8YwCItwSdyAgASVBNZSHXSbozf/9OSR0h9gWI\nN8oQqkccDgACAJwVVrh2Sf/TzA6Z2Y35tiXufjJ//wVJS8Lp2jwRThBVhTKEkeOSfLwMgddwOOJw\nABAA4KywwvVV7r5a0h9LusnM1k78oru7cgH8VczsRjPrM7O+06dPV6Cr0yCcIMooQ6gunMgIALES\nSrh290z+9pSkeyVdIelFM7tYkvK3p2b43q+6e4u7tyxevLhSXZ6McIIoowyhunDsOwDESsV3CzGz\n8yX9G3f/df7+eyXtkHS/pBskdeVv76t03+aNcIIoq1ua/9RlmnaEgwOAACA2wpi5XiLpUTP7saQf\nSXrQ3b+jXKi+xsyOSnpP/nF1okYSUTZdGYIk/csrlDYBAFCiis9cu/vPJa2apv2MpLZK92dB2rZN\n3pdWokYS0VGYIf0ft0jZl8fbsy+zvzIAACWqpq34ooMaSURd00bpNee/up21AwAAlCSUExpjgRpJ\nRB1rBwCEpKc/o137B3ViOKv6dEqd7Y3qaG4Iu1tAIJi5BpKKtQMAQtDTn9HWfUeUGc7KJWWGs9q6\n74h6+jNhdw0IBOEaSCr2VwYQgl37B5UdHZvUlh0d0679gyH1CAgW4RpIKtYOAAjBieFsUe1A1FBz\nDSQZawcAVFh9OqXMNEG6Pj3NFqFABDFzDQBAFenpz6i1q1fLtzyo1q7e2NUid7Y3KlVbM6ktVVuj\nzvbGkHoEBIuZawAAqkRhsV+hJrmw2E9SbHbTKPw52C0EcUW4BgCgSsy22C9O4bOjuSFWfx5gIspC\nkmagW7pthbQ9nbvluGsAqBos9gOij3CdJAPdueOtR45L8tztA5sJ2JgdF2RAxcy0qI/FfkB0EK6T\n5MCO3PHWE3HcNWbDBRkWKqEXZaUuRmSxHxB9hOsk4bhrFIsLMixEQi/Kgjh5sKO5QTs3rFRDOiWT\n1JBOaeeGldQnAxHCgsYkqVuaf7Obph2YDhdkWIjZLspivK96UIsRWewHRBsz10nCcdco1kwXXlyQ\nYTYJvShjMSIAiXCdLBx3jWJxQYaFSOhFGYsRAUiUhSQPx11jLgPduY/vR4ZyYWjVh6SjD40/btvG\nawiza9uWq7GeWBqSgIuyzvbGSQfASCxGRHXr6c9wmE8ZEK4BjCssRCuEopHj0o+/ziccKE7htTLx\nIi0BF2WcPIgoScJpoGExdw+7DwvW0tLifX19YXcDiI/bVsyw6HWZdPOTle8PAERMVGaDW7t6lZlm\nPUBDOqXHtqwLoUfVz8wOuXvLXM9j5hrAuIQuRAOAIERpNpgFuOXDgkYA4xK6EA0AgjDbdozVhgW4\n5UO4BjCO3UGAxCn1VEmMi9JsMKeBlg9lIQDGJXQhGpBUUSpjiIL6dGraOuZqnA1mAW75sKARAICE\nYlFbsKZerEi52WCOsI8HFjQCAIBZRamMIQqYDYZEuI6PqQd/8FE+AGAOUSpjiIqO5gbCdMKxoDEO\nCgd/jByX5LnbBzbn2gEAmAGL2oDgEa5LNdCdO3hjezp3G0agPbBj8jHDUu7xgR2V7wsAIDI6mhu0\nc8NKNaRTMuVqrakPBkpDWUgppjsq+oHNufuVLMng4A8AwAJRxgAEi5nrUlTLjDEHfwAAYoS9txFl\nhOtSlGvGuNhSEw7+AADERGE7u8xwVq7xvbejHrC5YEgOwnUpyjFjvJDFiU0bpfW7pbplkix3u343\nu4UAACInSkeIz1dcLxgwPcJ1KcoxY7zQUpOmjdLNT0rbh3O3BGsAQATFce/tOF4wYGaE61KUY8aY\nxYkAoqYadk1C+AJ6Hcy0x3aU996O4wUDZsZuIaVq2hjsLHHd0nxJyDTtAFBtqmXXJIQrwNdBZ3vj\ntEeIR3nvbQ7rSRZmrqsNixOB6EvSTG617JqEcAX4Oojj3tsc1pMszFxXm8IVPkeZA9GUtJlcStkg\nBf46iNve24U/y679gzoxnFV9OqXO9sZY/RkxjnBdjYIuNQFQObPN4MXx3zWlbJB4HcxD3C4YMDPK\nQgAgSEmbyaWUDRKvA2ACwjUABClpJ6ayzz4kXgfABObuYfdhwVpaWryvry/sbgDAuKk111JuBo+g\nEW8D3ayVAWLOzA65e8tcz6PmGgCCxKLk5EnaIlYAsyJcA0DQWJScLElbxApgVtRcAwDmlqS9u4uV\ntEWsAGZFuAYAzK5Q9jByXJKPlz0QsHOStogVwKwI1wCA2XEK4+zYhm4cn3AA1FwDAOZA2cPsWMSa\nw8JOQBLhGgAwF07fmxuLWFnYCeRRFgIAmB1lD5gPPuEAJIUYrs2sxsz6zezb+ccXmtnDZnY0f3tB\nWH0DAEzA6XuYDxZ2ApLCnbn+lKRnJjzeIumAu79J0oH8YwBANWjaKN38pLR9OHdLsMZUfMIBSAop\nXJvZUkn/XtLXJjRfJ+nO/P07JXVUul8AAGCB+IQDkBTegsa/lfRXkl4/oW2Ju5/M339B0pLpvtHM\nbpR0oyRdcskl5ewjAAAoBgs7gcrPXJvZ+ySdcvdDMz3H3V2Sz/C1r7p7i7u3LF68uFzdBBAl7K0L\nAKgSYcxct0p6v5n9iaRzJb3BzP67pBfN7GJ3P2lmF0s6FULfAEQNe+sCAKpIxWeu3X2ruy9190sl\nbZLU6+4fkXS/pBvyT7tB0n2V7huACOL0QABAFammfa67JF1jZkclvSf/GABmx966AJKMsriqE+oJ\nje7+PUnfy98/I6ktzP4AiCBODwSQVJTFVaVqmrkGgOKxty6ApKIsrioRrgFEG3vrAkiqSpTFUXZS\ntFDLQgAgEOytCyCJyl0WR9nJgjBzDQAAEEXlLouj7GRBCNcAAABRVO6yOHZjWhDKQgAAAKKqnGVx\n7Ma0IMxcAwAA4NXYjWlBCNcAAAB4NXZjWhDKQgAAADA9dmMqGjPXAAAAQEAI1wAAAEBACNcAAABA\nQAjXAAAAQEAI1wAAAEBACNcAAABAQAjXAAAAQEAI1wCiYaBbum2FtD2dux3oDrtHAAC8CofIAKh+\nA93SA5ul0Wzu8cjx3GOJww0AAFWFmWsA1e/AjvFgXTCazbUDAFBFCNcAqt/IUHHtAACEhHANoPrV\nLS2uHQCAkBCuAVS/tm1SbWpyW20q1w4AQBUhXAOofk0bpfW7pbplkix3u343ixkBAFWH3UIAREPT\nRsI0AKDqMXMNAAAABIRwDQAAAASEcA0AAAAEhHANAAAABIRwDQAAAASEcA0AAAAEhHANAAAABIRw\nDQAAAASEcA0AAAAEhHANAAAABIRwDQAAAASEcA0AAAAEhHANAAAABIRwDQAAAASEcA0AAAAEhHAN\nAAAABIRwDQAAAASEcA0AAAAEhHANAAAABIRwDQAAAASEcA0AAAAEhHANAEBQBrql21ZI29O524Hu\nsHsEoMLOCbsDAADEwkC39MBmaTSbezxyPPdYkpo2htcvABXFzDUAAEE4sGM8WBeMZnPtABKj4uHa\nzM41sx+Z2Y/N7Ckz+8/59gvN7GEzO5q/vaDSfQMAYMFGhoprBxBLYcxc/07SOndfJWm1pGvN7B2S\ntkg64O5vknQg/xgAgGioW1pcO4BYqni49pzf5B/W5n+5pOsk3Zlvv1NSR6X7BgDAgrVtk2pTk9tq\nU7l2AIkRSs21mdWY2WFJpyQ97O4/lLTE3U/mn/KCpCUzfO+NZtZnZn2nT5+uUI8BAJhD00Zp/W6p\nbpkky92u381iRiBhzN3D+83N0pLulfRJSY+6e3rC137p7rPWXbe0tHhfX1+ZewkAAICkM7ND7t4y\n1/NC3S3E3YclfVfStZJeNLOLJSl/eyrMvgEAAADFCmO3kMX5GWuZWUrSNZKelXS/pBvyT7tB0n2V\n7hsAAABQijAOkblY0p1mVqNcuO9292+b2eOSus3sE5Kek0SRGgAAACKl4uHa3QckNU/TfkZSW6X7\nAwAAAASFExoBAACAgBCuAQAAgIAQrgEAAICAEK4BAACAgBCuAQAAgIAQrgEAAICAEK4BAACAgBCu\nAQAAgIAQrgEAAICAEK4BAACAgBCuAQAAgIAQrgEAAICAEK4BAACAgJi7h92HBTOz05KeC7kbF0l6\nKeQ+xBVjWz6MbXkwruXD2JYPY1s+jG35hDG2/4u7L57rSZEO19XAzPrcvSXsfsQRY1s+jG15MK7l\nw9iWD2NbPoxt+VTz2FIWAgAAAASEcA0AAAAEhHBduq+G3YEYY2zLh7EtD8a1fBjb8mFsy4exLZ+q\nHVtqrgEAAICAMHMNAAAABIRwDQAAAASEcD1PZnaumf3IzH5sZk+Z2X/Ot19oZg+b2dH87QVh9zWq\nzKzGzPrN7Nv5x4xtAMzsmJkdMbPDZtaXb2NsA2BmaTPba2bPmtkzZvZOxrZ0ZtaYf70Wfv3KzD7N\n2JbOzG7Ov4c9aWZ78u9tjGsAzOxT+XF9ysw+nW9jbBfIzP6rmZ0ysycntM04nma21cx+amaDZtYe\nTq9zCNfz9ztJ69x9laTVkq41s3dI2iLpgLu/SdKB/GMszKckPTPhMWMbnHe7++oJe4IytsH4oqTv\nuPu/k7RKudcvY1sidx/Mv15XS7pc0j9LuleMbUnMrEHSZkkt7r5CUo2kTWJcS2ZmKyT9R0lXKPd/\nwfvM7H8VY1uK/ybp2ilt046nmb1FudfyW/Pf81/MrKZyXZ2McD1PnvOb/MPa/C+XdJ2kO/Ptd0rq\nCKF7kWdmSyX9e0lfm9DM2JYPY1siM6uTtFbS7ZLk7v/i7sNibIPWJuln7v6cGNsgnCMpZWbnSDpP\n0gkxrkF4s6Qfuvs/u/vvJX1f0gYxtgvm7o9IenlK80zjeZ2ke9z9d+7+C0k/Ve5CJxSE6yLkyxYO\nSzol6WF3/6GkJe5+Mv+UFyQtCa2D0fa3kv5K0r9OaGNsg+GS/qeZHTKzG/NtjG3plks6LemOfDnT\n18zsfDG2QdskaU/+PmNbAnfPSLpV0vOSTkoacfeHxLgG4UlJa8xskZmdJ+lPJC0TYxu0mcazQdLx\nCc8byreFgnBdBHcfy39MuVTSFfmPgSZ+3ZULMiiCmb1P0il3PzTTcxjbklyVf93+saSbzGztxC8y\ntgt2jqS3SfqKuzdLekVTPvJlbEtjZq+R9H5J35z6Nca2ePn61OuUuzCsl3S+mX1k4nMY14Vx92ck\n/T+SHpL0HUmHJY1NeQ5jG6BqHk/C9QLkP/r9rnJ1PS+a2cWSlL89FWbfIqpV0vvN7JikeyStM7P/\nLsY2EPnZKrn7KeXqVq8QYxuEIUlD+U+wJGmvcmGbsQ3OH0v6J3d/Mf+YsS3NeyT9wt1Pu/uopH2S\nrhTjGgh3v93dL3f3tZJ+KeknYmyDNtN4ZpT7pKBgab4tFITreTKzxWaWzt9PSbpG0rOS7pd0Q/5p\nN0i6L5weRpe7b3X3pe5+qXIfAfe6+0fE2JbMzM43s9cX7kt6r3IfXzK2JXL3FyQdN7PGfFObpKfF\n2AbpgxovCZEY21I9L+kdZnaemZlyr9lnxLgGwsz+bf72EuXqrb8uxjZoM43n/ZI2mdlrzWy5pDdJ\n+lEI/ZPECY3zZmZNyhXP1yh3UdLt7jvMbJGkbkmXSHpO0kZ3n1qAj3kys6sl/R/u/j7GtnRmdply\ns9VSrozh6+7+ecY2GGa2WrlFuK+R9HNJf6b8/w9ibEuSvxh8XtJl7j6Sb+N1WyLLbSP7HyT9XlK/\npD+X9DoxriUzs4OSFkkalfQZdz/Aa3bhzGyPpKslXSTpRUmfk9SjGcbTzP5a0seVe21/2t3/Rwjd\nlkS4BgAAAAJDWQgAAAAQEMI1AAAAEBDCNQAAABAQwjUAAAAQEMI1AAAAEBDCNQDEmJmNmdnhCb+2\n5Nv/0sx+amZuZheF3U8AiAu24gOAGDOz37j766Zpb1buFLnvSWpx95cq3TcAiKNzwu4AAKDy3L1f\nknIH9QEAgkJZCADEW2pKWch/CLtDABBnzFwDQLxl3X112J0AgKRg5hoAAAAICOEaAAAACAjhGgDi\nbWrNdZckmdlmMxuStFTSgJl9LdxuAkA8sBUfAAAAEBBmrgEAAICAEK4BAACAgBCuAQAAgIAQrgEA\nAICAEK4BAACAgBCuAQAAgIAQrgEAAICA/P+ZOcL0lIb/FQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1af54a9ebe0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "positive = data[data[\"Allowed\"] == 1]\n",
    "negative = data[data[\"Allowed\"] == 0]\n",
    "\n",
    "fig,ax = plt.subplots(figsize=(12,8))\n",
    "ax.scatter(positive[\"E1\"], positive[\"E2\"],label='Allowed')\n",
    "ax.scatter(negative[\"E1\"], negative[\"E2\"], label='Not Allowed')\n",
    "ax.legend()\n",
    "ax.set_xlabel(\"E1\")\n",
    "ax.set_ylabel(\"E2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data.drop(\"Allowed\", axis =1)\n",
    "y = data[\"Allowed\"]\n",
    "\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "            X, y, test_size=0.20, random_state=42)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "\n",
    "class LogisticRegression():\n",
    "    def __init__(self, X, y , X_valid, y_valid, lr,epochs, reg = False,useoptimizer = False,):\n",
    "        self.X,self.y,  self.X_valid,self.y_valid,self.lr ,self.epochs,self.reg  = X, y , X_valid, y_valid, lr, epochs, reg\n",
    "        self.optim = GradientDesent( X, y , X_valid, y_valid, lr,epochs, reg,useoptimizer, self.sigmoid)\n",
    "   \n",
    "    def sigmoid(self, z):    \n",
    "        return 1/(1+np.exp(-z))  \n",
    "    \n",
    "    def fit(self):\n",
    "        w = self.optim.optimize()\n",
    "        h = sigmoid(np.dot(self.X, w))\n",
    "        h_valid = sigmoid(np.dot(self.X_valid, w))\n",
    "        train_preds = [1 if i > 0.5 else 0 for i in h]\n",
    "        val_preds = [1 if i > 0.5 else 0 for i in h_valid]\n",
    "        train_accuracy =  accuracy_score(self.y , train_preds)\n",
    "        val_accuracy =  accuracy_score(self.y_valid , val_preds)\n",
    "        \n",
    "        print(f'Train Accuracy: {train_accuracy} Val Accuracy: {val_accuracy}')      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import scipy.optimize as opt\n",
    "class GradientDesent():\n",
    "    \n",
    "    def __init__(self, X, y , X_valid, y_valid, lr, epochs, reg = False, useoptimizer = False, activate_fn = None):\n",
    "        self.X,self.y,  self.X_valid,self.y_valid,self.lr ,self.epochs, self.reg, self.useoptimizer, self.activate_fn  = X, y , X_valid, y_valid, lr,epochs, reg,useoptimizer,activate_fn\n",
    "            \n",
    "    def optimize(self):\n",
    "        w = np.zeros(X.shape[1])\n",
    "              \n",
    "        \n",
    "        if(self.useoptimizer):\n",
    "            result = opt.fmin_tnc(func=self.cost, x0=w, fprime=self.gradient,args=(self.X, self.y)) \n",
    "            train_loss= self.cost(result[0], self.X, self.y)\n",
    "            val_loss = self.cost(result[0], self.X_valid, self.y_valid)  \n",
    "            print(f'Runned Spicy Optimizer > Train Loss: {train_loss} Val Loss: {val_loss}')          \n",
    "            return result[0]             \n",
    "            \n",
    "        train_cost = np.zeros(self.epochs)\n",
    "        val_cost = np.zeros(self.epochs)      \n",
    "\n",
    "        \n",
    "        for i in range(self.epochs):\n",
    "            grad = self.gradient(w, self.X, self.y)\n",
    "            w = np.subtract(w , np.multiply(grad , self.lr))\n",
    "            train_loss= self.cost(w, self.X, self.y)\n",
    "            val_loss = self.cost(w, self.X_valid, self.y_valid)         \n",
    "            train_cost[i] = train_loss\n",
    "            val_cost[i] = val_loss\n",
    "            print(f'Epochs: {i} Train Loss: {train_loss} Val Loss: {val_loss}')   \n",
    "            \n",
    "        \n",
    "        return w\n",
    "        \n",
    "    def cost(self,w, X, y):\n",
    "        h = np.dot(X, w)\n",
    "        m = len(X)       \n",
    "        if(self.activate_fn != None):  h = self.activate_fn(h)\n",
    "        loss= (np.dot(-y, np.log(h))) - (np.dot(1-y, np.log(1-h)))\n",
    "        loss = np.sum(loss)/m\n",
    "        \n",
    "        if(self.reg):           \n",
    "            loss = loss + ((self.lr/2*m) * np.sum(np.power(w,2)))\n",
    "        return loss\n",
    "            \n",
    "    \n",
    "    def gradient(self,w,X, y):\n",
    "        h = np.dot(X, w)      \n",
    "        if(self.activate_fn != None):  h = self.activate_fn(h)     \n",
    "        m = len(X)\n",
    "        loss= h - y\n",
    "        gradient = np.dot(X.T,loss)/m\n",
    "        if(self.reg): gradient = gradient + np.dot((self.lr/m), w)      \n",
    "        return gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = LogisticRegression(X_train, y_train,X_valid,y_valid,  lr = 0.001, epochs =5000, reg = True, useoptimizer=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Runned Spicy Optimizer > Train Loss: 0.6251428381136805 Val Loss: 0.6022619343596847\n",
      "Train Accuracy: 0.6 Val Accuracy: 0.6\n"
     ]
    }
   ],
   "source": [
    "model.fit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let see how to handle the non linear data with polynomial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bias</th>\n",
       "      <th>Test1</th>\n",
       "      <th>Test2</th>\n",
       "      <th>Accepted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.051267</td>\n",
       "      <td>0.69956</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.092742</td>\n",
       "      <td>0.68494</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.213710</td>\n",
       "      <td>0.69225</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.375000</td>\n",
       "      <td>0.50219</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.513250</td>\n",
       "      <td>0.46564</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Bias     Test1    Test2  Accepted\n",
       "0     1  0.051267  0.69956         1\n",
       "1     1 -0.092742  0.68494         1\n",
       "2     1 -0.213710  0.69225         1\n",
       "3     1 -0.375000  0.50219         1\n",
       "4     1 -0.513250  0.46564         1"
      ]
     },
     "execution_count": 336,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"data/ex2data2.txt\", header = None, names = [\"Test1\",\"Test2\",\"Accepted\"])\n",
    "data.insert(0 , \"Bias\",  1)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x1af55a0c630>"
      ]
     },
     "execution_count": 337,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtUAAAHVCAYAAAA+Wwf2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X9sHPd55/HPcwwDbNOAm9SqY5Hy2QUMAjmZsXILJ4iC\nNrGS0HahWjEOhHNFkusVJ+TQ1BfjSkC6ADqB/cO6Cq1QHdLk3NSoU7TJ8VCFsWO3vJg6wFcH6ZkK\nE0pOysR1UkgrxVbSkOmPBUILz/0xs+KS2iV3dmZ3fr1fwGJ3vzsjfXc4u3z4nef7fM3dBQAAAKB3\n/yLtDgAAAAB5R1ANAAAAxERQDQAAAMREUA0AAADERFANAAAAxERQDQAAAMREUA0AAADERFANAAAA\nxERQDQAAAMT0urQ70IubbrrJb7vttrS7AQAAgII7d+7cD919107b5TKovu2227S4uJh2NwAAAFBw\nZvZ33WxH+gcAAAAQE0E1AAAAEBNBNQAAABATQTUAAAAQE0E1AAAAEBNBNQAAABATQTUAAAAQE0E1\nAAAAEBNBNQAAABATQTUAAAAQE0E1AAAAEBNBNQAAABATQTUAAAAQE0E1AAAAEBNBNQAAABATQTUA\nAAAQE0E1AAAAEBNBNYDiW56VTu2VjleD++XZtHsEACiY16XdAQDoq+VZ6amHpfVG8HztYvBckiam\n0usXAKBQGKkGUGwLMxsBddN6I2gHACAhBNUAim3tUrR2AAB6QFANoNhGxqK1AwDQA4JqAMV24Jg0\nXNncNlwJ2pFfTD4FkDFMVARQbM3JiAszQcrHyFgQUDNJMb+YfAoggwiqARTfxBTBVpFsN/mUnzOA\nlJD+AQDIFyafAsgggmoAQL4w+RRABhFUAwDyhcmnADKIoBoAkC8TU9LB09LIHkkW3B88TT41gFQx\nUREAkD9MPgWQMYxUAwAAADERVAMAAAAxEVQDAAAAMRFUAwAAADERVAMAAAAxEVQDAAAAMRFUAwAA\nADERVAMAAAAxEVQDAAAAMRFUAwAAADElElSb2eNm9qqZXejwupnZaTN7ycyWzeztLa/da2Yr4WtH\nkugPsMnyrHRqr3S8Gtwvz6bdIxQJ5xcAQMmNVP+xpHu3ef0+SXeEt8OSPi1JZjYk6VPh62+V9CEz\ne2tCfQKCAOeph6W1i5I8uH/qYQKfokkrsOX8AgCEEgmq3f05SX+/zSYPSPqcB74mqWpmt0i6W9JL\n7v6yu/9U0hfCbYFkLMxI643NbeuNoB3FkGZgy/kFAAgNKqd6VNLFlueXwrZO7Tcws8Nmtmhmi1ev\nXu1bR1Ewa5eitSN/0gxsOb8AAKHcTFR098fcvebutV27dqXdHeTFyFi0duRPmoEt5xcAIDSooLou\naU/L87GwrVM7kIwDx6Thyua24UrQjmJIM7Dl/AIAhAYVVD8p6SNhFZB3Slpz9yuSXpB0h5ndbmav\nl/RQuC2QjIkp6eBpaWSPJAvuD54O2lEMaQa2nF8AgNDrkvhHzOzzkt4j6SYzuyTpv0oaliR3/4yk\nZyTdL+klSf8s6dfC114zs49Lmpc0JOlxd38xiT4B101MEeQUWfNnuzATpHyMjAUB9aB+5pxfAABJ\n5u5p9yGyWq3mi4uLaXcDyLW5pbpOzq/o8mpDu6sVTU+O69C+tvOEAQAoLTM75+61nbZLZKQaQL7M\nLdV19Mx5NdavSZLqqw0dPXNekgisAQDoQW6qfwBIzsn5lesBdVNj/ZpOzq+k1CMAAPKNoBooocur\njUjtAABgewTVQAntrlYitQMAgO0RVAMlND05rsrw0Ka2yvCQpifHU+oRAAD5xkRFoISakxGp/oGe\nLM+mV8IQADKKoBooqUP7RgmiEd3yrPTUw9J6mH+/djF4LhFYAyg10j8AAN1bmNkIqJvWG0E7AJQY\nQTUwCMuz0qm90vFqcL88m3aPkGVZPl/WLkVrB4CSIP0D6DculyOKrJ8vI2NBn9q1A0CJMVIN9BuX\nyxFF1s+XA8ek4S2lF4crQTsAlBhBNYqBy+UoiqyfLxNT0sHT0sgeSRbcHzydjVF0AEgR6R/IPy6X\no0jycL5MTGXjswUAGcJINfKPy+UoEs4XAMglgmrkH5fLUSScLyiDLKfsAT0i/QP5x+VyFA3nC4os\n6yl7QI8YqUb+cbkcAPIj6yl7QI8IqpF/XC4HgPzIesoe0CPSP1AMXC4HgHzIQ8oe0ANGqgEAwOCQ\nsoeCIqgGAACDQ8oeCor0DwAAMFik7KGAGKkGAAAAYiKoBgAAAGIiqAYAAABiIqgGAAAAYiKoBgAA\nAGIiqAYAII7lWenUXul4Nbhfnk27RwBSQEk9AAB6tTwrPfWwtN4Inq9dDJ5LlIwDSoagGgBazC3V\ndXJ+RZdXG9pdrWh6clyH9o2m3S1k1cLMRkDdtN4I2gmqgVIhqAaA0NxSXUfPnFdj/Zokqb7a0NEz\n5yWJwBrtrV2K1g6gsMipBoDQyfmV6wF1U2P9mk7Or6TUI2TeyFi0dgCFRVANIDVzS3XtP3FWtx95\nWvtPnNXcUj3V/lxebURqB3TgmDRc2dw2XAnaAZQKQTWAVDRTLeqrDbk2Ui3SDKx3VyuR2gFNTEkH\nT0sjeyRZcH/wNPnUQAmRUw0gFdulWqSVvzw9Ob4pp1qSKsNDmp4cT6U/yImJKYJoAATVANKRxVSL\nZjBP9Q8AQFQE1QBSsbtaUb1NAJ12qsWhfaME0QCAyMipBgooaxMA25meHFdleGhTG6kWMWV5Zb8s\n9w0AEsBINVAweam1TKpFwrK8sl+W+wYACTF3T7sPkdVqNV9cXEy7G0Am7T9xtm1axWi1oueP3JNC\njzAQp/YGwepWI3ukRy4Mvj+tstw3ANiBmZ1z99pO25H+ARRMFicAYgCyvLJflvsGAAkhqAYKhlrL\nJZXllf2y3LciIW8dSBVBNVAwTAAsqSyv7JflvhVFM2997aIk38hbJ7AGBiaRoNrM7jWzFTN7ycyO\ntHl92sy+Ed4umNk1M3tz+Nr3zex8+BqJ0kBMh/aN6tEH79RotSJTkEv96IN3MgGw6LK8sl+W+1YU\nCzMbE0Gb1htBO4CBiD1R0cyGJH1H0vslXZL0gqQPufu3Omx/UNIj7n5P+Pz7kmru/sNu/08mKgIA\n0OJ4VVK73+cmHV8ddG+AQhnkRMW7Jb3k7i+7+08lfUHSA9ts/yFJn0/g/wUAABJ560AGJBFUj0pq\nrZV0KWy7gZn9jKR7Jf15S7NLetbMzpnZ4U7/iZkdNrNFM1u8evVqAt0GAKAgyFsHUjfoiYoHJT3v\n7n/f0vZud79L0n2SfsPMfrHdju7+mLvX3L22a9euQfQVAIB8yFreOpVIUEJJrKhYl7Sn5flY2NbO\nQ9qS+uHu9fD+VTP7ooJ0kucS6BcAAOUxMZWNyZ+soImSSmKk+gVJd5jZ7Wb2egWB85NbNzKzEUm/\nJOlLLW1vMLM3Nh9L+oAkltcCACCvqESCkoo9Uu3ur5nZxyXNSxqS9Li7v2hmHwtf/0y46Qcl/W93\n/6eW3W+W9EUza/blz9z9L+P2CQAApIQVNFFSSaR/yN2fkfTMlrbPbHn+x5L+eEvby5LelkQfAABA\nBoyMhYvQtGkHCowVFZE9eZ3gktd+A0CSqESCkkpkpBpITF4nuOS13wM2t1TXyfkVXV5taHe1ounJ\ncVZ6bMHxQSE0v/MWZoKUj5GxIKDmuxAFF3tFxTSwomKBndrb4bLhHumRDM9hzWu/B2huqa6jZ86r\nsX7teltleIgl1EMcHwDIpm5XVGSkGtmS1wkuee13TFFGVk/Or2wKGCWpsX5NJ+dXCBrF8eknrgAA\nGASC6qJans3npbe8TnDJa79j2DqyWl9t6OiZ85LUNmC5vNq4oW279rh9y1sQNcjjUyZRz1MA6BUT\nFYuomd+7dlGSb+T35mHiXF4nuOS13zFsN7Lazu5qJVJ7r5pBVH21IddGEDW31GlNqmwY1PEpm6jn\nKQD0iqC6iPJceD9rS+12K6/9jiHqyOr05Lgqw0Ob2irDQ5qeHE+0X3kNogZ1fMqGKwAABoX0jyLK\ne35vVpbajSqv/e7R7mpF9TaBSaeR1eal9n6nZeQ1iBrU8SmbqOfpIOUxTQlAZwTVRVTC/F4M3vTk\neNtqFduNrB7aN9r3oCHLQdROBnF8yqaX83QQyPUGiof0jyIqYX4vBu/QvlE9+uCdGq1WZJJGq5VM\nlH8jjQKtsnqe5jVNCRGxKFipMFJdRBTex4BkcWSVNApslcXzNK9pSoiARcFKh6C6qEqW3wu0ymIQ\nBbTKc5oSurRd0QB+PxcS6R8ABmZuqa79J87q9iNPa/+Js5kvcwf0C2lKJZD3ogGIjJFqAAPBxCxg\nA2lKEbCYGXKCoBrAQLAMN7AZaUpdyHNe8oFjm/suUTSg4AiqURrUhE0XE7MARJbnvGSKBpQOQTVK\ngdSD9DExC0Bkec9LpmhAqTBREaVATdj0MTELQGSd8o/JS0YGEVSjFEg9SF9WF+FAQbDIRjGxmBly\nhPQPlAKpB9nAxCz0RZ4ns2F75CUjRwiqUQrTk+ObcqolUg+AwsjzZDbsjLxk5ARBNUqBmrBAgeV9\nMhuAQiCoRmKyXrKO1AOgoFhkIxvyukgLkBAmKiIRzZJ19dWGXBsl61iGGkDfMZktfc289rWLknwj\nr50JoygRgmokgpJ1AFIzMSUdPC2N7JFkwf3B04ySDtJ2ee1ASZD+gURQsg5AqpjMli7y2gGCaiSD\nknUAsirr8z0Kgbx2gPQPJIPV8oB8mVuqa/+Js7r9yNPaf+JsYec/MN9jQMhrBwiqkQxWywPyo0yB\nJvM9BoS8doD0DySHknVAPmwXaBbtM8x8jwEirx0lx0g1AJRMmQLNTvM6mO8BIGkE1QBQMmUKNJnv\nAWBQCKoBFE5ZJuH1qkyBJvM9AAwKOdUAMqnXMmjNSXjNnOHmJDxJBFKh5nEoS5k55nsAGASCagCZ\nEycwLtMkvDgINAEgWaR/5NXyrHRqr3S8Gtwvz6bdIyAxccqglWkSXr+RRgMA3WOkOo+WZ6WnHpbW\nwyBh7WLwXKKcEQohTmDM6p7JII0GAKJhpDqPFmY2Auqm9UbQDhRAnOoUZZqE108smgIA0RBU59Ha\npWjtQM7ECYyp9pCMpNNoSCUBUHSkf+TRyFiQ8tGuHSiAuNUpmIQXX5JpNKSSACgDguo8OnBsc061\nJA1XgnagIAiM0zU9Ob4pEJZ6T6OhIguAMiCozqPmZMSFmSDlY2QsCKiZpAggIUnWsqYiC4AySCSo\nNrN7Jf2+pCFJn3X3E1tef4+kL0n6Xth0xt1nutkXHUxMEUTnUK8LmgBpSOpqARVZkDvLswxcIbLY\nExXNbEjSpyTdJ+mtkj5kZm9ts+n/dfe7wttMxH2B3GvmldZXG3Jt5JUyYQtFR0UW5EqzbO3aRUm+\nUbaW9SCwgySqf9wt6SV3f9ndfyrpC5IeGMC+yDoWqNmEEmUoKyqyIFcoW4seJZH+MSqptRTFJUnv\naLPdu8xsWVJd0m+5+4sR9kXesEDNDcgrRZkx8TSbSElrg7K16NGg6lR/XdKt7j4h6b9Lmov6D5jZ\nYTNbNLPFq1evJt5BJIy/9G8QZ0ETAEgaKWkddCpPS9la7CCJoLouaU/L87Gw7Tp3/4m7/2P4+BlJ\nw2Z2Uzf7tvwbj7l7zd1ru3btSqDb6Cv+0r8BeaUAsoSUtA4OHAvK1LaibC26kET6xwuS7jCz2xUE\nxA9J+retG5jZWyS94u5uZncrCOZ/JGl1p32RUyxQc4MkS5RFxSVeAFuRktYBZWvRo9hBtbu/ZmYf\nlzSvoCze4+7+opl9LHz9M5L+jaT/aGavSWpIesjdXVLbfeP2CRnAAjVtpZFXymp2ANqh1OE2KFuL\nHlgQ2+ZLrVbzxcXFtLuBnQygzicjsDvbf+Js21+co9WKnj9yTwo9ApAFW//gloKUNCqzAJuZ2Tl3\nr+20HSsqon/6/Jc+I7Dd4RIvgHbSTEkDioigGrm13SQbfils4BIvgE4odQgkZ1Al9YDEMQLbHaqO\nAADQf4xUI7cYge0Ol3gBlBlzbzAoBNXIrenJ8baTbBiBvRGXeAGUEXNvMEikfyC3Du0b1aMP3qnR\nakWmoJoFs9YBAE0scINBYqQaucYILACgE+beYJAYqQYAAIXUaY4Nc2/QDwTVAACgkKh+hEEi/QMA\nABQS1Y8wSATVAACgsJh7g0Eh/QMAAACIiaAaAAAAiIn0DwCIgdXaAAASQTUA9IzV2gAATaR/AECP\nWK0NANBEUA0APWK1NgBAE0E1APSI1doAAE0E1QDQI1ZrAwA0MVERAHrEam0AgCaCagCIgdXaAGTa\n8qy0MCOtXZJGxqQDx6SJqbR7VUgE1QAAAEW0PCs99bC0Hk6eXrsYPJcIrPuAoBpALCx+AgAZtTCz\nEVA3rTeCdoLqxBFUA+gZi58AQIatXYrWjlio/gGgZyx+AgAZNjIWrR2xEFQD6BmLnwBAhh04Jg1v\nqZs/XAnakTjSPxAJ+bNotbtaUb1NAM3iJwCQAc28aap/DARBNbpG/iy2mp4c33ROSCx+AgCZMjFF\nED0gpH+ga+TPYqtD+0b16IN3arRakUkarVb06IN38kcWAKB0GKlG18ifRTssfgIAACPViKBTniz5\nswAAoOwIqqNanpVO7ZWOV4P75dm0ezQw05PjqgwPbWojfzaeuaW69p84q9uPPK39J85qbqmedpcA\nAEAPSP+IouTLfTYv8VP9IxlM/AQAoDjM3dPuQ2S1Ws0XFxcH/x+f2hsE0luN7JEeuTD4/iDX9p84\n27Yc3Wi1oueP3JNCjwAAwFZmds7dazttR/pHFCz3iQQx8RMAgOIgqI6i38t9ljhfu4yY+AkAQHEQ\nVEfRz+U+m/naaxcl+Ua+NoF1YTHxEwCA4iCojmJiSjp4OsihlgX3B08nM0lxYWZjAmTTeiNoRyGx\ncAoAAMVB9Y+o+rXcJ/napcTCKQBQYMuzweDY2qUgVfTAsVJUCysrRqqzot/52gAAYHBI6ywdguqs\n6Ge+NgAAPWKRqh6R1lk6pH9kRfNyEJeJAAAZwSJVMZDWWToE1VnSr3xtAAB6cHJ+5XpA3dRYv6aT\n8ysE1TsZGeuwYBxpnUWVSPqHmd1rZitm9pKZHWnz+q+a2bKZnTezr5rZ21pe+37Y/g0zS2GZRAAA\n0A6LVMVAWmfpxB6pNrMhSZ+S9H5JlyS9YGZPuvu3Wjb7nqRfcvcfm9l9kh6T9I6W19/r7j+M2xcA\nAJCc3dWK6m0CaBap6gJpnaWTRPrH3ZJecveXJcnMviDpAUnXg2p3/2rL9l+TxLUPAAAybnpyfFNO\ntcQiVZGQ1lkqSaR/jEpqTRq6FLZ18uuS/qLluUt61szOmdnhBPoDAAASwCJVQPcGOlHRzN6rIKh+\nd0vzu929bmY/L+krZvY37v5cm30PSzosSbfeeutA+gsAQNmxSBXQnSRGquuS9rQ8HwvbNjGzCUmf\nlfSAu/+o2e7u9fD+VUlfVJBOcgN3f8zda+5e27VrVwLdBgAAAJKRRFD9gqQ7zOx2M3u9pIckPdm6\ngZndKumMpA+7+3da2t9gZm9sPpb0AUkXEugTAAAAMDCx0z/c/TUz+7ikeUlDkh539xfN7GPh65+R\ndEzSz0n6AzOTpNfcvSbpZklfDNteJ+nP3P0v4/YJSMrcUl0n51d0ebWh3dWKpifHuQwKAABuYO6e\ndh8iq9VqvrhISWv019aVxKRg1juTdAAAKA8zOxcOBm+LFRWRO4MaPWYlMQAA0C2CauTK1tHj+mpD\nR8+cl6TEA11WEgMAAN1KZJlyYFC2Gz1OWqcVw1hJDAAAbEVQjVwZ5Ojx9OS4KsNDm9pYSQwAALRD\nUI1cGeToMSuJAQCAbpFTjVyZnhxvW5GjX6PHrCQGAAC6QVCNXGkGuNSOBgAAWUJQjdxh9BgAAGQN\nQTWAWFh1EgAAgmoAMQyybjgAAFlG9Q8APRtk3XAAALKMkWoAPWPVSQDdIE0MZcBINYCeseokgJ00\n08Tqqw25NtLE5pbqaXcNSBRBNYCeseokgJ2QJoayIP0DQM+oGw5gJ6SJoSwIqgHEQt1wANvZXa2o\n3iaAJk0MRUP6BwAA6BvSxFAWjFQDAIC+IU0MZUFQDQAA+oo0MZQB6R8AAABATATVAAAAQEwE1QAA\nAEBMBNUAAABATATVAAAAyJblWenUXul4Nbhfnk27Rzui+gcAAACyY3lWeuphaT1cNGjtYvBckiam\n0uvXDhipBgAAQHYszGwE1E3rjaA9wwiqAQAAkJ2Ui7VL0dozgqAaAACg7JopF2sXJflGykUagfXI\nWLT2jCCoBgAAKLsspVwcOCYNVza3DVeC9gwjqAYAACi7LKVcTExJB09LI3skWXB/8HSmJylKVP8A\nAADAyFiY+tGmPQ0TU5kPordipBoAAKDscppykSUE1QAAAGWX05SLLCH9AwAAALlMucgSRqoBAACA\nmAiqiyorBdwBAABKgPSPImoWcG/Wm2wWcJe4rDMAc0t1nZxf0eXVhnZXK5qeHNehfaNpdwsAAPQR\nI9VFlKUC7iUzt1TX0TPnVV9tyCXVVxs6eua85pbqaXcNAAD0ESPVRZSlAu4lc3J+RY31a5vaGuvX\ndHJ+ZSCj1YySAwCQDkaqi6hTofa0CriXyOXVRqT2JDFKDgBAegiqi4gC7qnZXa1Eak/SdqPkAACg\nvwiqi4gC7qmZnhxXZXhoU1tleEjTk+N9/7/THCUHkJy5pbr2nzir2488rf0nznK1CcmiOljfkFNd\nVBRwT0UzfzmNvObd1YrqbQLoQYySA0hGM42redWpmcYlifkRiI/qYH2VyEi1md1rZitm9pKZHWnz\nupnZ6fD1ZTN7e7f7Ir/KOtpyaN+onj9yj7534pf1/JF7BvaLMM1RcgDJII0LfUV1sL6KPVJtZkOS\nPiXp/ZIuSXrBzJ5092+1bHafpDvC2zskfVrSO7rcFznEaMvgpTlKDiAZpHGhr6gO1ldJpH/cLekl\nd39ZkszsC5IekNQaGD8g6XPu7pK+ZmZVM7tF0m1d7IscSru0XFkd2jfK8QVyjDQu9NXIWJDy0a4d\nsSWR/jEqqfUndCls62abbvZFDjHaAgDRkcaFvqI6WF/lZqKimR2WdFiSbr311pR7g50w2lJsLDID\n9AdpXOir5mTEhZkg5WNkLAiomaSYiCSC6rqkPS3Px8K2brYZ7mJfSZK7PybpMUmq1Woer8vot+nJ\n8U051RKjLUVBvjzQX6Rxoa+oDtY3SaR/vCDpDjO73cxeL+khSU9u2eZJSR8Jq4C8U9Kau1/pcl/k\n0KF9o3r0wTs1Wq3IJI1WK3r0wTv5RVEAVCcAAOBGsUeq3f01M/u4pHlJQ5Ied/cXzexj4eufkfSM\npPslvSTpnyX92nb7xu0TsoHRlmIiXx5Az5ZnST1AYSWSU+3uzygInFvbPtPy2CX9Rrf7Asgu8uUB\n9ISFR1BwLFMOIBKqEwDoCQuPoOByU/0DQDZQnQBAT1h4BAVHUA0gMvLlAUTGwiMoONI/AABA/7Hw\nCAqOoBoosuVZ6dRe6Xg1uF+eTbtHAMpqYko6eFoa2SPJgvuDp5mkiMIg/QMoKmbaA8gaFh5BgTFS\njfgYDc0mZtoDADAwjFQjHkZDs4uZ9gAADAwj1YiH0dDs6jSjnpn2AAAkjqA6j7KUbsFoaHYx0x4A\n8idLv+MRCekfeZO1dAvqjmZX83xYmAn+yBkZCwJq0nIAIJuy9jsekZi7p92HyGq1mi8uLqbdjXSc\n2tshiN0jPXJh8P3Z+gUgBaOhlEkCACCarP2OhyTJzM65e22n7Uj/yJuspVtQdxQAgGRk7Xc8IiH9\nI2+ymG5B3VEAAOLL4u94dI2R6rxh8hkAAMXE7/hcI6jOG9ItAAAoJn7H5xoTFQEAAIAOmKgIAAAA\nDAhBNQAAABATQTXKi1WrAABAQiiph3Ji1SoAAJAggmqU08LM5lUgpeD5wgxBNQAMwNxSXSfnV3R5\ntaHd1YqmJ8d1aN9o2t0CekZQjXJi1SoASM3cUl1Hz5xXY/2aJKm+2tDRM+clicAauUVONcqp0+pU\nrFoFAH13cn7lekDd1Fi/ppPzKyn1CIiPoBrlxKpVAJCay6uNSO1AHpD+gXJq5k0vzAQpHyNjQUBN\nPjUygnxTFNnuakX1NgH07mqlzdZAPhBUo7wmpgiikUnkm6LopifHN53jklQZHtL05HiKvQLiIf0D\nADKGfFMU3aF9o3r0wTs1Wq3IJI1WK3r0wTv5oxG5xkg1AGQM+aYog0P7RgmiUSiMVANAxnTKKyXf\nFACyi5FqAKWU5YmA5Juikyyft3nBMUS/EFQDKJ2sTwRs9oFf/GiV9fM2DziG6Cdz97T7EFmtVvPF\nxcW0uwFkBiMv0ew/cbZtOa/RakXPH7knhR4BO+O8jY9jiF6Y2Tl3r+20HSPVQM4x8hIdEwGRR5y3\n8XEM0U9MVARyjvJr0TEREHnEeRsfxxD9RFAN5BwjL9FNT46rMjy0qY2JgMg6ztv4CncMl2elU3ul\n49Xgfnk27R6VGukfCD6ELNedW/1c7reoudpMBEQecd7GV6hjuDwrPfWwtB5+/69dDJ5L/A5PCRMV\ny27rh1KShivSwdN8KHNia061FIy8xF2drF//LgAgAaf2BoH0ViN7pEcuDL4/BdbtREXSP8puYWZz\nQC0Fzxdm0ukPIuvXcr/kagNAhq1ditaOviP9o+z4UBZCP5b7JVcbeVDUFCVgRyNjHUaqxwbfF0hi\npBqdPnx8KEuPWfLIumaKUn21IddGOcm5pXraXQP678CxIF2z1XAlaEcqCKrLjg8lOijcLHkUDilK\nKLWJqWD+08geSRbcMx8qVaR/lF3zw0f1D2xRqFnyKCRSlPKLtJ2ETEzx+zpDYgXVZvZmSf9T0m2S\nvi9pyt1/vGWbPZI+J+lmSS7pMXf//fC145L+g6Sr4eb/xd2fidMn9IAPJTroR642kJR+lpNE/7AK\nLIoqbvrHEUkL7n6HpIXw+VavSfrP7v5WSe+U9Btm9taW10+5+13hjYAaANAVUpTyibQdFFXcoPoB\nSU+Ej58b3+A6AAASmUlEQVSQdGjrBu5+xd2/Hj7+B0nflsSfot1gpaR84+cH9FW/ykmiv0jbQVHF\nzam+2d2vhI9/oCDFoyMzu03SPkl/3dL8m2b2EUmLCka0f9xm1/JhpaR84+cHDAQpSvlD2g6KaseR\najN71swutLk90LqdB0szdlye0cx+VtKfS/qEu/8kbP60pF+QdJekK5J+d5v9D5vZopktXr16tdNm\nxcGiLPnGzw8A2iJtB0W140i1u7+v02tm9oqZ3eLuV8zsFkmvdthuWEFA/afufqbl336lZZs/lPTl\nbfrxmKTHpGCZ8p36nXssypJpO85c5+cXG9UBgGKishCKKm76x5OSPirpRHj/pa0bmJlJ+iNJ33b3\n39vy2i0t6SMflMRi9U2slJRZXc1c5+cXC9UBgGIjbQdFFHei4glJ7zez70p6X/hcZrbbzJqVPPZL\n+rCke8zsG+Ht/vC13zGz82a2LOm9kh6J2Z/iYFGWzOpq5jo/v1ioDgAAyJtYI9Xu/iNJB9q0X5Z0\nf/j4ryRZh/0/HOf/LzQWZcmsrmau8/OLheoAAIC8YUXFLGNRlkzqeuY6P7+eUR0AAJA3cdM/gNJh\n5nr/cYwBAHnDSDWKbXk28RQMZq73H8cYAJA3FpSXzpdareaLi4tpdwNZt3UBFimYLHjwNGkZAACg\nK2Z2zt1rO21H+geKiwVYAADAgBBUo7hYgAUAAAwIQTWKq9NCKyzAAgAAEkZQjeJiARYAADAgBNUo\nrompYFLiyB5JFtwzSREAAPQBJfVQbCzAAgAABoCRagAAACAmgmoAAAAgJoJqAAAAICaCagAAACAm\ngmoAAAAgJoJqAAAAICaCagDZsTwrndorHa8G98uzafcIAPqL773CoE41gGxYnpWeelhabwTP1y4G\nzyVqjQMoJr73CoWRagDZsDCz8Yulab0RtANAEfG9VygE1QCyYe1StHYAyDu+9wqFoBpANoyMRWsH\ngLzje69QCKoBZMOBY9JwZXPbcCVoB4Ai4nuvUJioCCAbmpNyFmaCS58jY8Evlg6TdeaW6jo5v6LL\nqw3trlY0PTmuQ/tGB9hhAIgp4vcess3cPe0+RFar1XxxcTHtbgBIydxSXUfPnFdj/dr1tsrwkB59\n8E4CawBAoszsnLvXdtqOkWoAuXNyfmVTQC1JjfVrOjm/krmgmhF1ACgHgmogruVZLt0N2OXVRqT2\ntGwdUa+vNnT0zHlJKm1gzR8ZAIqKiYpAHM3C/WsXJflG4X5WxOqr3dVKpPa0bDeiXkbNPzLqqw25\nNv7ImFuqp901AIiNoBrJK9OSqxTuT8X05Lgqw0Ob2irDQ5qeHE+pR+3lZUR9UPgjI6IyfZcCBUD6\nB5JVtiVXKdyfima6QNbTCHZXK6q3CaCzNqI+KPyREUHZvkuBAiCoRrK2G7kt4i+CkbEw9aNNO/rq\n0L7RzAXRW01PjretUpK1EfVB4Y+MCMr2XQoUAOkfSFbZRm4p3I9tHNo3qkcfvFOj1YpM0mi1Uuqy\nf3lJ28mEsn2XAgXASDWSVbaRWwr3Ywd5GFEflLyk7WRC2b5LgQJg8Rcka2seoBSM3B48TaAJAN3i\nuxTIjG4XfyH9A8mamAq+9Ef2SLLgnl8CiIMKCCgjvkuB3GGkGkB2MVoHAEgZI9UA8o864ACAnCCo\nBpBdVEAAAOQEQTWA7OpU6YAKCACAjCGoBpBd1AEHAOQEQTWA7KICAgAgJ1j8BUC2TUwRRAMAMo+R\nagAAACAmgmoAAAAgplhBtZm92cy+YmbfDe/f1GG775vZeTP7hpktRt0fAAAAyLK4I9VHJC24+x2S\nFsLnnbzX3e/asiJNlP0BAACATIobVD8g6Ynw8ROSDg14fwAAACB1cat/3OzuV8LHP5B0c4ftXNKz\nZnZN0v9w98ci7i8zOyzpsCTdeuutMbsNAOU0t1TXyfkVXV5taHe1ounJcR3aN5p2twAg93YMqs3s\nWUlvafPSJ1ufuLubmXf4Z97t7nUz+3lJXzGzv3H35yLsrzAQf0ySarVax+0AAO3NLdV19Mx5Ndav\nSZLqqw0dPXNekgisASCmHYNqd39fp9fM7BUzu8Xdr5jZLZJe7fBv1MP7V83si5LulvScpK72BwDE\nd3J+5XpA3dRYv6aT8ysE1SXAVYoULc9KCzPS2iVpZCxYFZb6+4UTN6f6SUkfDR9/VNKXtm5gZm8w\nszc2H0v6gKQL3e4PAEjG5dVGpHYUR/MqRX21IdfGVYq5pXraXSu+5VnpqYeltYuSPLh/6uGgHYUS\nN6g+Ien9ZvZdSe8Ln8vMdpvZM+E2N0v6KzP7pqT/J+lpd//L7fYHACRvd7USqR3Fsd1VCvTZwoy0\nvuUP1/VG0I5CiTVR0d1/JOlAm/bLku4PH78s6W1R9gcAJG96cnxTTrUkVYaHND05nmKv0EmS6Rp5\nvkqR+7SVtUvR2pFbcat/AAByohmI5DpAKYmkJ5XurlZUbxNAZ/0qRSEm146MhakfbdpRKCxTDgAl\ncmjfqJ4/co++d+KX9fyRe/ITmJRM0uka05PjqgwPbWrLw1WKQqStHDgmDW/542W4ErSjUBipBgAg\nY5JO18jrVYo8p61c16zyQfWPwiOoBgAgY/qRrnFo32jmg+itMp22EqVM3sQUQXQJkP4BAEDG5DVd\nI2mZPQ6UyUMbjFQDABDBIKpR5DVdI2mZPQ7blcljRLq0zD1/K37XajVfXFxMuxsAgJLZWo1CCkZO\nH33wzvQDPQzO8aqkdvGTScdXB90b9JmZnXP32k7bkf4BAECXClGNAvF1KodHmbxSI6gGgFbLs9Kp\nvcFI1Km95cmRLOv7bury/ReiGgXio0we2iCnGgCampOPmrmSzclHUrHzJMv6vpsivP9MV6PA4FAm\nD20wUg0ATdtNPiqysr7vpgjvP7PVKDB4E1PSIxeCHOpHLhBQg5FqALhu7VK09qIo6/tuivD+M1uN\nop0odZQBxEZQDQBNI2Nh3dk27UVW1vfdFPH952IRlbKn9AApIP0DAJrKOvkogfc9t1TX/hNndfuR\np7X/xFnNLdUT7mQfFfHnXvaUHiAFjFQDQFNZJx/FfN9bazfXVxs6eua8JGV/RFcq5s+97Ck9QApY\n/AUAEMv+E2fbVsQYrVb0/JF7UugRdGpvh5SWPcGkOgBdY/EXAMBAULs5g4qY0gJkHEE1ACCWTjWa\nqd2cookp6eDpYGRaFtwfPJ3vlBYg48ipBgDEMj05vimnWqJ2cyZMTBFEAwNEUA0AiCVXtZsBoE8I\nqgF0xuIR6FIuajcDQB8RVANoj8UjAADoGhMVAbTH4hEAAHSNoBpAeyweAQBA1wiqAbQ3MhatHQCA\nEiOoBtAei0cAANA1gmoA7bF4BAAAXaP6B4DOWDwCAICuMFINAAAAxERQDQAAAMREUA0AAADERFAN\nAAAAxERQDQAAAMREUA0AAADERFANAAAAxERQDQAAAMREUA0AAADERFANAAAAxERQDQAAAMREUA0A\nAADERFANAAAAxERQDQAAoluelU7tlY5Xg/vl2bR7BKTqdWl3AAAA5MzyrPTUw9J6I3i+djF4LkkT\nU+n1C0hRrJFqM3uzmX3FzL4b3r+pzTbjZvaNlttPzOwT4WvHzaze8tr9cfoDAAAGYGFmI6BuWm8E\n7UBJxU3/OCJpwd3vkLQQPt/E3Vfc/S53v0vSv5b0z5K+2LLJqebr7v5MzP4AAIB+W7sUrR0ogbhB\n9QOSnggfPyHp0A7bH5D0t+7+dzH/XwAoN/JZkaaRsWjtQAnEDapvdvcr4eMfSLp5h+0fkvT5LW2/\naWbLZvZ4u/SRJjM7bGaLZrZ49erVGF0GgJxr5rOuXZTkG/msBNYYlAPHpOHK5rbhStAOlNSOQbWZ\nPWtmF9rcHmjdzt1dkm/z77xe0q9I+l8tzZ+W9AuS7pJ0RdLvdtrf3R9z95q713bt2rVTtwGguMhn\nRdompqSDp6WRPZIsuD94mkmKKLUdq3+4+/s6vWZmr5jZLe5+xcxukfTqNv/UfZK+7u6vtPzb1x+b\n2R9K+nJ33QaAEiOfNRiVX5gJ3vPIWDBCSkA3WBNTHHOgRdz0jyclfTR8/FFJX9pm2w9pS+pHGIg3\nfVDShZj9AYDiK3s+K+kvADIoblB9QtL7zey7kt4XPpeZ7Taz65U8zOwNkt4v6cyW/X/HzM6b2bKk\n90p6JGZ/AKD4yp7PSvoLgAyKtfiLu/9IQUWPre2XJd3f8vyfJP1cm+0+HOf/B4BSal5yL2v6A+kv\nADKIFRUBII/KnM86MhamfrRpB4CUxE3/AABgsMqe/gIgkwiqAQD5Qjk3ABlE+gcAIH/KnP4CIJMY\nqQYAAABiIqgGAAAAYiKoBgAAAGIiqAYAAABiIqgGAAAAYiKoBgAAAGIiqAYAAABiIqgGAAAAYiKo\nBgAAAGIiqAYAAABiIqgGAAAAYiKoBgAAAGIiqAYAAABiIqgGAAAAYiKoBgAAAGIiqAYAAABiIqgG\nAAAAYjJ3T7sPkZnZVUl/l3Y/BuAmST9MuxM5xbGLh+PXO45dPBy/3nHs4uH49a7ox+5fuvuunTbK\nZVBdFma26O61tPuRRxy7eDh+vePYxcPx6x3HLh6OX+84dgHSPwAAAICYCKoBAACAmAiqs+2xtDuQ\nYxy7eDh+vePYxcPx6x3HLh6OX+84diKnGgAAAIiNkWoAAAAgJoJqAAAAICaC6pSZ2ZvN7Ctm9t3w\n/k1tthk3s2+03H5iZp8IXztuZvWW1+4f/LtIRzfHLtzu+2Z2Pjw+i1H3L6ouz709ZvZ/zOxbZvai\nmf2nltdKd+6Z2b1mtmJmL5nZkTavm5mdDl9fNrO3d7tv0XVx7H41PGbnzeyrZva2ltfafobLpIvj\n9x4zW2v5PB7rdt+i6+LYTbcctwtmds3M3hy+Vupzz8weN7NXzexCh9f5zmvl7txSvEn6HUlHwsdH\nJP23HbYfkvQDBYXIJem4pN9K+31k+dhJ+r6km+Ie+6Ldunn/km6R9Pbw8RslfUfSW8PnpTr3ws/e\n30r6BUmvl/TN5rFo2eZ+SX8hySS9U9Jfd7tvkW9dHrt3SXpT+Pi+5rELn7f9DJfl1uXxe4+kL/ey\nb5FvUd+/pIOSzrY8L/u594uS3i7pQofX+c5ruTFSnb4HJD0RPn5C0qEdtj8g6W/dvQwrSu4k6rFL\nev+82/H9u/sVd/96+PgfJH1b0ujAepgtd0t6yd1fdvefSvqCgmPY6gFJn/PA1yRVzeyWLvctsh3f\nv7t/1d1/HD79mqSxAfcxy+KcP5x70d7/hyR9fiA9ywF3f07S32+zCd95LQiq03ezu18JH/9A0s07\nbP+QbvzA/2Z42eXxkqUwdHvsXNKzZnbOzA73sH9RRXr/ZnabpH2S/rqluUzn3qikiy3PL+nGPzA6\nbdPNvkUW9f3/uoLRr6ZOn+Gy6Pb4vSv8PP6Fmf2riPsWVdfv38x+RtK9kv68pbns595O+M5r8bq0\nO1AGZvaspLe0eemTrU/c3c2sY41DM3u9pF+RdLSl+dOSflvBB/+3Jf2upH8ft89ZkdCxe7e7183s\n5yV9xcz+Jvzru9v9cyvBc+9nFfyi+YS7/yRsLvS5h3SY2XsVBNXvbmne8TMMfV3Sre7+j+H8hjlJ\nd6Tcp7w5KOl5d28dmeXcQ9cIqgfA3d/X6TUze8XMbnH3K+Elk1e3+afuk/R1d3+l5d++/tjM/lDS\nl5Poc1YkcezcvR7ev2pmX1RwWeo5SVGOfS4lcfzMbFhBQP2n7n6m5d8u9LnXRl3SnpbnY2FbN9sM\nd7FvkXVz7GRmE5I+K+k+d/9Rs32bz3BZ7Hj8Wv7Ylbs/Y2Z/YGY3dbNvwUV5/zdcCebc2xHfeS1I\n/0jfk5I+Gj7+qKQvbbPtDbleYTDU9EFJbWfoFtSOx87M3mBmb2w+lvQBbRyjKMe+iLo5fibpjyR9\n291/b8trZTv3XpB0h5ndHl41ekjBMWz1pKSPhDPi3ylpLUyx6WbfItvx/ZvZrZLOSPqwu3+npX27\nz3BZdHP83hJ+XmVmdyv4/f6jbvYtuK7ev5mNSPoltXwPcu51he+8VmnPlCz7TdLPSVqQ9F1Jz0p6\nc9i+W9IzLdu9QcEX5MiW/f9E0nlJywpO2FvSfk9ZOnYKZh5/M7y9KOmTO+1flluXx+/dCtI7liV9\nI7zdX9ZzT8FM9+8omNX+ybDtY5I+Fj42SZ8KXz8vqbbdvmW6dXHsPivpxy3n2WLY3vEzXKZbF8fv\n4+Hx+aaCiZ7v4tzr7tiFz/+dpC9s2a/0556CgbwrktYV5EX/Ot95nW8sUw4AAADERPoHAAAAEBNB\nNQAAABATQTUAAAAQE0E1AAAAEBNBNQAAABATQTUAAAAQE0E1AAAAENP/B0b9VKUIsnQ5AAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1af549ce550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "postive = data[data[\"Accepted\"] == 1]\n",
    "negative = data[data[\"Accepted\"] == 0]\n",
    "\n",
    "fig,ax = plt.subplots(figsize=(12,8))\n",
    "ax.scatter(postive[\"Test1\"], postive[\"Test2\"], label =\"Accepted\")\n",
    "ax.scatter(negative[\"Test1\"], negative[\"Test2\"], label = \"NotAccepted\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(95, 3)"
      ]
     },
     "execution_count": 317,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = data.drop(\"Accepted\", axis =1)\n",
    "y = data[\"Accepted\"]\n",
    "\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "            X, y, test_size=0.80, random_state=42)\n",
    "\n",
    "X_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Runned Spicy Optimizer > Train Loss: 0.6114304670894042 Val Loss: 0.8364360225740582\n",
      "Train Accuracy: 0.6086956521739131 Val Accuracy: 0.47368421052631576\n"
     ]
    }
   ],
   "source": [
    "model = LogisticRegression(X_train, y_train,X_valid,y_valid,  lr = 0.001, epochs =5000,useoptimizer=True)\n",
    "model.fit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add Polynomial Features to the input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bias</th>\n",
       "      <th>Accepted</th>\n",
       "      <th>x10</th>\n",
       "      <th>x01</th>\n",
       "      <th>x20</th>\n",
       "      <th>x02</th>\n",
       "      <th>x21</th>\n",
       "      <th>x12</th>\n",
       "      <th>x30</th>\n",
       "      <th>x03</th>\n",
       "      <th>...</th>\n",
       "      <th>x94</th>\n",
       "      <th>x49</th>\n",
       "      <th>x95</th>\n",
       "      <th>x59</th>\n",
       "      <th>x96</th>\n",
       "      <th>x69</th>\n",
       "      <th>x97</th>\n",
       "      <th>x79</th>\n",
       "      <th>x98</th>\n",
       "      <th>x89</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.051267</td>\n",
       "      <td>0.69956</td>\n",
       "      <td>0.002628</td>\n",
       "      <td>0.489384</td>\n",
       "      <td>0.035864</td>\n",
       "      <td>0.035864</td>\n",
       "      <td>0.000135</td>\n",
       "      <td>0.342354</td>\n",
       "      <td>...</td>\n",
       "      <td>8.481827e-08</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>8.481827e-08</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>6.215876e-09</td>\n",
       "      <td>0.000216</td>\n",
       "      <td>4.555282e-10</td>\n",
       "      <td>0.002941</td>\n",
       "      <td>3.338322e-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.092742</td>\n",
       "      <td>0.68494</td>\n",
       "      <td>0.008601</td>\n",
       "      <td>0.469143</td>\n",
       "      <td>-0.063523</td>\n",
       "      <td>-0.063523</td>\n",
       "      <td>-0.000798</td>\n",
       "      <td>0.321335</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.510054e-06</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>-1.510054e-06</td>\n",
       "      <td>-0.000082</td>\n",
       "      <td>2.044638e-07</td>\n",
       "      <td>0.000608</td>\n",
       "      <td>-2.768473e-08</td>\n",
       "      <td>-0.004493</td>\n",
       "      <td>3.748558e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.213710</td>\n",
       "      <td>0.69225</td>\n",
       "      <td>0.045672</td>\n",
       "      <td>0.479210</td>\n",
       "      <td>-0.147941</td>\n",
       "      <td>-0.147941</td>\n",
       "      <td>-0.009761</td>\n",
       "      <td>0.331733</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.023708e-04</td>\n",
       "      <td>0.000332</td>\n",
       "      <td>0.000332</td>\n",
       "      <td>-1.023708e-04</td>\n",
       "      <td>-0.001074</td>\n",
       "      <td>3.160370e-05</td>\n",
       "      <td>0.003479</td>\n",
       "      <td>-9.756630e-06</td>\n",
       "      <td>-0.011270</td>\n",
       "      <td>3.012047e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.375000</td>\n",
       "      <td>0.50219</td>\n",
       "      <td>0.140625</td>\n",
       "      <td>0.252195</td>\n",
       "      <td>-0.188321</td>\n",
       "      <td>-0.188321</td>\n",
       "      <td>-0.052734</td>\n",
       "      <td>0.126650</td>\n",
       "      <td>...</td>\n",
       "      <td>-4.716595e-04</td>\n",
       "      <td>0.000632</td>\n",
       "      <td>0.000632</td>\n",
       "      <td>-4.716595e-04</td>\n",
       "      <td>-0.000846</td>\n",
       "      <td>3.522020e-04</td>\n",
       "      <td>0.001133</td>\n",
       "      <td>-2.629995e-04</td>\n",
       "      <td>-0.001517</td>\n",
       "      <td>1.963895e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.513250</td>\n",
       "      <td>0.46564</td>\n",
       "      <td>0.263426</td>\n",
       "      <td>0.216821</td>\n",
       "      <td>-0.238990</td>\n",
       "      <td>-0.238990</td>\n",
       "      <td>-0.135203</td>\n",
       "      <td>0.100960</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.674349e-03</td>\n",
       "      <td>0.001519</td>\n",
       "      <td>0.001519</td>\n",
       "      <td>-1.674349e-03</td>\n",
       "      <td>-0.001378</td>\n",
       "      <td>1.845545e-03</td>\n",
       "      <td>0.001250</td>\n",
       "      <td>-2.034245e-03</td>\n",
       "      <td>-0.001134</td>\n",
       "      <td>2.242239e-03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 92 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Bias  Accepted       x10      x01       x20       x02       x21       x12  \\\n",
       "0     1         1  0.051267  0.69956  0.002628  0.489384  0.035864  0.035864   \n",
       "1     1         1 -0.092742  0.68494  0.008601  0.469143 -0.063523 -0.063523   \n",
       "2     1         1 -0.213710  0.69225  0.045672  0.479210 -0.147941 -0.147941   \n",
       "3     1         1 -0.375000  0.50219  0.140625  0.252195 -0.188321 -0.188321   \n",
       "4     1         1 -0.513250  0.46564  0.263426  0.216821 -0.238990 -0.238990   \n",
       "\n",
       "        x30       x03      ...                x94       x49       x95  \\\n",
       "0  0.000135  0.342354      ...       8.481827e-08  0.000001  0.000001   \n",
       "1 -0.000798  0.321335      ...      -1.510054e-06  0.000011  0.000011   \n",
       "2 -0.009761  0.331733      ...      -1.023708e-04  0.000332  0.000332   \n",
       "3 -0.052734  0.126650      ...      -4.716595e-04  0.000632  0.000632   \n",
       "4 -0.135203  0.100960      ...      -1.674349e-03  0.001519  0.001519   \n",
       "\n",
       "            x59       x96           x69       x97           x79       x98  \\\n",
       "0  8.481827e-08  0.000016  6.215876e-09  0.000216  4.555282e-10  0.002941   \n",
       "1 -1.510054e-06 -0.000082  2.044638e-07  0.000608 -2.768473e-08 -0.004493   \n",
       "2 -1.023708e-04 -0.001074  3.160370e-05  0.003479 -9.756630e-06 -0.011270   \n",
       "3 -4.716595e-04 -0.000846  3.522020e-04  0.001133 -2.629995e-04 -0.001517   \n",
       "4 -1.674349e-03 -0.001378  1.845545e-03  0.001250 -2.034245e-03 -0.001134   \n",
       "\n",
       "            x89  \n",
       "0  3.338322e-11  \n",
       "1  3.748558e-09  \n",
       "2  3.012047e-06  \n",
       "3  1.963895e-04  \n",
       "4  2.242239e-03  \n",
       "\n",
       "[5 rows x 92 columns]"
      ]
     },
     "execution_count": 319,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"data/ex2data2.txt\", header = None, names = [\"Test1\",\"Test2\",\"Accepted\"])\n",
    "data.insert(0 , \"Bias\",  1)\n",
    "\n",
    "degree = 10\n",
    "\n",
    "x1 = data[\"Test1\"]\n",
    "x2 = data[\"Test2\"]\n",
    "\n",
    "for i in range(1, degree):\n",
    "    for j in range(0,i):\n",
    "        data['x' + str(i) + str(j)] = np.power(x1, i - j) * np.power(x2, j)    \n",
    "        data['x' + str(j) + str(i)] = np.power(x2, i - j) * np.power(x1, j)     \n",
    "        \n",
    "data.drop('Test1', axis= 1, inplace =True)\n",
    "data.drop('Test2',axis= 1, inplace = True)\n",
    "\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = data.drop(\"Accepted\", axis =1)\n",
    "y = data[\"Accepted\"]\n",
    "\n",
    "\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "            X, y, test_size=0.20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Runned Spicy Optimizer > Train Loss: 0.6707540066462749 Val Loss: 0.6132621130254335\n",
      "Train Accuracy: 0.8404255319148937 Val Accuracy: 0.7916666666666666\n"
     ]
    }
   ],
   "source": [
    "model = LogisticRegression(X_train, y_train,X_valid,y_valid,  lr = 0.001, epochs =10000, reg =True,useoptimizer=True)\n",
    "model.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}