Logistic Regression with L2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "from builtins import range\n",
    "# Note: you may need to update your version of future\n",
    "# sudo pip install -U future\n",
    "\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 100\n",
    "D = 2\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "X = np.random.randn(N,D)\n",
    "\n",
    "# center the first 50 points at (-2,-2)\n",
    "X[:50,:] = X[:50,:] - 2*np.ones((50,D))\n",
    "\n",
    "# center the last 50 points at (2, 2)\n",
    "X[50:,:] = X[50:,:] + 2*np.ones((50,D))\n",
    "\n",
    "# labels: first 50 are 0, last 50 are 1\n",
    "T = np.array([0]*50 + [1]*50)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# add a column of ones\n",
    "# ones = np.array([[1]*N]).T\n",
    "ones = np.ones((N, 1))\n",
    "Xb = np.concatenate((ones, X), axis=1)\n",
    "\n",
    "# randomly initialize the weights\n",
    "w = np.random.randn(D + 1)\n",
    "\n",
    "# calculate the model output\n",
    "z = Xb.dot(w)\n",
    "\n",
    "def sigmoid(z):\n",
    "    return 1/(1 + np.exp(-z))\n",
    "\n",
    "\n",
    "Y = sigmoid(z)\n",
    "\n",
    "# calculate the cross-entropy error\n",
    "def cross_entropy(T, Y):\n",
    "    E = 0\n",
    "    for i in range(len(T)):\n",
    "        if T[i] == 1:\n",
    "            E -= np.log(Y[i])\n",
    "        else:\n",
    "            E -= np.log(1 - Y[i])\n",
    "    return E\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15.968656067504762\n",
      "0.2964098277436758\n",
      "0.2016914159781954\n",
      "0.15387972888820003\n",
      "0.12486398778579343\n",
      "0.10530967609864574\n",
      "0.0912034654444548\n",
      "0.08052828299325128\n",
      "0.07215709629029159\n",
      "0.065409764403649\n",
      "Final w: [-1.07090478  3.79747761  2.80399643]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "# let's do gradient descent 100 times\n",
    "learning_rate = 0.1\n",
    "for i in range(100):\n",
    "    if i % 10 == 0:\n",
    "        print(cross_entropy(T, Y))\n",
    "\n",
    "    \n",
    "    w += learning_rate * ( Xb.T.dot(T - Y))\n",
    "\n",
    "    # recalculate Y\n",
    "    Y = sigmoid(Xb.dot(w))\n",
    "\n",
    "\n",
    "print(\"Final w:\", w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.059851020208224526\n",
      "0.09023645370601975\n",
      "0.12663973510968907\n",
      "0.1648939792206466\n",
      "0.19971836329068376\n",
      "0.22729362471491935\n",
      "0.2467001351288679\n",
      "0.2592139299212979\n",
      "0.2668240221330389\n",
      "0.2712848135066528\n",
      "Final w: [-0.96112006  2.64157122  2.0512339 ]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "# let's do gradient descent 100 times\n",
    "learning_rate = 0.1\n",
    "for i in range(100):\n",
    "    if i % 10 == 0:\n",
    "        print(cross_entropy(T, Y))\n",
    "\n",
    "    # gradient descent weight udpate with regularization\n",
    "    w += learning_rate * ( Xb.T.dot(T - Y) - 0.1*w )\n",
    "\n",
    "    # recalculate Y\n",
    "    Y = sigmoid(Xb.dot(w))\n",
    "\n",
    "\n",
    "print(\"Final w:\", w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}