Dimensionality Reduction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA Principal Component Analysis\n",
    "* MOst popular dimension reduction algorithm\n",
    "* identifies the hyperplane that lies closest to data & then projects data onto it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "iris = load_iris()\n",
    "X = iris.data[:, 0:] # petal length and width\n",
    "y = iris.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pca = PCA(n_components = 2)\n",
    "X2D = pca.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-2.68420713,  0.32660731],\n",
       "       [-2.71539062, -0.16955685],\n",
       "       [-2.88981954, -0.13734561],\n",
       "       [-2.7464372 , -0.31112432],\n",
       "       [-2.72859298,  0.33392456],\n",
       "       [-2.27989736,  0.74778271],\n",
       "       [-2.82089068, -0.08210451],\n",
       "       [-2.62648199,  0.17040535],\n",
       "       [-2.88795857, -0.57079803],\n",
       "       [-2.67384469, -0.1066917 ],\n",
       "       [-2.50652679,  0.65193501],\n",
       "       [-2.61314272,  0.02152063],\n",
       "       [-2.78743398, -0.22774019],\n",
       "       [-3.22520045, -0.50327991],\n",
       "       [-2.64354322,  1.1861949 ],\n",
       "       [-2.38386932,  1.34475434],\n",
       "       [-2.6225262 ,  0.81808967],\n",
       "       [-2.64832273,  0.31913667],\n",
       "       [-2.19907796,  0.87924409],\n",
       "       [-2.58734619,  0.52047364],\n",
       "       [-2.3105317 ,  0.39786782],\n",
       "       [-2.54323491,  0.44003175],\n",
       "       [-3.21585769,  0.14161557],\n",
       "       [-2.30312854,  0.10552268],\n",
       "       [-2.35617109, -0.03120959],\n",
       "       [-2.50791723, -0.13905634],\n",
       "       [-2.469056  ,  0.13788731],\n",
       "       [-2.56239095,  0.37468456],\n",
       "       [-2.63982127,  0.31929007],\n",
       "       [-2.63284791, -0.19007583],\n",
       "       [-2.58846205, -0.19739308],\n",
       "       [-2.41007734,  0.41808001],\n",
       "       [-2.64763667,  0.81998263],\n",
       "       [-2.59715948,  1.10002193],\n",
       "       [-2.67384469, -0.1066917 ],\n",
       "       [-2.86699985,  0.0771931 ],\n",
       "       [-2.62522846,  0.60680001],\n",
       "       [-2.67384469, -0.1066917 ],\n",
       "       [-2.98184266, -0.48025005],\n",
       "       [-2.59032303,  0.23605934],\n",
       "       [-2.77013891,  0.27105942],\n",
       "       [-2.85221108, -0.93286537],\n",
       "       [-2.99829644, -0.33430757],\n",
       "       [-2.4055141 ,  0.19591726],\n",
       "       [-2.20883295,  0.44269603],\n",
       "       [-2.71566519, -0.24268148],\n",
       "       [-2.53757337,  0.51036755],\n",
       "       [-2.8403213 , -0.22057634],\n",
       "       [-2.54268576,  0.58628103],\n",
       "       [-2.70391231,  0.11501085],\n",
       "       [ 1.28479459,  0.68543919],\n",
       "       [ 0.93241075,  0.31919809],\n",
       "       [ 1.46406132,  0.50418983],\n",
       "       [ 0.18096721, -0.82560394],\n",
       "       [ 1.08713449,  0.07539039],\n",
       "       [ 0.64043675, -0.41732348],\n",
       "       [ 1.09522371,  0.28389121],\n",
       "       [-0.75146714, -1.00110751],\n",
       "       [ 1.04329778,  0.22895691],\n",
       "       [-0.01019007, -0.72057487],\n",
       "       [-0.5110862 , -1.26249195],\n",
       "       [ 0.51109806, -0.10228411],\n",
       "       [ 0.26233576, -0.5478933 ],\n",
       "       [ 0.98404455, -0.12436042],\n",
       "       [-0.174864  , -0.25181557],\n",
       "       [ 0.92757294,  0.46823621],\n",
       "       [ 0.65959279, -0.35197629],\n",
       "       [ 0.23454059, -0.33192183],\n",
       "       [ 0.94236171, -0.54182226],\n",
       "       [ 0.0432464 , -0.58148945],\n",
       "       [ 1.11624072, -0.08421401],\n",
       "       [ 0.35678657, -0.06682383],\n",
       "       [ 1.29646885, -0.32756152],\n",
       "       [ 0.92050265, -0.18239036],\n",
       "       [ 0.71400821,  0.15037915],\n",
       "       [ 0.89964086,  0.32961098],\n",
       "       [ 1.33104142,  0.24466952],\n",
       "       [ 1.55739627,  0.26739258],\n",
       "       [ 0.81245555, -0.16233157],\n",
       "       [-0.30733476, -0.36508661],\n",
       "       [-0.07034289, -0.70253793],\n",
       "       [-0.19188449, -0.67749054],\n",
       "       [ 0.13499495, -0.31170964],\n",
       "       [ 1.37873698, -0.42120514],\n",
       "       [ 0.58727485, -0.48328427],\n",
       "       [ 0.8072055 ,  0.19505396],\n",
       "       [ 1.22042897,  0.40803534],\n",
       "       [ 0.81286779, -0.370679  ],\n",
       "       [ 0.24519516, -0.26672804],\n",
       "       [ 0.16451343, -0.67966147],\n",
       "       [ 0.46303099, -0.66952655],\n",
       "       [ 0.89016045, -0.03381244],\n",
       "       [ 0.22887905, -0.40225762],\n",
       "       [-0.70708128, -1.00842476],\n",
       "       [ 0.35553304, -0.50321849],\n",
       "       [ 0.33112695, -0.21118014],\n",
       "       [ 0.37523823, -0.29162202],\n",
       "       [ 0.64169028,  0.01907118],\n",
       "       [-0.90846333, -0.75156873],\n",
       "       [ 0.29780791, -0.34701652],\n",
       "       [ 2.53172698, -0.01184224],\n",
       "       [ 1.41407223, -0.57492506],\n",
       "       [ 2.61648461,  0.34193529],\n",
       "       [ 1.97081495, -0.18112569],\n",
       "       [ 2.34975798, -0.04188255],\n",
       "       [ 3.39687992,  0.54716805],\n",
       "       [ 0.51938325, -1.19135169],\n",
       "       [ 2.9320051 ,  0.35237701],\n",
       "       [ 2.31967279, -0.24554817],\n",
       "       [ 2.91813423,  0.78038063],\n",
       "       [ 1.66193495,  0.2420384 ],\n",
       "       [ 1.80234045, -0.21615461],\n",
       "       [ 2.16537886,  0.21528028],\n",
       "       [ 1.34459422, -0.77641543],\n",
       "       [ 1.5852673 , -0.53930705],\n",
       "       [ 1.90474358,  0.11881899],\n",
       "       [ 1.94924878,  0.04073026],\n",
       "       [ 3.48876538,  1.17154454],\n",
       "       [ 3.79468686,  0.25326557],\n",
       "       [ 1.29832982, -0.76101394],\n",
       "       [ 2.42816726,  0.37678197],\n",
       "       [ 1.19809737, -0.60557896],\n",
       "       [ 3.49926548,  0.45677347],\n",
       "       [ 1.38766825, -0.20403099],\n",
       "       [ 2.27585365,  0.33338653],\n",
       "       [ 2.61419383,  0.55836695],\n",
       "       [ 1.25762518, -0.179137  ],\n",
       "       [ 1.29066965, -0.11642525],\n",
       "       [ 2.12285398, -0.21085488],\n",
       "       [ 2.3875644 ,  0.46251925],\n",
       "       [ 2.84096093,  0.37274259],\n",
       "       [ 3.2323429 ,  1.37052404],\n",
       "       [ 2.15873837, -0.21832553],\n",
       "       [ 1.4431026 , -0.14380129],\n",
       "       [ 1.77964011, -0.50146479],\n",
       "       [ 3.07652162,  0.68576444],\n",
       "       [ 2.14498686,  0.13890661],\n",
       "       [ 1.90486293,  0.04804751],\n",
       "       [ 1.16885347, -0.1645025 ],\n",
       "       [ 2.10765373,  0.37148225],\n",
       "       [ 2.31430339,  0.18260885],\n",
       "       [ 1.92245088,  0.40927118],\n",
       "       [ 1.41407223, -0.57492506],\n",
       "       [ 2.56332271,  0.2759745 ],\n",
       "       [ 2.41939122,  0.30350394],\n",
       "       [ 1.94401705,  0.18741522],\n",
       "       [ 1.52566363, -0.37502085],\n",
       "       [ 1.76404594,  0.07851919],\n",
       "       [ 1.90162908,  0.11587675],\n",
       "       [ 1.38966613, -0.28288671]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X2D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 5.1,  3.5,  1.4,  0.2],\n",
       "       [ 4.9,  3. ,  1.4,  0.2],\n",
       "       [ 4.7,  3.2,  1.3,  0.2],\n",
       "       [ 4.6,  3.1,  1.5,  0.2],\n",
       "       [ 5. ,  3.6,  1.4,  0.2],\n",
       "       [ 5.4,  3.9,  1.7,  0.4],\n",
       "       [ 4.6,  3.4,  1.4,  0.3],\n",
       "       [ 5. ,  3.4,  1.5,  0.2],\n",
       "       [ 4.4,  2.9,  1.4,  0.2],\n",
       "       [ 4.9,  3.1,  1.5,  0.1],\n",
       "       [ 5.4,  3.7,  1.5,  0.2],\n",
       "       [ 4.8,  3.4,  1.6,  0.2],\n",
       "       [ 4.8,  3. ,  1.4,  0.1],\n",
       "       [ 4.3,  3. ,  1.1,  0.1],\n",
       "       [ 5.8,  4. ,  1.2,  0.2],\n",
       "       [ 5.7,  4.4,  1.5,  0.4],\n",
       "       [ 5.4,  3.9,  1.3,  0.4],\n",
       "       [ 5.1,  3.5,  1.4,  0.3],\n",
       "       [ 5.7,  3.8,  1.7,  0.3],\n",
       "       [ 5.1,  3.8,  1.5,  0.3],\n",
       "       [ 5.4,  3.4,  1.7,  0.2],\n",
       "       [ 5.1,  3.7,  1.5,  0.4],\n",
       "       [ 4.6,  3.6,  1. ,  0.2],\n",
       "       [ 5.1,  3.3,  1.7,  0.5],\n",
       "       [ 4.8,  3.4,  1.9,  0.2],\n",
       "       [ 5. ,  3. ,  1.6,  0.2],\n",
       "       [ 5. ,  3.4,  1.6,  0.4],\n",
       "       [ 5.2,  3.5,  1.5,  0.2],\n",
       "       [ 5.2,  3.4,  1.4,  0.2],\n",
       "       [ 4.7,  3.2,  1.6,  0.2],\n",
       "       [ 4.8,  3.1,  1.6,  0.2],\n",
       "       [ 5.4,  3.4,  1.5,  0.4],\n",
       "       [ 5.2,  4.1,  1.5,  0.1],\n",
       "       [ 5.5,  4.2,  1.4,  0.2],\n",
       "       [ 4.9,  3.1,  1.5,  0.1],\n",
       "       [ 5. ,  3.2,  1.2,  0.2],\n",
       "       [ 5.5,  3.5,  1.3,  0.2],\n",
       "       [ 4.9,  3.1,  1.5,  0.1],\n",
       "       [ 4.4,  3. ,  1.3,  0.2],\n",
       "       [ 5.1,  3.4,  1.5,  0.2],\n",
       "       [ 5. ,  3.5,  1.3,  0.3],\n",
       "       [ 4.5,  2.3,  1.3,  0.3],\n",
       "       [ 4.4,  3.2,  1.3,  0.2],\n",
       "       [ 5. ,  3.5,  1.6,  0.6],\n",
       "       [ 5.1,  3.8,  1.9,  0.4],\n",
       "       [ 4.8,  3. ,  1.4,  0.3],\n",
       "       [ 5.1,  3.8,  1.6,  0.2],\n",
       "       [ 4.6,  3.2,  1.4,  0.2],\n",
       "       [ 5.3,  3.7,  1.5,  0.2],\n",
       "       [ 5. ,  3.3,  1.4,  0.2],\n",
       "       [ 7. ,  3.2,  4.7,  1.4],\n",
       "       [ 6.4,  3.2,  4.5,  1.5],\n",
       "       [ 6.9,  3.1,  4.9,  1.5],\n",
       "       [ 5.5,  2.3,  4. ,  1.3],\n",
       "       [ 6.5,  2.8,  4.6,  1.5],\n",
       "       [ 5.7,  2.8,  4.5,  1.3],\n",
       "       [ 6.3,  3.3,  4.7,  1.6],\n",
       "       [ 4.9,  2.4,  3.3,  1. ],\n",
       "       [ 6.6,  2.9,  4.6,  1.3],\n",
       "       [ 5.2,  2.7,  3.9,  1.4],\n",
       "       [ 5. ,  2. ,  3.5,  1. ],\n",
       "       [ 5.9,  3. ,  4.2,  1.5],\n",
       "       [ 6. ,  2.2,  4. ,  1. ],\n",
       "       [ 6.1,  2.9,  4.7,  1.4],\n",
       "       [ 5.6,  2.9,  3.6,  1.3],\n",
       "       [ 6.7,  3.1,  4.4,  1.4],\n",
       "       [ 5.6,  3. ,  4.5,  1.5],\n",
       "       [ 5.8,  2.7,  4.1,  1. ],\n",
       "       [ 6.2,  2.2,  4.5,  1.5],\n",
       "       [ 5.6,  2.5,  3.9,  1.1],\n",
       "       [ 5.9,  3.2,  4.8,  1.8],\n",
       "       [ 6.1,  2.8,  4. ,  1.3],\n",
       "       [ 6.3,  2.5,  4.9,  1.5],\n",
       "       [ 6.1,  2.8,  4.7,  1.2],\n",
       "       [ 6.4,  2.9,  4.3,  1.3],\n",
       "       [ 6.6,  3. ,  4.4,  1.4],\n",
       "       [ 6.8,  2.8,  4.8,  1.4],\n",
       "       [ 6.7,  3. ,  5. ,  1.7],\n",
       "       [ 6. ,  2.9,  4.5,  1.5],\n",
       "       [ 5.7,  2.6,  3.5,  1. ],\n",
       "       [ 5.5,  2.4,  3.8,  1.1],\n",
       "       [ 5.5,  2.4,  3.7,  1. ],\n",
       "       [ 5.8,  2.7,  3.9,  1.2],\n",
       "       [ 6. ,  2.7,  5.1,  1.6],\n",
       "       [ 5.4,  3. ,  4.5,  1.5],\n",
       "       [ 6. ,  3.4,  4.5,  1.6],\n",
       "       [ 6.7,  3.1,  4.7,  1.5],\n",
       "       [ 6.3,  2.3,  4.4,  1.3],\n",
       "       [ 5.6,  3. ,  4.1,  1.3],\n",
       "       [ 5.5,  2.5,  4. ,  1.3],\n",
       "       [ 5.5,  2.6,  4.4,  1.2],\n",
       "       [ 6.1,  3. ,  4.6,  1.4],\n",
       "       [ 5.8,  2.6,  4. ,  1.2],\n",
       "       [ 5. ,  2.3,  3.3,  1. ],\n",
       "       [ 5.6,  2.7,  4.2,  1.3],\n",
       "       [ 5.7,  3. ,  4.2,  1.2],\n",
       "       [ 5.7,  2.9,  4.2,  1.3],\n",
       "       [ 6.2,  2.9,  4.3,  1.3],\n",
       "       [ 5.1,  2.5,  3. ,  1.1],\n",
       "       [ 5.7,  2.8,  4.1,  1.3],\n",
       "       [ 6.3,  3.3,  6. ,  2.5],\n",
       "       [ 5.8,  2.7,  5.1,  1.9],\n",
       "       [ 7.1,  3. ,  5.9,  2.1],\n",
       "       [ 6.3,  2.9,  5.6,  1.8],\n",
       "       [ 6.5,  3. ,  5.8,  2.2],\n",
       "       [ 7.6,  3. ,  6.6,  2.1],\n",
       "       [ 4.9,  2.5,  4.5,  1.7],\n",
       "       [ 7.3,  2.9,  6.3,  1.8],\n",
       "       [ 6.7,  2.5,  5.8,  1.8],\n",
       "       [ 7.2,  3.6,  6.1,  2.5],\n",
       "       [ 6.5,  3.2,  5.1,  2. ],\n",
       "       [ 6.4,  2.7,  5.3,  1.9],\n",
       "       [ 6.8,  3. ,  5.5,  2.1],\n",
       "       [ 5.7,  2.5,  5. ,  2. ],\n",
       "       [ 5.8,  2.8,  5.1,  2.4],\n",
       "       [ 6.4,  3.2,  5.3,  2.3],\n",
       "       [ 6.5,  3. ,  5.5,  1.8],\n",
       "       [ 7.7,  3.8,  6.7,  2.2],\n",
       "       [ 7.7,  2.6,  6.9,  2.3],\n",
       "       [ 6. ,  2.2,  5. ,  1.5],\n",
       "       [ 6.9,  3.2,  5.7,  2.3],\n",
       "       [ 5.6,  2.8,  4.9,  2. ],\n",
       "       [ 7.7,  2.8,  6.7,  2. ],\n",
       "       [ 6.3,  2.7,  4.9,  1.8],\n",
       "       [ 6.7,  3.3,  5.7,  2.1],\n",
       "       [ 7.2,  3.2,  6. ,  1.8],\n",
       "       [ 6.2,  2.8,  4.8,  1.8],\n",
       "       [ 6.1,  3. ,  4.9,  1.8],\n",
       "       [ 6.4,  2.8,  5.6,  2.1],\n",
       "       [ 7.2,  3. ,  5.8,  1.6],\n",
       "       [ 7.4,  2.8,  6.1,  1.9],\n",
       "       [ 7.9,  3.8,  6.4,  2. ],\n",
       "       [ 6.4,  2.8,  5.6,  2.2],\n",
       "       [ 6.3,  2.8,  5.1,  1.5],\n",
       "       [ 6.1,  2.6,  5.6,  1.4],\n",
       "       [ 7.7,  3. ,  6.1,  2.3],\n",
       "       [ 6.3,  3.4,  5.6,  2.4],\n",
       "       [ 6.4,  3.1,  5.5,  1.8],\n",
       "       [ 6. ,  3. ,  4.8,  1.8],\n",
       "       [ 6.9,  3.1,  5.4,  2.1],\n",
       "       [ 6.7,  3.1,  5.6,  2.4],\n",
       "       [ 6.9,  3.1,  5.1,  2.3],\n",
       "       [ 5.8,  2.7,  5.1,  1.9],\n",
       "       [ 6.8,  3.2,  5.9,  2.3],\n",
       "       [ 6.7,  3.3,  5.7,  2.5],\n",
       "       [ 6.7,  3. ,  5.2,  2.3],\n",
       "       [ 6.3,  2.5,  5. ,  1.9],\n",
       "       [ 6.5,  3. ,  5.2,  2. ],\n",
       "       [ 6.2,  3.4,  5.4,  2.3],\n",
       "       [ 5.9,  3. ,  5.1,  1.8]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kernel PCA\n",
    "* Implicityly maps instances into a very high-dimensional space, enabling nonlinear classification & regression with Support vector machines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Same kernel PCA can be used in dimensionality reduction\n",
    "from sklearn.decomposition import KernelPCA\n",
    "\n",
    "rbf_pca = KernelPCA(n_components = 2, kernel=\"rbf\", gamma=0.04)\n",
    "X_reduced = rbf_pca.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.6166177 , -0.09180361],\n",
       "       [-0.61589654, -0.04889014],\n",
       "       [-0.64084145, -0.08454319],\n",
       "       [-0.61808503, -0.04323666],\n",
       "       [-0.62264344, -0.10107493],\n",
       "       [-0.54606665, -0.07465949],\n",
       "       [-0.63127689, -0.07966942],\n",
       "       [-0.60788291, -0.06619658],\n",
       "       [-0.62879281, -0.04841373],\n",
       "       [-0.61127893, -0.04677647],\n",
       "       [-0.58658632, -0.09645961],\n",
       "       [-0.60399481, -0.05046106],\n",
       "       [-0.62492192, -0.0564587 ],\n",
       "       [-0.66726739, -0.11688995],\n",
       "       [-0.59240851, -0.17531023],\n",
       "       [-0.54619918, -0.1598249 ],\n",
       "       [-0.60037042, -0.13527043],\n",
       "       [-0.61122583, -0.08566366],\n",
       "       [-0.52925637, -0.07582282],\n",
       "       [-0.59984853, -0.09813618],\n",
       "       [-0.55474848, -0.03630726],\n",
       "       [-0.5937633 , -0.08255957],\n",
       "       [-0.67737513, -0.16972789],\n",
       "       [-0.55341192, -0.0059123 ],\n",
       "       [-0.55987594, -0.00157958],\n",
       "       [-0.58478963, -0.01426117],\n",
       "       [-0.5825563 , -0.03729568],\n",
       "       [-0.59803375, -0.07580827],\n",
       "       [-0.60961715, -0.08290609],\n",
       "       [-0.60417576, -0.03340192],\n",
       "       [-0.59750211, -0.02377006],\n",
       "       [-0.57161915, -0.0561565 ],\n",
       "       [-0.60221804, -0.13982418],\n",
       "       [-0.58901319, -0.16123059],\n",
       "       [-0.61127893, -0.04677647],\n",
       "       [-0.63962514, -0.09909375],\n",
       "       [-0.60354338, -0.11030976],\n",
       "       [-0.61127893, -0.04677647],\n",
       "       [-0.64301258, -0.0735078 ],\n",
       "       [-0.60251298, -0.06620439],\n",
       "       [-0.62842241, -0.10222282],\n",
       "       [-0.60396655, -0.01666637],\n",
       "       [-0.64835698, -0.08944456],\n",
       "       [-0.5702742 , -0.03549487],\n",
       "       [-0.53448763, -0.03053908],\n",
       "       [-0.6149222 , -0.04322361],\n",
       "       [-0.59195465, -0.08841241],\n",
       "       [-0.63266034, -0.0687716 ],\n",
       "       [-0.59326811, -0.09549837],\n",
       "       [-0.61883179, -0.07388415],\n",
       "       [ 0.32591114, -0.02837132],\n",
       "       [ 0.25734343,  0.07937124],\n",
       "       [ 0.36613433, -0.03079491],\n",
       "       [ 0.0687828 ,  0.29004238],\n",
       "       [ 0.29256139,  0.09199182],\n",
       "       [ 0.18771142,  0.21228739],\n",
       "       [ 0.29455594,  0.06065124],\n",
       "       [-0.17531186,  0.29534072],\n",
       "       [ 0.28068416,  0.07624187],\n",
       "       [ 0.01933787,  0.28297864],\n",
       "       [-0.11001466,  0.32453684],\n",
       "       [ 0.15628696,  0.18439494],\n",
       "       [ 0.08744578,  0.25088914],\n",
       "       [ 0.27084395,  0.13503561],\n",
       "       [-0.02675196,  0.23671042],\n",
       "       [ 0.25259933,  0.05700377],\n",
       "       [ 0.19235993,  0.19890004],\n",
       "       [ 0.08308726,  0.23465749],\n",
       "       [ 0.25437451,  0.18703736],\n",
       "       [ 0.03282632,  0.27192119],\n",
       "       [ 0.2987511 ,  0.10470659],\n",
       "       [ 0.11529537,  0.1929816 ],\n",
       "       [ 0.33578273,  0.11258669],\n",
       "       [ 0.25442699,  0.15033337],\n",
       "       [ 0.20473634,  0.13009287],\n",
       "       [ 0.24779801,  0.08137249],\n",
       "       [ 0.34039699,  0.02794272],\n",
       "       [ 0.38879427, -0.01276503],\n",
       "       [ 0.23117087,  0.16196481],\n",
       "       [-0.06299751,  0.25144415],\n",
       "       [ 0.00264743,  0.28676679],\n",
       "       [-0.03009413,  0.28527691],\n",
       "       [ 0.05726942,  0.23785411],\n",
       "       [ 0.35316483,  0.11085345],\n",
       "       [ 0.17290159,  0.21829489],\n",
       "       [ 0.22734098,  0.10862048],\n",
       "       [ 0.31942891,  0.02432886],\n",
       "       [ 0.22491532,  0.1826006 ],\n",
       "       [ 0.08687684,  0.2242793 ],\n",
       "       [ 0.06534032,  0.27715513],\n",
       "       [ 0.1416538 ,  0.25646262],\n",
       "       [ 0.24922271,  0.13522927],\n",
       "       [ 0.0821539 ,  0.24384571],\n",
       "       [-0.16374953,  0.29903809],\n",
       "       [ 0.11571652,  0.24730266],\n",
       "       [ 0.10922495,  0.21279943],\n",
       "       [ 0.12115397,  0.22039697],\n",
       "       [ 0.18826288,  0.15710626],\n",
       "       [-0.21994481,  0.26234094],\n",
       "       [ 0.10096043,  0.23303065],\n",
       "       [ 0.51514634, -0.18193413],\n",
       "       [ 0.35718621,  0.1201525 ],\n",
       "       [ 0.53277195, -0.24212377],\n",
       "       [ 0.45757761, -0.03398602],\n",
       "       [ 0.50738174, -0.1343727 ],\n",
       "       [ 0.56288686, -0.43183209],\n",
       "       [ 0.14802132,  0.28703398],\n",
       "       [ 0.54843019, -0.31268497],\n",
       "       [ 0.50006253, -0.10188826],\n",
       "       [ 0.54024576, -0.36030511],\n",
       "       [ 0.40783611, -0.03043818],\n",
       "       [ 0.4323817 ,  0.00551857],\n",
       "       [ 0.48571299, -0.12774668],\n",
       "       [ 0.33922692,  0.1530915 ],\n",
       "       [ 0.38285874,  0.0760123 ],\n",
       "       [ 0.44496616, -0.06413059],\n",
       "       [ 0.45610547, -0.05813211],\n",
       "       [ 0.5412963 , -0.51090641],\n",
       "       [ 0.56141124, -0.47688325],\n",
       "       [ 0.32898482,  0.16176555],\n",
       "       [ 0.51357558, -0.20624035],\n",
       "       [ 0.31106798,  0.15442463],\n",
       "       [ 0.55972757, -0.44201263],\n",
       "       [ 0.35712627,  0.08153468],\n",
       "       [ 0.49719541, -0.16744522],\n",
       "       [ 0.52636591, -0.26929237],\n",
       "       [ 0.33107447,  0.09988268],\n",
       "       [ 0.33792808,  0.08536972],\n",
       "       [ 0.47957811, -0.0627687 ],\n",
       "       [ 0.50269347, -0.20897558],\n",
       "       [ 0.5426926 , -0.29584911],\n",
       "       [ 0.52688106, -0.4848302 ],\n",
       "       [ 0.48356918, -0.07020763],\n",
       "       [ 0.36750706,  0.06469011],\n",
       "       [ 0.41851442,  0.04255026],\n",
       "       [ 0.54939265, -0.38286162],\n",
       "       [ 0.47344424, -0.11867385],\n",
       "       [ 0.44852585, -0.05063019],\n",
       "       [ 0.31177565,  0.11037112],\n",
       "       [ 0.47601781, -0.13700511],\n",
       "       [ 0.50115428, -0.1571116 ],\n",
       "       [ 0.44457034, -0.10719769],\n",
       "       [ 0.35718621,  0.1201525 ],\n",
       "       [ 0.5277924 , -0.22230436],\n",
       "       [ 0.50882129, -0.19639015],\n",
       "       [ 0.45145717, -0.08046264],\n",
       "       [ 0.38101386,  0.0775144 ],\n",
       "       [ 0.42684021, -0.02676107],\n",
       "       [ 0.43982625, -0.06601883],\n",
       "       [ 0.35538245,  0.08839875]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_reduced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clf = Pipeline([\n",
    "        (\"kpca\", KernelPCA(n_components=2)),\n",
    "        (\"log_reg\", LogisticRegression())\n",
    "    ])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "param_grid = [{\n",
    "        \"kpca__gamma\": np.linspace(0.03, 0.05, 10),\n",
    "        \"kpca__kernel\": [\"rbf\", \"sigmoid\"]\n",
    "    }]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise',\n",
       "       estimator=Pipeline(steps=[('kpca', KernelPCA(alpha=1.0, coef0=1, copy_X=True, degree=3, eigen_solver='auto',\n",
       "     fit_inverse_transform=False, gamma=None, kernel='linear',\n",
       "     kernel_params=None, max_iter=None, n_components=2, n_jobs=1,\n",
       "     random_state=None, remove_zero_eig=False, tol=0)), ('log_reg', LogisticRegre...ty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False))]),\n",
       "       fit_params={}, iid=True, n_jobs=1,\n",
       "       param_grid=[{'kpca__kernel': ['rbf', 'sigmoid'], 'kpca__gamma': array([ 0.03   ,  0.03222,  0.03444,  0.03667,  0.03889,  0.04111,\n",
       "        0.04333,  0.04556,  0.04778,  0.05   ])}],\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search = GridSearchCV(clf, param_grid, cv=3)\n",
    "grid_search.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.90000000000000002"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the MNIST dataset and split it into a training set and a test set (take the first 60,000 instances for training, and the remaining 10,000 for testing). Train a Random Forest classifier on the dataset and time how long it takes, then evaluate the resulting model on the test set. Next, use PCA to reduce the dataset’s dimensionality, with an explained variance ratio of 95%. Train a new Random Forest classifier on the reduced dataset and see how long it takes. Was training much faster? Next evaluate the classifier on the test set: how does it compare to the previous classifier?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.read_sql_query"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}