{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# interpolate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "site_url = \"http://www.kyoritsu-pub.co.jp/bookdetail/9784320123656\"\n",
    "zip_url = \"http://www.kyoritsu-pub.co.jp/app/file/goods_contents/2377.zip\"\n",
    "pdf_url = \"http://www.kyoritsu-pub.co.jp/app/file/goods_contents/2324.pdf\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import io\n",
    "import shutil\n",
    "import zipfile\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IQ</th>\n",
       "      <th>JobPerformance</th>\n",
       "      <th>MCAR</th>\n",
       "      <th>MCAR.is.missing</th>\n",
       "      <th>MAR</th>\n",
       "      <th>MAR.is.missing</th>\n",
       "      <th>MNAR</th>\n",
       "      <th>MNAR.is.missing</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>20.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>16.00000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>15.00000</td>\n",
       "      <td>20.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>10.350000</td>\n",
       "      <td>10.56250</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>10.666667</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>11.40000</td>\n",
       "      <td>0.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>14.127241</td>\n",
       "      <td>2.680829</td>\n",
       "      <td>2.82769</td>\n",
       "      <td>0.410391</td>\n",
       "      <td>2.794553</td>\n",
       "      <td>0.444262</td>\n",
       "      <td>2.22967</td>\n",
       "      <td>0.444262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>78.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.00000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>90.000000</td>\n",
       "      <td>8.750000</td>\n",
       "      <td>8.75000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.00000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>97.500000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>11.00000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>109.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>12.25000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>12.50000</td>\n",
       "      <td>0.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>134.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>16.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>16.00000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               IQ  JobPerformance      MCAR  MCAR.is.missing        MAR  \\\n",
       "count   20.000000       20.000000  16.00000        20.000000  15.000000   \n",
       "mean   100.000000       10.350000  10.56250         0.200000  10.666667   \n",
       "std     14.127241        2.680829   2.82769         0.410391   2.794553   \n",
       "min     78.000000        7.000000   7.00000         0.000000   7.000000   \n",
       "25%     90.000000        8.750000   8.75000         0.000000   9.000000   \n",
       "50%     97.500000       10.000000  10.00000         0.000000  10.000000   \n",
       "75%    109.000000       12.000000  12.25000         0.000000  12.000000   \n",
       "max    134.000000       16.000000  16.00000         1.000000  16.000000   \n",
       "\n",
       "       MAR.is.missing      MNAR  MNAR.is.missing  \n",
       "count       20.000000  15.00000        20.000000  \n",
       "mean         0.250000  11.40000         0.250000  \n",
       "std          0.444262   2.22967         0.444262  \n",
       "min          0.000000   9.00000         0.000000  \n",
       "25%          0.000000  10.00000         0.000000  \n",
       "50%          0.000000  11.00000         0.000000  \n",
       "75%          0.250000  12.50000         0.250000  \n",
       "max          1.000000  16.00000         1.000000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_dir = \"suppl_20151001/data/missdata\"\n",
    "filename = \"employee_IQ_JP.csv\"\n",
    "file_path = data_dir + \"/\" + filename\n",
    "if not os.path.exists(file_path):\n",
    "    # headers = {'user-agent': 'my-agent'}\n",
    "    # r = requests.get(pdf_url, headers=headers)\n",
    "    # shutil.copyfileobj(io.BytesIO(r.content), open(pdf_url.split(\"/\")[-1], \"wb\"))\n",
    "\n",
    "    proxies={\n",
    "        \"http\": \"http://xxx.xxx.xxx.:xxxx\",\n",
    "        \"https\": \"http://xxx.xxx.xxx.:xxxx\",\n",
    "    }\n",
    "    r = requests.get(zip_url, proxies=proxies)\n",
    "    with zipfile.ZipFile(io.BytesIO(r.content)) as z:\n",
    "        z.extractall()\n",
    "\n",
    "df = pd.read_csv(file_path)\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# http://ejje.weblio.jp/content/interpolate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "s = pd.Series([0, 1, np.nan, 3])\n",
    "# help(s.interpolate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['linear',\n",
       " 'index',\n",
       " 'values',\n",
       " 'nearest',\n",
       " 'zero',\n",
       " 'slinear',\n",
       " 'barycentric',\n",
       " 'krogh',\n",
       " 'pchip']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "methods = ['linear', 'time', 'index', 'values', 'nearest', 'zero',\n",
    "              'slinear', 'quadratic', 'cubic', 'barycentric', 'krogh',\n",
    "              'polynomial', 'spline', 'piecewise_polynomial', 'pchip']\n",
    "methods.remove(\"time\")\n",
    "methods.remove(\"quadratic\")\n",
    "methods.remove(\"cubic\")\n",
    "methods.remove(\"polynomial\")\n",
    "methods.remove(\"spline\")\n",
    "methods.remove(\"piecewise_polynomial\")\n",
    "\n",
    "methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>linear</th>\n",
       "      <th>index</th>\n",
       "      <th>values</th>\n",
       "      <th>nearest</th>\n",
       "      <th>zero</th>\n",
       "      <th>slinear</th>\n",
       "      <th>barycentric</th>\n",
       "      <th>krogh</th>\n",
       "      <th>pchip</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   linear  index  values  nearest  zero  slinear  barycentric  krogh  pchip\n",
       "0       0      0       0        0     0        0            0      0      0\n",
       "1       1      1       1        1     1        1            1      1      1\n",
       "2       2      2       2        1     1        2            2      2      2\n",
       "3       3      3       3        3     3        3            3      3      3"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def _interpolate(m):\n",
    "    _s = s.interpolate(method=m)\n",
    "    _s.name = m\n",
    "    return _s\n",
    "pd.concat(map(_interpolate, methods), axis=1)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}