example/eda/criteo_eda.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": "   y   I1   I2    I3    I4      I5    I6    I7   I8     I9  ...       C17  \\\n0  0  1.0    1   5.0   0.0  1382.0   4.0  15.0  2.0  181.0  ...  e5ba7672   \n1  0  2.0    0  44.0   1.0   102.0   8.0   2.0  2.0    4.0  ...  07c540c4   \n2  0  2.0    0   1.0  14.0   767.0  89.0   4.0  2.0  245.0  ...  8efede7f   \n3  0  1.0  893   5.0   4.0  4392.0  33.0   0.0  0.0    0.0  ...  1e88c74f   \n4  0  3.0   -1   5.0   0.0     2.0   0.0   3.0  0.0    0.0  ...  1e88c74f   \n\n        C18       C19       C20       C21       C22       C23       C24  \\\n0  f54016b9  21ddcdc9  b1252a9d  07b5194c         0  3a171ecb  c5c50484   \n1  b04e4670  21ddcdc9  5840adea  60f6221e         0  3a171ecb  43f13e8b   \n2  3412118d         0         0  e587c466  ad3062eb  3a171ecb  3b183c5c   \n3  74ef3502         0         0  6b3a5ca6         0  3a171ecb  9117a34a   \n4  26b3c7a7         0         0  21c9516a         0  32c7478e  b34f3128   \n\n        C25       C26  \n0  e8b83407  9727dd16  \n1  e8b83407  731c3655  \n2         0         0  \n3         0         0  \n4         0         0  \n\n[5 rows x 40 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>y</th>\n      <th>I1</th>\n      <th>I2</th>\n      <th>I3</th>\n      <th>I4</th>\n      <th>I5</th>\n      <th>I6</th>\n      <th>I7</th>\n      <th>I8</th>\n      <th>I9</th>\n      <th>...</th>\n      <th>C17</th>\n      <th>C18</th>\n      <th>C19</th>\n      <th>C20</th>\n      <th>C21</th>\n      <th>C22</th>\n      <th>C23</th>\n      <th>C24</th>\n      <th>C25</th>\n      <th>C26</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1.0</td>\n      <td>1</td>\n      <td>5.0</td>\n      <td>0.0</td>\n      <td>1382.0</td>\n      <td>4.0</td>\n      <td>15.0</td>\n      <td>2.0</td>\n      <td>181.0</td>\n      <td>...</td>\n      <td>e5ba7672</td>\n      <td>f54016b9</td>\n      <td>21ddcdc9</td>\n      <td>b1252a9d</td>\n      <td>07b5194c</td>\n      <td>0</td>\n      <td>3a171ecb</td>\n      <td>c5c50484</td>\n      <td>e8b83407</td>\n      <td>9727dd16</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>2.0</td>\n      <td>0</td>\n      <td>44.0</td>\n      <td>1.0</td>\n      <td>102.0</td>\n      <td>8.0</td>\n      <td>2.0</td>\n      <td>2.0</td>\n      <td>4.0</td>\n      <td>...</td>\n      <td>07c540c4</td>\n      <td>b04e4670</td>\n      <td>21ddcdc9</td>\n      <td>5840adea</td>\n      <td>60f6221e</td>\n      <td>0</td>\n      <td>3a171ecb</td>\n      <td>43f13e8b</td>\n      <td>e8b83407</td>\n      <td>731c3655</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>2.0</td>\n      <td>0</td>\n      <td>1.0</td>\n      <td>14.0</td>\n      <td>767.0</td>\n      <td>89.0</td>\n      <td>4.0</td>\n      <td>2.0</td>\n      <td>245.0</td>\n      <td>...</td>\n      <td>8efede7f</td>\n      <td>3412118d</td>\n      <td>0</td>\n      <td>0</td>\n      <td>e587c466</td>\n      <td>ad3062eb</td>\n      <td>3a171ecb</td>\n      <td>3b183c5c</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0</td>\n      <td>1.0</td>\n      <td>893</td>\n      <td>5.0</td>\n      <td>4.0</td>\n      <td>4392.0</td>\n      <td>33.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>1e88c74f</td>\n      <td>74ef3502</td>\n      <td>0</td>\n      <td>0</td>\n      <td>6b3a5ca6</td>\n      <td>0</td>\n      <td>3a171ecb</td>\n      <td>9117a34a</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0</td>\n      <td>3.0</td>\n      <td>-1</td>\n      <td>5.0</td>\n      <td>0.0</td>\n      <td>2.0</td>\n      <td>0.0</td>\n      <td>3.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>1e88c74f</td>\n      <td>26b3c7a7</td>\n      <td>0</td>\n      <td>0</td>\n      <td>21c9516a</td>\n      <td>0</td>\n      <td>32c7478e</td>\n      <td>b34f3128</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 40 columns</p>\n</div>"
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from example.loader.criteo_loader import load_data,missing_values_process\n",
    "data=load_data('../../dataset/criteo_sample_10k.txt')\n",
    "data=missing_values_process(data)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "counts=data['I1'].value_counts()\n",
    "counts=counts.sort_index()\n",
    "counts_ratio=counts/len(data)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.260558\n",
      "0 0 0 0.260558\n",
      "1 0.521462\n",
      "1 1 1 0.521462\n",
      "2 0.054293\n",
      "2 2 2 0.054293\n",
      "3 0.033641\n",
      "3 3 3 0.033641\n",
      "4 0.024655\n",
      "5 0.017578\n",
      "4 5 4 0.042233\n",
      "6 0.013687\n",
      "7 0.010403\n",
      "8 0.008458\n",
      "6 8 5 0.032548\n",
      "9 0.006793\n",
      "10 0.005751\n",
      "11 0.004659\n",
      "12 0.00406\n",
      "13 0.003334\n",
      "14 0.003063\n",
      "15 0.002627\n",
      "9 15 6 0.030287\n",
      "0.9750219999999999\n"
     ]
    }
   ],
   "source": [
    "total_acc=0\n",
    "acc=0\n",
    "idx=0\n",
    "left=0\n",
    "right=0\n",
    "for i in range(len(counts_ratio)):\n",
    "    total_acc+=counts_ratio[i]\n",
    "    acc+=counts_ratio[i]\n",
    "    print(i,counts_ratio[i])\n",
    "\n",
    "    if acc>0.03:\n",
    "        right=i\n",
    "        print(left,right,idx,acc)\n",
    "        idx+=1\n",
    "        left=i+1\n",
    "        acc=0\n",
    "        if total_acc>0.97:\n",
    "            print(total_acc)\n",
    "            break\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}