data_sampling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "88752c58-cac2-4954-a5c1-6dc51e4e17fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as ps\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d3d94406-a136-474c-aae7-73d447418a10",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = ps.read_csv('data/goodreads_interactions.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "33e1ce29-ffcb-4e23-8555-96bef6e389df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>user_id</th><th>book_id</th><th>is_read</th><th>rating</th><th>is_reviewed</th></tr><tr><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>0</td><td>948</td><td>1</td><td>5</td><td>0</td></tr><tr><td>0</td><td>947</td><td>1</td><td>5</td><td>1</td></tr><tr><td>0</td><td>946</td><td>1</td><td>5</td><td>0</td></tr><tr><td>0</td><td>945</td><td>1</td><td>5</td><td>0</td></tr><tr><td>0</td><td>944</td><td>1</td><td>5</td><td>0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 5)\n",
       "┌─────────┬─────────┬─────────┬────────┬─────────────┐\n",
       "│ user_id ┆ book_id ┆ is_read ┆ rating ┆ is_reviewed │\n",
       "│ ---     ┆ ---     ┆ ---     ┆ ---    ┆ ---         │\n",
       "│ i64     ┆ i64     ┆ i64     ┆ i64    ┆ i64         │\n",
       "╞═════════╪═════════╪═════════╪════════╪═════════════╡\n",
       "│ 0       ┆ 948     ┆ 1       ┆ 5      ┆ 0           │\n",
       "│ 0       ┆ 947     ┆ 1       ┆ 5      ┆ 1           │\n",
       "│ 0       ┆ 946     ┆ 1       ┆ 5      ┆ 0           │\n",
       "│ 0       ┆ 945     ┆ 1       ┆ 5      ┆ 0           │\n",
       "│ 0       ┆ 944     ┆ 1       ┆ 5      ┆ 0           │\n",
       "└─────────┴─────────┴─────────┴────────┴─────────────┘"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "775b1b22-745f-4c0e-b274-2d0a0e561635",
   "metadata": {},
   "outputs": [],
   "source": [
    "users = df.select([ps.col('user_id'), ps.col('is_read')]).group_by('user_id').sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8b79d0e1-2eea-4ec8-94ac-a4ccab55d152",
   "metadata": {},
   "outputs": [],
   "source": [
    "books = df.select([ps.col('book_id'), ps.col('is_read')]).group_by('book_id').sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1079b77b-eef2-4c06-9afd-963f2da22926",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>statistic</th><th>user_id</th><th>is_read</th></tr><tr><td>str</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;count&quot;</td><td>876145.0</td><td>876145.0</td></tr><tr><td>&quot;null_count&quot;</td><td>0.0</td><td>0.0</td></tr><tr><td>&quot;mean&quot;</td><td>438072.0</td><td>127.982472</td></tr><tr><td>&quot;std&quot;</td><td>252921.420137</td><td>241.612867</td></tr><tr><td>&quot;min&quot;</td><td>0.0</td><td>0.0</td></tr><tr><td>&quot;25%&quot;</td><td>219036.0</td><td>19.0</td></tr><tr><td>&quot;50%&quot;</td><td>438072.0</td><td>54.0</td></tr><tr><td>&quot;75%&quot;</td><td>657108.0</td><td>142.0</td></tr><tr><td>&quot;max&quot;</td><td>876144.0</td><td>38895.0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9, 3)\n",
       "┌────────────┬───────────────┬────────────┐\n",
       "│ statistic  ┆ user_id       ┆ is_read    │\n",
       "│ ---        ┆ ---           ┆ ---        │\n",
       "│ str        ┆ f64           ┆ f64        │\n",
       "╞════════════╪═══════════════╪════════════╡\n",
       "│ count      ┆ 876145.0      ┆ 876145.0   │\n",
       "│ null_count ┆ 0.0           ┆ 0.0        │\n",
       "│ mean       ┆ 438072.0      ┆ 127.982472 │\n",
       "│ std        ┆ 252921.420137 ┆ 241.612867 │\n",
       "│ min        ┆ 0.0           ┆ 0.0        │\n",
       "│ 25%        ┆ 219036.0      ┆ 19.0       │\n",
       "│ 50%        ┆ 438072.0      ┆ 54.0       │\n",
       "│ 75%        ┆ 657108.0      ┆ 142.0      │\n",
       "│ max        ┆ 876144.0      ┆ 38895.0    │\n",
       "└────────────┴───────────────┴────────────┘"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2a50c288-cd1d-42d1-988d-fd21579724cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>user_id</th><th>is_read</th></tr><tr><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>445813</td><td>65</td></tr><tr><td>123882</td><td>38</td></tr><tr><td>565217</td><td>20</td></tr><tr><td>4070</td><td>510</td></tr><tr><td>167523</td><td>64</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌─────────┬─────────┐\n",
       "│ user_id ┆ is_read │\n",
       "│ ---     ┆ ---     │\n",
       "│ i64     ┆ i64     │\n",
       "╞═════════╪═════════╡\n",
       "│ 445813  ┆ 65      │\n",
       "│ 123882  ┆ 38      │\n",
       "│ 565217  ┆ 20      │\n",
       "│ 4070    ┆ 510     │\n",
       "│ 167523  ┆ 64      │\n",
       "└─────────┴─────────┘"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6a2da6b-2bf7-4f3b-b719-31cde54817a1",
   "metadata": {},
   "source": [
    "75% of users have read 19 or more books. In order to not skew the data toward very heavy readers. Users that have read less than 25 books will be dropped and a random subset of readers than have read 25 or more will be selected."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dab00445-ca53-4706-ac09-e9de9b191624",
   "metadata": {},
   "outputs": [],
   "source": [
    "users = users.filter(ps.col('is_read')>=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "18bbf952-70e9-4c8a-a513-5a7707119ed1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>statistic</th><th>user_id</th><th>is_read</th></tr><tr><td>str</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;count&quot;</td><td>610196.0</td><td>610196.0</td></tr><tr><td>&quot;null_count&quot;</td><td>0.0</td><td>0.0</td></tr><tr><td>&quot;mean&quot;</td><td>348066.765087</td><td>179.893926</td></tr><tr><td>&quot;std&quot;</td><td>234447.416438</td><td>273.704438</td></tr><tr><td>&quot;min&quot;</td><td>0.0</td><td>25.0</td></tr><tr><td>&quot;25%&quot;</td><td>156351.0</td><td>50.0</td></tr><tr><td>&quot;50%&quot;</td><td>313556.0</td><td>95.0</td></tr><tr><td>&quot;75%&quot;</td><td>501091.0</td><td>202.0</td></tr><tr><td>&quot;max&quot;</td><td>876144.0</td><td>38895.0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9, 3)\n",
       "┌────────────┬───────────────┬────────────┐\n",
       "│ statistic  ┆ user_id       ┆ is_read    │\n",
       "│ ---        ┆ ---           ┆ ---        │\n",
       "│ str        ┆ f64           ┆ f64        │\n",
       "╞════════════╪═══════════════╪════════════╡\n",
       "│ count      ┆ 610196.0      ┆ 610196.0   │\n",
       "│ null_count ┆ 0.0           ┆ 0.0        │\n",
       "│ mean       ┆ 348066.765087 ┆ 179.893926 │\n",
       "│ std        ┆ 234447.416438 ┆ 273.704438 │\n",
       "│ min        ┆ 0.0           ┆ 25.0       │\n",
       "│ 25%        ┆ 156351.0      ┆ 50.0       │\n",
       "│ 50%        ┆ 313556.0      ┆ 95.0       │\n",
       "│ 75%        ┆ 501091.0      ┆ 202.0      │\n",
       "│ max        ┆ 876144.0      ┆ 38895.0    │\n",
       "└────────────┴───────────────┴────────────┘"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "54fc3631-3009-4f0e-9d1b-d4a7838eb904",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_ids = users.select(ps.col('user_id')).sample(n=50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "bf21c6dd-6da4-4edc-9f97-b6c8126a8256",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_ids.write_csv('user_subset.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8bc7c25e-257e-4cce-89d6-e90f06022173",
   "metadata": {},
   "source": [
    "## Books"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "da06aab7-eddd-4107-ac4d-f404343e4065",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>book_id</th><th>is_read</th></tr><tr><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>1730387</td><td>4</td></tr><tr><td>2282891</td><td>1</td></tr><tr><td>751482</td><td>8</td></tr><tr><td>639286</td><td>13</td></tr><tr><td>950204</td><td>2</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌─────────┬─────────┐\n",
       "│ book_id ┆ is_read │\n",
       "│ ---     ┆ ---     │\n",
       "│ i64     ┆ i64     │\n",
       "╞═════════╪═════════╡\n",
       "│ 1730387 ┆ 4       │\n",
       "│ 2282891 ┆ 1       │\n",
       "│ 751482  ┆ 8       │\n",
       "│ 639286  ┆ 13      │\n",
       "│ 950204  ┆ 2       │\n",
       "└─────────┴─────────┘"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "books.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2acdd0eb-450b-4791-9d0d-b80c90af7e21",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>statistic</th><th>book_id</th><th>is_read</th></tr><tr><td>str</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;count&quot;</td><td>2.36065e6</td><td>2.36065e6</td></tr><tr><td>&quot;null_count&quot;</td><td>0.0</td><td>0.0</td></tr><tr><td>&quot;mean&quot;</td><td>1180324.5</td><td>47.500139</td></tr><tr><td>&quot;std&quot;</td><td>681461.100819</td><td>848.946093</td></tr><tr><td>&quot;min&quot;</td><td>0.0</td><td>0.0</td></tr><tr><td>&quot;25%&quot;</td><td>590162.0</td><td>2.0</td></tr><tr><td>&quot;50%&quot;</td><td>1.180325e6</td><td>5.0</td></tr><tr><td>&quot;75%&quot;</td><td>1.770487e6</td><td>14.0</td></tr><tr><td>&quot;max&quot;</td><td>2.360649e6</td><td>285698.0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9, 3)\n",
       "┌────────────┬───────────────┬────────────┐\n",
       "│ statistic  ┆ book_id       ┆ is_read    │\n",
       "│ ---        ┆ ---           ┆ ---        │\n",
       "│ str        ┆ f64           ┆ f64        │\n",
       "╞════════════╪═══════════════╪════════════╡\n",
       "│ count      ┆ 2.36065e6     ┆ 2.36065e6  │\n",
       "│ null_count ┆ 0.0           ┆ 0.0        │\n",
       "│ mean       ┆ 1180324.5     ┆ 47.500139  │\n",
       "│ std        ┆ 681461.100819 ┆ 848.946093 │\n",
       "│ min        ┆ 0.0           ┆ 0.0        │\n",
       "│ 25%        ┆ 590162.0      ┆ 2.0        │\n",
       "│ 50%        ┆ 1.180325e6    ┆ 5.0        │\n",
       "│ 75%        ┆ 1.770487e6    ┆ 14.0       │\n",
       "│ max        ┆ 2.360649e6    ┆ 285698.0   │\n",
       "└────────────┴───────────────┴────────────┘"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "books.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a461daae-3a78-4a48-af1f-9a52f2fb406c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>statistic</th><th>book_id</th><th>is_read</th></tr><tr><td>str</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;count&quot;</td><td>125672.0</td><td>125672.0</td></tr><tr><td>&quot;null_count&quot;</td><td>0.0</td><td>0.0</td></tr><tr><td>&quot;mean&quot;</td><td>168269.505888</td><td>703.613987</td></tr><tr><td>&quot;std&quot;</td><td>160942.264963</td><td>3616.45257</td></tr><tr><td>&quot;min&quot;</td><td>3.0</td><td>100.0</td></tr><tr><td>&quot;25%&quot;</td><td>50370.0</td><td>138.0</td></tr><tr><td>&quot;50%&quot;</td><td>118719.0</td><td>214.0</td></tr><tr><td>&quot;75%&quot;</td><td>240830.0</td><td>438.0</td></tr><tr><td>&quot;max&quot;</td><td>1.510246e6</td><td>285698.0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9, 3)\n",
       "┌────────────┬───────────────┬────────────┐\n",
       "│ statistic  ┆ book_id       ┆ is_read    │\n",
       "│ ---        ┆ ---           ┆ ---        │\n",
       "│ str        ┆ f64           ┆ f64        │\n",
       "╞════════════╪═══════════════╪════════════╡\n",
       "│ count      ┆ 125672.0      ┆ 125672.0   │\n",
       "│ null_count ┆ 0.0           ┆ 0.0        │\n",
       "│ mean       ┆ 168269.505888 ┆ 703.613987 │\n",
       "│ std        ┆ 160942.264963 ┆ 3616.45257 │\n",
       "│ min        ┆ 3.0           ┆ 100.0      │\n",
       "│ 25%        ┆ 50370.0       ┆ 138.0      │\n",
       "│ 50%        ┆ 118719.0      ┆ 214.0      │\n",
       "│ 75%        ┆ 240830.0      ┆ 438.0      │\n",
       "│ max        ┆ 1.510246e6    ┆ 285698.0   │\n",
       "└────────────┴───────────────┴────────────┘"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "books = books.filter(ps.col('is_read')>=100)\n",
    "books.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "42cbfbf3-8bf2-4a63-986b-2903d957fb13",
   "metadata": {},
   "outputs": [],
   "source": [
    "book_ids = books.select(ps.col('book_id')).sample(n=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "4c525080-a959-47b0-81f3-374c1b9995f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1_000, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>book_id</th></tr><tr><td>i64</td></tr></thead><tbody><tr><td>323461</td></tr><tr><td>170046</td></tr><tr><td>101</td></tr><tr><td>164649</td></tr><tr><td>162702</td></tr><tr><td>&hellip;</td></tr><tr><td>742248</td></tr><tr><td>415406</td></tr><tr><td>223497</td></tr><tr><td>6204</td></tr><tr><td>65013</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1_000, 1)\n",
       "┌─────────┐\n",
       "│ book_id │\n",
       "│ ---     │\n",
       "│ i64     │\n",
       "╞═════════╡\n",
       "│ 323461  │\n",
       "│ 170046  │\n",
       "│ 101     │\n",
       "│ 164649  │\n",
       "│ 162702  │\n",
       "│ …       │\n",
       "│ 742248  │\n",
       "│ 415406  │\n",
       "│ 223497  │\n",
       "│ 6204    │\n",
       "│ 65013   │\n",
       "└─────────┘"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "book_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "9c875f15-f32b-4de6-94f9-f037c64fab3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "book_ids.write_csv('books_subset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "163743bf-94f8-41a1-b4d5-499122ae7a49",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.filter((ps.col('book_id').is_in(books.select(ps.col('book_id')))) & (ps.col('user_id').is_in(users.select(ps.col('user_id')))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "ef21d2b6-2f28-4604-9511-745c9d172955",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.write_csv('interactions_subset_large.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d2d1328c-992e-417c-b16e-2edf219415d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = ps.read_csv('interactions_subset_large.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7fc75029-a94e-4fbd-a5f4-4611bc20e701",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.select([ps.col(x) for x in ['user_id', 'book_id', 'rating']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f150d8c2-f407-46d3-9139-f53599d6cc35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>user_id</th><th>book_id</th><th>rating</th></tr><tr><td>i64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>0</td><td>948</td><td>5</td></tr><tr><td>0</td><td>947</td><td>5</td></tr><tr><td>0</td><td>946</td><td>5</td></tr><tr><td>0</td><td>945</td><td>5</td></tr><tr><td>0</td><td>944</td><td>5</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌─────────┬─────────┬────────┐\n",
       "│ user_id ┆ book_id ┆ rating │\n",
       "│ ---     ┆ ---     ┆ ---    │\n",
       "│ i64     ┆ i64     ┆ i64    │\n",
       "╞═════════╪═════════╪════════╡\n",
       "│ 0       ┆ 948     ┆ 5      │\n",
       "│ 0       ┆ 947     ┆ 5      │\n",
       "│ 0       ┆ 946     ┆ 5      │\n",
       "│ 0       ┆ 945     ┆ 5      │\n",
       "│ 0       ┆ 944     ┆ 5      │\n",
       "└─────────┴─────────┴────────┘"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "28103a5a-18cb-4c7d-bbe6-6ceed1a73080",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "125672"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.select(ps.col('book_id')).n_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "732e4e96-a2df-4b9b-80ae-a1956cc35b26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "610076"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.select(ps.col('user_id')).n_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "93d02d96-77aa-47e8-8d5d-72519aff85d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.with_columns(ps.col('rating').cast(ps.Int8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "50ffaa48-7b10-4fb7-be51-05b20eb59b20",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (161_666_559, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>user_id</th><th>book_id</th><th>rating</th></tr><tr><td>i64</td><td>i64</td><td>i8</td></tr></thead><tbody><tr><td>0</td><td>948</td><td>5</td></tr><tr><td>0</td><td>947</td><td>5</td></tr><tr><td>0</td><td>946</td><td>5</td></tr><tr><td>0</td><td>945</td><td>5</td></tr><tr><td>0</td><td>944</td><td>5</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>876144</td><td>38802</td><td>0</td></tr><tr><td>876144</td><td>38968</td><td>0</td></tr><tr><td>876144</td><td>23847</td><td>4</td></tr><tr><td>876144</td><td>23950</td><td>3</td></tr><tr><td>876144</td><td>374106</td><td>5</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (161_666_559, 3)\n",
       "┌─────────┬─────────┬────────┐\n",
       "│ user_id ┆ book_id ┆ rating │\n",
       "│ ---     ┆ ---     ┆ ---    │\n",
       "│ i64     ┆ i64     ┆ i8     │\n",
       "╞═════════╪═════════╪════════╡\n",
       "│ 0       ┆ 948     ┆ 5      │\n",
       "│ 0       ┆ 947     ┆ 5      │\n",
       "│ 0       ┆ 946     ┆ 5      │\n",
       "│ 0       ┆ 945     ┆ 5      │\n",
       "│ 0       ┆ 944     ┆ 5      │\n",
       "│ …       ┆ …       ┆ …      │\n",
       "│ 876144  ┆ 38802   ┆ 0      │\n",
       "│ 876144  ┆ 38968   ┆ 0      │\n",
       "│ 876144  ┆ 23847   ┆ 4      │\n",
       "│ 876144  ┆ 23950   ┆ 3      │\n",
       "│ 876144  ┆ 374106  ┆ 5      │\n",
       "└─────────┴─────────┴────────┘"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ec6e3453-babf-48ce-9645-39f4a372e181",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>statistic</th><th>user_id</th><th>book_id</th><th>rating</th></tr><tr><td>str</td><td>f64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;count&quot;</td><td>1.61666559e8</td><td>1.61666559e8</td><td>1.61666559e8</td></tr><tr><td>&quot;null_count&quot;</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>&quot;mean&quot;</td><td>245188.905232</td><td>62067.768136</td><td>1.991295</td></tr><tr><td>&quot;std&quot;</td><td>165654.703148</td><td>97845.859061</td><td>2.09733</td></tr><tr><td>&quot;min&quot;</td><td>0.0</td><td>3.0</td><td>0.0</td></tr><tr><td>&quot;25%&quot;</td><td>112520.0</td><td>7161.0</td><td>0.0</td></tr><tr><td>&quot;50%&quot;</td><td>231378.0</td><td>21266.0</td><td>1.0</td></tr><tr><td>&quot;75%&quot;</td><td>354887.0</td><td>73911.0</td><td>4.0</td></tr><tr><td>&quot;max&quot;</td><td>876144.0</td><td>1.510246e6</td><td>5.0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9, 4)\n",
       "┌────────────┬───────────────┬──────────────┬──────────────┐\n",
       "│ statistic  ┆ user_id       ┆ book_id      ┆ rating       │\n",
       "│ ---        ┆ ---           ┆ ---          ┆ ---          │\n",
       "│ str        ┆ f64           ┆ f64          ┆ f64          │\n",
       "╞════════════╪═══════════════╪══════════════╪══════════════╡\n",
       "│ count      ┆ 1.61666559e8  ┆ 1.61666559e8 ┆ 1.61666559e8 │\n",
       "│ null_count ┆ 0.0           ┆ 0.0          ┆ 0.0          │\n",
       "│ mean       ┆ 245188.905232 ┆ 62067.768136 ┆ 1.991295     │\n",
       "│ std        ┆ 165654.703148 ┆ 97845.859061 ┆ 2.09733      │\n",
       "│ min        ┆ 0.0           ┆ 3.0          ┆ 0.0          │\n",
       "│ 25%        ┆ 112520.0      ┆ 7161.0       ┆ 0.0          │\n",
       "│ 50%        ┆ 231378.0      ┆ 21266.0      ┆ 1.0          │\n",
       "│ 75%        ┆ 354887.0      ┆ 73911.0      ┆ 4.0          │\n",
       "│ max        ┆ 876144.0      ┆ 1.510246e6   ┆ 5.0          │\n",
       "└────────────┴───────────────┴──────────────┴──────────────┘"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9de0ff46-c1c0-42d9-b599-a0a5396fd267",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.with_columns([ps.col(column).cast(ps.Int32) for column in ['user_id', 'book_id']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "edeca869-3ea5-4a88-a256-0d1134c559b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>user_id</th><th>book_id</th><th>rating</th></tr><tr><td>i32</td><td>i32</td><td>i8</td></tr></thead><tbody><tr><td>0</td><td>948</td><td>5</td></tr><tr><td>0</td><td>947</td><td>5</td></tr><tr><td>0</td><td>946</td><td>5</td></tr><tr><td>0</td><td>945</td><td>5</td></tr><tr><td>0</td><td>944</td><td>5</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌─────────┬─────────┬────────┐\n",
       "│ user_id ┆ book_id ┆ rating │\n",
       "│ ---     ┆ ---     ┆ ---    │\n",
       "│ i32     ┆ i32     ┆ i8     │\n",
       "╞═════════╪═════════╪════════╡\n",
       "│ 0       ┆ 948     ┆ 5      │\n",
       "│ 0       ┆ 947     ┆ 5      │\n",
       "│ 0       ┆ 946     ┆ 5      │\n",
       "│ 0       ┆ 945     ┆ 5      │\n",
       "│ 0       ┆ 944     ┆ 5      │\n",
       "└─────────┴─────────┴────────┘"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1863a7b3-b46a-49cf-92a8-c11d3d7bc026",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.pivot(index='user_id', columns='book_id', values='rating')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e0bdce16-4026-4860-a040-f96de1c12432",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pathlib\n",
    "path: pathlib.Path = \"ratings_subset_large.parquet\"\n",
    "df.write_parquet('ratings_subset_large.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3ef74e77-dccf-4019-88a3-9130bab30456",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 125_673)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>user_id</th><th>948</th><th>947</th><th>946</th><th>945</th><th>944</th><th>943</th><th>942</th><th>941</th><th>940</th><th>939</th><th>938</th><th>937</th><th>936</th><th>935</th><th>933</th><th>932</th><th>931</th><th>930</th><th>929</th><th>928</th><th>927</th><th>924</th><th>923</th><th>921</th><th>920</th><th>919</th><th>918</th><th>916</th><th>915</th><th>914</th><th>912</th><th>909</th><th>908</th><th>907</th><th>906</th><th>905</th><th>&hellip;</th><th>1168157</th><th>927527</th><th>1181800</th><th>1183906</th><th>1184277</th><th>1184258</th><th>1185901</th><th>1185862</th><th>1188021</th><th>1190981</th><th>1204475</th><th>1215785</th><th>1218934</th><th>1221111</th><th>1223627</th><th>1223611</th><th>1225459</th><th>1226479</th><th>1227128</th><th>1229072</th><th>1234489</th><th>1238445</th><th>1238830</th><th>1240358</th><th>1243608</th><th>1248448</th><th>1274292</th><th>1295869</th><th>1303636</th><th>1307598</th><th>1333095</th><th>1340190</th><th>1352127</th><th>1380905</th><th>1415581</th><th>1451636</th><th>1510246</th></tr><tr><td>i32</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>&hellip;</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td></tr></thead><tbody><tr><td>0</td><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>4</td><td>4</td><td>4</td><td>4</td><td>4</td><td>5</td><td>2</td><td>4</td><td>4</td><td>5</td><td>5</td><td>5</td><td>4</td><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>4</td><td>5</td><td>4</td><td>4</td><td>0</td><td>0</td><td>4</td><td>&hellip;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td></tr><tr><td>1</td><td>null</td><td>null</td><td>null</td><td>null</td><td>4</td><td>5</td><td>null</td><td>5</td><td>null</td><td>4</td><td>4</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>&hellip;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td></tr><tr><td>2</td><td>null</td><td>5</td><td>null</td><td>null</td><td>null</td><td>3</td><td>null</td><td>3</td><td>null</td><td>3</td><td>3</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>3</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>&hellip;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td></tr><tr><td>3</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>&hellip;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td></tr><tr><td>4</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>&hellip;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 125_673)\n",
       "┌─────────┬──────┬──────┬──────┬───┬─────────┬─────────┬─────────┬─────────┐\n",
       "│ user_id ┆ 948  ┆ 947  ┆ 946  ┆ … ┆ 1380905 ┆ 1415581 ┆ 1451636 ┆ 1510246 │\n",
       "│ ---     ┆ ---  ┆ ---  ┆ ---  ┆   ┆ ---     ┆ ---     ┆ ---     ┆ ---     │\n",
       "│ i32     ┆ i8   ┆ i8   ┆ i8   ┆   ┆ i8      ┆ i8      ┆ i8      ┆ i8      │\n",
       "╞═════════╪══════╪══════╪══════╪═══╪═════════╪═════════╪═════════╪═════════╡\n",
       "│ 0       ┆ 5    ┆ 5    ┆ 5    ┆ … ┆ null    ┆ null    ┆ null    ┆ null    │\n",
       "│ 1       ┆ null ┆ null ┆ null ┆ … ┆ null    ┆ null    ┆ null    ┆ null    │\n",
       "│ 2       ┆ null ┆ 5    ┆ null ┆ … ┆ null    ┆ null    ┆ null    ┆ null    │\n",
       "│ 3       ┆ null ┆ null ┆ null ┆ … ┆ null    ┆ null    ┆ null    ┆ null    │\n",
       "│ 4       ┆ null ┆ null ┆ null ┆ … ┆ null    ┆ null    ┆ null    ┆ null    │\n",
       "└─────────┴──────┴──────┴──────┴───┴─────────┴─────────┴─────────┴─────────┘"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3168a59a-2853-412f-ad76-7a4bc9a73bce",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sparse = df.drop('user_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8df436a9-e0ed-40ba-bfc4-38c940f407a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f2837492-c98e-4189-8bdb-9fea11ab9d42",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 125_672)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>948</th><th>947</th><th>946</th><th>945</th><th>944</th><th>943</th><th>942</th><th>941</th><th>940</th><th>939</th><th>938</th><th>937</th><th>936</th><th>935</th><th>933</th><th>932</th><th>931</th><th>930</th><th>929</th><th>928</th><th>927</th><th>924</th><th>923</th><th>921</th><th>920</th><th>919</th><th>918</th><th>916</th><th>915</th><th>914</th><th>912</th><th>909</th><th>908</th><th>907</th><th>906</th><th>905</th><th>904</th><th>&hellip;</th><th>1168157</th><th>927527</th><th>1181800</th><th>1183906</th><th>1184277</th><th>1184258</th><th>1185901</th><th>1185862</th><th>1188021</th><th>1190981</th><th>1204475</th><th>1215785</th><th>1218934</th><th>1221111</th><th>1223627</th><th>1223611</th><th>1225459</th><th>1226479</th><th>1227128</th><th>1229072</th><th>1234489</th><th>1238445</th><th>1238830</th><th>1240358</th><th>1243608</th><th>1248448</th><th>1274292</th><th>1295869</th><th>1303636</th><th>1307598</th><th>1333095</th><th>1340190</th><th>1352127</th><th>1380905</th><th>1415581</th><th>1451636</th><th>1510246</th></tr><tr><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>&hellip;</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td></tr></thead><tbody><tr><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>4</td><td>4</td><td>4</td><td>4</td><td>4</td><td>5</td><td>2</td><td>4</td><td>4</td><td>5</td><td>5</td><td>5</td><td>4</td><td>5</td><td>5</td><td>5</td><td>5</td><td>5</td><td>4</td><td>5</td><td>4</td><td>4</td><td>0</td><td>0</td><td>4</td><td>0</td><td>&hellip;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td></tr><tr><td>null</td><td>null</td><td>null</td><td>null</td><td>4</td><td>5</td><td>null</td><td>5</td><td>null</td><td>4</td><td>4</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>&hellip;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td></tr><tr><td>null</td><td>5</td><td>null</td><td>null</td><td>null</td><td>3</td><td>null</td><td>3</td><td>null</td><td>3</td><td>3</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>3</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>&hellip;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td></tr><tr><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>&hellip;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td></tr><tr><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>&hellip;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 125_672)\n",
       "┌──────┬──────┬──────┬──────┬───┬─────────┬─────────┬─────────┬─────────┐\n",
       "│ 948  ┆ 947  ┆ 946  ┆ 945  ┆ … ┆ 1380905 ┆ 1415581 ┆ 1451636 ┆ 1510246 │\n",
       "│ ---  ┆ ---  ┆ ---  ┆ ---  ┆   ┆ ---     ┆ ---     ┆ ---     ┆ ---     │\n",
       "│ i8   ┆ i8   ┆ i8   ┆ i8   ┆   ┆ i8      ┆ i8      ┆ i8      ┆ i8      │\n",
       "╞══════╪══════╪══════╪══════╪═══╪═════════╪═════════╪═════════╪═════════╡\n",
       "│ 5    ┆ 5    ┆ 5    ┆ 5    ┆ … ┆ null    ┆ null    ┆ null    ┆ null    │\n",
       "│ null ┆ null ┆ null ┆ null ┆ … ┆ null    ┆ null    ┆ null    ┆ null    │\n",
       "│ null ┆ 5    ┆ null ┆ null ┆ … ┆ null    ┆ null    ┆ null    ┆ null    │\n",
       "│ null ┆ null ┆ null ┆ null ┆ … ┆ null    ┆ null    ┆ null    ┆ null    │\n",
       "│ null ┆ null ┆ null ┆ null ┆ … ┆ null    ┆ null    ┆ null    ┆ null    │\n",
       "└──────┴──────┴──────┴──────┴───┴─────────┴─────────┴─────────┴─────────┘"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sparse.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5cc6377-7596-44de-84cd-13935434d99a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sparse = df_sparse.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "799409c5-6d97-4615-a6a8-204a99b87877",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sparse"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}