mana_manee/reading_and_visualizing_books.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "protagonists = ['ปิติ', 'มานะ', 'มานี', 'ชูใจ', 'วีระ', 'เพชร', 'กล้า', 'สีเทา', 'โต', 'ไพลิน']\n",
    "protagonists_dict = {v: k for k, v in enumerate(protagonists)}\n",
    "\n",
    "def read_book(file_path, page_start='ป.4 เล่ม 1'):\n",
    "    f = open(file_path)\n",
    "    lines = f.readlines()\n",
    "    lines = [line.strip() for line in lines if not line.strip() == '']\n",
    "    pages = []\n",
    "    page = []\n",
    "    page.append(lines[0])\n",
    "    for line in lines[1::]:\n",
    "        if page_start in line:\n",
    "            pages.append(page)\n",
    "            page = []\n",
    "        page.append(line)\n",
    "    return pages\n",
    "\n",
    "def count_characters(pages):\n",
    "    \"\"\"\n",
    "    Count characters\n",
    "    \"\"\"\n",
    "    d = defaultdict(int)\n",
    "    characters_all_pages = []\n",
    "    for page in pages:\n",
    "        page_text = ' '.join(page)\n",
    "        characters = []\n",
    "        for i, p in enumerate(protagonists):\n",
    "            if p in page_text:\n",
    "                d[p] += 1\n",
    "                characters.append(p)\n",
    "        if len(characters) >= 2:\n",
    "            characters_all_pages.append(characters)\n",
    "    return d, characters_all_pages\n",
    "\n",
    "def create_network_matrix(characters_all_pages):\n",
    "    \"\"\"\n",
    "    Create network of \n",
    "    \"\"\"\n",
    "    n = len(protagonists)\n",
    "    W = np.zeros((n, n))\n",
    "    for characters in characters_all_pages:\n",
    "        for c1, c2 in itertools.permutations(characters, 2):\n",
    "            W[protagonists_dict[c1], protagonists_dict[c2]] += 1\n",
    "    W = pd.DataFrame(W, index=protagonists, columns=protagonists).astype(int)\n",
    "    return W"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = ['ปอ 1 เล่ม 1', 'ปอ 1 เล่ม 2', \n",
    "           'ป.๒', 'ป.๒', 'ป. ๓ เล่ม ๑', \n",
    "           'ป.3', 'ป.4 เล่ม 1', 'ป.๔ เล่ม ๒', \n",
    "           'ป.๕ เล่ม ๑', 'ป.๕ เล่ม ๒', \n",
    "           'ป.๖ เล่ม', 'ป.6 ล.2']\n",
    "path_tuple = list(zip(sorted(glob('books/*.txt')), headers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "book_titles = []\n",
    "for i in range(1, 7):\n",
    "    for j in range(1, 3):\n",
    "        book_titles.append('ป.%s เล่ม %s' % (str(i), str(j)))\n",
    "\n",
    "protagonists_ = [\n",
    "    'ปิติ',\n",
    "    'มานะ',\n",
    "    'มานี',\n",
    "    'ชูใจ',\n",
    "    'วีระ',\n",
    "    'เพชร',\n",
    "    'กล้า',\n",
    "    'สีเทา',\n",
    "    'โต',\n",
    "    'ครูไพลิน'\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Calculating occurence**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "social = []\n",
    "pages_list = []\n",
    "for file_path, line_header in path_tuple:\n",
    "    pages = read_book(file_path, line_header)\n",
    "    d, characters_all_pages = count_characters(pages)\n",
    "    d = dict(d)\n",
    "    W = create_network_matrix(characters_all_pages)\n",
    "    social.append([file_path, line_header, d, characters_all_pages, W])\n",
    "    pages_list.append(pages)\n",
    "O_df = pd.DataFrame([s[2] for s in social]).fillna(0).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lightning import Lightning\n",
    "lgn = Lightning(local=True, ipython=True)\n",
    "\n",
    "viz = lgn.matrix(O_df.as_matrix(), colormap='Reds', \n",
    "                 column_labels=list(O_df.columns), \n",
    "                 row_labels=book_titles,\n",
    "                 numbers=True, width=500, height=700)\n",
    "viz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Example of social network**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# book number 11 >> ป. 5 เล่ม 1\n",
    "viz = lgn.matrix(social[11][-1].as_matrix(), colormap='Reds', \n",
    "                 row_labels=protagonists_, \n",
    "                 column_labels=protagonists_, \n",
    "                 numbers=True, width=500, height=500)\n",
    "viz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Issue of the HTML file\n",
    "\n",
    "What make it does not render properly within `{% include .html %}` is at \n",
    "\n",
    "```\n",
    "...<div id=A5YE3C47OE data-type=\"matrix\"...\n",
    "```\n",
    "\n",
    "`id` has no quote `\"`.\n",
    "\n",
    "## What to do\n",
    "\n",
    "Find `id` with no quote and quote them, that's it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "# This function expects only 1 occurrence of un-quoted id\n",
    "# if it errors in the future, check this function.\n",
    "\n",
    "def quoteID(baseString):\n",
    "    tmpRe = re.findall('id=(\\w+)', baseString)[0]\n",
    "    return baseString.replace(tmpRe, '\"{0}\"'.format(tmpRe))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Suggestion\n",
    "\n",
    "- wrap this as module :p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "js = viz.load_embed()\n",
    "base = viz._html\n",
    "\n",
    "postName = !basename $(pwd)\n",
    "# magic return list\n",
    "postName = postName[0]\n",
    "\n",
    "plotName = 'plots/example_plot'\n",
    "\n",
    "# use this quotedBase instead of unquoted\n",
    "quotedBase = quoteID(base)\n",
    "\n",
    "# append ref to js into html first\n",
    "quotedBase+='\\n<script src=\"https://rawgit.com/tupleblog/tuple_code/master/{0}/{1}.js\"></script>'.format(postName, plotName)\n",
    "\n",
    "with open(\"./{0}.html\".format(plotName), \"wb\") as f:\n",
    "    f.write(quotedBase.encode('utf-8'))\n",
    "    \n",
    "with open(\"./{0}.js\".format(plotName), \"wb\") as f:\n",
    "    f.write(js.encode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}