explore_text.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "there are 270 txt files in C:\\Users\\amrul\\programming\\various_projects\\scribd_parsing\\steven_king_books\\doctor_sleep\n"
     ]
    }
   ],
   "source": [
    "import pathlib\n",
    "\n",
    "author_name=\"steven_king_books\"\n",
    "book_name=\"doctor_sleep\"\n",
    "folder = pathlib.Path(r\"C:\\Users\\amrul\\programming\\various_projects\\scribd_parsing\")/author_name/book_name\n",
    "files = [file for file in folder.iterdir() if \"txt\" in file.suffix]\n",
    "print(f\"there are {len(files)} txt files in {folder}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_number(number):\n",
    "    return f\"{number:,.2f}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all_text length is 931,301.00\n"
     ]
    }
   ],
   "source": [
    "all_text_list = [file.read_text(encoding=\"utf-8\") for file in files]\n",
    "all_text = \" \".join(all_text_list)\n",
    "print(f\"all_text length is {format_number(len(all_text))}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "if you were to whisper all text you would pay : 13.97$\n"
     ]
    }
   ],
   "source": [
    "# trying to estimate how much would it cost me if I were to use OpenAI Whisper TTS model to convert the whole text into speech\n",
    "price_per_thousand_chars=0.015\n",
    "total_price = (len(all_text)/1000)*price_per_thousand_chars\n",
    "print(f\"if you were to whisper all text you would pay : {format_number(total_price)}$\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found 1,004.00 matches\n"
     ]
    }
   ],
   "source": [
    "# I know that some words in my text are split by hyphen to leak to next page. \n",
    "# I want to find out how many cases are there\n",
    "import re\n",
    "all_hyphen_words = re.findall(\"\\w+\\-\\s+\\w+\",all_text)\n",
    "print(f\"found {format_number(len(all_hyphen_words))} matches\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this matches two words separated with hyphen and spaces\n",
    "# it will utilize matching groups to concatenate two groups represented by two words\n",
    "repaired_text = re.sub(r\"(\\w+)\\-\\s+(\\w+)\",r\"\\1\\2\",all_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove duplicate spaces\n",
    "repaired_text = re.sub(r\"\\s+\",\" \",repaired_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"repaired text length is {format_number(len(repaired_text))}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's try to use nltk sentence tokenizer to tokenize our text into sentences\n",
    "import nltk\n",
    "from nltk.tokenize import sent_tokenize\n",
    "\n",
    "nltk.download('punkt')\n",
    "\n",
    "sentences = sent_tokenize(repaired_text)\n",
    "\n",
    "print(f\"tokenized into total of {format_number(len(sentences))} sentences\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\n\".join(sentences[200:300]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let's extract nouns only with nltk and see their frequencies in the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nltk.download('punkt')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_of_words = [word_tokenize(sentence) for sentence in sentences]\n",
    "flat_words = [word.lower() for sublist in list_of_words for word in sublist if word.isalpha()]\n",
    "print(f\"extracted total of {format_number(len(flat_words))} flat words\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = set(stopwords.words('english'))\n",
    "print(f\"there are total of {len(stop_words)} stop words in english\")\n",
    "filtered_words = [word for word in flat_words if word not in stop_words]\n",
    "print(f\"I filtered {format_number(len(filtered_words))} non stopwords\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"some filtered words : {filtered_words[:10]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tagged_words = nltk.pos_tag(filtered_words)\n",
    "print(f\"tagged words have length {format_number(len(tagged_words))}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nouns = [word for word,tag in tagged_words if tag in ('NN','NNS','NNP','NNPS')]\n",
    "print(f\"extracted {format_number(len(nouns))} nouns\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "noun_freq = Counter(nouns)\n",
    "num=100\n",
    "print(f\"{num} most frequent nouns : {noun_freq.most_common(num)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "all_words = all_text.split()\n",
    "word_counter=Counter(all_words)\n",
    "num=100\n",
    "print(f\"counter has {len(word_counter)} elements vs all words length of {len(all_words)} with ratio {len(word_counter)/len(all_words)}\")\n",
    "print(f\"most frequent {num} words : {word_counter.most_common(num)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_line_splits = all_text.split(\"\\n\")\n",
    "print(f\"new line splits count : {len(new_line_splits)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "sentences = re.split(\"[.!?]\",all_text)\n",
    "print(f\"total of {len(sentences)} sentences\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_chars = list(set(list(all_text)))\n",
    "print(f\"total of {len(all_chars)} distinct characters in the text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allcharcounter = Counter(list(all_text))\n",
    "for key,val in allcharcounter.most_common(len(allcharcounter)):\n",
    "    print(f\"{key} : {val}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dl_311_venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}