20 - PDF Analyzer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "959ffae8",
   "metadata": {},
   "source": [
    "# PDF Analyzer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b685aac0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install pinecone-client\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8a95ca0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import os \n",
    "from langchain.llms import AzureOpenAI\n",
    "import pinecone\n",
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain.vectorstores import Pinecone\n",
    "from langchain.chains.question_answering import load_qa_chain\n",
    "from langchain.document_loaders import PyPDFLoader\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()\n",
    "openai.api_key = os.getenv('OPENAI_API_KEY')\n",
    "openai.api_base= os.getenv('OPENAI_API_BASE')\n",
    "openai.api_type= \"azure\"\n",
    "openai.api_version = os.getenv('OPENAI_API_VERSION')\n",
    "deployment:str=os.getenv('CHATGPT_MODEL')\n",
    "embeddings_deployment:str=os.getenv('EMBEDDINGS_MODEL')\n",
    "\n",
    "llm = AzureOpenAI(\n",
    "    deployment_name=deployment,\n",
    "    model_name=deployment\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a35095c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "PINECONE_API_KEY = os.getenv('PINECONE_API_KEY')\n",
    "PINECONE_ENV = os.getenv('PINECONE_ENV')\n",
    "print(PINECONE_ENV)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "633f61d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "loader = PyPDFLoader(\"gen_ai.pdf\")\n",
    "data = loader.load()\n",
    "pages = len(data)\n",
    "pdf_content = ''\n",
    "\n",
    "for x in range(pages):\n",
    "    pdf_content = pdf_content + data[x].page_content\n",
    "\n",
    "print(pdf_content)    \n",
    "\n",
    "import pinecone      \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76e338d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size = 2000,\n",
    "    chunk_overlap = 0,\n",
    "    length_function = len\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21d071a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "split_pdf_content = text_splitter.create_documents([pdf_content])\n",
    "print(len(split_pdf_content))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d81da4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(split_pdf_content[3])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8f36e3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "pinecone.init(\n",
    "    api_key = PINECONE_API_KEY,\n",
    "    environment = PINECONE_ENV\n",
    ")\n",
    "\n",
    "index_name = 'langchainclass'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0e0fb72",
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = OpenAIEmbeddings(\n",
    "    deployment=embeddings_deployment,\n",
    "    model=embeddings_deployment,\n",
    "    engine=embeddings_deployment,\n",
    "    chunk_size=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6085671d",
   "metadata": {},
   "outputs": [],
   "source": [
    "if index_name not in pinecone.list_indexes():\n",
    "    print(\"index does not exist\", index_name)\n",
    "\n",
    "pdf_docsearch = Pinecone.from_texts(\n",
    "    [t.page_content for t in split_pdf_content],\n",
    "    embeddings,\n",
    "    index_name = index_name\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97d7838a",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"Who is Sarah Guo?\"\n",
    "\n",
    "docs = pdf_docsearch.similarity_search(question)\n",
    "\n",
    "\n",
    "chain = load_qa_chain(llm, chain_type=\"stuff\")\n",
    "chain.run(input_documents=docs, question=question)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a9965c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"Who is John Smith?\"\n",
    "\n",
    "docs = pdf_docsearch.similarity_search(question)\n",
    "\n",
    "\n",
    "chain = load_qa_chain(llm, chain_type=\"stuff\")\n",
    "chain.run(input_documents=docs, question=question)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71677c0e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}