OCR_PDFs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check your jupyter notebook is in the correct directory\n",
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## libraries\n",
    "import os, ocrmypdf, time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## OCR program parameters\n",
    "\n",
    "keep_processed_PDFs = True  # If TRUE, when the program finds a 'processed' version of the original PDF \n",
    "                            # it will skip the original PDF and not overwrite the existing 'processed' PDF.\n",
    "                            # If FALSE, will overwrite all processed PDFs files for all original PDFs. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Stationkeeping variables\n",
    "starting_time = time.time() # time program started\n",
    "files_in_directory = !ls    # list of all files in directory\n",
    "directory = !pwd            # full directory name and path \n",
    "PDF_processed_counter = 0   # Number of PDFs processed\n",
    "PDF_examined_counter = 0    # Number of PDFs examined, including PDFs processed\n",
    "\n",
    "## Output number and location of PDFs at script start\n",
    "total_number_of_PDFs = !ls | grep '\\.pdf$' | wc -l\n",
    "processed_number_of_PDFs = !ls | grep '\\.processed.pdf$' | wc -l\n",
    "PDFs_to_OCR = str(int(total_number_of_PDFs[0]) - int(processed_number_of_PDFs[0]))\n",
    "if keep_processed_PDFs == True:\n",
    "  PDFs_to_OCR = str(int(total_number_of_PDFs[0]) - int(processed_number_of_PDFs[0])*2)\n",
    "  print('OCRing ' + PDFs_to_OCR + ' PDFs in ' + directory[0] + '\\n')\n",
    "else:    \n",
    "  print('OCRing ' + PDFs_to_OCR + ' PDFs in ' + directory[0] + '\\n')\n",
    "\n",
    "\n",
    "## Execute OCR on PDF files\n",
    "for filename in files_in_directory:    \n",
    "    ## only look at non-processed PDF files \n",
    "    if os.path.splitext(filename)[1] == '.pdf' and os.path.splitext(filename)[0][-10:] != '.processed': \n",
    "        running_time = (time.time()-starting_time)/60\n",
    "        print('PDF ' + str(PDF_examined_counter+1) +': '+ filename + ', '+ str(running_time) + ' minutes.')\n",
    "        \n",
    "        ## generate filenames for PDF  \n",
    "        processed_filename = os.path.splitext(filename)[0] + '.processed.pdf'\n",
    "        original_OCR_txt_filename = os.path.splitext(filename)[0] + '.original.txt'\n",
    "        processed_OCR_txt_filename = os.path.splitext(filename)[0] + '.ocr.txt'\n",
    "        \n",
    "        ## Case 1: Ignore PDFs we have previously processed. Do not perform OCR on PDF.\n",
    "        ## keep_processed_PDFs==True\n",
    "        if os.path.isfile(processed_filename) == True and keep_processed_PDFs == True:\n",
    "            print('PDF already processed. keep_processed_PDFs = True. Moving to next PDF...\\n') \n",
    "            PDF_examined_counter+=1\n",
    "        \n",
    "        ## Case 2: Overwrite processed PDFs by reOCRing original PDF, or OCR PDF if not processed.\n",
    "        ## Save original OCR in unprocessed PDF, if it exists, to txt file. keep_processed_PDFs==False.\n",
    "        if (os.path.isfile(processed_filename) == True and keep_processed_PDFs == False) or os.path.isfile(processed_filename) == False:\n",
    "            try:\n",
    "                ## execute OCR on PDF, and catch exception if OCR text already exists\n",
    "                ocrmypdf.ocr(input_file = filename, output_file = processed_filename, \n",
    "                             deskew = True, \n",
    "                             clean_final = True, \n",
    "                             rotate_pages = True,\n",
    "                             remove_background = True, \n",
    "                             sidecar = processed_OCR_txt_filename)\n",
    "                PDF_processed_counter+=1\n",
    "                PDF_examined_counter+=1\n",
    "                \n",
    "                \n",
    "            except ocrmypdf.exceptions.PriorOcrFoundError as error:\n",
    "                ## Unprocessed PDF already has OCR and needs to be reOCRed \n",
    "                !pdftotext \"$filename\" \"$original_OCR_txt_filename\" \n",
    "                print('PDF already has OCR text. Saved original OCR text to a txt file.')\n",
    "                \n",
    "                               \n",
    "                ## OCR unprocessed PDF, & generate new processed PDF\n",
    "                print('Overwriting existing OCR, and saving in ' + processed_filename)\n",
    "                ocrmypdf.ocr(input_file = filename, output_file = processed_filename, \n",
    "                             deskew = True, \n",
    "                             clean_final = True, \n",
    "                             rotate_pages = True,\n",
    "                             remove_background = True, \n",
    "                             sidecar = processed_OCR_txt_filename,\n",
    "                             force_ocr = True)  \n",
    "                print('moving to next PDF....\\n')\n",
    "                PDF_processed_counter+=1\n",
    "                PDF_examined_counter+=1\n",
    "                \n",
    "# Concluding Stats                \n",
    "print('')\n",
    "print('Processed PDFs in ' + directory[0] + ' are OCRed.')\n",
    "print('PDFs processed: ' + str(PDF_processed_counter))\n",
    "print('PDFs examined, including processed: ' + str(PDF_examined_counter))\n",
    "running_time = (time.time()-starting_time)/60\n",
    "print('Program Completed in ' + str(running_time) + ' minutes.')"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "FOR REFERENCE ONLY. \n",
    "\n",
    "ocrmypdf.ocr(input_file: Union[BinaryIO, os.PathLike, str, bytes], \n",
    "             output_file: Union[BinaryIO, os.PathLike, str, bytes], *, \n",
    "             language: Iterable[str] = None, \n",
    "             image_dpi: int = None, \n",
    "             output_type=None, \n",
    "             sidecar: os.PathLike = None, \n",
    "             jobs: int = None, \n",
    "             use_threads: bool = None, \n",
    "             title: str = None, \n",
    "             author: str = None, \n",
    "             subject: str = None, \n",
    "             keywords: str = None, \n",
    "             rotate_pages: bool = None, \n",
    "             remove_background: bool = None, \n",
    "             deskew: bool = None, \n",
    "             clean: bool = None, \n",
    "             clean_final: bool = None, \n",
    "             unpaper_args: str = None, \n",
    "             oversample: int = None, \n",
    "             remove_vectors: bool = None, \n",
    "             threshold: bool = None, \n",
    "             force_ocr: bool = None, \n",
    "             skip_text: bool = None, \n",
    "             redo_ocr: bool = None, \n",
    "             skip_big: float = None, \n",
    "             optimize: int = None, \n",
    "             jpg_quality: int = None, \n",
    "             png_quality: int = None, \n",
    "             jbig2_lossy: bool = None, \n",
    "             jbig2_page_group_size: int = None, \n",
    "             pages: str = None, \n",
    "             max_image_mpixels: float = None, \n",
    "             tesseract_config: Iterable[str] = None, \n",
    "             tesseract_pagesegmode: int = None, \n",
    "             tesseract_oem: int = None, \n",
    "             pdf_renderer=None, \n",
    "             tesseract_timeout: float = None, \n",
    "             rotate_pages_threshold: float = None, \n",
    "            pdfa_image_compression=None, \n",
    "             user_words: os.PathLike = None, \n",
    "             user_patterns: os.PathLike = None, \n",
    "             fast_web_view: float = None, \n",
    "             plugins: Iterable[str] = None, \n",
    "             keep_temporary_files: bool = None, \n",
    "             progress_bar: bool = None, \n",
    "             **kwargs)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}