Support ingesting DOCX files using pandoc

Dockerfile

@@ -51,7 +51,7 @@ FROM base
 
 # Install packages needed for deployment
 RUN apt-get update -qq && \
-    apt-get install --no-install-recommends -y curl libsqlite3-0 libvips netcat-traditional libpq5 poppler-utils && \
+    apt-get install --no-install-recommends -y curl libsqlite3-0 libvips netcat-traditional libpq5 poppler-utils pandoc && \
     rm -rf /var/lib/apt/lists /var/cache/apt/archives
 
 # Copy built artifacts: gems, application

app/services/parsers.rb

@@ -2,6 +2,7 @@ module Parsers
   def self.parser_for(filename)
     name_locase = filename.downcase
     return Parsers::Pdf if name_locase.end_with?(".pdf")
+    return Parsers::Docx if name_locase.end_with?(".docx")
     return Parsers::Text if name_locase.end_with?(".txt", ".html", ".md")
 
     raise StandardError, "Unsupported file extension: '#{filename.slice(/\.\w+$/)}'"

app/services/parsers/docx.rb

@@ -0,0 +1,22 @@
+module Parsers
+  class Docx
+    include BasicTextChunker
+
+    def initialize(document)
+      @document = document
+    end
+
+    def text
+      # NOTE: Using -raw opt causes text to be broken up a lot; but not using raw
+      #       may cause tables to be "pretty" in text which may not be ideal for chunking.
+      #       Not specifying works best for rotated pages, so doing that for now
+      cmd = 'pandoc -f docx  --to=commonmark -'
+      txt, serr, status = Open3.capture3(cmd, stdin_data: @document.contents, binmode: true)
+      return txt if status.success?
+
+      Rails.logger.error("Error running '#{cmd}' on DOCX: #{@document.filename}\n#{serr}")
+
+      raise StandardError, "Error converting DPCX to text: #{@document.filename}'"
+    end
+  end
+end

ops.yml

@@ -2,6 +2,7 @@ dependencies:
   brew:
     - overmind
     - poppler
+    - pandoc
   custom:
     - bundle config --local path vendor/bundle
     - bundle config set --local build.pg ${PG_OPTS}