Improve fax backend (totallylegitco#115)

* ftp from inside a k8s cluster is... not ideal

* Addd migration

* Only include claim id and fax name if it's more than two char

.dockerignore

@@ -7,6 +7,7 @@ __pycache__
 __pycache__/
 *.py[cod]
 *$py.class
+*.pyc
 
 # C extensions
 *.so

CombinedDockerfile

@@ -7,6 +7,10 @@ FROM holdenk/ray:${RAY_VERSION}
 RUN sudo apt-get update && sudo apt-get upgrade -y && sudo apt-get install vim emacs libssl-dev python3-opencv libgl1 tesseract-ocr nano nfs-common iputils-ping hylafax-client build-essential python3-dev pkg-config pandoc texlive -y
 RUN sudo apt-get install -y libmariadb-dev-compat
 
+# Copy hylafax settings
+RUN mkdir -p /etc/hylafax
+COPY hylafax-settings/* /etc/hylafax
+
 # Installed seperately because only used in prod
 RUN pip install mysqlclient
 COPY requirements.txt ./

build.sh

@@ -8,7 +8,7 @@ mypy -p fighthealthinsurance
 ./manage.py collectstatic --no-input
 
 pushd ./static/js; npm i; npm run build; popd
-FHI_VERSION=v0.8.4a
+FHI_VERSION=v0.8.6d
 export FHI_VERSION
 # Build ray cluster first so that the cluster can come up before the job that registers the workers
 source build_ray.sh

cluster.yaml

@@ -18,7 +18,7 @@ spec:
       spec:
         affinity: {}
         containers:
-          - image: holdenk/fhi-ray:v0.8.4a
+          - image: holdenk/fhi-ray:v0.8.6d
             env:
               - name: RAY_gcs_server_request_timeout_seconds
                 value: "20"
@@ -30,21 +30,29 @@ spec:
             resources:
               limits:
                 cpu: "1"
-                memory: 4G
+                memory: 13G
               requests:
                 cpu: "1"
-                memory: 4G
+                memory: 12G
             volumeMounts:
               - mountPath: /tmp/ray
                 name: log-volume
               - name: uploads
                 mountPath: /external_data
+              - name: ssh-privatekey
+                readOnly: true
+                # container will see /root/.ssh/id_rsa as usual:
+                mountPath: "/home/ray/.ssh"
         volumes:
         - emptyDir: {}
           name: log-volume
         - name: uploads
           persistentVolumeClaim:
             claimName: uploads
+        - name: ssh-privatekey
+          secret:
+            secretName: ssh-privatekey
+            defaultMode: 0755
   workerGroupSpecs:
   - groupName: workergroup
     maxReplicas: 4
@@ -58,7 +66,7 @@ spec:
           app.kubernetes.io/instance: raycluster
       spec:
         containers:
-          - image: holdenk/fhi-ray:v0.8.4a
+          - image: holdenk/fhi-ray:v0.8.6d
             env:
               - name: RAY_gcs_server_request_timeout_seconds
                 value: "20"
@@ -70,19 +78,26 @@ spec:
             resources:
               limits:
                 cpu: "1"
-                memory: 4G
+                memory: 13G
               requests:
                 cpu: "1"
-                memory: 4G
-            securityContext: {}
+                memory: 12G
             volumeMounts:
             - mountPath: /tmp/ray
               name: log-volume
             - name: uploads
               mountPath: /external_data
+            - name: ssh-privatekey
+              readOnly: true
+              # container will see /root/.ssh/id_rsa as usual:
+              mountPath: "/home/ray/.ssh"
         volumes:
         - emptyDir: {}
           name: log-volume
         - name: uploads
           persistentVolumeClaim:
             claimName: uploads
+        - name: ssh-privatekey
+          secret:
+            secretName: ssh-privatekey
+            defaultMode: 0755

deploy.yaml

@@ -25,7 +25,7 @@ spec:
         special: web-primary-pod
     spec:
       containers:
-        - image: holdenk/fight-health-insurance:v0.8.4a
+        - image: holdenk/fight-health-insurance:v0.8.6d
           name: totallylegitco
           env:
           - name: PRIMARY
@@ -64,7 +64,7 @@ spec:
           matchLabels:
             app: fight-health-insurance
       containers:
-        - image: holdenk/fight-health-insurance:v0.8.4a
+        - image: holdenk/fight-health-insurance:v0.8.6d
           name: totallylegitco
           envFrom:
           - secretRef:

fighthealthinsurance/combined_storage.py

@@ -19,17 +19,22 @@ def open(self, *args, **kwargs):
                 with Timeout(2.0) as timeout_ctx:
                     return backend.open(*args, **kwargs)
             except Exception as e:
-                print(f"Error {e}")
+                print(
+                    f"Error opening from {args} {kwargs} on backend {backend} from {self.backends}: {e}"
+                )
                 last_error = e
         if last_error is not None:
+            print(
+                f"Opening failed on all backends -- {self.backends} -- raising {last_error}"
+            )
             raise last_error
 
     def delete(self, *args, **kwargs):
         last_error: Optional[BaseException] = None
         for backend in self.backends:
             try:
-                with Timeout(2.0) as timeout_ctx:
-                    return backend.open(*args, **kwargs)
+                with Timeout(1.0) as timeout_ctx:
+                    return backend.delete(*args, **kwargs)
             except Exception as e:
                 print(f"Error {e}")
                 last_error = e
@@ -39,7 +44,7 @@ def delete(self, *args, **kwargs):
     def save(self, *args, **kwargs):
         for backend in self.backends:
             try:
-                with Timeout(2.0) as timeout_ctx:
+                with Timeout(4.0) as timeout_ctx:
                     l = backend.save(*args, **kwargs)
             except Exception as e:
                 print(f"Error saving {e} to {backend}")

fighthealthinsurance/common_view_logic.py

@@ -260,6 +260,14 @@ def blocking_dosend_all(cls, count) -> int:
             c = c + 1
         return c
 
+    @classmethod
+    def resend(cls, fax_phone, uuid, hashed_email) -> bool:
+        f = FaxesToSend.objects.filter(hashed_email=hashed_email, uuid=uuid).get()
+        f.destination = fax_phone
+        f.save()
+        future = fax_actor_ref.get.do_send_fax.remote(hashed_email, uuid)
+        return True
+
     @classmethod
     def remote_send_fax(cls, hashed_email, uuid) -> bool:
         """Send a fax using ray non-blocking"""

fighthealthinsurance/core_forms.py

@@ -78,6 +78,8 @@ class FaxForm(DenialRefForm):
 
 class FaxResendForm(forms.Form):
     fax_phone = forms.CharField(required=True)
+    uuid = forms.UUIDField(required=True, widget=forms.HiddenInput)
+    hashed_email = forms.CharField(required=True, widget=forms.HiddenInput)
 
 
 class PostInferedForm(DenialRefForm):

fighthealthinsurance/email_polling_actor.py

@@ -1,30 +1,37 @@
 import os
 
 import ray
-from fighthealthinsurance.ray import *
 import asyncio
+import time
 
 name = "EmailPollingActor"
 
 
-@ray.remote
+@ray.remote(max_restarts=-1, max_task_retries=-1)
 class EmailPollingActor:
     def __init__(self):
+        print(f"Starting actor")
+        time.sleep(1)
         # This is a bit of a hack but we do this so we have the app configured
         from configurations.wsgi import get_wsgi_application
 
         os.environ.setdefault("DJANGO_SETTINGS_MODULE", "fighthealthinsurance.settings")
         application = get_wsgi_application()
+        print(f"wsgi started")
         # Now we can import the follow up e-mails logic
-        from fighthealthinsurance.followup_emails import *
+        from fighthealthinsurance.followup_emails import FollowUpEmailSender
 
-        self.sender = FollowUpEmailSender
+        self.sender = FollowUpEmailSender()
+        print(f"Sender started")
 
-    async def run(self):
+    def run(self):
+        print(f"Starting run")
         self.running = True
         while self.running:
             try:
-                print(self.sender.find_candidates)
-                await asyncio.sleep(1)
+                print(f"Top candidates: {self.sender.find_candidates()[0:4]}")
+                time.sleep(10)
             except Exception as e:
                 print(f"Error {e} while checking messages.")
+
+    print(f"Done running? what?")

fighthealthinsurance/email_polling_actor_ref.py

@@ -2,16 +2,15 @@
 from functools import cached_property
 
 import ray
-from fighthealthinsurance.email_polling_actor import *
-from fighthealthinsurance.ray import *
+from fighthealthinsurance.email_polling_actor import EmailPollingActor
 
 
 class EmailPollingActorRef:
     """A reference to the email polling actor."""
 
     @cached_property
-    def get(self) -> str:
-        name = "fax_polling_actor"
+    def get(self):
+        name = "email_polling_actor"
         # Shut down existing actor
         try:
             a = ray.get_actor(name, namespace="fhi")
@@ -27,8 +26,9 @@ def get(self) -> str:
             name=name, lifetime="detached", namespace="fhi"
         ).remote()
         # Kick of the remote task
-        email_polling_actor.run.remote()
-        return email_polling_actor
+        rr = email_polling_actor.run.remote()
+        print(f"Remote run of email actor {rr}")
+        return (email_polling_actor, rr)
 
 
 email_polling_actor_ref = EmailPollingActorRef()

fighthealthinsurance/fax_actor.py

@@ -3,17 +3,19 @@
 from fighthealthinsurance.fax_utils import *
 from django.utils import timezone
 from datetime import datetime, timedelta
-from fighthealthinsurance.models import Denial, FaxesToSend
+import time
 
 
-@ray.remote
+@ray.remote(max_restarts=-1, max_task_retries=-1)
 class FaxActor:
     def __init__(self):
+        time.sleep(1)
         # This is a bit of a hack but we do this so we have the app configured
         from configurations.wsgi import get_wsgi_application
 
         os.environ.setdefault("DJANGO_SETTINGS_MODULE", "fighthealthinsurance.settings")
-        self.application = get_wsgi_application()
+        get_wsgi_application()
+        from fighthealthinsurance.models import FaxesToSend
 
     def hi(self):
         return "ok"
@@ -31,7 +33,7 @@ def send_delayed_faxes(self) -> Tuple[int, int]:
         from fighthealthinsurance.models import FaxesToSend
 
         target_time = timezone.now() - timedelta(hours=1)
-        print(f"Target: {target_time}")
+        print(f"Sending faxes older than target: {target_time}")
 
         delayed_faxes = FaxesToSend.objects.filter(
             should_send=True, sent=False, date__lt=target_time
@@ -40,11 +42,12 @@ def send_delayed_faxes(self) -> Tuple[int, int]:
         f = 0
         for fax in delayed_faxes:
             try:
+                print(f"Attempting to send fax {fax}")
                 t = t + 1
                 self.do_send_fax_object(fax)
+                print(f"Sent fax {fax}")
             except Exception as e:
                 print(f"Error sending fax {fax}: {e}")
-                raise e
                 f = f + 1
         return (t, f)
 
@@ -63,16 +66,25 @@ def do_send_fax_object(self, fax) -> bool:
         if fax.destination is None:
             return False
         extra = ""
-        if denial.claim_id is not None:
-            extra += "This is regarding claim id {denial.claim_id}."
-        if fax.name is not None:
-            extra += "This fax is sent on behalf of {fax.name}."
+        if denial.claim_id is not None and len(denial.claim_id) > 2:
+            extra += f"This is regarding claim id {denial.claim_id}."
+        if fax.name is not None and len(fax.name) > 2:
+            extra += f"This fax is sent on behalf of {fax.name}."
+        print(f"Recording attempt to send time")
+        fax.attempting_to_send_as_of = timezone.now()
+        fax.save()
+        print(f"Kicking of fax sending")
         fax_sent = flexible_fax_magic.send_fax(
             input_paths=[fax.get_temporary_document_path()],
             extra=extra,
             destination=fax.destination,
             blocking=True,
         )
+        print(f"Fax send command returned :)")
+        fax.sent = True
+        fax.fax_success = fax_sent
+        fax.save()
+        print(f"Notifing user of result {fax.fax_success}")
         fax_redo_link = "https://www.fighthealthinsurance.com" + reverse(
             "fax-followup",
             kwargs={
@@ -105,4 +117,5 @@ def do_send_fax_object(self, fax) -> bool:
         )
         msg.attach_alternative(html_content, "text/html")
         msg.send()
+        print(f"E-mail sent!")
         return True

fighthealthinsurance/fax_polling_actor.py

@@ -6,14 +6,17 @@
 import asyncio
 
 
-@ray.remote
+@ray.remote(max_restarts=-1, max_task_retries=-1)
 class FaxPollingActor:
     def __init__(self, i=60):
         # This is seperate from the global one
         name = "fpa-worker"
+        print(f"Starting fax polling actor")
+        time.sleep(1)
         self.fax_actor = FaxActor.options(  # type: ignore
             name=name, namespace="fhi"
         ).remote()
+        print(f"Created fpa-worker {self.fax_actor}")
         self.interval = i
         self.c = 0
         self.e = 0
@@ -23,6 +26,7 @@ async def hello(self) -> str:
         return "Hi"
 
     async def run(self) -> bool:
+        print(f"Starting run")
         self.running = True
         while self.running:
             # Like yield
@@ -36,7 +40,9 @@ async def run(self) -> bool:
                 self.aec += 1
             finally:
                 # Success or failure we wait.
-                await asyncio.sleep(1)
+                print(f"Waiting for next run")
+                await asyncio.sleep(5)
+        print(f"Done running? what?")
         return True
 
     async def count(self) -> int: