Add an option for folks to upload plan documents (totallylegitco#80)

* Pass through shell params to runserver_plus so you can specify the ip to listen on

* Allow for upload of plan documents & save them to a PVC

* Add the forms & fields to allow for file uploading (TODO: how do we handle that on the REST side? idk)

* Skip plan_documents for rest for now.

* MYPY settings checking can be a bit off and we added custom fields to it (see https://github.com/typeddjango/django-stubs#how-to-use-a-custom-library-to-handle-django-settings)

* Fix sorting

* Style cleanup

* Drop isort conflicts with black

* Update setup.cfg to match mypy.ini

* Oops wrong location

* Try using mypy.ini

.gitignore

@@ -136,4 +136,8 @@ reports/
 /static/
 
 fighthealthinsurance/static/js/node_modules
-fighthealthinsurance/static/js/dist
+fighthealthinsurance/static/js/dist
+
+# Ignore external storage
+external_storage
+external_data

Dockerfile

@@ -20,6 +20,7 @@ RUN chown -R www-data:www-data /opt/fighthealthinsurance
 # We hope requirements has not changed so we can use the cache
 COPY requirements.txt /opt/fighthealthinsurance/
 RUN pip install --upgrade pip && pip install -r /opt/fighthealthinsurance/requirements.txt
+RUN mkdir -p /external_data
 # We copy static early ish since it could also be cached nicely
 ADD --chown=www-data:www-data static /opt/fighthealthinsurance/static
 ADD --chown=www-data:www-data fighthealthinsurance /opt/fighthealthinsurance/fighthealthinsurance

build.sh

@@ -8,6 +8,6 @@ mypy -p fighthealthinsurance
 ./manage.py collectstatic --no-input
 
 pushd ./static/js; npm i; npm run build; popd
-IMAGE=holdenk/fight-health-insurance:v0.3.0a
+IMAGE=holdenk/fight-health-insurance:v0.4.0b
 docker pull "${IMAGE}" || docker buildx build --platform=linux/amd64,linux/arm64 -t "${IMAGE}" . --push
 kubectl apply -f deploy.yaml

deploy.yaml

@@ -1,3 +1,16 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: uploads
+  namespace: totallylegitco
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 200Gi
+---
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -12,7 +25,7 @@ spec:
         special: web-primary-pod
     spec:
       containers:
-        - image: holdenk/fight-health-insurance:v0.3.0a
+        - image: holdenk/fight-health-insurance:v0.4.0b
           name: totallylegitco
           env:
           - name: PRIMARY
@@ -51,7 +64,7 @@ spec:
           matchLabels:
             app: fight-health-insurance
       containers:
-        - image: holdenk/fight-health-insurance:v0.3.0a
+        - image: holdenk/fight-health-insurance:v0.4.0b
           name: totallylegitco
           envFrom:
           - secretRef:
@@ -60,6 +73,9 @@ spec:
           ports:
             - containerPort: 80
               name: web
+          volumeMounts:
+            - name: uploads
+              mountPath: /external_data
           livenessProbe:
             httpGet:
               httpHeaders:
@@ -77,6 +93,10 @@ spec:
               port: 80
             periodSeconds: 10
             failureThreshold: 30
+      volumes:
+        - name: uploads
+          persistentVolumeClaim:
+            claimName: uploads
 ---
 apiVersion: v1
 kind: Service

fighthealthinsurance/common_view_logic.py

@@ -1,18 +1,17 @@
 import json
-
+from dataclasses import dataclass
+from string import Template
 from typing import Any, Tuple
 
-from django.forms import Form
 from django.core.validators import validate_email
+from django.forms import Form
 from django.http import StreamingHttpResponse
-from string import Template
+
+import uszipcode
 
 from fighthealthinsurance.forms import *
-from fighthealthinsurance.models import *
 from fighthealthinsurance.generate_appeal import *
-from dataclasses import dataclass
-
-import uszipcode
+from fighthealthinsurance.models import *
 
 appealGenerator = AppealGenerator()
 
@@ -244,6 +243,7 @@ def create_denial(
         privacy=False,
         use_external_models=False,
         store_raw_email=False,
+        plan_documents=None,
     ):
         hashed_email = Denial.get_hashed_email(email)
         # If they ask us to store their raw e-mail we do
@@ -260,6 +260,12 @@ def create_denial(
             health_history=health_history,
         )
 
+        for plan_document in plan_documents:
+            pd = PlanDocuments.objects.create(
+                plan_document=plan_document, denial=denial
+            )
+            pd.save()
+
         # Try and guess at the denial types
         denial_types = cls.regex_denial_processor.get_denialtype(denial_text)
         denial_type = []

fighthealthinsurance/forms.py

@@ -2,8 +2,31 @@
 
 from django import forms
 
-from django_recaptcha.fields import ReCaptchaField, ReCaptchaV3, ReCaptchaV2Checkbox
-from fighthealthinsurance.models import DenialTypes, PlanType, PlanSource
+from django_recaptcha.fields import ReCaptchaField, ReCaptchaV2Checkbox, ReCaptchaV3
+
+from fighthealthinsurance.models import DenialTypes, PlanSource, PlanType
+
+
+# See https://docs.djangoproject.com/en/5.1/topics/http/file-uploads/
+class MultipleFileInput(forms.ClearableFileInput):
+    allow_multiple_selected = True
+
+
+class MultipleFileField(forms.FileField):
+    def __init__(self, *args, **kwargs):
+        kwargs.setdefault("widget", MultipleFileInput())
+        super().__init__(*args, **kwargs)
+
+    def clean(self, data, initial=None):
+        single_file_clean = super().clean
+        if isinstance(data, (list, tuple)):
+            result = [single_file_clean(d, initial) for d in data]
+        else:
+            result = [single_file_clean(data, initial)]
+        return result
+
+
+# Actual forms
 
 
 class DeleteDataForm(forms.Form):
@@ -32,6 +55,7 @@ class DenialForm(forms.Form):
     denial_text = forms.CharField(required=True)
     health_history = forms.CharField(required=False)
     email = forms.EmailField(required=True)
+    plan_documents = MultipleFileField(required=False)
 
 
 class DenialRefForm(forms.Form):

fighthealthinsurance/generate_appeal.py

@@ -1,17 +1,19 @@
-import itertools
 import concurrent
-from concurrent.futures import Future
 import csv
+import itertools
 import os
-from functools import cache, lru_cache
-from typing import Tuple, List, Optional
-import traceback
 import time
-
-from typing_extensions import reveal_type
+import traceback
+from concurrent.futures import Future
+from functools import cache, lru_cache
+from typing import List, Optional, Tuple
 
 import icd10
 import requests
+from typing_extensions import reveal_type
+
+from fighthealthinsurance.exec import *
+from fighthealthinsurance.ml_models import *
 from fighthealthinsurance.models import (
     AppealTemplates,
     DenialTypes,
@@ -21,8 +23,6 @@
     Regulator,
 )
 from fighthealthinsurance.process_denial import *
-from fighthealthinsurance.ml_models import *
-from fighthealthinsurance.exec import *
 
 
 class AppealTemplateGenerator(object):

fighthealthinsurance/management/commands/ensure_adminuser.py

@@ -1,6 +1,7 @@
 # See https://stackoverflow.com/questions/39744593/how-to-create-a-django-superuser-if-it-doesnt-exist-non-interactively
 # Covered by https://stackoverflow.com/help/licensing
 import os
+
 from django.contrib.auth import get_user_model
 from django.core.management.base import BaseCommand
 

fighthealthinsurance/migrations/0009_appealtemplates_diagnosis_regex.py

@@ -1,8 +1,9 @@
 # Generated by Django 5.0.8 on 2024-08-29 01:04
 
-import regex_field.fields
 from django.db import migrations
 
+import regex_field.fields
+
 
 class Migration(migrations.Migration):
 

fighthealthinsurance/migrations/0011_plansource_plansourcerelation_denial_plan_source.py

@@ -1,9 +1,10 @@
 # Generated by Django 5.0.8 on 2024-09-08 05:27
 
 import django.db.models.deletion
-import regex_field.fields
 from django.db import migrations, models
 
+import regex_field.fields
+
 
 class Migration(migrations.Migration):
 

fighthealthinsurance/migrations/0012_denial_semi_sekret.py

@@ -1,8 +1,9 @@
 # Generated by Django 5.0.8 on 2024-09-12 06:40
 
-import fighthealthinsurance.models
 from django.db import migrations, models
 
+import fighthealthinsurance.models
+
 
 class Migration(migrations.Migration):
 

fighthealthinsurance/migrations/0016_denial_plan_documents.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.1.1 on 2024-09-24 02:11
+
+import django.core.files.storage
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("fighthealthinsurance", "0015_rename_medical_context_denial_health_history"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="denial",
+            name="plan_documents",
+            field=models.FileField(
+                null=True,
+                storage=django.core.files.storage.FileSystemStorage(
+                    location="external_data"
+                ),
+                upload_to="",
+            ),
+        ),
+    ]

fighthealthinsurance/migrations/0017_remove_denial_plan_documents_plandocuments.py

@@ -0,0 +1,45 @@
+# Generated by Django 5.1.1 on 2024-09-25 03:54
+
+import django.core.files.storage
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("fighthealthinsurance", "0016_denial_plan_documents"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="denial",
+            name="plan_documents",
+        ),
+        migrations.CreateModel(
+            name="PlanDocuments",
+            fields=[
+                (
+                    "plan_document_id",
+                    models.AutoField(primary_key=True, serialize=False),
+                ),
+                (
+                    "plan_document",
+                    models.FileField(
+                        null=True,
+                        storage=django.core.files.storage.FileSystemStorage(
+                            location="external_data"
+                        ),
+                        upload_to="",
+                    ),
+                ),
+                (
+                    "denial",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="fighthealthinsurance.denial",
+                    ),
+                ),
+            ],
+        ),
+    ]

fighthealthinsurance/ml_models.py

@@ -1,19 +1,20 @@
-import itertools
 import concurrent
-from concurrent.futures import Future
 import csv
+import itertools
 import os
 import re
+import time
+import traceback
 from abc import ABC, abstractmethod
+from concurrent.futures import Future
 from functools import cache, lru_cache
-from typing import Any, Tuple, List, Optional, Iterable, Union, Callable
-import traceback
-import time
-
-from typing_extensions import reveal_type
+from typing import Any, Callable, Iterable, List, Optional, Tuple, Union
 
 import icd10
 import requests
+from typing_extensions import reveal_type
+
+from fighthealthinsurance.exec import *
 from fighthealthinsurance.models import (
     AppealTemplates,
     DenialTypes,
@@ -22,7 +23,6 @@
     Procedures,
     Regulator,
 )
-from fighthealthinsurance.exec import *
 
 
 class RemoteModel(object):

fighthealthinsurance/models.py

@@ -1,14 +1,15 @@
 import hashlib
+import os
 import re
 import sys
 from typing import Optional
+from uuid import UUID
 
+from django.conf import settings
 from django.db import models
+from django.db.models.functions import Now
 
 from regex_field.fields import RegexField
-from django.db.models.functions import Now
-from uuid import UUID
-import os
 
 
 class FollowUpType(models.Model):
@@ -175,6 +176,14 @@ def sekret_gen():
     return str(UUID(bytes=os.urandom(16), version=4))
 
 
+class PlanDocuments(models.Model):
+    plan_document_id = models.AutoField(primary_key=True)
+    plan_document = models.FileField(null=True, storage=settings.EXTERNAL_STORAGE)
+    # If the denial is deleted it's either SPAM or a removal request in either case
+    # we cascade the delete
+    denial = models.ForeignKey("Denial", on_delete=models.CASCADE)
+
+
 class Denial(models.Model):
     denial_id = models.AutoField(primary_key=True)
     hashed_email = models.CharField(max_length=300, primary_key=False)