document oracle test tool and check for valid_insecure emails

README.md

@@ -6,7 +6,7 @@
 
 * Verification of DKIM signatures (https://datatracker.ietf.org/doc/html/rfc6376)
 * No boolean result: Additional detection of DKIM security pitfalls
-* Query DMARC infos 
+* Query DMARC infos
 
 ## API
 
@@ -62,3 +62,19 @@ public struct DKIMResult: Equatable {
   var info: DKIMInfo?
 }
 ```
+
+## Oracle Test Tool
+
+### Setup
+
+```
+python3 -m pip install dkimpy
+npm install -g mailauth
+```
+
+### Run
+
+```
+./Tools/gen_oracle_reports.sh ../../emails 50 > oracle_log.csv
+./Tools/analyze_oracle_report.py oracle_log.csv
+```

Tools/analyze_oracle_report.py

@@ -14,13 +14,17 @@
 total_emails = 0
 valid_emails = 0
 no_signature_emails = 0
+valid_insecure_emails = 0
 
 reader = csv.DictReader(args.report)
 for row in reader:
     total_emails += 1
     if row["dkimpy"] == "signature ok" and "pass" in row["mailauth"] and "Valid" in row["dkimverifier"]:
-        valid_emails += 1
-        continue
+        if "Valid" not in row["dkimverifier"].replace("Valid_Insecure", ""):
+            valid_insecure_emails += 1
+        else:
+            valid_emails += 1
+            continue
     if (
         row["dkimpy"] == "signature verification failed"
         and "message not signed" in row["mailauth"]
@@ -33,5 +37,6 @@
 
 print("total_emails: ", total_emails)
 print("valid_emails: ", valid_emails)
+print("valid_insecure_emails: ", valid_insecure_emails)
 print("no_signature_emails: ", no_signature_emails)
-print("fail_emails: ", total_emails - valid_emails - no_signature_emails)
+print("fail_emails: ", total_emails - valid_emails - no_signature_emails - valid_insecure_emails)