Merge pull request hodcroftlab#154 from hodcroftlab/fix/missing-ack

Warn about missing acknowledgements on data update

scripts/convert_to_web_app_json.py

@@ -4,7 +4,7 @@
 """
 Converts cluster data to a format suitable for consumption by the web app
 """
-
+import glob
 import json
 import os
 import re
@@ -396,6 +396,36 @@ def compare_dates(left_date: str, right_date: str, comp):
     return format_date(dt)
 
 
+def check_acknowledgements(output_path: str):
+    with open(os.path.join(output_path, "acknowledgements", "acknowledgements_keys.json"), "r") as f:
+        acknowledgements_keys = json.load(f)
+
+        for cluster in clusters:
+            build_name = cluster["build_name"]
+            ack_dir = os.path.join(output_path, 'acknowledgements', build_name)
+
+            warnings = []
+
+            if not os.path.isdir(ack_dir):
+                warnings.append(f" * does not have acknowledgements directory ('{ack_dir}')")
+
+            if build_name not in acknowledgements_keys['acknowledgements']:
+                warnings.append(f" * not in 'acknowledgements_keys.json'")
+            else:
+                num_chunks = acknowledgements_keys['acknowledgements'][build_name]["numChunks"]
+                chunks = set(glob.glob(os.path.join(ack_dir, "*.json")))
+                for i in range(num_chunks):
+                    filename = "{0:03}.json".format(i)
+                    chunk = os.path.join(ack_dir, filename)
+                    if not chunk in chunks:
+                        warnings.append(f" * 'acknowledgements_keys.json' has 'numChunks' "
+                                        f"set to {num_chunks}, but chunk '{filename}' was not found")
+
+            if len(warnings) > 0:
+                warnings_str = '\n    '.join(warnings)
+                print(f"\nWarning: cluster {build_name}:\n    {warnings_str}")
+
+
 if __name__ == "__main__":
     os.makedirs(output_path, exist_ok=True)
 
@@ -444,3 +474,5 @@ def compare_dates(left_date: str, right_date: str, comp):
     name_table_data = {"nameTable": name_table}
     with open(os.path.join(output_path, "nameTable.json"), "w") as fh:
         json.dump(name_table_data, fh, indent=2, sort_keys=True)
+
+    check_acknowledgements(output_path)

web/src/components/Acknowledgements/AcknowledgementsPage.tsx

@@ -4,6 +4,7 @@ import styled from 'styled-components'
 import { Col, Container, Row } from 'reactstrap'
 
 import { getClusters } from 'src/io/getClusters'
+import { getAcknowledgementsKeys } from 'src/io/getClusterEpiIslsNumChunks'
 import { Layout } from 'src/components/Layout/Layout'
 import { AcknowledgementsCard } from 'src/components/Acknowledgements/AcknowledgementsCard'
 
@@ -16,7 +17,11 @@ export const AcknowledgementsPageContainer = styled(Container)`
 
 const clusters = getClusters()
 
+const acknowledgementsKeys = getAcknowledgementsKeys()
+
 export function AcknowledgementsPage() {
+  const clustersWithAcks = clusters.filter((cluster) => acknowledgementsKeys.has(cluster.build_name))
+
   return (
     <Layout>
       <AcknowledgementsPageContainer>
@@ -34,7 +39,7 @@ export function AcknowledgementsPage() {
 
         <Row>
           <Col>
-            {clusters.map((cluster) => (
+            {clustersWithAcks.map((cluster) => (
               <AcknowledgementsCard key={cluster.build_name} cluster={cluster} />
             ))}
           </Col>

web/src/io/getClusterEpiIslsNumChunks.ts

@@ -2,6 +2,16 @@ import { get } from 'lodash'
 
 import acknowledgements from 'src/../data/acknowledgements/acknowledgements_keys.json'
 
+export function getAcknowledgementsKeys(): Set<string> {
+  const acks = get(acknowledgements, 'acknowledgements')
+  if (!acks) {
+    console.warn(`Acknowledgements not found`)
+    return new Set()
+  }
+
+  return new Set(Object.keys(acknowledgements.acknowledgements))
+}
+
 export function getClusterEpiIslsNumChunks(clusterBuildName: string): number {
   const acks = get(acknowledgements, 'acknowledgements')
   if (!acks) {