GL-1692. Make analysis_version_number optional (broadinstitute#422)

* GL-1692 Modified Arrays pipeline to automatically generate the analysis_version_number if it is not supplied as an input. Updated all picard-private accessing tasks to use latest docker
kew24 · Sep 8, 2021 · 28bd37c · 28bd37c
1 parent b1db5d8
commit 28bd37c
Show file tree

Hide file tree

Showing 10 changed files with 83 additions and 20 deletions.
diff --git a/dockers/broad/arrays_picard_private/build_arrays_picard_private_docker.sh b/dockers/broad/arrays_picard_private/build_arrays_picard_private_docker.sh
@@ -9,7 +9,7 @@ declare -r DOCKER_IMAGE_VERSION=4.0.10
 # Update this when there is a new release of picard-private to use as the
 # default jar.
 #
-declare -r PICARD_PRIVATE_VERSION=61af9bff4587783e5981a496f422ea36102482b5
+declare -r PICARD_PRIVATE_VERSION=257537c72dae29257b09bacf413505eed295ac32
 
 declare -r ARTIFACTORY=https://broadinstitute.jfrog.io/broadinstitute
 declare -r LIBS_SNAPSHOT_LOCAL=$ARTIFACTORY/libs-snapshot-local

diff --git a/dockers/broad/arrays_picard_private/build_arrays_picard_private_docker_version.tsv b/dockers/broad/arrays_picard_private/build_arrays_picard_private_docker_version.tsv
@@ -10,3 +10,4 @@ us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.8-1591037419	a9453e2477762
 us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.8-1591647180	62ec51fa1c8dcee4efb2c60fcdb58c2e6efd6098
 us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.9-1593023734	bca9362254e7cca14c79c1fd8833042a07f133d5
 us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1602016912	61af9bff4587783e5981a496f422ea36102482b5
+us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1631039849	257537c72dae29257b09bacf413505eed295ac32
diff --git a/pipelines/broad/arrays/single_sample/Arrays.changelog.md b/pipelines/broad/arrays/single_sample/Arrays.changelog.md
@@ -1,3 +1,8 @@
+# 2.4.2
+2021-09-07
+
+* Modified pipeline to automatically generate the analysis_version_number if it is not supplied as an input.
+
 # 2.4.1
 2021-08-30
 

diff --git a/pipelines/broad/arrays/single_sample/Arrays.wdl b/pipelines/broad/arrays/single_sample/Arrays.wdl
@@ -22,7 +22,7 @@ import "../../../../tasks/broad/InternalTasks.wdl" as InternalTasks
 
 workflow Arrays {
 
-  String pipeline_version = "2.4.1"
+  String pipeline_version = "2.4.2"
 
   input {
 
@@ -31,7 +31,7 @@ workflow Arrays {
     String autocall_version = "3.0.0"
 
     String chip_well_barcode
-    Int analysis_version_number
+    Int? analysis_version_number
 
     String sample_alias
     String? sample_id
@@ -176,6 +176,20 @@ workflow Arrays {
     }
   }
 
+  if (!defined(analysis_version_number)) {
+    # If the analysis_version_number is not provided as an input, we will query the ARRAYS_QC table to get the
+    # next value.
+    call InternalArraysTasks.GetNextArraysQcAnalysisVersionNumber {
+      input:
+        chip_well_barcode = chip_well_barcode,
+        preemptible_tries = preemptible_tries,
+        authentication = authentication_block,
+        service_account_filename = service_account_filename
+    }
+  }
+
+  Int analysis_version = select_first([analysis_version_number, GetNextArraysQcAnalysisVersionNumber.analysis_version_number])
+
   if (read_fingerprint_from_mercury) {
     call InternalTasks.MakeSafeFilename {
       input:
@@ -212,7 +226,7 @@ workflow Arrays {
     input:
       autocall_version = autocall_version,
       sample_alias = sample_alias,
-      analysis_version_number = analysis_version_number,
+      analysis_version_number = analysis_version,
       call_rate_threshold = call_rate_threshold,
       reported_gender = reported_gender,
       chip_well_barcode = chip_well_barcode,
@@ -254,7 +268,7 @@ workflow Arrays {
         autocall_version = autocall_version,
         output_metrics_basename = sample_alias,
         cluster_filename = cluster_filename,
-        analysis_version_number = analysis_version_number,
+        analysis_version_number = analysis_version,
         preemptible_tries = preemptible_tries
     }
 
@@ -270,7 +284,7 @@ workflow Arrays {
     call InternalArraysTasks.BlacklistBarcode as BlacklistFailedNormalization {
       input:
         upload_metrics_output = UploadEmptyArraysMetrics.upload_metrics_empty_file,
-        analysis_version = analysis_version_number,
+        analysis_version_number = analysis_version,
         chip_well_barcode = chip_well_barcode,
         preemptible_tries = preemptible_tries,
         reason = "DATA_QUALITY",
@@ -334,7 +348,7 @@ workflow Arrays {
       call InternalArraysTasks.BlacklistBarcode as BlacklistFailedGenotypeConcordance {
         input:
           upload_metrics_output = UploadArraysMetrics.upload_metrics_empty_file,
-          analysis_version = analysis_version_number,
+          analysis_version_number = analysis_version,
           chip_well_barcode = chip_well_barcode,
           preemptible_tries = preemptible_tries,
           reason = "GENOTYPE_CONCORDANCE",
@@ -349,7 +363,7 @@ workflow Arrays {
       call InternalArraysTasks.BlacklistBarcode as BlacklistFailedFingerprint {
         input:
           upload_metrics_output = UploadArraysMetrics.upload_metrics_empty_file,
-          analysis_version = analysis_version_number,
+          analysis_version_number = analysis_version,
           chip_well_barcode = chip_well_barcode,
           preemptible_tries = preemptible_tries,
           reason = "SAMPLE_MIXUP",

diff --git a/...lines/broad/arrays/single_sample/test_inputs/Scientific/200598830050_R01C01_90C04566.json b/...lines/broad/arrays/single_sample/test_inputs/Scientific/200598830050_R01C01_90C04566.json
@@ -1,6 +1,5 @@
 {
   "Arrays.chip_well_barcode": "200598830050_R01C01",
-  "Arrays.analysis_version_number": 1,
 
   "Arrays.sample_alias": "90C04566",
   "Arrays.sample_id": "SM-CMPZK",

diff --git a/pipelines/broad/arrays/single_sample/test_inputs/Scientific/README.md b/pipelines/broad/arrays/single_sample/test_inputs/Scientific/README.md
@@ -21,6 +21,9 @@
     - NOT Pulling fingerprint from Mercury
         - Using pre-existing fingerprint file (uploaded previously to GCS)
     - Pushing fingerprint to Mercury    
+- 200598830050_R01C01_90C04566
+    - Non-control sample
+    - NOT defining the analysis_version_number.  Test that the wrapper wdl will generate it.
 - 201651080129_R05C01_S8_N4B3GY
     - Non-control sample
     - Sample has a space in its name

diff --git a/pipelines/broad/arrays/validate_chip/ValidateChip.changelog.md b/pipelines/broad/arrays/validate_chip/ValidateChip.changelog.md
@@ -1,3 +1,8 @@
+# 1.13.4
+2021-09-07
+
+* Task wdls used by Validate chip were updated with changes that don't affect ValidateChip wdl
+
 # 1.13.3
 2021-08-31
 

diff --git a/pipelines/broad/arrays/validate_chip/ValidateChip.wdl b/pipelines/broad/arrays/validate_chip/ValidateChip.wdl
@@ -21,7 +21,7 @@ import "../../../../tasks/broad/InternalArraysTasks.wdl" as InternalTasks
 
 workflow ValidateChip {
 
-  String pipeline_version = "1.13.3"
+  String pipeline_version = "1.13.4"
 
   input {
     String sample_alias

diff --git a/tasks/broad/InternalArraysTasks.wdl b/tasks/broad/InternalArraysTasks.wdl
@@ -27,7 +27,7 @@ task GenerateEmptyVariantCallingMetricsFile {
   >>>
 
   runtime {
-    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1602016912"
+    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1631039849"
     memory: "3.5 GiB"
     preemptible: preemptible_tries
   }
@@ -41,7 +41,7 @@ task BlacklistBarcode {
   input {
     File upload_metrics_output
     String chip_well_barcode
-    Int analysis_version
+    Int analysis_version_number
     Int preemptible_tries
     Array[String] authentication
     String service_account_filename
@@ -62,7 +62,7 @@ task BlacklistBarcode {
     java -Xms2g -Dpicard.useLegacyParser=false -jar /usr/gitc/picard-private.jar \
                   ArraysManualBlacklistUpdate \
                   --CHIP_WELL_BARCODE ~{chip_well_barcode} \
-                  --ANALYSIS_VERSION ~{analysis_version} \
+                  --ANALYSIS_VERSION ~{analysis_version_number} \
                   --REASON ~{reason} \
                   --DB_USERNAME_FILE cloudsql.db_user.txt \
                   --DB_PASSWORD_FILE cloudsql.db_password.txt \
@@ -71,7 +71,7 @@ task BlacklistBarcode {
   >>>
 
   runtime {
-    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1602016912"
+    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1631039849"
     memory: "3.5 GiB"
     preemptible: preemptible_tries
   }
@@ -122,7 +122,7 @@ task VcfToMercuryFingerprintJson {
   >>>
 
   runtime {
-    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1602016912"
+    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1631039849"
     disks: "local-disk " + disk_size + " HDD"
     memory: "3.5 GiB"
     preemptible: preemptible_tries
@@ -150,7 +150,7 @@ task CreateBafRegressMetricsFile {
       --OUTPUT ~{output_metrics_basefilename}
   }
   runtime {
-    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1602016912"
+    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1631039849"
     disks: "local-disk " + disk_size + " HDD"
     memory: "3.5 GiB"
     preemptible: preemptible_tries
@@ -227,7 +227,7 @@ task UploadArraysMetrics {
   >>>
 
   runtime {
-    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1602016912"
+    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1631039849"
     disks: "local-disk " + disk_size + " HDD"
     memory: "3.5 GiB"
     preemptible: preemptible_tries
@@ -324,9 +324,45 @@ task UpdateChipWellBarcodeIndex {
   >>>
 
   runtime {
-    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1602016912"
+    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1631039849"
     disks: "local-disk " + disk_size + " HDD"
     memory: "3.5 GiB"
     preemptible: preemptible_tries
   }
 }
+
+task GetNextArraysQcAnalysisVersionNumber {
+  input {
+    String chip_well_barcode
+    Int preemptible_tries
+    Array[String] authentication
+    String service_account_filename
+  }
+
+  meta {
+    volatile: true
+  }
+
+  command <<<
+    set -eo pipefail
+
+    AUTH=~{write_lines(authentication)} && chmod +x $AUTH && $AUTH
+    export GOOGLE_APPLICATION_CREDENTIALS=/cromwell_root/~{service_account_filename}
+
+    java -Xms2g -Dpicard.useLegacyParser=false -jar /usr/gitc/picard-private.jar \
+      GetNextArraysQcAnalysisVersionNumber \
+        --CHIP_WELL_BARCODE ~{chip_well_barcode} \
+        --DB_USERNAME_FILE cloudsql.db_user.txt \
+        --DB_PASSWORD_FILE cloudsql.db_password.txt \
+        --DB_JDBC_FILE cloudsql.db_jdbc.txt
+  >>>
+
+  runtime {
+    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1631039849"
+    memory: "3.5 GiB"
+    preemptible: preemptible_tries
+  }
+  output {
+    Int analysis_version_number = read_int(stdout())
+  }
+}
diff --git a/tasks/broad/InternalTasks.wdl b/tasks/broad/InternalTasks.wdl
@@ -100,7 +100,7 @@ task DownloadGenotypes {
   >>>
 
   runtime {
-    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1602016912"
+    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1631039849"
     memory: "3.5 GiB"
     preemptible: preemptible_tries
   }
@@ -145,7 +145,7 @@ task UploadFingerprintToMercury {
   >>>
 
   runtime {
-    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1602016912"
+    docker: "us.gcr.io/broad-arrays-prod/arrays-picard-private:4.0.10-1631039849"
     memory: "3.5 GiB"
     preemptible: preemptible_tries
   }
-Original file line number
+Diff line change
@@ -1,3 +1,8 @@
+    # 2.4.2
+-09-07
+    * Modified pipeline to automatically generate the analysis_version_number if it is not supplied as an input.
     # 2.4.1
 -08-30
@@ Expand Down @@