Merge branch 'develop' into np_gl-1394_turn_on_pullapprove

sickler-alex · Feb 24, 2021 · c3b193e · c3b193e
2 parents 5e516a2 + c1f20c4
commit c3b193e
Show file tree

Hide file tree

Showing 16 changed files with 45 additions and 74 deletions.
diff --git a/...oad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.changelog.md b/...oad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.changelog.md
@@ -1,3 +1,8 @@
+# 2.4.2
+2021-02-22
+
+* Added SORTING_COLLECTION_SIZE_RATIO as an optional task input to MarkDuplicates
+
 # 2.4.1
 2020-12-21
 

diff --git a/pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.wdl b/pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.wdl
@@ -40,7 +40,7 @@ import "../../../../../../structs/dna_seq/DNASeqStructs.wdl"
 workflow ExomeGermlineSingleSample {
 
 
-  String pipeline_version = "2.4.1"
+  String pipeline_version = "2.4.2"
 
   input {
     PapiSettings papi_settings

diff --git a/...dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.changelog.md b/...dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.changelog.md
@@ -1,3 +1,8 @@
+# 2.3.2
+2021-02-22
+
+* Added SORTING_COLLECTION_SIZE_RATIO as an optional task input to MarkDuplicates
+
 # 2.3.1
 2020-12-21
 

diff --git a/pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.wdl b/pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.wdl
@@ -38,7 +38,7 @@ import "../../../../../../structs/dna_seq/DNASeqStructs.wdl"
 # WORKFLOW DEFINITION
 workflow WholeGenomeGermlineSingleSample {
 
-  String pipeline_version = "2.3.1"
+  String pipeline_version = "2.3.2"
 
   input {
     SampleAndUnmappedBams sample_and_unmapped_bams

diff --git a/...s/broad/reprocessing/cram_to_unmapped_bams/test_inputs/Scientific/G96830.NA12878.WGS.json b/...s/broad/reprocessing/cram_to_unmapped_bams/test_inputs/Scientific/G96830.NA12878.WGS.json
@@ -2,7 +2,7 @@
   "CramToUnmappedBams.input_cram": "gs://broad-gotc-test-storage/germline_single_sample/wgs/scientific/truth/{TRUTH_BRANCH}/G96830.NA12878/NA12878.cram",
   "CramToUnmappedBams.output_map": "gs://broad-gotc-test-storage/germline_single_sample/wgs/scientific/bams/G96830.NA12878/readgroupid_to_bamfilename_map.txt",
 
-  "CramToUnmappedBams.base_file_name": "G96830.NA12878",
+  "CramToUnmappedBams.base_file_name": "G96830.NA12878.WGS",
   "CramToUnmappedBams.unmapped_bam_suffix": ".unmapped.bam",
   "CramToUnmappedBams.ref_fasta": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta",
   "CramToUnmappedBams.ref_fasta_index": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta.fai"

diff --git a/pipelines/broad/reprocessing/exome/ExomeReprocessing.changelog.md b/pipelines/broad/reprocessing/exome/ExomeReprocessing.changelog.md
@@ -1,3 +1,8 @@
+# 2.4.4
+2021-02-22
+
+* Added SORTING_COLLECTION_SIZE_RATIO as an optional task input to MarkDuplicates
+
 # 2.4.3
 2021-02-08
 

diff --git a/pipelines/broad/reprocessing/exome/ExomeReprocessing.wdl b/pipelines/broad/reprocessing/exome/ExomeReprocessing.wdl
@@ -6,7 +6,7 @@ import "../../../../structs/dna_seq/DNASeqStructs.wdl"
 
 workflow ExomeReprocessing {
 
-  String pipeline_version = "2.4.3"
+  String pipeline_version = "2.4.4"
 
   input {
     File? input_cram

diff --git a/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.changelog.md b/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.changelog.md
@@ -1,3 +1,8 @@
+# 2.4.4
+2021-02-22
+
+* Added SORTING_COLLECTION_SIZE_RATIO as an optional task input to MarkDuplicates
+
 # 2.4.3
 2021-02-08
 

diff --git a/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.plumbing.input.json b/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.plumbing.input.json
diff --git a/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.wdl b/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.wdl
@@ -5,7 +5,7 @@ import "../../../../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow ExternalExomeReprocessing {
 
-  String pipeline_version = "2.4.3"
+  String pipeline_version = "2.4.4"
 
   input {
     File? input_cram

diff --git a/...es/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.changelog.md b/...es/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.changelog.md
@@ -1,3 +1,8 @@
+# 1.3.4
+2021-02-22
+
+* Added SORTING_COLLECTION_SIZE_RATIO as an optional task input to MarkDuplicates
+
 # 1.3.3
 2021-02-08
 

diff --git a/pipelines/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.wdl b/pipelines/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.wdl
@@ -5,7 +5,7 @@ import "../../../../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow ExternalWholeGenomeReprocessing {
 
-  String pipeline_version = "1.3.3"
+  String pipeline_version = "1.3.4"
 
   input {
     File? input_cram

diff --git a/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.changelog.md b/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.changelog.md
@@ -1,3 +1,8 @@
+# 2.3.4
+2021-02-22
+
+* Added SORTING_COLLECTION_SIZE_RATIO as an optional task input to MarkDuplicates
+
 # 2.3.3
 2021-02-08
 

diff --git a/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.wdl b/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.wdl
@@ -6,7 +6,7 @@ import "../../../../structs/dna_seq/DNASeqStructs.wdl"
 
 workflow WholeGenomeReprocessing {
 
-  String pipeline_version = "2.3.3"
+  String pipeline_version = "2.3.4"
 
   input {
     File? input_cram

diff --git a/tasks/broad/BamProcessing.wdl b/tasks/broad/BamProcessing.wdl
@@ -108,6 +108,8 @@ task MarkDuplicates {
     String? read_name_regex
     Int memory_multiplier = 1
     Int additional_disk = 20
+
+    Float? sorting_collection_size_ratio
   }
 
   # The merged bam will be smaller than the sum of the parts so we need to account for the unmerged inputs and the merged output.
@@ -130,6 +132,7 @@ task MarkDuplicates {
       METRICS_FILE=~{metrics_filename} \
       VALIDATION_STRINGENCY=SILENT \
       ~{"READ_NAME_REGEX=" + read_name_regex} \
+      ~{"SORTING_COLLECTION_SIZE_RATIO=" + sorting_collection_size_ratio} \
       OPTICAL_DUPLICATE_PIXEL_DISTANCE=2500 \
       ASSUME_SORT_ORDER="queryname" \
       CLEAR_DT="false" \

diff --git a/...t/src/main/scala/org/broadinstitute/dsp/pipelines/tester/ExternalReprocessingTester.scala b/...t/src/main/scala/org/broadinstitute/dsp/pipelines/tester/ExternalReprocessingTester.scala
@@ -60,13 +60,17 @@ class ExternalReprocessingTester(testerConfig: GermlineCloudWorkflowConfig)(
     )
   }
 
+  // Note - we are explicitly setting the google_project here so that when running in a non-dev environment,
+  // The workflow can still access the test data AND can then read from the vault
   override def readTestOptions(
       releaseDir: File,
       environment: CromwellEnvironment
   ): String = {
     val defaultOptions = Array(
       "read_from_cache" -> testerConfig.useCallCaching.asJson,
-      "backend" -> testerConfig.papiVersion.entryName.asJson
+      "backend" -> testerConfig.papiVersion.entryName.asJson,
+      "monitoring_script" -> "gs://broad-gotc-test-storage/cromwell_monitoring_script.sh".asJson,
+      "google_project" -> "broad-exomes-dev1".asJson
     )
 
     val optionsJson = defaultOptions ++ environment.environmentOptions
-Original file line number
+Diff line change
@@ -1,3 +1,8 @@
+    # 2.4.2
+-02-22
+    * Added SORTING_COLLECTION_SIZE_RATIO as an optional task input to MarkDuplicates
     # 2.4.1
 -12-21
@@ Expand Down @@