Feature/dragen rna fusion 420 (#588)

* Deleted VS launch settings and updated Nirvana to .NET Core 3.1 * Fixed file permissions * Updated file permissions * Fixed SaCommon.IndexSuffix spelling. Initial implementation of FusionCatcher file parser. * Added the gene fusion reader and writer. Also included end-to-end tests for the GeneFusionReader & GeneFusionWriter. * Add MakeUserError utility method. Add capability to write gene fusions. * Updated cytogenetic band support to skip lookup if the chromosome is unknown * Added new SAUtils command for creating COSMIC gene fusions * Created the GeneFusionJsonReader & Writer and some end-to-end unit tests. * Created a new gene fusions section in VariantAnnotation. Updated the gene fusions readers to inherit from IGeneFusionSaReader. Added gene relationships to GeneFusionSources. Added a new interface called ISaMetadata that contains Assembly, Version, and JsonKey. Moved SA provider after transcript annotation provider. * Forgot some stragglers. Nirvana is in a runnable state. * Removed unnecessary using statements. Removed HasStopCodonNotation from PositionOffset (HGVS). Removed extra reference bases from HGVS c. & g. duplications & inversions. Added lots of GeneFusionCaller unit tests. * Updated GeneFusionSource relationship entries. Added relationships to GeneFusionSourceWriterTests.cs. * Add support for calculating the codon position and detecting in-frame gene fusions. * Added support for disabling in-frame gene fusion detection if the IMPRECISE flag is provided in the VCF. * Created an InfoDataBuilder to improve INFO field parsing. Added legacy support for INV3 & INV5 handling in gene fusions. * Fixed issues with FusionCatcher SA creation. Finished integrating FusionCatcher with the NsaProvider. Suffering from high memory usage. * Dramatically reduced memory usage for FusionCatcher sources in Nirvana. Only using 102 MB at this time. * Added gene symbols to the FusionCatcher output. * Removed the coding pre-requisite for gene fusions. * Added unit tests - 95% coverage for VariantAnnotation and 59% for SAUtils. * Minor Resharper cleanup * Fixed an issue that caused the GeneFusionKey.Create method to error out if one of the Ensembl Gene IDs was null. * Fixed an issue that resulted from not factoring in the adjacency strand when evaluating the origin strand.
Illumina · May 18, 2021 · 2d7f93f · 2d7f93f
1 parent 2a53306
commit 2d7f93f
Show file tree

Hide file tree

Showing 167 changed files with 4,146 additions and 1,910 deletions.
diff --git a/AnnotationLambda/AnnotationLambda.csproj b/AnnotationLambda/AnnotationLambda.csproj
@@ -1,17 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
     <AWSProjectType>Lambda</AWSProjectType>
-    <OutputPath>bin\$(Configuration)</OutputPath>
+    <OutputPath>bin\$(Configuration)</OutputPath>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Amazon.Lambda.Core" Version="1.1.0" />
-    <PackageReference Include="Amazon.Lambda.Serialization.Json" Version="1.7.0" />
-    <PackageReference Include="AWSSDK.Lambda" Version="3.3.106.4" />
-    <PackageReference Include="AWSSDK.S3" Version="3.3.109.4" />
-    <PackageReference Include="AWSSDK.SimpleNotificationService" Version="3.3.101.100" />
+    <PackageReference Include="Amazon.Lambda.Core" Version="2.0.0" />
+    <PackageReference Include="Amazon.Lambda.Serialization.Json" Version="2.0.0" />
+    <PackageReference Include="AWSSDK.Lambda" Version="3.7.0.15" />
+    <PackageReference Include="AWSSDK.S3" Version="3.7.0.17" />
+    <PackageReference Include="AWSSDK.SimpleNotificationService" Version="3.7.1.1" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Cloud\Cloud.csproj" />

diff --git a/AnnotationLambda/Properties/launchSettings.json b/AnnotationLambda/Properties/launchSettings.json
diff --git a/CacheUtils/CacheUtils.csproj b/CacheUtils/CacheUtils.csproj
@@ -1,8 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
-    <OutputPath>..\bin\$(Configuration)</OutputPath>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <OutputPath>..\bin\$(Configuration)</OutputPath>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <Import Project="..\CommonAssemblyInfo.props" />
@@ -12,9 +12,9 @@
     </Content>
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="3.1.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="3.1.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="3.1.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="5.0.0" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\CommandLine\CommandLine.csproj" />

diff --git a/CacheUtils/Properties/launchSettings.json b/CacheUtils/Properties/launchSettings.json
diff --git a/CacheUtils/TranscriptCache/TranscriptCacheUtilities.cs b/CacheUtils/TranscriptCache/TranscriptCacheUtilities.cs
@@ -20,13 +20,13 @@ public static List<ITranscript> GetTranscripts(DataBundle bundle, IChromosomeInt
 
         public static IntervalArray<T>[] ToIntervalArrays<T>(this IEnumerable<T> items, int numRefSeqs) where T : IChromosomeInterval
         {
-            var intervalArrays = new IntervalArray<T>[numRefSeqs];
-            var itemsByRef     = items.GetMultiValueDict(x => x.Chromosome.Index);
+            var                         intervalArrays = new IntervalArray<T>[numRefSeqs];
+            Dictionary<ushort, List<T>> itemsByRef     = items.GetMultiValueDict(x => x.Chromosome.Index);
 
             foreach (ushort refIndex in itemsByRef.Keys.OrderBy(x => x))
             {
-                var unsortedItems = itemsByRef[refIndex];
-                var intervals     = unsortedItems.OrderBy(x => x.Start).ThenBy(x => x.End).ToIntervals(unsortedItems.Count);
+                List<T>       unsortedItems = itemsByRef[refIndex];
+                Interval<T>[] intervals     = unsortedItems.OrderBy(x => x.Start).ThenBy(x => x.End).ToIntervals(unsortedItems.Count);
                 intervalArrays[refIndex] = new IntervalArray<T>(intervals);
             }
 

diff --git a/Cloud/Cloud.csproj b/Cloud/Cloud.csproj
@@ -1,15 +1,15 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <OutputPath>..\bin\$(Configuration)</OutputPath>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Amazon.Lambda.Core" Version="1.1.0" />
-    <PackageReference Include="Amazon.Lambda.Serialization.Json" Version="1.7.0" />
-    <PackageReference Include="AWSSDK.S3" Version="3.3.109.4" />
-    <PackageReference Include="AWSSDK.SimpleNotificationService" Version="3.3.101.100" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="Amazon.Lambda.Core" Version="2.0.0" />
+    <PackageReference Include="Amazon.Lambda.Serialization.Json" Version="2.0.0" />
+    <PackageReference Include="AWSSDK.S3" Version="3.7.0.17" />
+    <PackageReference Include="AWSSDK.SimpleNotificationService" Version="3.7.1.1" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Genome\Genome.csproj" />

diff --git a/CommandLine/CommandLine.csproj b/CommandLine/CommandLine.csproj
@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <OutputPath>..\bin\$(Configuration)</OutputPath>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>

diff --git a/CommonAssemblyInfo.props b/CommonAssemblyInfo.props
@@ -1,10 +1,10 @@
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
     <Company>Illumina</Company>
-    <Copyright>© 2020 Illumina, Inc.</Copyright>
-    <FileVersion>3.13.0</FileVersion>
-    <AssemblyVersion>3.13.0</AssemblyVersion>
-    <Version>3.13.0</Version>
+    <Copyright>© 2021 Illumina, Inc.</Copyright>
+    <FileVersion>3.15.0</FileVersion>
+    <AssemblyVersion>3.15.0</AssemblyVersion>
+    <Version>3.15.0</Version>
     <Authors>Stromberg, Roy, Lajugie, Jiang, Li, and Kang</Authors>
   </PropertyGroup>
 </Project>
diff --git a/Compression/Compression.csproj b/Compression/Compression.csproj
@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <OutputPath>..\bin\$(Configuration)</OutputPath>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>

diff --git a/Compression/Utilities/BlockExtensions.cs b/Compression/Utilities/BlockExtensions.cs
@@ -0,0 +1,45 @@
+using System;
+using System.Buffers;
+using System.IO;
+using Compression.Algorithms;
+
+namespace Compression.Utilities
+{
+    public static class BlockExtensions
+    {
+        private static readonly Zstandard Zstd = new(21);
+
+        public static byte[] ReadCompressedByteArray(this BinaryReader reader, ArrayPool<byte> bytePool)
+        {
+            int uncompressedSize = reader.ReadInt32();
+            int compressedSize   = reader.ReadInt32();
+
+            byte[] compressedBuffer   = bytePool.Rent(compressedSize);
+            byte[] uncompressedBuffer = bytePool.Rent(uncompressedSize);
+            int    numRead            = reader.Read(compressedBuffer, 0, compressedSize);
+
+            int len = Zstd.Decompress(compressedBuffer, compressedSize, uncompressedBuffer, uncompressedBuffer.Length);
+
+            bytePool.Return(compressedBuffer);
+            return uncompressedBuffer;
+        }
+
+        public static void WriteCompressedByteArray(this BinaryWriter writer, byte[] uncompressed, int uncompressedSize)
+        {
+            ArrayPool<byte> bytePool             = ArrayPool<byte>.Shared;
+            int             compressedBufferSize = Zstd.GetCompressedBufferBounds(uncompressedSize);
+            byte[]          compressedBuffer     = bytePool.Rent(compressedBufferSize);
+
+            int compressedSize = Zstd.Compress(uncompressed, uncompressedSize, compressedBuffer, compressedBuffer.Length);
+
+            writer.Write(uncompressedSize);
+            writer.Write(compressedSize);
+            writer.Write(compressedBuffer, 0, compressedSize);
+
+            double percentCompression = compressedSize / (double) uncompressedSize * 100.0;
+            Console.WriteLine($"uncompressed: {uncompressedSize:N0}, compressed: {compressedSize:N0}, {percentCompression:0.0}%");
+
+            bytePool.Return(compressedBuffer);
+        }
+    }
+}
diff --git a/CustomAnnotationLambda/CustomAnnotationLambda.csproj b/CustomAnnotationLambda/CustomAnnotationLambda.csproj
@@ -1,15 +1,15 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
     <AWSProjectType>Lambda</AWSProjectType>
     <OutputPath>bin\$(Configuration)</OutputPath>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Amazon.Lambda.Core" Version="1.1.0" />
-    <PackageReference Include="Amazon.Lambda.Serialization.Json" Version="1.7.0" />
-    <PackageReference Include="AWSSDK.SimpleNotificationService" Version="3.3.101.100" />
+    <PackageReference Include="Amazon.Lambda.Core" Version="2.0.0" />
+    <PackageReference Include="Amazon.Lambda.Serialization.Json" Version="2.0.0" />
+    <PackageReference Include="AWSSDK.SimpleNotificationService" Version="3.7.1.1" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Cloud\Cloud.csproj" />

diff --git a/CustomAnnotationLambda/GeneAnnotationCreator.cs b/CustomAnnotationLambda/GeneAnnotationCreator.cs
@@ -2,7 +2,6 @@
 using System.IO;
 using System.Security.Cryptography;
 using Cloud;
-using Cloud.Messages;
 using Cloud.Messages.Custom;
 using Cloud.Utilities;
 using Compression.Utilities;

diff --git a/CustomAnnotationLambda/Properties/launchSettings.json b/CustomAnnotationLambda/Properties/launchSettings.json
diff --git a/CustomAnnotationLambda/VariantAnnotationCreator.cs b/CustomAnnotationLambda/VariantAnnotationCreator.cs
@@ -2,7 +2,6 @@
 using System.IO;
 using System.Security.Cryptography;
 using Cloud;
-using Cloud.Messages;
 using Cloud.Messages.Custom;
 using Cloud.Utilities;
 using Compression.Utilities;
@@ -25,7 +24,7 @@ public static CustomResult Create(CustomConfig config, string inputFileName, Cus
             string inputBaseName   = inputFileName.TrimEndFromFirst(".tsv");
             string nsaFileName     = inputBaseName + SaCommon.SaFileSuffix;
             string localNsaPath    = Path.Combine(tempPath, nsaFileName);
-            string localIndexPath  = localNsaPath + SaCommon.IndexSufix;
+            string localIndexPath  = localNsaPath + SaCommon.IndexSuffix;
             string localSchemaPath = localNsaPath + SaCommon.JsonSchemaSuffix;
             int    variantCount    = 0;
 
@@ -80,7 +79,7 @@ public static CustomResult Create(CustomConfig config, string inputFileName, Cus
                 if (nsaItemsCount > 0)
                 {
                     string nsaS3Path    = string.Join('/', config.outputDir.path.Trim('/'), nsaFileName);
-                    string indexS3Path  = nsaS3Path + SaCommon.IndexSufix;
+                    string indexS3Path  = nsaS3Path + SaCommon.IndexSuffix;
                     string schemaS3Path = nsaS3Path + SaCommon.JsonSchemaSuffix;
 
                     s3Client.DecryptUpload(config.outputDir.bucketName, nsaS3Path, localNsaPath, aes, nsaMetadata);
@@ -90,7 +89,7 @@ public static CustomResult Create(CustomConfig config, string inputFileName, Cus
                         schemaMetadata);
 
                     outputFiles.Add(nsaFileName);
-                    outputFiles.Add(nsaFileName + SaCommon.IndexSufix);
+                    outputFiles.Add(nsaFileName + SaCommon.IndexSuffix);
                     outputFiles.Add(nsaFileName + SaCommon.JsonSchemaSuffix);
                 }
 

diff --git a/CustomStrValidationLambda/CustomStrValidationLambda.csproj b/CustomStrValidationLambda/CustomStrValidationLambda.csproj
@@ -1,12 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
     <AWSProjectType>Lambda</AWSProjectType>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Amazon.Lambda.Core" Version="1.1.0" />
-    <PackageReference Include="Amazon.Lambda.Serialization.Json" Version="1.7.0" />
+    <PackageReference Include="Amazon.Lambda.Core" Version="2.0.0" />
+    <PackageReference Include="Amazon.Lambda.Serialization.Json" Version="2.0.0" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Cloud\Cloud.csproj" />

diff --git a/CustomStrValidationLambda/Properties/launchSettings.json b/CustomStrValidationLambda/Properties/launchSettings.json
diff --git a/Downloader/Downloader.csproj b/Downloader/Downloader.csproj
@@ -1,15 +1,15 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <OutputPath>..\bin\$(Configuration)</OutputPath>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <Import Project="..\CommonAssemblyInfo.props" />
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="3.1.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="3.1.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="3.1.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="5.0.0" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\CommandLine\CommandLine.csproj" />

diff --git a/Downloader/Properties/launchSettings.json b/Downloader/Properties/launchSettings.json
diff --git a/ErrorHandling/ErrorHandling.csproj b/ErrorHandling/ErrorHandling.csproj
@@ -1,11 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <OutputPath>..\bin\$(Configuration)</OutputPath>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <Import Project="..\CommonAssemblyInfo.props" />
   <ItemGroup>
-    <PackageReference Include="AWSSDK.S3" Version="3.3.109.4" />
+    <PackageReference Include="AWSSDK.S3" Version="3.7.0.17" />
   </ItemGroup>
 </Project>