Merge branch 'bug_fixes'

BenLangmead · Oct 13, 2023 · f62f2b4 · f62f2b4
2 parents 521c323 + 288f50d
commit f62f2b4
Show file tree

Hide file tree

Showing 14 changed files with 3,920 additions and 105 deletions.
diff --git a/BOWTIE2_VERSION b/BOWTIE2_VERSION
@@ -1 +1 @@
-2.5.1
+2.5.2
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -21,7 +21,9 @@ option(BOWTIE_SHARED_MEM "enable shared memory mapping" OFF)
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-
+if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "")
+   set(CMAKE_BUILD_TYPE "Release" CACHE STRING "" FORCE)
+endif()
 set(USE_SRA ${USE_SRA})
 set(WITH_THREAD_PROFILING ${WITH_THREAD_PROFILING})
 set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
@@ -151,8 +153,8 @@ elseif (${ARCH} MATCHES amd64|x64_64)
 endif()
 
 if (USE_SRA)
-  set(NGS_VER 2.9.2)
-  set(NCBI_VDB_VER 2.9.2-1)
+  set(NGS_VER 2.10.2)
+  set(NCBI_VDB_VER 2.10.2)
 
   find_package(Java COMPONENTS Development REQUIRED)
   find_package(PythonInterp REQUIRED)
@@ -237,7 +239,9 @@ endif()
 
 include_directories(${PROJECT_SOURCE_DIR})
 get_directory_property(COMPILER_DEFS COMPILE_DEFINITIONS)
-add_definitions(-DCOMPILER_OPTIONS="${CMAKE_CXX_FLAGS}")
+string(REPLACE ";" " -D" COMPILER_DEFS "${COMPILER_DEFS}")
+string(REPLACE "\"" "" COMPILER_DEFS "${COMPILER_DEFS}")
+add_definitions(-DCOMPILER_OPTIONS="${CMAKE_CXX_FLAGS} -D${COMPILER_DEFS}")
 
 add_executable(bowtie2-align-s ${SEARCH_CPPS} ${SHARED_CPPS})
 add_executable(bowtie2-align-l ${SEARCH_CPPS} ${SHARED_CPPS})
@@ -246,8 +250,9 @@ add_executable(bowtie2-build-l ${BUILD_CPPS} ${SHARED_CPPS})
 add_executable(bowtie2-inspect-s ${INSPECT_CPPS} ${SHARED_CPPS})
 add_executable(bowtie2-inspect-l ${INSPECT_CPPS} ${SHARED_CPPS})
 
-set_target_properties(bowtie2-align-l bowtie2-build-l bowtie2-inspect-l PROPERTIES COMPILE_FLAGS "-DBOWTIE2_64BIT_INDEX")
-set_target_properties(bowtie2-inspect-s bowtie2-inspect-l PROPERTIES COMPILE_FLAGS "-DBOWTIE_INSPECT_MAIN")
+set_target_properties(bowtie2-align-l bowtie2-build-l PROPERTIES COMPILE_FLAGS "-DBOWTIE_64BIT_INDEX")
+set_target_properties(bowtie2-inspect-l PROPERTIES COMPILE_FLAGS "-DBOWTIE_64BIT_INDEX -DBOWTIE_INSPECT_MAIN")
+set_target_properties(bowtie2-inspect-s PROPERTIES COMPILE_FLAGS "-DBOWTIE_INSPECT_MAIN")
 
 install(TARGETS ${BOWTIE2_BIN_LIST} DESTINATION bin)
 install(FILES ${BOWTIE2_WRAPPER_SCRIPTS}
@@ -257,9 +262,17 @@ install(FILES ${BOWTIE2_WRAPPER_SCRIPTS}
 add_test(NAME simple-align
   WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
   COMMAND bowtie2 -x example/index/lambda_virus example/reads/longreads.fq -u 10)
-add_test(NAME simple-build
+# Small index
+add_test(NAME simple-build-small
   WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
   COMMAND bowtie2-build-s -c GGGCGGCGACCTCGCGGGTTTTCGCTA out)
-add_test(NAME simple-inspect
+add_test(NAME simple-inspect-small
   WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
   COMMAND bowtie2-inspect-s out && rm out*)
+# Large index
+add_test(NAME simple-build-large
+  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+  COMMAND bowtie2-build-l -c GGGCGGCGACCTCGCGGGTTTTCGCTA out)
+add_test(NAME simple-inspect-large
+  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+  COMMAND bowtie2-inspect-l out && rm out*)
diff --git a/Makefile b/Makefile
@@ -41,9 +41,15 @@ MINGW :=
 ifneq (,$(findstring mingw,$(shell $(CXX) --version)))
   WINDOWS := 1
   MINGW := 1
-  # POSIX memory-mapped files not currently supported on Windows
+
+  CXX ?= x86_64-w64-mingw32-g++-posix
+  CC  ?= x86_64-w64-mingw32-gcc-posix
+  AR  ?= x86_64-w64-mingw32-ar
+  RC  ?= x86_64-w64-mingw32-windres
+  STRIP ?= x86_64-w64-mingw32-strip
 endif
 
+# POSIX memory-mapped files not currently supported on Windows
 ifeq (1, $(WINDOWS))
   BOWTIE_MM :=
   BOWTIE_SHARED_MEM :=
@@ -544,6 +550,7 @@ perl-deps:
 	eval `perl -I $(CURDIR)/.tmp/lib/perl5 -Mlocal::lib=$(CURDIR)/.tmp` ; \
 	$(CURDIR)/.tmp/bin/cpanm --force File::Which Math::Random Clone Test::Deep Sys::Info ; \
 
+#make -f win32/Makefile.gcc CC=x86_64-w64-mingw32-gcc-posix AR=x86_64-w64-mingw32-ar RC=x86_64-w64-mingw32-windres STRIP=x86_64-w64-mingw32-strip
 .PHONY: static-libs
 static-libs:
 	if [ ! -d "$(CURDIR)/.tmp" ] ; then \
@@ -555,17 +562,18 @@ static-libs:
 	fi ; \
 	cd $(CURDIR)/.tmp ; \
 	DL=$$( ( which wget >/dev/null 2>&1 && echo "wget --no-check-certificate" ) || echo "curl -LOk") ; \
-	if [ ! -f "$(CURDIR)/.tmp/include/zlib.h" ] ; then \
-		$$DL https://zlib.net/zlib-1.2.13.tar.gz && tar xzf zlib-1.2.13.tar.gz && cd zlib-1.2.13 ; \
-		$(if $(MINGW), mingw32-make -f win32/Makefile.gcc, ./configure --static && make) ; \
+	if [ ! -f "$(CURDIR)/.tmp/lib/libz.a" ] ; then \
+		$$DL http://zlib.net/zlib-1.3.tar.gz && tar xzf zlib-1.3.tar.gz && cd zlib-1.3 ; \
+		$(if $(MINGW), $(MAKE) -f win32/Makefile.gcc, ./configure --static && make) && \
 		cp zlib.h zconf.h $(CURDIR)/.tmp/include && cp libz.a $(CURDIR)/.tmp/lib ; \
-		rm -f zlib-1.2.13 ; \
+		rm -rf zlib-1.3* ; \
 	fi ; \
-        if [ ! -f "$(CURDIR)/.tmp/include/zstd.h" ]; then \
+        if [ ! -f "$(CURDIR)/.tmp/lib/libzstd.a" ]; then \
                 cd $(CURDIR)/.tmp ; \
-                $$DL https://github.com/facebook/zstd/releases/download/v1.5.1/zstd-1.5.1.tar.gz && tar xzf zstd-1.5.1.tar.gz ; \
-                cd zstd-1.5.1 && $(MAKE) lib ; \
-                cd $(CURDIR)/.tmp/zstd-1.5.1/lib && cp zstd.h $(CURDIR)/.tmp/include && cp libzstd.a $(CURDIR)/.tmp/lib ; \
+                $$DL https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz && tar xzf zstd-1.5.5.tar.gz ; \
+                cd zstd-1.5.5 && $(MAKE) lib ; \
+                cd $(CURDIR)/.tmp/zstd-1.5.5/lib && cp zstd.h $(CURDIR)/.tmp/include && cp libzstd.a $(CURDIR)/.tmp/lib ; \
+		rm -rf zstd-1.5.5* ; \
         fi
 
 .PHONY: sra-deps

diff --git a/NEWS b/NEWS
@@ -4,7 +4,7 @@ Bowtie 2 NEWS
 Bowtie 2 is available for download from the project website,
 http://bowtie-bio.sf.net/bowtie2 and on Github,
 https://github.com/BenLangmead/bowtie2/releases.  2.0.0-beta1 is
-the first version released to the public and 2.5.1 is the latest
+the first version released to the public and 2.5.2 is the latest
 version.  Bowtie 2 is licensed under the GPLv3 license.  See `LICENSE'
 file for details.
 
@@ -19,7 +19,15 @@ Please report any issues to the Bowtie 2 Github page or using the Sourceforge bu
 Version Release History
 =======================
 
-## Version 2.5.1 - Jan 17, 2023 ##
+## Version 2.5.2 - Oct 13, 2023 ##
+
+### bowtie2 ###
+  * Fixed an issue preventing `bowtie2` from utilizing the specified number of CPU cores
+  * Fixed an issue causing `bowtie2` to segfault when reading compressed inputs on Windows
+  * Fixed an issue causing `bowtie2` to segfault while parsing interleaved reads
+  * Overhauled the FASTQ parser with better tolerance for empty lines
+
+## Version 2.5.1 - Feb 17, 2023 ##
 
 ### bowtie2 ###
   * Fixed an issue affecting `bowtie2` alignment accuracy.

diff --git a/bowtie2-build b/bowtie2-build
@@ -22,12 +22,12 @@
 import os
 import sys
 import gzip
-import struct
 import inspect
 import logging
 import argparse
 import subprocess
 
+
 def get_gunzip_size(fn):
     size = 0
     with gzip.open(fn) as f:
@@ -66,48 +66,61 @@ def main():
         logging.getLogger().setLevel(logging.INFO)
 
     if script_options.debug:
-        build_bin_spec += '-debug'
-        build_bin_l += '-debug'
+        build_bin_spec += "-debug"
+        build_bin_l += "-debug"
 
     if script_options.sanitized:
-        build_bin_spec += '-sanitized'
-        build_bin_l += '-sanitized'
+        build_bin_spec += "-sanitized"
+        build_bin_l += "-sanitized"
 
     fastas = []
-    if '-c' not in argv and len(argv) >= 2:
+    if "-c" not in argv and len(argv) >= 2:
         for index in reversed(range(len(argv) - 1)):
             arg = argv[index]
-            if arg.startswith('-') or arg.isdigit():
+            if arg.startswith("-") or arg.isdigit():
                 break
             fastas.insert(0, arg)
             argv.remove(arg)
         if fastas:
-            fastas = ','.join(fastas)
+            fastas = ",".join(fastas)
             argv.insert(len(argv) - 1, fastas)
 
     if script_options.large_index:
-        build_bin_spec = os.path.join(ex_path,build_bin_l)
+        build_bin_spec = os.path.join(ex_path, build_bin_l)
     elif fastas:
         tot_size = 0
-        for fn in fastas.split(','):
+        for fn in fastas.split(","):
             if os.path.exists(fn):
-                if fn.endswith('.gz') or fn.endswith(".Z"):
+                if fn.endswith(".gz"):
                     tot_size += get_gunzip_size(fn)
+                elif fn.endswith(".zst"):
+                    sys.stderr.write(
+                        "{} cannot determine the uncompressed size of ZSTD compressed files.\
+                        Please make sure that the uncompressed file size is suitable for\
+                        building the current index".format(
+                            curr_script
+                        )
+                    )
                 else:
                     statinfo = os.stat(fn)
                     tot_size += statinfo.st_size
         if tot_size > small_index_max_size:
-            build_bin_spec = os.path.join(ex_path,build_bin_l)
+            build_bin_spec = os.path.join(ex_path, build_bin_l)
 
     if not os.path.exists(build_bin_spec):
-        sys.stderr.write('{0} does not exist, try running `[g]make {0}\'\n'.format(os.path.basename(build_bin_spec)))
+        sys.stderr.write(
+            "{0} does not exist, try running `[g]make {0}'\n".format(
+                os.path.basename(build_bin_spec)
+            )
+        )
         sys.exit(1)
 
-    argv.insert(0, 'basic-0')
-    argv.insert(0, '--wrapper')
+    argv.insert(0, "basic-0")
+    argv.insert(0, "--wrapper")
     argv.insert(0, build_bin_spec)
-    logging.info('Command: %s' % ' '.join(argv))
+    logging.info("Command: %s" % " ".join(argv))
     sys.exit(subprocess.call(argv))
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     main()
diff --git a/bt2_search.cpp b/bt2_search.cpp
@@ -4695,6 +4695,10 @@ static void multiseedSearch(
 	EList<int> tids;
 	EList<std::thread*> threads(nthreads);
 	EList<thread_tracking_pair> tps;
+	// Important: Need at least nthreads+1 elements, more is OK
+	PatternSourceReadAheadFactory readahead_factory(patsrc,pp,4*nthreads+1);
+	multiseed_readahead_factory = &readahead_factory;
+
 	tps.resize(std::max(nthreads, thread_ceiling));
 	threads.reserveExact(std::max(nthreads, thread_ceiling));
 	tids.reserveExact(std::max(nthreads, thread_ceiling));
@@ -4727,9 +4731,6 @@ static void multiseedSearch(
 			startVerbose);
 	}
 
-	// Important: Need at least nthreads+1 elements, more is OK
-	PatternSourceReadAheadFactory readahead_factory(patsrc,pp,2*nthreads+1);
-	multiseed_readahead_factory = &readahead_factory;
 
 	// Start the metrics thread