Support -fsanitize=address with --enable-asan

When --enable-asan is provided to configure then build all user
space components with fsanitize=address.  For kernel support
use the Linux KASAN feature instead.

https://github.com/google/sanitizers/wiki/AddressSanitizer

When using gcc version 4.8 any test case which intentionally
generates a core dump will fail when using --enable-asan.
The default behavior is to disable core dumps and only newer
versions allow this behavior to be controled at run time with
the ASAN_OPTIONS environment variable.

Additionally, this patch includes some build system cleanup.

* Rules.am updated to set the minimum AM_CFLAGS, AM_CPPFLAGS,
  and AM_LDFLAGS.  Any additional flags should be added on a
  per-Makefile basic.  The --enable-debug and --enable-asan
  options apply to all user space binaries and libraries.

* Compiler checks consolidated in always-compiler-options.m4
  and renamed for consistency.

* -fstack-check compiler flag was removed, this functionality
  is provided by asan when configured with --enable-asan.

* Split DEBUG_CFLAGS in to DEBUG_CFLAGS, DEBUG_CPPFLAGS, and
  DEBUG_LDFLAGS.

* Moved default kernel build flags in to module/Makefile.in and
  split in to ZFS_MODULE_CFLAGS and ZFS_MODULE_CPPFLAGS.  These
  flags are set with the standard ccflags-y kbuild mechanism.

* -Wframe-larger-than checks applied only to binaries or
  libraries which include source files which are built in
  both user space and kernel space.  This restriction is
  relaxed for user space only utilities.

* -Wno-unused-but-set-variable applied only to libzfs and
  libzpool.  The remaining warnings are the result of an
  ASSERT using a variable when is always declared.

* -D_POSIX_PTHREAD_SEMANTICS and -D__EXTENSIONS__ dropped
  because they are Solaris specific and thus not needed.

* Ensure $GDB is defined as gdb by default in zloop.sh.

Signed-off-by: DHE <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Closes openzfs#7027

cmd/raidz_test/Makefile.am

@@ -1,7 +1,10 @@
 include $(top_srcdir)/config/Rules.am
 
-AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN)
-AM_CPPFLAGS += -DDEBUG
+# Includes kernel code, generate warnings for large stack frames
+AM_CFLAGS += $(FRAME_LARGER_THAN)
+
+# Unconditionally enable ASSERTs
+AM_CPPFLAGS += -DDEBUG -UNDEBUG
 
 DEFAULT_INCLUDES += \
 	-I$(top_srcdir)/include \

cmd/zdb/Makefile.am

@@ -1,6 +1,7 @@
 include $(top_srcdir)/config/Rules.am
 
-AM_CPPFLAGS += -DDEBUG
+# Unconditionally enable debugging for zdb
+AM_CPPFLAGS += -DDEBUG -UNDEBUG
 
 DEFAULT_INCLUDES += \
 	-I$(top_srcdir)/include \

cmd/ztest/Makefile.am

@@ -1,8 +1,12 @@
 include $(top_srcdir)/config/Rules.am
 
-# -Wnoformat-truncation to get rid of compiler warning for unchecked
-#  truncating snprintfs on gcc 7.1.1.
-AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN) $(NO_FORMAT_TRUNCATION)
+# Get rid of compiler warning for unchecked truncating snprintfs on gcc 7.1.1
+AM_CFLAGS += $(NO_FORMAT_TRUNCATION)
+
+# Includes kernel code, generate warnings for large stack frames
+AM_CFLAGS += $(FRAME_LARGER_THAN)
+
+# Unconditionally enable ASSERTs
 AM_CPPFLAGS += -DDEBUG -UNDEBUG
 
 DEFAULT_INCLUDES += \

config/Rules.am

@@ -1,18 +1,29 @@
+#
+# Default build rules for all user space components, every Makefile.am
+# should include these rules and override or extend them as needed.
+#
+
 DEFAULT_INCLUDES = -include ${top_builddir}/zfs_config.h
 
 AM_LIBTOOLFLAGS = --silent
-AM_CFLAGS  = ${DEBUG_CFLAGS} -Wall -Wstrict-prototypes
-AM_CFLAGS += ${NO_UNUSED_BUT_SET_VARIABLE}
-AM_CFLAGS += ${NO_BOOL_COMPARE}
-AM_CFLAGS += -fno-strict-aliasing
-AM_CFLAGS += -std=gnu99
+
+AM_CFLAGS  = -std=gnu99 -Wall -Wstrict-prototypes -fno-strict-aliasing
+AM_CFLAGS += $(DEBUG_CFLAGS)
+AM_CFLAGS += $(ASAN_CFLAGS)
 AM_CFLAGS += $(CODE_COVERAGE_CFLAGS)
-AM_CPPFLAGS  = -D_GNU_SOURCE -D__EXTENSIONS__ -D_REENTRANT
-AM_CPPFLAGS += -D_POSIX_PTHREAD_SEMANTICS -D_FILE_OFFSET_BITS=64
-AM_CPPFLAGS += -D_LARGEFILE64_SOURCE -DHAVE_LARGE_STACKS=1
+
+AM_CPPFLAGS  = -D_GNU_SOURCE
+AM_CPPFLAGS += -D_REENTRANT
+AM_CPPFLAGS += -D_FILE_OFFSET_BITS=64
+AM_CPPFLAGS += -D_LARGEFILE64_SOURCE
+AM_CPPFLAGS += -DHAVE_LARGE_STACKS=1
 AM_CPPFLAGS += -DTEXT_DOMAIN=\"zfs-linux-user\"
 AM_CPPFLAGS += -DLIBEXECDIR=\"$(libexecdir)\"
 AM_CPPFLAGS += -DRUNSTATEDIR=\"$(runstatedir)\"
 AM_CPPFLAGS += -DSBINDIR=\"$(sbindir)\"
 AM_CPPFLAGS += -DSYSCONFDIR=\"$(sysconfdir)\"
+AM_CPPFLAGS += $(DEBUG_CPPFLAGS)
 AM_CPPFLAGS += $(CODE_COVERAGE_CPPFLAGS)
+
+AM_LDFLAGS  = $(DEBUG_LDFLAGS)
+AM_LDFLAGS += $(ASAN_LDFLAGS)

config/always-compiler-options.m4

@@ -0,0 +1,141 @@
+dnl #
+dnl # Enabled -fsanitize=address if supported by gcc.
+dnl #
+dnl # LDFLAGS needs -fsanitize=address at all times so libraries compiled with
+dnl # it will be linked successfully. CFLAGS will vary by binary being built.
+dnl #
+dnl # The ASAN_OPTIONS environment variable can be used to further control
+dnl # the behavior of binaries and libraries build with -fsanitize=address.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_ASAN], [
+	AC_MSG_CHECKING([whether to build with -fsanitize=address support])
+	AC_ARG_ENABLE([asan],
+		[AS_HELP_STRING([--enable-asan],
+		[Enable -fsanitize=address support  @<:@default=no@:>@])],
+		[],
+		[enable_asan=no])
+
+	AM_CONDITIONAL([ASAN_ENABLED], [test x$enable_asan = xyes])
+	AC_SUBST([ASAN_ENABLED], [$enable_asan])
+	AC_MSG_RESULT($enable_asan)
+
+	AS_IF([ test "$enable_asan" = "yes" ], [
+		AC_MSG_CHECKING([whether $CC supports -fsanitize=address])
+		saved_cflags="$CFLAGS"
+		CFLAGS="$CFLAGS -fsanitize=address"
+		AC_LINK_IFELSE([
+			AC_LANG_SOURCE([[ int main() { return 0; } ]])
+		], [
+			ASAN_CFLAGS="-fsanitize=address"
+			ASAN_LDFLAGS="-fsanitize=address"
+			ASAN_ZFS="_with_asan"
+			AC_MSG_RESULT([yes])
+		], [
+			AC_MSG_ERROR([$CC does not support -fsanitize=address])
+		])
+		CFLAGS="$saved_cflags"
+	], [
+		ASAN_CFLAGS=""
+		ASAN_LDFLAGS=""
+		ASAN_ZFS="_without_asan"
+	])
+
+	AC_SUBST([ASAN_CFLAGS])
+	AC_SUBST([ASAN_LDFLAGS])
+	AC_SUBST([ASAN_ZFS])
+])
+
+dnl #
+dnl # Check if gcc supports -Wframe-larger-than=<size> option.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_FRAME_LARGER_THAN], [
+	AC_MSG_CHECKING([whether $CC supports -Wframe-larger-than=<size>])
+
+	saved_flags="$CFLAGS"
+	CFLAGS="$CFLAGS -Wframe-larger-than=4096"
+
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
+		FRAME_LARGER_THAN="-Wframe-larger-than=4096"
+		AC_MSG_RESULT([yes])
+	], [
+		FRAME_LARGER_THAN=""
+		AC_MSG_RESULT([no])
+	])
+
+	CFLAGS="$saved_flags"
+	AC_SUBST([FRAME_LARGER_THAN])
+])
+
+dnl #
+dnl # Check if gcc supports -Wno-format-truncation option.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_TRUNCATION], [
+	AC_MSG_CHECKING([whether $CC supports -Wno-format-truncation])
+
+	saved_flags="$CFLAGS"
+	CFLAGS="$CFLAGS -Wno-format-truncation"
+
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
+		NO_FORMAT_TRUNCATION=-Wno-format-truncation
+		AC_MSG_RESULT([yes])
+	], [
+		NO_FORMAT_TRUNCATION=
+		AC_MSG_RESULT([no])
+	])
+
+	CFLAGS="$saved_flags"
+	AC_SUBST([NO_FORMAT_TRUNCATION])
+])
+
+
+dnl #
+dnl # Check if gcc supports -Wno-bool-compare option.
+dnl #
+dnl # We actually invoke gcc with the -Wbool-compare option
+dnl # and infer the 'no-' version does or doesn't exist based upon
+dnl # the results.  This is required because when checking any of
+dnl # no- prefixed options gcc always returns success.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_BOOL_COMPARE], [
+	AC_MSG_CHECKING([whether $CC supports -Wno-bool-compare])
+
+	saved_flags="$CFLAGS"
+	CFLAGS="$CFLAGS -Wbool-compare"
+
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
+		NO_BOOL_COMPARE=-Wno-bool-compare
+		AC_MSG_RESULT([yes])
+	], [
+		NO_BOOL_COMPARE=
+		AC_MSG_RESULT([no])
+	])
+
+	CFLAGS="$saved_flags"
+	AC_SUBST([NO_BOOL_COMPARE])
+])
+
+dnl #
+dnl # Check if gcc supports -Wno-unused-but-set-variable option.
+dnl #
+dnl # We actually invoke gcc with the -Wunused-but-set-variable option
+dnl # and infer the 'no-' version does or doesn't exist based upon
+dnl # the results.  This is required because when checking any of
+dnl # no- prefixed options gcc always returns success.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_UNUSED_BUT_SET_VARIABLE], [
+	AC_MSG_CHECKING([whether $CC supports -Wno-unused-but-set-variable])
+
+	saved_flags="$CFLAGS"
+	CFLAGS="$CFLAGS -Wunused-but-set-variable"
+
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
+		NO_UNUSED_BUT_SET_VARIABLE=-Wno-unused-but-set-variable
+		AC_MSG_RESULT([yes])
+	], [
+		NO_UNUSED_BUT_SET_VARIABLE=
+		AC_MSG_RESULT([no])
+	])
+
+	CFLAGS="$saved_flags"
+	AC_SUBST([NO_UNUSED_BUT_SET_VARIABLE])
+])

config/kernel.m4

@@ -125,21 +125,10 @@ AC_DEFUN([ZFS_AC_CONFIG_KERNEL], [
 	ZFS_AC_KERNEL_VM_NODE_STAT
 
 	AS_IF([test "$LINUX_OBJ" != "$LINUX"], [
-		KERNELMAKE_PARAMS="$KERNELMAKE_PARAMS O=$LINUX_OBJ"
+		KERNEL_MAKE="$KERNEL_MAKE O=$LINUX_OBJ"
 	])
-	AC_SUBST(KERNELMAKE_PARAMS)
 
-
-	dnl # -Wall -fno-strict-aliasing -Wstrict-prototypes and other
-	dnl # compiler options are added by the kernel build system.
-	KERNELCPPFLAGS="$KERNELCPPFLAGS -std=gnu99"
-	KERNELCPPFLAGS="$KERNELCPPFLAGS -Wno-declaration-after-statement"
-	KERNELCPPFLAGS="$KERNELCPPFLAGS $NO_UNUSED_BUT_SET_VARIABLE"
-	KERNELCPPFLAGS="$KERNELCPPFLAGS $NO_BOOL_COMPARE"
-	KERNELCPPFLAGS="$KERNELCPPFLAGS -DHAVE_SPL -D_KERNEL"
-	KERNELCPPFLAGS="$KERNELCPPFLAGS -DTEXT_DOMAIN=\\\"zfs-linux-kernel\\\""
-
-	AC_SUBST(KERNELCPPFLAGS)
+	AC_SUBST(KERNEL_MAKE)
 ])
 
 dnl #

config/user.m4

@@ -14,11 +14,9 @@ AC_DEFUN([ZFS_AC_CONFIG_USER], [
 	ZFS_AC_CONFIG_USER_LIBATTR
 	ZFS_AC_CONFIG_USER_LIBUDEV
 	ZFS_AC_CONFIG_USER_LIBSSL
-	ZFS_AC_CONFIG_USER_FRAME_LARGER_THAN
 	ZFS_AC_CONFIG_USER_RUNSTATEDIR
 	ZFS_AC_CONFIG_USER_MAKEDEV_IN_SYSMACROS
 	ZFS_AC_CONFIG_USER_MAKEDEV_IN_MKDEV
-	ZFS_AC_CONFIG_USER_NO_FORMAT_TRUNCATION
 
 	ZFS_AC_TEST_FRAMEWORK